From 98ab545e89241e73842ccccc63563b4b9030bed5 Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Wed, 3 Jun 2026 12:23:22 -0700 Subject: [PATCH 1/2] Replace fluentd with fluent-bit in the operator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Render the calico-fluent-bit DaemonSet (and its Windows variant) in place of fluentd, migrating the resource identity and wiring up a working fluent-bit configuration. - Namespace tigera-fluentd -> calico-system; DaemonSet/ServiceAccount fluentd-node -> calico-fluent-bit; TLS secret -> calico-fluent-bit-tls; image ComponentFluentd -> ComponentFluentBit; metrics port 9081 -> 2020 (fluent-bit's built-in HTTP server). - Config is rendered in fluent-bit's YAML schema into per-OS ConfigMaps (calico-fluent-bit-conf and -windows — a shared name would make the two renders overwrite each other on mixed clusters), subPath-mounted on Linux and directory-mounted on Windows (which cannot mount single files), and started with `-c`. It loads the Go plugins via plugins_file, defines parsers inline, applies the record_transformer lua filter, and inlines user-provided fluent-bit YAML filter lists. A hash of the rendered config on the pod template rolls the daemonset on config-only changes. - Tail inputs use the producing components' real paths (waf/, runtime-security/report.log, audit/tsee-audit.log, ids/events.log, the compliance.*.reports.log glob, policy/policy_activity.log) with SQLite offsets and filesystem buffering under /var/log/calico/calico-fluent-bit. The pos-migrator init container (Linux and Windows) seeds offsets from the fluentd .pos files and pre-creates the tailed directories so glob inputs don't error while a feature's log dir is absent. Windows tails the fluentd-windows types (flows, audit.tsee, audit.kube) against the C:\fluent-bit image layout. - The linseed output matches only Linseed-bound tags (match_regex; IDS events and compliance reports are not Linseed-bound), posts with ca_file/cert_file/key_file (Go proxy plugins reject the native tls.* namespace) and the in-cluster ServiceAccount token, and retries without limit against the bounded filesystem buffer. S3, Splunk and Syslog outputs mirror fluentd's per-type fan-out: standard AWS credential env vars, endpoint scheme honored, and syslog packs the whole record as JSON via a per-output lua processor with TLS actually enabled (mode alone only selects framing) and the trusted-bundle CA when a user syslog certificate is configured. - NonClusterHost renders the :9880 http input with client-certificate verification (voltron presents its internal certificate, matching fluentd's client_cert_auth), and the input Service is cleaned up when the resource is removed. - eks-log-forwarder runs the fluent-bit image with a rendered in_eks -> linseed pipeline and health probes; the fluentd-era startup init container is gone (the plugin resolves its resume point from Linseed) and FetchInterval maps to EKS_CLOUDWATCH_POLL_INTERVAL. - Health probes hit :2020/api/v1/health (health_check on). The ServiceMonitor scrapes plain HTTP — fluent-bit's monitoring server has no TLS, unlike fluentd's mTLS exporter — with access restricted by the component NetworkPolicy; legacy fluentd monitors are removed. - The LogCollector controller no longer creates or owns the calico-system namespace (deleting the LogCollector must not garbage-collect it), the deprecated fluentdDaemonSet override is honored as an alias with container-name translation, deepcopy and the embedded LogCollector CRD are regenerated, and the legacy tigera-fluentd resources — the namespace last — are cleaned up idempotently on every reconcile. - API: CalicoFluentBitDaemonSet added (FluentdDaemonSet deprecated); golden policy fixtures and enterprise_versions.yml updated. Co-Authored-By: Claude Fable 5 --- api/v1/fluentd_daemonset_types.go | 12 +- api/v1/logcollector_types.go | 13 +- api/v1/zz_generated.deepcopy.go | 5 + config/enterprise_versions.yml | 8 +- hack/gen-versions/enterprise.go.tpl | 12 +- pkg/components/enterprise.go | 12 +- .../logcollector/logcollector_controller.go | 102 +- .../logcollector_controller_test.go | 141 +- .../logstorage/secrets/secret_controller.go | 6 +- .../secrets/secret_controller_test.go | 12 +- pkg/controller/monitor/monitor_controller.go | 4 +- .../monitor/monitor_controller_test.go | 24 +- pkg/controller/monitor/prometheus.go | 8 +- pkg/controller/tiers/tiers_controller.go | 3 +- .../operator.tigera.io_logcollectors.yaml | 269 ++- .../applicationlayer/applicationlayer.go | 2 +- pkg/render/common/elasticsearch/service.go | 8 +- pkg/render/fluentbit.go | 1772 +++++++++++++++++ .../{fluentd_test.go => fluentbit_test.go} | 913 ++++----- pkg/render/fluentd.go | 1372 ------------- pkg/render/guardian.go | 2 +- pkg/render/intrusion_detection.go | 4 +- pkg/render/logstorage/esgateway/esgateway.go | 2 +- pkg/render/logstorage/linseed/linseed.go | 2 +- pkg/render/manager.go | 4 +- pkg/render/manager_test.go | 4 +- pkg/render/monitor/monitor.go | 38 +- pkg/render/monitor/monitor_test.go | 45 +- .../testutils/expected_policies/dns.json | 2 +- .../testutils/expected_policies/dns_ocp.json | 2 +- .../expected_policies/es-gateway.json | 6 +- .../expected_policies/es-gateway_ocp.json | 6 +- ...td_managed.json => fluentbit_managed.json} | 8 +- ...nmanaged.json => fluentbit_unmanaged.json} | 8 +- ..._ocp.json => fluentbit_unmanaged_ocp.json} | 8 +- .../testutils/expected_policies/guardian.json | 4 +- .../expected_policies/guardian_ocp.json | 4 +- .../testutils/expected_policies/linseed.json | 6 +- .../linseed_dpi_enabled.json | 6 +- .../expected_policies/linseed_ocp.json | 6 +- .../linseed_ocp_dpi_enabled.json | 6 +- .../node_local_dns_dual.json | 2 +- .../node_local_dns_ipv4.json | 2 +- .../node_local_dns_ipv6.json | 2 +- pkg/render/tiers/tiers_test.go | 1 - 45 files changed, 2738 insertions(+), 2140 deletions(-) create mode 100644 pkg/render/fluentbit.go rename pkg/render/{fluentd_test.go => fluentbit_test.go} (53%) delete mode 100644 pkg/render/fluentd.go rename pkg/render/testutils/expected_policies/{fluentd_managed.json => fluentbit_managed.json} (81%) rename pkg/render/testutils/expected_policies/{fluentd_unmanaged.json => fluentbit_unmanaged.json} (88%) rename pkg/render/testutils/expected_policies/{fluentd_unmanaged_ocp.json => fluentbit_unmanaged_ocp.json} (90%) diff --git a/api/v1/fluentd_daemonset_types.go b/api/v1/fluentd_daemonset_types.go index 1db6e847be..2721f428cb 100644 --- a/api/v1/fluentd_daemonset_types.go +++ b/api/v1/fluentd_daemonset_types.go @@ -61,9 +61,9 @@ type FluentdDaemonSetPodSpec struct { // FluentdDaemonSetContainer is a Fluentd DaemonSet container. type FluentdDaemonSetContainer struct { - // Name is an enum which identifies the Fluentd DaemonSet container by name. - // Supported values are: fluentd - // +kubebuilder:validation:Enum=fluentd + // Name is an enum which identifies the Fluent Bit DaemonSet container by name. + // Supported values are: calico-fluent-bit + // +kubebuilder:validation:Enum=calico-fluent-bit Name string `json:"name"` // Resources allows customization of limits and requests for compute resources such as cpu and memory. @@ -85,9 +85,9 @@ type FluentdDaemonSetContainer struct { // FluentdDaemonSetInitContainer is a Fluentd DaemonSet init container. type FluentdDaemonSetInitContainer struct { - // Name is an enum which identifies the Fluentd DaemonSet init container by name. - // Supported values are: tigera-fluentd-prometheus-tls-key-cert-provisioner - // +kubebuilder:validation:Enum=tigera-fluentd-prometheus-tls-key-cert-provisioner + // Name is an enum which identifies the Fluent Bit DaemonSet init container by name. + // Supported values are: calico-fluent-bit-tls-key-cert-provisioner + // +kubebuilder:validation:Enum=calico-fluent-bit-tls-key-cert-provisioner Name string `json:"name"` // Resources allows customization of limits and requests for compute resources such as cpu and memory. diff --git a/api/v1/logcollector_types.go b/api/v1/logcollector_types.go index 0deffdadde..849c60409d 100644 --- a/api/v1/logcollector_types.go +++ b/api/v1/logcollector_types.go @@ -43,8 +43,19 @@ type LogCollectorSpec struct { MultiTenantManagementClusterNamespace string `json:"multiTenantManagementClusterNamespace,omitempty"` // FluentdDaemonSet configures the Fluentd DaemonSet. + // + // Deprecated: use CalicoFluentBitDaemonSet instead. This field is retained + // as an alias for one release during the Fluentd → Fluent Bit migration; + // when both are set, CalicoFluentBitDaemonSet takes precedence. + // +optional FluentdDaemonSet *FluentdDaemonSet `json:"fluentdDaemonSet,omitempty"` + // CalicoFluentBitDaemonSet configures the calico-fluent-bit DaemonSet, the + // Fluent Bit replacement for the Fluentd DaemonSet. Pod-template override + // semantics are unchanged from the deprecated FluentdDaemonSet field. + // +optional + CalicoFluentBitDaemonSet *FluentdDaemonSet `json:"calicoFluentBitDaemonSet,omitempty"` + // EKSLogForwarderDeployment configures the EKSLogForwarderDeployment Deployment. // +optional EKSLogForwarderDeployment *EKSLogForwarderDeployment `json:"eksLogForwarderDeployment,omitempty"` @@ -222,7 +233,7 @@ type LogCollectorStatus struct { // +kubebuilder:resource:scope=Cluster // LogCollector installs the components required for Tigera flow and DNS log collection. At most one instance -// of this resource is supported. It must be named "tigera-secure". When created, this installs fluentd on all nodes +// of this resource is supported. It must be named "tigera-secure". When created, this installs fluent-bit on all nodes // configured to collect Tigera log data and export it to Tigera's Elasticsearch cluster as well as any additionally configured destinations. // // +kubebuilder:validation:XValidation:rule="self.metadata.name == 'tigera-secure'",message="resource name must be 'tigera-secure'" diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index dead9aa130..816884ea0c 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -7243,6 +7243,11 @@ func (in *LogCollectorSpec) DeepCopyInto(out *LogCollectorSpec) { *out = new(FluentdDaemonSet) (*in).DeepCopyInto(*out) } + if in.CalicoFluentBitDaemonSet != nil { + in, out := &in.CalicoFluentBitDaemonSet, &out.CalicoFluentBitDaemonSet + *out = new(FluentdDaemonSet) + (*in).DeepCopyInto(*out) + } if in.EKSLogForwarderDeployment != nil { in, out := &in.EKSLogForwarderDeployment, &out.EKSLogForwarderDeployment *out = new(EKSLogForwarderDeployment) diff --git a/config/enterprise_versions.yml b/config/enterprise_versions.yml index e5761a627a..03581c6d2b 100644 --- a/config/enterprise_versions.yml +++ b/config/enterprise_versions.yml @@ -15,11 +15,11 @@ components: node-windows: image: node-windows version: master - fluentd: - image: fluentd + fluent-bit: + image: fluent-bit version: master - fluentd-windows: - image: fluentd-windows + fluent-bit-windows: + image: fluent-bit-windows version: master dex: image: dex diff --git a/hack/gen-versions/enterprise.go.tpl b/hack/gen-versions/enterprise.go.tpl index 7ed9089073..c13c1ec61e 100644 --- a/hack/gen-versions/enterprise.go.tpl +++ b/hack/gen-versions/enterprise.go.tpl @@ -91,8 +91,8 @@ var ( variant: enterpriseVariant, } {{- end }} -{{ with .Components.fluentd }} - ComponentFluentd = Component{ +{{ with index .Components "fluent-bit" }} + ComponentFluentBit = Component{ Version: "{{ .Version }}", Image: "{{ .Image }}", Registry: "{{ .Registry }}", @@ -100,8 +100,8 @@ var ( variant: enterpriseVariant, } {{- end }} -{{ with index .Components "fluentd-windows" }} - ComponentFluentdWindows = Component{ +{{ with index .Components "fluent-bit-windows" }} + ComponentFluentBitWindows = Component{ Version: "{{ .Version }}", Image: "{{ .Image }}", Registry: "{{ .Registry }}", @@ -306,8 +306,8 @@ var ( ComponentElasticTseeInstaller, ComponentElasticsearch, ComponentElasticsearchOperator, - ComponentFluentd, - ComponentFluentdWindows, + ComponentFluentBit, + ComponentFluentBitWindows, ComponentIntrusionDetectionController, ComponentKibana, ComponentManager, diff --git a/pkg/components/enterprise.go b/pkg/components/enterprise.go index 3753ca105d..f4329cfa08 100644 --- a/pkg/components/enterprise.go +++ b/pkg/components/enterprise.go @@ -83,17 +83,17 @@ var ( variant: enterpriseVariant, } - ComponentFluentd = Component{ + ComponentFluentBit = Component{ Version: "master", - Image: "fluentd", + Image: "fluent-bit", Registry: "", imagePath: "", variant: enterpriseVariant, } - ComponentFluentdWindows = Component{ + ComponentFluentBitWindows = Component{ Version: "master", - Image: "fluentd-windows", + Image: "fluent-bit-windows", Registry: "", imagePath: "", variant: enterpriseVariant, @@ -273,8 +273,8 @@ var ( ComponentElasticTseeInstaller, ComponentElasticsearch, ComponentElasticsearchOperator, - ComponentFluentd, - ComponentFluentdWindows, + ComponentFluentBit, + ComponentFluentBitWindows, ComponentIntrusionDetectionController, ComponentKibana, ComponentManager, diff --git a/pkg/controller/logcollector/logcollector_controller.go b/pkg/controller/logcollector/logcollector_controller.go index 60e88431fe..74b051542a 100644 --- a/pkg/controller/logcollector/logcollector_controller.go +++ b/pkg/controller/logcollector/logcollector_controller.go @@ -79,7 +79,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { go utils.WaitToAddLicenseKeyWatch(c, opts.K8sClientset, log, licenseAPIReady) go utils.WaitToAddTierWatch(networkpolicy.CalicoTierName, c, opts.K8sClientset, log, tierWatchReady) go utils.WaitToAddNetworkPolicyWatches(c, opts.K8sClientset, log, []types.NamespacedName{ - {Name: render.FluentdPolicyName, Namespace: render.LogCollectorNamespace}, + {Name: render.FluentBitPolicyName, Namespace: render.LogCollectorNamespace}, }) if opts.MultiTenant { @@ -130,16 +130,16 @@ func add(mgr manager.Manager, c ctrlruntime.Controller) error { for _, secretName := range []string{ render.ElasticsearchEksLogForwarderUserSecret, - render.S3FluentdSecretName, render.EksLogForwarderSecret, - render.SplunkFluentdTokenSecretName, monitor.PrometheusClientTLSSecretName, - render.FluentdPrometheusTLSSecretName, render.TigeraLinseedSecret, render.VoltronLinseedPublicCert, render.EKSLogForwarderTLSSecretName, + render.S3FluentBitSecretName, render.EksLogForwarderSecret, + render.SplunkFluentBitTokenSecretName, monitor.PrometheusClientTLSSecretName, + render.FluentBitTLSSecretName, render.TigeraLinseedSecret, render.VoltronLinseedPublicCert, render.EKSLogForwarderTLSSecretName, } { if err = utils.AddSecretsWatch(c, secretName, common.OperatorNamespace()); err != nil { return fmt.Errorf("log-collector-controller failed to watch the Secret resource(%s): %v", secretName, err) } } - for _, configMapName := range []string{render.FluentdFilterConfigMapName, relasticsearch.ClusterConfigConfigMapName} { + for _, configMapName := range []string{render.FluentBitFilterConfigMapName, relasticsearch.ClusterConfigConfigMapName} { if err = utils.AddConfigMapWatch(c, configMapName, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil { return fmt.Errorf("logcollector-controller failed to watch ConfigMap %s: %v", configMapName, err) } @@ -369,9 +369,9 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - // fluentdKeyPair is the key pair fluentd presents to identify itself - httpInputServiceNames := dns.GetServiceDNSNames(render.FluentdInputService, render.LogCollectorNamespace, r.opts.ClusterDomain) - fluentdKeyPair, err := certificateManager.GetOrCreateKeyPair(r.client, render.FluentdPrometheusTLSSecretName, common.OperatorNamespace(), append([]string{render.FluentdPrometheusTLSSecretName}, httpInputServiceNames...)) + // fluentBitKeyPair is the key pair fluent-bit presents to identify itself + httpInputServiceNames := dns.GetServiceDNSNames(render.FluentBitInputService, render.LogCollectorNamespace, r.opts.ClusterDomain) + fluentBitKeyPair, err := certificateManager.GetOrCreateKeyPair(r.client, render.FluentBitTLSSecretName, common.OperatorNamespace(), append([]string{render.FluentBitTLSSecretName}, httpInputServiceNames...)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating TLS certificate", err, reqLogger) return reconcile.Result{}, err @@ -430,7 +430,7 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, nil } - // Fluentd needs to mount system certificates in the case where Splunk, Syslog or AWS are used. + // Fluent Bit needs to mount system certificates in the case where Splunk, Syslog or AWS are used. trustedBundle, err := certificateManager.CreateTrustedBundleWithSystemRootCertificates(prometheusCertificate, linseedCertificate) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create tigera-ca-bundle configmap", err, reqLogger) @@ -524,28 +524,18 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile } } - filters, err := getFluentdFilters(r.client) + filters, err := getFluentBitFilters(r.client) if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error retrieving Fluentd filters", err, reqLogger) + r.status.SetDegraded(operatorv1.ResourceReadError, "Error retrieving Fluent Bit filters", err, reqLogger) return reconcile.Result{}, err } var eksConfig *render.EksCloudwatchLogConfig - var esClusterConfig *relasticsearch.ClusterConfig var eksLogForwarderKeyPair certificatemanagement.KeyPairInterface if installationSpec.KubernetesProvider.IsEKS() { log.Info("Managed kubernetes EKS found, getting necessary credentials and config") if instance.Spec.AdditionalSources != nil { if instance.Spec.AdditionalSources.EksCloudwatchLog != nil { - esClusterConfig, err = utils.GetElasticsearchClusterConfig(ctx, r.client) - if err != nil { - if errors.IsNotFound(err) { - r.status.SetDegraded(operatorv1.ResourceNotReady, "Elasticsearch cluster configuration is not available, waiting for it to become available", err, reqLogger) - return reconcile.Result{}, nil - } - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get the elasticsearch cluster configuration", err, reqLogger) - return reconcile.Result{}, err - } eksConfig, err = getEksCloudwatchLogConfig(r.client, instance.Spec.AdditionalSources.EksCloudwatchLog.FetchInterval, instance.Spec.AdditionalSources.EksCloudwatchLog.Region, @@ -588,9 +578,8 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile // Create a component handler to manage the rendered component. handler := utils.NewComponentHandler(log, r.client, r.scheme, instance) - fluentdCfg := &render.FluentdConfiguration{ + fluentBitCfg := &render.FluentBitConfiguration{ LogCollector: instance, - ESClusterConfig: esClusterConfig, S3Credential: s3Credential, SplkCredential: splunkCredential, Filters: filters, @@ -599,7 +588,7 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile Installation: installationSpec, ClusterDomain: r.opts.ClusterDomain, OSType: rmeta.OSTypeLinux, - FluentdKeyPair: fluentdKeyPair, + FluentBitKeyPair: fluentBitKeyPair, TrustedBundle: trustedBundle, ManagedCluster: managedCluster, UseSyslogCertificate: useSyslogCertificate, @@ -610,14 +599,14 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile NonClusterHost: nonclusterhost, LicenseExpired: licenseExpired, } - // Render the fluentd component for Linux - comp := render.Fluentd(fluentdCfg) + // Render the fluent-bit component for Linux + comp := render.FluentBit(fluentBitCfg) certificateComponent := rcertificatemanagement.Config{ Namespace: render.LogCollectorNamespace, - ServiceAccounts: []string{render.FluentdNodeName}, + ServiceAccounts: []string{render.FluentBitNodeName}, KeyPairOptions: []rcertificatemanagement.KeyPairOption{ - rcertificatemanagement.NewKeyPairOption(fluentdKeyPair, true, true), + rcertificatemanagement.NewKeyPairOption(fluentBitKeyPair, true, true), }, TrustedBundle: trustedBundle, } @@ -632,12 +621,16 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile } setUp := render.NewSetup(&render.SetUpConfiguration{ - OpenShift: r.opts.DetectedProvider.IsOpenShift(), - Installation: installationSpec, - PullSecrets: pullSecrets, - Namespace: render.LogCollectorNamespace, - PSS: render.PSSPrivileged, - CreateNamespace: true, + OpenShift: r.opts.DetectedProvider.IsOpenShift(), + Installation: installationSpec, + PullSecrets: pullSecrets, + Namespace: render.LogCollectorNamespace, + PSS: render.PSSPrivileged, + // calico-system is created and owned by the core Installation + // controller. Owning it from the LogCollector CR would let a routine + // `kubectl delete logcollector` garbage-collect the entire namespace — + // calico-node included. + CreateNamespace: false, }) components := []render.Component{ setUp, @@ -657,16 +650,15 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile } } - // Render a fluentd component for Windows if the cluster has Windows nodes. + // Render a fluent-bit component for Windows if the cluster has Windows nodes. hasWindowsNodes, err := common.HasWindowsNodes(r.client) if err != nil { return reconcile.Result{}, err } if hasWindowsNodes { - fluentdCfg = &render.FluentdConfiguration{ + fluentBitCfg = &render.FluentBitConfiguration{ LogCollector: instance, - ESClusterConfig: esClusterConfig, S3Credential: s3Credential, SplkCredential: splunkCredential, Filters: filters, @@ -678,11 +670,11 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile TrustedBundle: trustedBundle, ManagedCluster: managedCluster, UseSyslogCertificate: useSyslogCertificate, - FluentdKeyPair: fluentdKeyPair, + FluentBitKeyPair: fluentBitKeyPair, EKSLogForwarderKeyPair: eksLogForwarderKeyPair, LicenseExpired: licenseExpired, } - comp = render.Fluentd(fluentdCfg) + comp = render.FluentBit(fluentBitCfg) if err = imageset.ApplyImageSet(ctx, r.client, variant, comp); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error with images from ImageSet", err, reqLogger) @@ -706,8 +698,8 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile // Check BYO certificate expiry warnings. certificatemanagement.CheckKeyPairWarnings(map[string]certificatemanagement.KeyPairInterface{ - render.FluentdPrometheusTLSSecretName: fluentdKeyPair, - render.EKSLogForwarderTLSSecretName: eksLogForwarderKeyPair, + render.FluentBitTLSSecretName: fluentBitKeyPair, + render.EKSLogForwarderTLSSecretName: eksLogForwarderKeyPair, }, r.status) // Clear the degraded bit if we've reached this far. @@ -730,26 +722,26 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile func getS3Credential(client client.Client) (*render.S3Credential, error) { secret := &corev1.Secret{} secretNamespacedName := types.NamespacedName{ - Name: render.S3FluentdSecretName, + Name: render.S3FluentBitSecretName, Namespace: common.OperatorNamespace(), } if err := client.Get(context.Background(), secretNamespacedName, secret); err != nil { if errors.IsNotFound(err) { return nil, nil } - return nil, fmt.Errorf("failed to read secret %q: %s", render.S3FluentdSecretName, err) + return nil, fmt.Errorf("failed to read secret %q: %s", render.S3FluentBitSecretName, err) } var ok bool var kId []byte if kId, ok = secret.Data[render.S3KeyIdName]; !ok || len(kId) == 0 { return nil, fmt.Errorf("expected secret %q to have a field named %q", - render.S3FluentdSecretName, render.S3KeyIdName) + render.S3FluentBitSecretName, render.S3KeyIdName) } var kSecret []byte if kSecret, ok = secret.Data[render.S3KeySecretName]; !ok || len(kSecret) == 0 { return nil, fmt.Errorf("expected secret %q to have a field named %q", - render.S3FluentdSecretName, render.S3KeySecretName) + render.S3FluentBitSecretName, render.S3KeySecretName) } return &render.S3Credential{ @@ -761,20 +753,20 @@ func getS3Credential(client client.Client) (*render.S3Credential, error) { func getSplunkCredential(client client.Client) (*render.SplunkCredential, error) { tokenSecret := &corev1.Secret{} tokenNamespacedName := types.NamespacedName{ - Name: render.SplunkFluentdTokenSecretName, + Name: render.SplunkFluentBitTokenSecretName, Namespace: common.OperatorNamespace(), } if err := client.Get(context.Background(), tokenNamespacedName, tokenSecret); err != nil { if errors.IsNotFound(err) { return nil, nil } - return nil, fmt.Errorf("failed to read secret %q: %s", render.SplunkFluentdTokenSecretName, err) + return nil, fmt.Errorf("failed to read secret %q: %s", render.SplunkFluentBitTokenSecretName, err) } - token, ok := tokenSecret.Data[render.SplunkFluentdSecretTokenKey] + token, ok := tokenSecret.Data[render.SplunkFluentBitSecretTokenKey] if !ok || len(token) == 0 { return nil, fmt.Errorf("expected secret %q to have a field named %q", - render.SplunkFluentdTokenSecretName, render.SplunkFluentdSecretTokenKey) + render.SplunkFluentBitTokenSecretName, render.SplunkFluentBitSecretTokenKey) } return &render.SplunkCredential{ @@ -782,22 +774,22 @@ func getSplunkCredential(client client.Client) (*render.SplunkCredential, error) }, nil } -func getFluentdFilters(client client.Client) (*render.FluentdFilters, error) { +func getFluentBitFilters(client client.Client) (*render.FluentBitFilters, error) { cm := &corev1.ConfigMap{} cmNamespacedName := types.NamespacedName{ - Name: render.FluentdFilterConfigMapName, + Name: render.FluentBitFilterConfigMapName, Namespace: common.OperatorNamespace(), } if err := client.Get(context.Background(), cmNamespacedName, cm); err != nil { if errors.IsNotFound(err) { return nil, nil } - return nil, fmt.Errorf("failed to read ConfigMap %q: %s", render.FluentdFilterConfigMapName, err) + return nil, fmt.Errorf("failed to read ConfigMap %q: %s", render.FluentBitFilterConfigMapName, err) } - return &render.FluentdFilters{ - Flow: cm.Data[render.FluentdFilterFlowName], - DNS: cm.Data[render.FluentdFilterDNSName], + return &render.FluentBitFilters{ + Flow: cm.Data[render.FluentBitFilterFlowName], + DNS: cm.Data[render.FluentBitFilterDNSName], }, nil } diff --git a/pkg/controller/logcollector/logcollector_controller_test.go b/pkg/controller/logcollector/logcollector_controller_test.go index f97af34c4f..c4f48462c5 100644 --- a/pkg/controller/logcollector/logcollector_controller_test.go +++ b/pkg/controller/logcollector/logcollector_controller_test.go @@ -28,6 +28,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -77,6 +78,7 @@ var _ = Describe("LogCollector controller tests", func() { mockStatus.On("AddCronJobs", mock.Anything) mockStatus.On("RemoveCertificateSigningRequests", mock.Anything).Return() mockStatus.On("RemoveDaemonsets", mock.Anything).Return() + mockStatus.On("RemoveDeployments", mock.Anything).Return() mockStatus.On("AddCertificateSigningRequests", mock.Anything).Return() mockStatus.On("IsAvailable").Return(true) mockStatus.On("OnCRFound").Return() @@ -174,7 +176,7 @@ var _ = Describe("LogCollector controller tests", func() { ds := appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "fluentd-node", + Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace, }, } @@ -185,16 +187,16 @@ var _ = Describe("LogCollector controller tests", func() { Expect(node.Image).To(Equal( fmt.Sprintf("some.registry.org/%s%s:%s", components.TigeraImagePath, - components.ComponentFluentd.Image, - components.ComponentFluentd.Version))) + components.ComponentFluentBit.Image, + components.ComponentFluentBit.Version))) }) It("should use images from imageset", func() { Expect(c.Create(ctx, &operatorv1.ImageSet{ ObjectMeta: metav1.ObjectMeta{Name: "enterprise-" + components.EnterpriseRelease}, Spec: operatorv1.ImageSetSpec{ Images: []operatorv1.Image{ - {Image: "tigera/fluentd", Digest: "sha256:fluentdhash"}, - {Image: "tigera/fluentd-windows", Digest: "sha256:fluentdwindowshash"}, + {Image: "tigera/fluent-bit", Digest: "sha256:fluentbithash"}, + {Image: "tigera/fluent-bit-windows", Digest: "sha256:fluentbitwindowshash"}, {Image: "tigera/calico", Digest: "sha256:deadbeef0123456789"}, }, }, @@ -215,7 +217,7 @@ var _ = Describe("LogCollector controller tests", func() { ds := appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "fluentd-node", + Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace, }, } @@ -226,10 +228,10 @@ var _ = Describe("LogCollector controller tests", func() { Expect(node.Image).To(Equal( fmt.Sprintf("some.registry.org/%s%s@%s", components.TigeraImagePath, - components.ComponentFluentd.Image, - "sha256:fluentdhash"))) + components.ComponentFluentBit.Image, + "sha256:fluentbithash"))) - ds.Name = "fluentd-node-windows" + ds.Name = "calico-fluent-bit-windows" Expect(test.GetResource(c, &ds)).To(BeNil()) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) node = ds.Spec.Template.Spec.Containers[0] @@ -237,14 +239,14 @@ var _ = Describe("LogCollector controller tests", func() { Expect(node.Image).To(Equal( fmt.Sprintf("some.registry.org/%s%s@%s", components.TigeraImagePath, - components.ComponentFluentdWindows.Image, - "sha256:fluentdwindowshash"))) + components.ComponentFluentBitWindows.Image, + "sha256:fluentbitwindowshash"))) }) Context("Forward to S3", func() { s3Vars := []corev1.EnvVar{ { - Name: "AWS_KEY_ID", + Name: "AWS_ACCESS_KEY_ID", Value: "", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ @@ -256,7 +258,7 @@ var _ = Describe("LogCollector controller tests", func() { }, }, { - Name: "AWS_SECRET_KEY", + Name: "AWS_SECRET_ACCESS_KEY", Value: "", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ @@ -267,11 +269,6 @@ var _ = Describe("LogCollector controller tests", func() { }, }, }, - {Name: "S3_STORAGE", Value: "true"}, - {Name: "S3_BUCKET_NAME", Value: "s3Bucket"}, - {Name: "AWS_REGION", Value: "s3Region"}, - {Name: "S3_BUCKET_PATH", Value: "s3Path"}, - {Name: "S3_FLUSH_INTERVAL", Value: "5s"}, } BeforeEach(func() { @@ -314,7 +311,7 @@ var _ = Describe("LogCollector controller tests", func() { ds := appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "fluentd-node", + Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace, }, } @@ -323,6 +320,18 @@ var _ = Describe("LogCollector controller tests", func() { node := ds.Spec.Template.Spec.Containers[0] Expect(node).ToNot(BeNil()) Expect(node.Env).To(ContainElements(s3Vars)) + + // The bucket settings live in the rendered config rather than + // env vars. + cm := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}, + } + Expect(test.GetResource(c, &cm)).To(BeNil()) + conf := cm.Data["fluent-bit.yaml"] + Expect(conf).To(ContainSubstring(`"name": "s3"`)) + Expect(conf).To(ContainSubstring(`"bucket": "s3Bucket"`)) + Expect(conf).To(ContainSubstring(`"region": "s3Region"`)) }) Context("Disable feature via license", func() { @@ -341,7 +350,7 @@ var _ = Describe("LogCollector controller tests", func() { ds := appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "fluentd-node", + Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace, }, } @@ -370,13 +379,6 @@ var _ = Describe("LogCollector controller tests", func() { }, }, }, - {Name: "SPLUNK_FLOW_LOG", Value: "true"}, - {Name: "SPLUNK_AUDIT_LOG", Value: "true"}, - {Name: "SPLUNK_DNS_LOG", Value: "true"}, - {Name: "SPLUNK_HEC_HOST", Value: "localhost"}, - {Name: "SPLUNK_HEC_PORT", Value: "1234"}, - {Name: "SPLUNK_PROTOCOL", Value: "https"}, - {Name: "SPLUNK_FLUSH_INTERVAL", Value: "5s"}, } BeforeEach(func() { @@ -416,7 +418,7 @@ var _ = Describe("LogCollector controller tests", func() { ds := appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "fluentd-node", + Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace, }, } @@ -425,6 +427,18 @@ var _ = Describe("LogCollector controller tests", func() { node := ds.Spec.Template.Spec.Containers[0] Expect(node).ToNot(BeNil()) Expect(node.Env).To(ContainElements(splunkVars)) + + // The endpoint settings live in the rendered config rather than + // env vars. + cm := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}, + } + Expect(test.GetResource(c, &cm)).To(BeNil()) + conf := cm.Data["fluent-bit.yaml"] + Expect(conf).To(ContainSubstring(`"name": "splunk"`)) + Expect(conf).To(ContainSubstring(`"host": "localhost"`)) + Expect(conf).To(ContainSubstring(`"splunk_token": "${SPLUNK_HEC_TOKEN}"`)) }) Context("Disable feature via license", func() { @@ -444,7 +458,7 @@ var _ = Describe("LogCollector controller tests", func() { ds := appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "fluentd-node", + Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace, }, } @@ -461,30 +475,6 @@ var _ = Describe("LogCollector controller tests", func() { }) Context("Forward to Syslog", func() { - syslogVars := []corev1.EnvVar{ - {Name: "SYSLOG_HOST", Value: "localhost"}, - {Name: "SYSLOG_PORT", Value: "1234"}, - {Name: "SYSLOG_PROTOCOL", Value: "https"}, - {Name: "SYSLOG_FLUSH_INTERVAL", Value: "5s"}, - { - Name: "SYSLOG_HOSTNAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - { - Name: "SYSLOG_PACKET_SIZE", - Value: "0", - }, - {Name: "SYSLOG_AUDIT_EE_LOG", Value: "true"}, - {Name: "SYSLOG_AUDIT_KUBE_LOG", Value: "true"}, - {Name: "SYSLOG_DNS_LOG", Value: "true"}, - {Name: "SYSLOG_FLOW_LOG", Value: "true"}, - {Name: "SYSLOG_IDS_EVENT_LOG", Value: "true"}, - } - BeforeEach(func() { By("Specify splunk log storage") Expect(c.Delete(ctx, &operatorv1.LogCollector{ @@ -495,7 +485,7 @@ var _ = Describe("LogCollector controller tests", func() { Spec: operatorv1.LogCollectorSpec{ AdditionalStores: &operatorv1.AdditionalLogStoreSpec{ Syslog: &operatorv1.SyslogStoreSpec{ - Endpoint: "https://localhost:1234", + Endpoint: "tcp://localhost:1234", PacketSize: new(int32), LogTypes: []operatorv1.SyslogLogType{ operatorv1.SyslogLogAudit, @@ -520,15 +510,25 @@ var _ = Describe("LogCollector controller tests", func() { ds := appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "fluentd-node", + Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace, }, } Expect(test.GetResource(c, &ds)).To(BeNil()) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) - node := ds.Spec.Template.Spec.Containers[0] - Expect(node).ToNot(BeNil()) - Expect(node.Env).To(ContainElements(syslogVars)) + + // Syslog forwarding is fully config-driven (no env contract). + cm := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}, + } + Expect(test.GetResource(c, &cm)).To(BeNil()) + conf := cm.Data["fluent-bit.yaml"] + Expect(conf).To(ContainSubstring(`"name": "syslog"`)) + Expect(conf).To(ContainSubstring(`"host": "localhost"`)) + Expect(conf).To(ContainSubstring(`"mode": "tcp"`)) + // The whole record ships as one JSON MSG via the lua packer. + Expect(conf).To(ContainSubstring(`"call": "syslog_pack"`)) }) Context("Disable feature via license", func() { @@ -548,7 +548,7 @@ var _ = Describe("LogCollector controller tests", func() { ds := appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "fluentd-node", + Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace, }, } @@ -812,15 +812,16 @@ var _ = Describe("LogCollector controller tests", func() { Expect(err).NotTo(HaveOccurred()) Expect(result.RequeueAfter).To(Equal(0 * time.Second)) - // Expect namespace to be created + // The calico-system namespace is created and owned by the core + // Installation controller, NOT this reconciler — owning it here would + // let `kubectl delete logcollector` garbage-collect the whole + // namespace. namespace := corev1.Namespace{ TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, } - Expect(c.Get(ctx, client.ObjectKey{ + Expect(errors.IsNotFound(c.Get(ctx, client.ObjectKey{ Name: render.LogCollectorNamespace, - }, &namespace)).NotTo(HaveOccurred()) - Expect(namespace.Labels["pod-security.kubernetes.io/enforce"]).To(Equal("privileged")) - Expect(namespace.Labels["pod-security.kubernetes.io/enforce-version"]).To(Equal("latest")) + }, &namespace))).To(BeTrue()) // Expect operator rolebinding to be created rb := rbacv1.RoleBinding{ @@ -849,8 +850,8 @@ var _ = Describe("LogCollector controller tests", func() { }) Context("License expiry", func() { - It("should set degraded status and delete fluentd DaemonSet when license is expired", func() { - // First reconcile to create fluentd resources. + It("should set degraded status and delete fluent-bit DaemonSet when license is expired", func() { + // First reconcile to create fluent-bit resources. _, err := r.Reconcile(ctx, reconcile.Request{}) Expect(err).ShouldNot(HaveOccurred()) @@ -858,7 +859,7 @@ var _ = Describe("LogCollector controller tests", func() { ds := appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "fluentd-node", + Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace, }, } @@ -884,7 +885,7 @@ var _ = Describe("LogCollector controller tests", func() { ds = appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "fluentd-node", + Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace, }, } @@ -892,7 +893,7 @@ var _ = Describe("LogCollector controller tests", func() { }) It("should requeue when license is in the grace period", func() { - // First reconcile to create fluentd resources. + // First reconcile to create fluent-bit resources. _, err := r.Reconcile(ctx, reconcile.Request{}) Expect(err).ShouldNot(HaveOccurred()) @@ -918,7 +919,7 @@ var _ = Describe("LogCollector controller tests", func() { ds := appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "fluentd-node", + Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace, }, } diff --git a/pkg/controller/logstorage/secrets/secret_controller.go b/pkg/controller/logstorage/secrets/secret_controller.go index c24269f219..d124241ddc 100644 --- a/pkg/controller/logstorage/secrets/secret_controller.go +++ b/pkg/controller/logstorage/secrets/secret_controller.go @@ -470,8 +470,8 @@ func (r *SecretSubController) collectUpstreamCerts(log logr.Logger, helper utils // Get certificate for ui-apis, which Linseed and es-gateway need to trust. render.ManagerInternalTLSSecretName: helper.TruthNamespace(), - // Get certificate for fluentd, which Linseed needs to trust in a standalone or management cluster. - render.FluentdPrometheusTLSSecretName: common.OperatorNamespace(), + // Get certificate for fluent-bit, which Linseed needs to trust in a standalone or management cluster. + render.FluentBitTLSSecretName: common.OperatorNamespace(), // Get certificate for intrusion detection controller, which Linseed needs to trust in a standalone or management cluster. render.IntrusionDetectionTLSSecretName: helper.TruthNamespace(), @@ -620,7 +620,7 @@ type keyPairCollection struct { // Certificates that need to be added to the trusted bundle, but // aren't actually generated by this controller. - // Prometheus, ui-apis, fluentd, all compliance components. + // Prometheus, ui-apis, fluent-bit, all compliance components. upstreamCerts []certificatemanagement.CertificateInterface // Key pairs that are generated by this controller. These need to be diff --git a/pkg/controller/logstorage/secrets/secret_controller_test.go b/pkg/controller/logstorage/secrets/secret_controller_test.go index b86aec2de8..fe436a6c03 100644 --- a/pkg/controller/logstorage/secrets/secret_controller_test.go +++ b/pkg/controller/logstorage/secrets/secret_controller_test.go @@ -270,7 +270,7 @@ var _ = Describe("LogStorage Secrets controller", func() { ls.Status.State = operatorv1.TigeraStatusReady CreateLogStorage(cli, ls) - By("Creating a fluentd certificate secret without all necessary usages") + By("Creating a fluent-bit certificate secret without all necessary usages") cryptoCA, err := tls.MakeCA(rmeta.TigeraOperatorCAIssuerPrefix) Expect(err).NotTo(HaveOccurred()) tlsCfg, err := cryptoCA.MakeServerCertForDuration(sets.New[string]("test"), tls.DefaultCertificateDuration, tls.SetServerAuth) @@ -278,15 +278,15 @@ var _ = Describe("LogStorage Secrets controller", func() { keyContent, crtContent := &bytes.Buffer{}, &bytes.Buffer{} Expect(tlsCfg.WriteCertConfig(crtContent, keyContent)).NotTo(HaveOccurred()) privateKeyPEM, certificatePEM := keyContent.Bytes(), crtContent.Bytes() - fluentdCert, err := certificateManager.GetOrCreateKeyPair(cli, render.FluentdPrometheusTLSSecretName, common.OperatorNamespace(), []string{""}) + fluentBitCert, err := certificateManager.GetOrCreateKeyPair(cli, render.FluentBitTLSSecretName, common.OperatorNamespace(), []string{""}) Expect(err).NotTo(HaveOccurred()) - fluentdSecret := fluentdCert.Secret(common.OperatorNamespace()) - fluentdSecret.Data[corev1.TLSCertKey] = certificatePEM - fluentdSecret.Data[corev1.TLSPrivateKeyKey] = privateKeyPEM + fluentBitSecret := fluentBitCert.Secret(common.OperatorNamespace()) + fluentBitSecret.Data[corev1.TLSCertKey] = certificatePEM + fluentBitSecret.Data[corev1.TLSPrivateKeyKey] = privateKeyPEM Expect(err).NotTo(HaveOccurred()) r, err := NewSecretControllerWithShims(cli, scheme, mockStatus, operatorv1.ProviderNone, dns.DefaultClusterDomain) Expect(err).ShouldNot(HaveOccurred()) - Expect(r.client.Create(ctx, fluentdSecret)).NotTo(HaveOccurred()) + Expect(r.client.Create(ctx, fluentBitSecret)).NotTo(HaveOccurred()) By("reconciling the controller after a bad secret was created, we expect no problems, because bad secrets should be skipped") _, err = r.Reconcile(ctx, reconcile.Request{}) diff --git a/pkg/controller/monitor/monitor_controller.go b/pkg/controller/monitor/monitor_controller.go index 028c85059e..2814507be5 100644 --- a/pkg/controller/monitor/monitor_controller.go +++ b/pkg/controller/monitor/monitor_controller.go @@ -149,7 +149,7 @@ func add(_ manager.Manager, c ctrlruntime.Controller) error { certificatemanagement.CASecretName, esmetrics.ElasticsearchMetricsServerTLSSecret, monitor.PrometheusServerTLSSecretName, - render.FluentdPrometheusTLSSecretName, + render.FluentBitTLSSecretName, render.NodePrometheusTLSServerSecret, kubecontrollers.KubeControllerPrometheusTLSSecret, render.EKSLogForwarderTLSSecretName, @@ -338,7 +338,7 @@ func (r *ReconcileMonitor) Reconcile(ctx context.Context, request reconcile.Requ trustedBundle := certificateManager.CreateTrustedBundle() for _, certificateName := range []string{ esmetrics.ElasticsearchMetricsServerTLSSecret, - render.FluentdPrometheusTLSSecretName, + render.FluentBitTLSSecretName, render.NodePrometheusTLSServerSecret, render.CalicoAPIServerTLSSecretName, kubecontrollers.KubeControllerPrometheusTLSSecret, diff --git a/pkg/controller/monitor/monitor_controller_test.go b/pkg/controller/monitor/monitor_controller_test.go index 2f24492f2b..80f887d9e1 100644 --- a/pkg/controller/monitor/monitor_controller_test.go +++ b/pkg/controller/monitor/monitor_controller_test.go @@ -171,7 +171,7 @@ var _ = Describe("Monitor controller tests", func() { Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.TigeraPrometheusRule, Namespace: common.TigeraPrometheusNamespace}, pr)).To(HaveOccurred()) Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.CalicoNodeMonitor, Namespace: common.TigeraPrometheusNamespace}, sm)).To(HaveOccurred()) Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.ElasticsearchMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).To(HaveOccurred()) - Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentdMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).To(HaveOccurred()) + Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentBitMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).To(HaveOccurred()) }) It("should create Prometheus related resources", func() { @@ -184,7 +184,7 @@ var _ = Describe("Monitor controller tests", func() { Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.TigeraPrometheusRule, Namespace: common.TigeraPrometheusNamespace}, pr)).NotTo(HaveOccurred()) Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.CalicoNodeMonitor, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.ElasticsearchMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) - Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentdMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) + Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentBitMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) // Verify the recommended labels are correct on these resources. Expect(p.Labels).To(Equal(map[string]string{ @@ -207,7 +207,7 @@ var _ = Describe("Monitor controller tests", func() { }) It("should create Prometheus related resources even when a cert with missing key usages is configured for other components", func() { - By("Creating a fluentd certificate secret without all necessary usages") + By("Creating a fluent-bit certificate secret without all necessary usages") cryptoCA, err := tls.MakeCA(rmeta.TigeraOperatorCAIssuerPrefix) Expect(err).NotTo(HaveOccurred()) tlsCfg, err := cryptoCA.MakeServerCertForDuration(sets.New[string]("test"), tls.DefaultCertificateDuration, tls.SetServerAuth) @@ -215,13 +215,13 @@ var _ = Describe("Monitor controller tests", func() { keyContent, crtContent := &bytes.Buffer{}, &bytes.Buffer{} Expect(tlsCfg.WriteCertConfig(crtContent, keyContent)).NotTo(HaveOccurred()) privateKeyPEM, certificatePEM := keyContent.Bytes(), crtContent.Bytes() - fluentdCert, err := certificateManager.GetOrCreateKeyPair(cli, render.FluentdPrometheusTLSSecretName, common.OperatorNamespace(), []string{""}) + fluentBitCert, err := certificateManager.GetOrCreateKeyPair(cli, render.FluentBitTLSSecretName, common.OperatorNamespace(), []string{""}) Expect(err).NotTo(HaveOccurred()) - fluentdSecret := fluentdCert.Secret(common.OperatorNamespace()) - fluentdSecret.Data[corev1.TLSCertKey] = certificatePEM - fluentdSecret.Data[corev1.TLSPrivateKeyKey] = privateKeyPEM + fluentBitSecret := fluentBitCert.Secret(common.OperatorNamespace()) + fluentBitSecret.Data[corev1.TLSCertKey] = certificatePEM + fluentBitSecret.Data[corev1.TLSPrivateKeyKey] = privateKeyPEM Expect(err).NotTo(HaveOccurred()) - Expect(r.client.Create(ctx, fluentdSecret)).NotTo(HaveOccurred()) + Expect(r.client.Create(ctx, fluentBitSecret)).NotTo(HaveOccurred()) By("reconciling the controller after a bad secret was created, we expect no problems, because bad secrets should be skipped") _, err = r.Reconcile(ctx, reconcile.Request{}) @@ -288,7 +288,7 @@ var _ = Describe("Monitor controller tests", func() { Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.TigeraPrometheusRule, Namespace: common.TigeraPrometheusNamespace}, pr)).NotTo(HaveOccurred()) Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.CalicoNodeMonitor, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.ElasticsearchMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) - Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentdMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) + Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentBitMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) // External Prometheus related objects should be rendered after reconciliation. serviceMonitor := &monitoringv1.ServiceMonitor{} @@ -677,7 +677,7 @@ var _ = Describe("Monitor controller tests", func() { sm := &monitoringv1.ServiceMonitor{} Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.CalicoNodeMonitor, Namespace: common.TigeraPrometheusNamespace}, sm)).To(HaveOccurred()) Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.ElasticsearchMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).To(HaveOccurred()) - Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentdMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).To(HaveOccurred()) + Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentBitMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).To(HaveOccurred()) // Grace period warning should be cleared when license is fully expired. mockStatus.AssertCalled(GinkgoT(), "ClearWarning", "license-grace-period") @@ -700,7 +700,7 @@ var _ = Describe("Monitor controller tests", func() { sm := &monitoringv1.ServiceMonitor{} Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.CalicoNodeMonitor, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.ElasticsearchMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) - Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentdMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) + Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentBitMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) }) It("should requeue when license is in the grace period", func() { @@ -729,7 +729,7 @@ var _ = Describe("Monitor controller tests", func() { sm := &monitoringv1.ServiceMonitor{} Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.CalicoNodeMonitor, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.ElasticsearchMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) - Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentdMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) + Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.FluentBitMetrics, Namespace: common.TigeraPrometheusNamespace}, sm)).NotTo(HaveOccurred()) }) }) }) diff --git a/pkg/controller/monitor/prometheus.go b/pkg/controller/monitor/prometheus.go index 1c4233a4dd..2ac5f8af52 100644 --- a/pkg/controller/monitor/prometheus.go +++ b/pkg/controller/monitor/prometheus.go @@ -66,10 +66,10 @@ func addServiceMonitorElasticsearchWatch(c ctrlruntime.Controller) error { }, &handler.EnqueueRequestForObject{}) } -func addServiceMonitorFluentdWatch(c ctrlruntime.Controller) error { +func addServiceMonitorFluentBitWatch(c ctrlruntime.Controller) error { return utils.AddNamespacedWatch(c, &monitoringv1.ServiceMonitor{ TypeMeta: metav1.TypeMeta{Kind: monitoringv1.ServiceMonitorsKind, APIVersion: monitor.MonitoringAPIVersion}, - ObjectMeta: metav1.ObjectMeta{Name: monitor.FluentdMetrics, Namespace: common.TigeraPrometheusNamespace}, + ObjectMeta: metav1.ObjectMeta{Name: monitor.FluentBitMetrics, Namespace: common.TigeraPrometheusNamespace}, }, &handler.EnqueueRequestForObject{}) } @@ -102,8 +102,8 @@ func addWatch(c ctrlruntime.Controller) error { return fmt.Errorf("failed to watch ServiceMonitor elasticsearch-metrics resource: %w", err) } - if err = addServiceMonitorFluentdWatch(c); err != nil { - return fmt.Errorf("failed to watch ServiceMonitor fluentd-metrics resource: %w", err) + if err = addServiceMonitorFluentBitWatch(c); err != nil { + return fmt.Errorf("failed to watch ServiceMonitor fluent-bit-metrics resource: %w", err) } if err = addServiceMonitorKubeControllerWatch(c); err != nil { diff --git a/pkg/controller/tiers/tiers_controller.go b/pkg/controller/tiers/tiers_controller.go index e7e7bcbb43..0d6ab03f44 100644 --- a/pkg/controller/tiers/tiers_controller.go +++ b/pkg/controller/tiers/tiers_controller.go @@ -162,11 +162,12 @@ func (r *ReconcileTiers) prepareTiersConfig(ctx context.Context, reqLogger logr. common.CalicoNamespace, } if r.opts.EnterpriseCRDExists { + // The log collector (fluent-bit) runs in common.CalicoNamespace, which is + // already in the list. namespaces = append(namespaces, render.ComplianceNamespace, render.DexNamespace, render.ElasticsearchNamespace, - render.LogCollectorNamespace, render.IntrusionDetectionNamespace, kibana.Namespace, eck.OperatorNamespace, diff --git a/pkg/imports/crds/operator/operator.tigera.io_logcollectors.yaml b/pkg/imports/crds/operator/operator.tigera.io_logcollectors.yaml index f244015211..b350350559 100644 --- a/pkg/imports/crds/operator/operator.tigera.io_logcollectors.yaml +++ b/pkg/imports/crds/operator/operator.tigera.io_logcollectors.yaml @@ -18,7 +18,7 @@ spec: openAPIV3Schema: description: |- LogCollector installs the components required for Tigera flow and DNS log collection. At most one instance - of this resource is supported. It must be named "tigera-secure". When created, this installs fluentd on all nodes + of this resource is supported. It must be named "tigera-secure". When created, this installs fluent-bit on all nodes configured to collect Tigera log data and export it to Tigera's Elasticsearch cluster as well as any additionally configured destinations. properties: apiVersion: @@ -183,6 +183,255 @@ spec: - logTypes type: object type: object + calicoFluentBitDaemonSet: + description: |- + CalicoFluentBitDaemonSet configures the calico-fluent-bit DaemonSet, the + Fluent Bit replacement for the Fluentd DaemonSet. Pod-template override + semantics are unchanged from the deprecated FluentdDaemonSet field. + properties: + spec: + description: Spec is the specification of the Fluentd DaemonSet. + properties: + template: + description: + Template describes the Fluentd DaemonSet pod + that will be created. + properties: + spec: + description: Spec is the Fluentd DaemonSet's PodSpec. + properties: + containers: + description: |- + Containers is a list of Fluentd DaemonSet containers. + If specified, this overrides the specified Fluentd DaemonSet containers. + If omitted, the Fluentd DaemonSet will use its default values for its containers. + items: + description: + FluentdDaemonSetContainer is a Fluentd + DaemonSet container. + properties: + livenessProbe: + description: |- + LivenessProbe allows customization of the liveness probe timing parameters. + The probe handler is set by the operator and cannot be overridden. + properties: + failureThreshold: + description: |- + FailureThreshold is the minimum consecutive failures for the probe + to be considered failed after having succeeded. + format: int32 + type: integer + initialDelaySeconds: + description: |- + InitialDelaySeconds is the number of seconds after the container + starts before the probe is initiated. + format: int32 + type: integer + periodSeconds: + description: + PeriodSeconds is how often + (in seconds) to perform the probe. + format: int32 + type: integer + timeoutSeconds: + description: + TimeoutSeconds is the number + of seconds after which the probe times + out. + format: int32 + type: integer + type: object + name: + description: |- + Name is an enum which identifies the Fluent Bit DaemonSet container by name. + Supported values are: calico-fluent-bit + enum: + - calico-fluent-bit + type: string + readinessProbe: + description: |- + ReadinessProbe allows customization of the readiness probe timing parameters. + The probe handler is set by the operator and cannot be overridden. + properties: + failureThreshold: + description: |- + FailureThreshold is the minimum consecutive failures for the probe + to be considered failed after having succeeded. + format: int32 + type: integer + initialDelaySeconds: + description: |- + InitialDelaySeconds is the number of seconds after the container + starts before the probe is initiated. + format: int32 + type: integer + periodSeconds: + description: + PeriodSeconds is how often + (in seconds) to perform the probe. + format: int32 + type: integer + timeoutSeconds: + description: + TimeoutSeconds is the number + of seconds after which the probe times + out. + format: int32 + type: integer + type: object + resources: + description: |- + Resources allows customization of limits and requests for compute resources such as cpu and memory. + If specified, this overrides the named Fluentd DaemonSet container's resources. + If omitted, the Fluentd DaemonSet will use its default value for this container's resources. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + This field depends on the + DynamicResourceAllocation feature gate. + This field is immutable. It can only be set for containers. + items: + description: + ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + required: + - name + type: object + type: array + initContainers: + description: |- + InitContainers is a list of Fluentd DaemonSet init containers. + If specified, this overrides the specified Fluentd DaemonSet init containers. + If omitted, the Fluentd DaemonSet will use its default values for its init containers. + items: + description: + FluentdDaemonSetInitContainer is a + Fluentd DaemonSet init container. + properties: + name: + description: |- + Name is an enum which identifies the Fluent Bit DaemonSet init container by name. + Supported values are: calico-fluent-bit-tls-key-cert-provisioner + enum: + - calico-fluent-bit-tls-key-cert-provisioner + type: string + resources: + description: |- + Resources allows customization of limits and requests for compute resources such as cpu and memory. + If specified, this overrides the named Fluentd DaemonSet init container's resources. + If omitted, the Fluentd DaemonSet will use its default value for this init container's resources. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + This field depends on the + DynamicResourceAllocation feature gate. + This field is immutable. It can only be set for containers. + items: + description: + ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + required: + - name + type: object + type: array + type: object + type: object + type: object + type: object collectProcessPath: description: |- Configuration for enabling/disabling process path collection in flowlogs. @@ -445,7 +694,11 @@ spec: type: object type: object fluentdDaemonSet: - description: FluentdDaemonSet configures the Fluentd DaemonSet. + description: |- + FluentdDaemonSet configures the Fluentd DaemonSet. + Deprecated: use CalicoFluentBitDaemonSet instead. This field is retained + as an alias for one release during the Fluentd → Fluent Bit migration; + when both are set, CalicoFluentBitDaemonSet takes precedence. properties: spec: description: Spec is the specification of the Fluentd DaemonSet. @@ -501,10 +754,10 @@ spec: type: object name: description: |- - Name is an enum which identifies the Fluentd DaemonSet container by name. - Supported values are: fluentd + Name is an enum which identifies the Fluent Bit DaemonSet container by name. + Supported values are: calico-fluent-bit enum: - - fluentd + - calico-fluent-bit type: string readinessProbe: description: |- @@ -615,10 +868,10 @@ spec: properties: name: description: |- - Name is an enum which identifies the Fluentd DaemonSet init container by name. - Supported values are: tigera-fluentd-prometheus-tls-key-cert-provisioner + Name is an enum which identifies the Fluent Bit DaemonSet init container by name. + Supported values are: calico-fluent-bit-tls-key-cert-provisioner enum: - - tigera-fluentd-prometheus-tls-key-cert-provisioner + - calico-fluent-bit-tls-key-cert-provisioner type: string resources: description: |- diff --git a/pkg/render/applicationlayer/applicationlayer.go b/pkg/render/applicationlayer/applicationlayer.go index 812e85c5c8..7823fc9220 100644 --- a/pkg/render/applicationlayer/applicationlayer.go +++ b/pkg/render/applicationlayer/applicationlayer.go @@ -455,7 +455,7 @@ func (c *component) volumes() []corev1.Volume { // Needed for Coraza library - contains rule set. if c.config.PerHostWAFEnabled || c.config.SidecarInjectionEnabled { - // WAF logs need HostPath volume - logs to be consumed by fluentd. + // WAF logs need HostPath volume - logs to be consumed by fluent-bit. volumes = append(volumes, corev1.Volume{ Name: CalicoLogsVolumeName, VolumeSource: corev1.VolumeSource{ diff --git a/pkg/render/common/elasticsearch/service.go b/pkg/render/common/elasticsearch/service.go index bc2e70569b..ac88a1486a 100644 --- a/pkg/render/common/elasticsearch/service.go +++ b/pkg/render/common/elasticsearch/service.go @@ -25,16 +25,16 @@ const ( httpsFQDNEndpoint = "https://tigera-secure-es-gateway-http.%s.svc.%s:9200" ) -func LinseedEndpoint(osType rmeta.OSType, clusterDomain, namespace string, isManagedCluster bool, isFluentd bool) string { +func LinseedEndpoint(osType rmeta.OSType, clusterDomain, namespace string, isManagedCluster bool, isFluentBit bool) string { switch { // In a managed cluster, all Elasticsearch requests to Linseed are redirected via the Guardian service. // Clients using the Linseed client are automatically configured with the correct SNI for certificate validation. - // Since Fluentd doesn't use the Linseed client, we expose an external service named "tigera-linseed" that redirects to Guardian. + // Since Fluent Bit doesn't use the Linseed client, we expose an external service named "tigera-linseed" that redirects to Guardian. // The Linseed certificate is already configured to accept connections with SNI set to "tigera-linseed". - case isManagedCluster && isFluentd: + case isManagedCluster && isFluentBit: return "https://tigera-linseed" - // Non-Fluentd components in the managed cluster forward traffic to Guardian + // Non-Fluent-Bit components in the managed cluster forward traffic to Guardian case isManagedCluster && osType == rmeta.OSTypeWindows: return fmt.Sprintf("https://guardian.calico-system.svc.%s", clusterDomain) case isManagedCluster: diff --git a/pkg/render/fluentbit.go b/pkg/render/fluentbit.go new file mode 100644 index 0000000000..deee776246 --- /dev/null +++ b/pkg/render/fluentbit.go @@ -0,0 +1,1772 @@ +// Copyright (c) 2019-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render + +import ( + "crypto/x509" + "encoding/json" + "fmt" + "strings" + + "sigs.k8s.io/yaml" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + rcomponents "github.com/tigera/operator/pkg/render/common/components" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/resourcequota" + "github.com/tigera/operator/pkg/render/common/secret" + "github.com/tigera/operator/pkg/render/common/securitycontext" + "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" + "github.com/tigera/operator/pkg/tls/certificatemanagement" + "github.com/tigera/operator/pkg/tls/certkeyusage" + "github.com/tigera/operator/pkg/url" +) + +const ( + LogCollectorNamespace = "calico-system" + FluentBitFilterConfigMapName = "fluent-bit-filters" + FluentBitFilterFlowName = "flow" + FluentBitFilterDNSName = "dns" + S3FluentBitSecretName = "log-collector-s3-credentials" + S3KeyIdName = "key-id" + S3KeySecretName = "key-secret" + + // FluentBitTLSSecretName is the TLS secret used on all connections served by fluent-bit. + FluentBitTLSSecretName = "calico-fluent-bit-tls" + FluentBitMetricsService = "calico-fluent-bit-metrics" + FluentBitMetricsServiceWindows = "calico-fluent-bit-metrics-windows" + FluentBitInputService = "calico-fluent-bit-http-input" + FluentBitMetricsPortName = "fluent-bit-metrics-port" + FluentBitMetricsPort = 2020 + FluentBitInputPortName = "fluent-bit-input-port" + FluentBitInputPort = 9880 + FluentBitPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "allow-calico-fluent-bit" + configHashAnnotation = "hash.operator.tigera.io/fluent-bit-config" + s3CredentialHashAnnotation = "hash.operator.tigera.io/s3-credentials" + splunkCredentialHashAnnotation = "hash.operator.tigera.io/splunk-credentials" + eksCloudwatchLogCredentialHashAnnotation = "hash.operator.tigera.io/eks-cloudwatch-log-credentials" + fluentBitDefaultFlush = "5s" + ElasticsearchEksLogForwarderUserSecret = "tigera-eks-log-forwarder-elasticsearch-access" + EksLogForwarderSecret = "tigera-eks-log-forwarder-secret" + EksLogForwarderAwsId = "aws-id" + EksLogForwarderAwsKey = "aws-key" + SplunkFluentBitTokenSecretName = "logcollector-splunk-credentials" + SplunkFluentBitSecretTokenKey = "token" + SplunkFluentBitSecretCertificateKey = "ca.pem" + SysLogPublicCADir = "/etc/pki/tls/certs/" + SysLogPublicCertKey = "ca-bundle.crt" + SysLogPublicCAPath = SysLogPublicCADir + SysLogPublicCertKey + SyslogCAConfigMapName = "syslog-ca" + + // Constants for Linseed token volume mounting in managed clusters. + LinseedTokenVolumeName = "linseed-token" + LinseedTokenKey = "token" + LinseedTokenSubPath = "token" + LinseedTokenSecret = "%s-tigera-linseed-token" + LinseedVolumeMountPath = "/var/run/secrets/tigera.io/linseed/" + LinseedTokenPath = "/var/run/secrets/tigera.io/linseed/token" + + FluentBitConfConfigMapName = "calico-fluent-bit-conf" + EKSLogForwarderConfConfigMapName = "eks-log-forwarder-conf" + + legacyFluentdNamespace = "tigera-fluentd" + + fluentBitName = "calico-fluent-bit" + fluentBitWindowsName = "calico-fluent-bit-windows" + + FluentBitNodeName = "calico-fluent-bit" + fluentBitNodeWindowsName = "calico-fluent-bit-windows" + + EKSLogForwarderName = "eks-log-forwarder" + EKSLogForwarderTLSSecretName = "tigera-eks-log-forwarder-tls" + + PacketCaptureAPIRole = "packetcapture-api-role" + PacketCaptureAPIRoleBinding = "packetcapture-api-role-binding" +) + +var FluentBitSourceEntityRule = v3.EntityRule{ + NamespaceSelector: fmt.Sprintf("name == '%s'", LogCollectorNamespace), + Selector: networkpolicy.KubernetesAppSelector(FluentBitNodeName, fluentBitNodeWindowsName), +} + +var EKSLogForwarderEntityRule = networkpolicy.CreateSourceEntityRule(LogCollectorNamespace, EKSLogForwarderName) + +// Register secret/certs that need Server and Client Key usage +func init() { + certkeyusage.SetCertKeyUsage(FluentBitTLSSecretName, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}) + certkeyusage.SetCertKeyUsage(EKSLogForwarderTLSSecretName, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}) +} + +type FluentBitFilters struct { + Flow string + DNS string +} + +type S3Credential struct { + KeyId []byte + KeySecret []byte +} + +type SplunkCredential struct { + Token []byte +} + +func FluentBit(cfg *FluentBitConfiguration) Component { + return &fluentBitComponent{ + cfg: cfg, + probeTimeout: 10, + probePeriod: 60, + } +} + +type EksCloudwatchLogConfig struct { + AwsId []byte + AwsKey []byte + AwsRegion string + GroupName string + StreamPrefix string + FetchInterval int32 +} + +// FluentBitConfiguration contains all the config information needed to render the component. +type FluentBitConfiguration struct { + LogCollector *operatorv1.LogCollector + S3Credential *S3Credential + SplkCredential *SplunkCredential + Filters *FluentBitFilters + EKSConfig *EksCloudwatchLogConfig + PullSecrets []*corev1.Secret + Installation *operatorv1.InstallationSpec + ClusterDomain string + OSType rmeta.OSType + FluentBitKeyPair certificatemanagement.KeyPairInterface + TrustedBundle certificatemanagement.TrustedBundle + ManagedCluster bool + + // Set if running as a multi-tenant management cluster. Configures the management cluster's + // own fluent-bit daemonset. + Tenant *operatorv1.Tenant + ExternalElastic bool + + // Whether to use User provided certificate or not. + UseSyslogCertificate bool + + // EKSLogForwarderKeyPair contains the certificate presented by EKS LogForwarder when communicating with Linseed + EKSLogForwarderKeyPair certificatemanagement.KeyPairInterface + + PacketCapture *operatorv1.PacketCaptureAPI + + NonClusterHost *operatorv1.NonClusterHost + + // LicenseExpired indicates the license has expired and fluent-bit DaemonSet should be removed. + LicenseExpired bool +} + +type fluentBitComponent struct { + cfg *FluentBitConfiguration + image string + probeTimeout int32 + probePeriod int32 +} + +func (c *fluentBitComponent) ResolveImages(is *operatorv1.ImageSet) error { + reg := c.cfg.Installation.Registry + path := c.cfg.Installation.ImagePath + prefix := c.cfg.Installation.ImagePrefix + + if c.cfg.OSType == rmeta.OSTypeWindows { + var err error + c.image, err = components.GetReference(components.ComponentFluentBitWindows, reg, path, prefix, is) + return err + } + + var err error + c.image, err = components.GetReference(components.ComponentFluentBit, reg, path, prefix, is) + if err != nil { + return err + } + return err +} + +func (c *fluentBitComponent) SupportedOSType() rmeta.OSType { + return c.cfg.OSType +} + +func (c *fluentBitComponent) fluentBitName() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return fluentBitWindowsName + } + return fluentBitName +} + +func (c *fluentBitComponent) fluentBitNodeName() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return fluentBitNodeWindowsName + } + return FluentBitNodeName +} + +// Use different service names depending on the OS type ("calico-fluent-bit-metrics" +// vs "calico-fluent-bit-metrics-windows") in order to help identify which OS daemonset +// we are referring to. +func (c *fluentBitComponent) fluentBitMetricsServiceName() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return FluentBitMetricsServiceWindows + } + return FluentBitMetricsService +} + +func (c *fluentBitComponent) healthProbeHandler() corev1.ProbeHandler { + return corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/api/v1/health", + Port: intstr.FromInt(FluentBitMetricsPort), + }, + } +} + +func (c *fluentBitComponent) securityContext(privileged bool) *corev1.SecurityContext { + if c.cfg.OSType == rmeta.OSTypeWindows { + return nil + } + return securitycontext.NewRootContext(privileged) +} + +func (c *fluentBitComponent) volumeHostPath() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return "c:/TigeraCalico" + } + return "/var/log/calico" +} + +func (c *fluentBitComponent) path(path string) string { + if c.cfg.OSType == rmeta.OSTypeWindows { + // Use c: path prefix for windows. + return "c:" + path + } + // For linux just leave the path as-is. + return path +} + +// Image-layout paths. The Linux image ships fluent-bit at /usr/bin with its +// support files under /etc/fluent-bit; the Windows image (Dockerfile-windows) +// ships everything under C:\fluent-bit. +func (c *fluentBitComponent) binPath() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return "c:/fluent-bit/fluent-bit.exe" + } + return "/usr/bin/fluent-bit" +} + +func (c *fluentBitComponent) pluginsFilePath() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return "c:/fluent-bit/plugins.conf" + } + return "/etc/fluent-bit/plugins.conf" +} + +func (c *fluentBitComponent) luaScriptPath() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return "c:/fluent-bit/record_transformer.lua" + } + return "/etc/fluent-bit/record_transformer.lua" +} + +// configPath is where the container reads the rendered config: a subPath file +// mount on Linux, a directory mount on Windows (which cannot mount single +// files). +func (c *fluentBitComponent) configPath() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return "c:/etc/fluent-bit/conf/fluent-bit.yaml" + } + return "/etc/fluent-bit/fluent-bit.yaml" +} + +// fluentBitConfConfigMapName is OS-suffixed: on mixed clusters the Linux and +// Windows components each render their own config (different paths and input +// sets), and a shared name would make the two renders overwrite each other. +func (c *fluentBitComponent) fluentBitConfConfigMapName() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return FluentBitConfConfigMapName + "-windows" + } + return FluentBitConfConfigMapName +} + +func (c *fluentBitComponent) Objects() ([]client.Object, []client.Object) { + var objs, toDelete []client.Object + objs = append(objs, c.calicoSystemPolicy()) + objs = append(objs, c.metricsService()) + objs = append(objs, c.fluentBitConfigMap()) + + // allow-tigera Tier was renamed to calico-system + toDelete = append(toDelete, networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("allow-calico-fluent-bit", LogCollectorNamespace)) + + // Clean up legacy fluentd resources from the old namespace. + toDelete = append(toDelete, + &appsv1.DaemonSet{TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: legacyFluentdNamespace}}, + &appsv1.DaemonSet{TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node-windows", Namespace: legacyFluentdNamespace}}, + &appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder", Namespace: legacyFluentdNamespace}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-metrics", Namespace: legacyFluentdNamespace}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-http-input", Namespace: legacyFluentdNamespace}}, + &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: legacyFluentdNamespace}}, + &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node-windows", Namespace: legacyFluentdNamespace}}, + &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder", Namespace: legacyFluentdNamespace}}, + &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}}, + &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd-windows"}}, + &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}}, + &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd-windows"}}, + &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder"}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-metrics-windows", Namespace: legacyFluentdNamespace}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-linseed", Namespace: legacyFluentdNamespace}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd-prometheus-tls", Namespace: legacyFluentdNamespace}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: EKSLogForwarderTLSSecretName, Namespace: legacyFluentdNamespace}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: EksLogForwarderSecret, Namespace: legacyFluentdNamespace}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: S3FluentBitSecretName, Namespace: legacyFluentdNamespace}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: SplunkFluentBitTokenSecretName, Namespace: legacyFluentdNamespace}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf(LinseedTokenSecret, "fluentd-node"), Namespace: legacyFluentdNamespace}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf(LinseedTokenSecret, "eks-log-forwarder"), Namespace: legacyFluentdNamespace}}, + &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-filters", Namespace: legacyFluentdNamespace}}, + &rbacv1.Role{TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: PacketCaptureAPIRole, Namespace: legacyFluentdNamespace}}, + &rbacv1.RoleBinding{TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: PacketCaptureAPIRoleBinding, Namespace: legacyFluentdNamespace}}, + &corev1.ResourceQuota{TypeMeta: metav1.TypeMeta{Kind: "ResourceQuota", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: resourcequota.TigeraCriticalResourceQuotaName, Namespace: legacyFluentdNamespace}}, + // The namespace itself goes last so the resources above are removed + // individually first (no finalizer surprises) on clusters upgrading from + // the fluentd era. + &corev1.Namespace{TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: legacyFluentdNamespace}}, + ) + + if c.cfg.Installation.KubernetesProvider.IsGKE() { + // We do this only for GKE as other providers don't (yet?) + // automatically add resource quota that constrains whether + // components that are marked cluster or node critical + // can be scheduled. + objs = append(objs, c.fluentBitResourceQuota()) + } + if c.cfg.S3Credential != nil { + objs = append(objs, c.s3CredentialSecret()) + } + if c.cfg.SplkCredential != nil { + objs = append(objs, secret.ToRuntimeObjects(secret.CopyToNamespace(LogCollectorNamespace, c.splunkCredentialSecret()...)...)...) + } + // User filters are inlined into the rendered config (addUserFilters); the + // copy of the filters ConfigMap an earlier iteration rendered into + // calico-system is no longer used. + toDelete = append(toDelete, + &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: FluentBitFilterConfigMapName, Namespace: LogCollectorNamespace}}) + if c.cfg.EKSConfig != nil && c.cfg.OSType == rmeta.OSTypeLinux { + objs = append(objs, + c.eksLogForwarderClusterRole(), + c.eksLogForwarderClusterRoleBinding()) + + objs = append(objs, c.eksLogForwarderServiceAccount(), + c.eksLogForwarderSecret(), + c.eksConfigMap(), + c.eksLogForwarderDeployment()) + } + + // Add in the cluster role and binding. + objs = append(objs, + c.fluentBitClusterRole(), + c.fluentBitClusterRoleBinding(), + ) + if c.cfg.ManagedCluster { + objs = append(objs, c.externalLinseedService()) + objs = append(objs, c.externalLinseedRoleBinding()) + } else { + toDelete = append(toDelete, c.externalLinseedService()) + toDelete = append(toDelete, c.externalLinseedRoleBinding()) + } + + objs = append(objs, c.fluentBitServiceAccount()) + if c.cfg.PacketCapture != nil { + objs = append(objs, c.packetCaptureApiRole(), c.packetCaptureApiRoleBinding()) + } + + if c.cfg.LicenseExpired { + toDelete = append(toDelete, c.daemonset()) + } else { + objs = append(objs, c.daemonset()) + } + + if c.cfg.OSType == rmeta.OSTypeLinux { + if c.cfg.NonClusterHost != nil { + objs = append(objs, c.nonClusterHostInputService()) + } else { + // Clean up the input service when the NonClusterHost resource is + // removed; the rendered config drops the http input at the same time. + toDelete = append(toDelete, c.nonClusterHostInputService()) + } + } + + return objs, toDelete +} + +func (c *fluentBitComponent) nonClusterHostInputService() *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: FluentBitInputService, + Namespace: LogCollectorNamespace, + Labels: map[string]string{"k8s-app": c.fluentBitNodeName()}, + }, + // We do not treat this service as a headless service, as we want to ensure traffic is load-balanced. This is because: + // - We have no guarantee that the client (voltron) will perform load balancing across the returned records. The + // golang dialer implementation appears to prefer the first record returned (see dialSerial in the go SDK) + // - We have no guarantee that the DNS server will perform load-balancing or randomize the order of records returned + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": c.fluentBitNodeName()}, + Ports: []corev1.ServicePort{ + { + Name: FluentBitInputPortName, + Port: int32(FluentBitInputPort), + TargetPort: intstr.FromInt(FluentBitInputPort), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} + +func (c *fluentBitComponent) externalLinseedRoleBinding() *rbacv1.RoleBinding { + // For managed clusters, we must create a role binding to allow Linseed to manage access token secrets + // in our namespace. + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-linseed", + Namespace: LogCollectorNamespace, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: TigeraLinseedSecretsClusterRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: GuardianServiceAccountName, + Namespace: GuardianNamespace, + }, + }, + } +} + +func (c *fluentBitComponent) externalLinseedService() *corev1.Service { + // For managed clusters, we must create an external service for fluent-bit to forward requests to guardian. + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-linseed", + Namespace: LogCollectorNamespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + ExternalName: fmt.Sprintf("%s.%s.svc.%s", GuardianServiceName, GuardianNamespace, c.cfg.ClusterDomain), + }, + } +} + +func (c *fluentBitComponent) Ready() bool { + return true +} + +func (c *fluentBitComponent) fluentBitResourceQuota() *corev1.ResourceQuota { + criticalPriorityClasses := []string{NodePriorityClassName} + return resourcequota.ResourceQuotaForPriorityClassScope(resourcequota.TigeraCriticalResourceQuotaName, LogCollectorNamespace, criticalPriorityClasses) +} + +func (c *fluentBitComponent) s3CredentialSecret() *corev1.Secret { + if c.cfg.S3Credential == nil { + return nil + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: S3FluentBitSecretName, + Namespace: LogCollectorNamespace, + }, + Data: map[string][]byte{ + S3KeyIdName: c.cfg.S3Credential.KeyId, + S3KeySecretName: c.cfg.S3Credential.KeySecret, + }, + } +} + +func (c *fluentBitComponent) splunkCredentialSecret() []*corev1.Secret { + if c.cfg.SplkCredential == nil { + return nil + } + return []*corev1.Secret{ + { + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: SplunkFluentBitTokenSecretName, + Namespace: LogCollectorNamespace, + }, + Data: map[string][]byte{ + SplunkFluentBitSecretTokenKey: c.cfg.SplkCredential.Token, + }, + }, + } +} + +func (c *fluentBitComponent) fluentBitServiceAccount() *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: c.fluentBitNodeName(), Namespace: LogCollectorNamespace}, + } +} + +// packetCaptureApiRole creates a role in calico-system to allow pod/exec +// only from fluent-bit pods. Created by the operator for the PacketCapture API. +func (c *fluentBitComponent) packetCaptureApiRole() *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: PacketCaptureAPIRole, + Namespace: LogCollectorNamespace, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"pods/exec"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + }, + } +} + +// packetCaptureApiRoleBinding creates a role binding in calico-system for the PacketCapture API. +func (c *fluentBitComponent) packetCaptureApiRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: PacketCaptureAPIRoleBinding, + Namespace: LogCollectorNamespace, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: PacketCaptureAPIRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: PacketCaptureServiceAccountName, + Namespace: PacketCaptureNamespace, + }, + }, + } +} + +func (c *fluentBitComponent) daemonset() *appsv1.DaemonSet { + var terminationGracePeriod int64 = 0 + // The rationale for this setting is that while there is no need for fluent-bit to be available, we want to avoid + // potentially negative consequences of an immediate roll-out on huge clusters. + maxUnavailable := intstr.FromInt(10) + + annots := c.cfg.TrustedBundle.HashAnnotations() + + if c.cfg.FluentBitKeyPair != nil { + annots[c.cfg.FluentBitKeyPair.HashAnnotationKey()] = c.cfg.FluentBitKeyPair.HashAnnotationValue() + } + if c.cfg.S3Credential != nil { + annots[s3CredentialHashAnnotation] = rmeta.AnnotationHash(c.cfg.S3Credential) + } + if c.cfg.SplkCredential != nil { + annots[splunkCredentialHashAnnotation] = rmeta.AnnotationHash(c.cfg.SplkCredential) + } + // Most LogCollector spec changes only alter the rendered ConfigMap (and the + // config is subPath-mounted, which kubelet never live-updates), so hash the + // rendered config into the pod template to force a rollout on change. This + // also covers user filters, which are inlined into the config. + annots[configHashAnnotation] = rmeta.AnnotationHash(c.renderFluentBitConf()) + var initContainers []corev1.Container + if c.cfg.OSType == rmeta.OSTypeLinux { + initContainers = append(initContainers, corev1.Container{ + Name: "pos-migrator", + Image: c.image, + Command: []string{"/usr/bin/pos-migrator"}, + Env: []corev1.EnvVar{ + {Name: "LOG_DIRS", Value: c.logDirsCSV()}, + }, + SecurityContext: c.securityContext(false), + VolumeMounts: []corev1.VolumeMount{ + {MountPath: "/var/log/calico", Name: "var-log-calico"}, + }, + }) + } else { + // Windows fluentd also kept tail positions (.pos files under the same + // mounted log dir), so the cutover migration applies there too. The + // Windows image ships the cross-compiled migrator; env overrides point + // it at the c:-prefixed mounts. + initContainers = append(initContainers, corev1.Container{ + Name: "pos-migrator", + Image: c.image, + Command: []string{c.path("/fluent-bit/pos-migrator.exe")}, + Env: []corev1.EnvVar{ + {Name: "POS_DIR", Value: c.path("/var/log/calico")}, + {Name: "DB_DIR", Value: c.path("/var/log/calico/calico-fluent-bit")}, + {Name: "LOG_DIRS", Value: c.logDirsCSV()}, + }, + SecurityContext: c.securityContext(false), + VolumeMounts: []corev1.VolumeMount{ + {MountPath: c.path("/var/log/calico"), Name: "var-log-calico"}, + }, + }) + } + if c.cfg.FluentBitKeyPair != nil && c.cfg.FluentBitKeyPair.UseCertificateManagement() { + initContainers = append(initContainers, c.cfg.FluentBitKeyPair.InitContainer(LogCollectorNamespace, c.container().SecurityContext)) + } + + podTemplate := &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annots, + }, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{}, + Tolerations: rmeta.TolerateAll, + ImagePullSecrets: secret.GetReferenceList(c.cfg.PullSecrets), + TerminationGracePeriodSeconds: &terminationGracePeriod, + InitContainers: initContainers, + Containers: []corev1.Container{c.container()}, + Volumes: c.volumes(), + ServiceAccountName: c.fluentBitNodeName(), + }, + } + + ds := &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: c.fluentBitNodeName(), + Namespace: LogCollectorNamespace, + }, + Spec: appsv1.DaemonSetSpec{ + Template: *podTemplate, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: &maxUnavailable, + }, + }, + }, + } + if c.cfg.LogCollector != nil { + overrides := c.cfg.LogCollector.Spec.CalicoFluentBitDaemonSet + if overrides == nil { + // Deprecated alias: fluentdDaemonSet is honored for one release when + // the new field is unset, per the API contract. Container entries + // stored under the legacy "fluentd" name are translated to the + // renamed container so resource overrides keep applying. + overrides = translateLegacyFluentdOverrides(c.cfg.LogCollector.Spec.FluentdDaemonSet) //nolint:staticcheck // deliberate use of the deprecated alias field + } + if overrides != nil { + rcomponents.ApplyDaemonSetOverrides(ds, overrides) + } + } + setNodeCriticalPod(&(ds.Spec.Template)) + return ds +} + +// translateLegacyFluentdOverrides maps a deprecated fluentdDaemonSet override +// onto the renamed calico-fluent-bit pod: container/init-container entries that +// still carry fluentd-era names (stored before the CRD enum was updated) are +// renamed so ApplyDaemonSetOverrides matches them. +func translateLegacyFluentdOverrides(legacy *operatorv1.FluentdDaemonSet) *operatorv1.FluentdDaemonSet { + if legacy == nil { + return nil + } + translated := legacy.DeepCopy() + if translated.Spec == nil || translated.Spec.Template == nil || translated.Spec.Template.Spec == nil { + return translated + } + for i, container := range translated.Spec.Template.Spec.Containers { + if container.Name == "fluentd" { + translated.Spec.Template.Spec.Containers[i].Name = "calico-fluent-bit" + } + } + for i, container := range translated.Spec.Template.Spec.InitContainers { + translated.Spec.Template.Spec.InitContainers[i].Name = strings.Replace(container.Name, "tigera-fluentd-prometheus-tls", FluentBitTLSSecretName, 1) + } + return translated +} + +func (c *fluentBitComponent) container() corev1.Container { + envs := c.envvars() + volumeMounts := []corev1.VolumeMount{ + {MountPath: c.path("/var/log/calico"), Name: "var-log-calico"}, + {MountPath: c.path("/etc/fluent-bit/certs"), Name: certificatemanagement.TrustedCertConfigMapName}, + } + if c.cfg.OSType == rmeta.OSTypeWindows { + // Windows containers cannot mount a single file (no subPath file + // mounts), so mount the whole ConfigMap as a directory. The Windows + // image keeps its own files under C:\fluent-bit, so c:\etc\fluent-bit + // shadows nothing. + volumeMounts = append(volumeMounts, + corev1.VolumeMount{MountPath: c.path("/etc/fluent-bit/conf"), Name: "fluent-bit-conf", ReadOnly: true}) + } else { + // Mount only the rendered config as a single file (SubPath) so it does + // not shadow the image's /etc/fluent-bit directory, which ships + // plugins.conf, record_transformer.lua and the loaded plugins. + volumeMounts = append(volumeMounts, + corev1.VolumeMount{MountPath: "/etc/fluent-bit/fluent-bit.yaml", Name: "fluent-bit-conf", SubPath: "fluent-bit.yaml", ReadOnly: true}) + } + + volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) + + if c.cfg.FluentBitKeyPair != nil { + volumeMounts = append(volumeMounts, c.cfg.FluentBitKeyPair.VolumeMount(c.SupportedOSType())) + } + + if c.cfg.ManagedCluster { + volumeMounts = append(volumeMounts, + corev1.VolumeMount{ + Name: LinseedTokenVolumeName, + MountPath: c.path(LinseedVolumeMountPath), + }) + } + + return corev1.Container{ + Name: "calico-fluent-bit", + Image: c.image, + Command: []string{c.binPath()}, + Args: []string{"-c", c.configPath()}, + Env: envs, + // On OpenShift Fluent Bit needs privileged access to access logs on host path volume + SecurityContext: c.securityContext(c.cfg.Installation.KubernetesProvider.IsOpenShift()), + VolumeMounts: volumeMounts, + StartupProbe: c.startup(), + LivenessProbe: c.liveness(), + ReadinessProbe: c.readiness(), + Ports: []corev1.ContainerPort{{ + Name: "metrics-port", + ContainerPort: FluentBitMetricsPort, + }}, + } +} + +func (c *fluentBitComponent) metricsService() *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: c.fluentBitMetricsServiceName(), + Namespace: LogCollectorNamespace, + Labels: map[string]string{"k8s-app": c.fluentBitNodeName()}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": c.fluentBitNodeName()}, + // Important: "None" tells Kubernetes that we want a headless service with + // no kube-proxy load balancer. If we omit this then kube-proxy will render + // a huge set of iptables rules for this service since there's an instance + // on every node. + ClusterIP: "None", + Ports: []corev1.ServicePort{ + { + Name: FluentBitMetricsPortName, + Port: int32(FluentBitMetricsPort), + TargetPort: intstr.FromInt(FluentBitMetricsPort), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} + +func (c *fluentBitComponent) envvars() []corev1.EnvVar { + envs := []corev1.EnvVar{ + {Name: "NODENAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}}, + } + // Additional stores are Linux-only (the Windows pipeline ships to Linseed + // only, matching the fluentd Windows variant), so their credentials are too. + if c.cfg.LogCollector.Spec.AdditionalStores != nil && c.cfg.OSType == rmeta.OSTypeLinux { + if s3 := c.cfg.LogCollector.Spec.AdditionalStores.S3; s3 != nil { + // The standard AWS credential env vars, which fluent-bit's native s3 + // output reads via the AWS credential chain (the legacy AWS_KEY_ID / + // AWS_SECRET_KEY names were fluentd-config-only and are read by + // nothing in fluent-bit). + envs = append(envs, + corev1.EnvVar{ + Name: "AWS_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: S3FluentBitSecretName}, + Key: S3KeyIdName, + }, + }, + }, + corev1.EnvVar{ + Name: "AWS_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: S3FluentBitSecretName}, + Key: S3KeySecretName, + }, + }, + }, + ) + } + if splunk := c.cfg.LogCollector.Spec.AdditionalStores.Splunk; splunk != nil { + envs = append(envs, + corev1.EnvVar{ + Name: "SPLUNK_HEC_TOKEN", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: SplunkFluentBitTokenSecretName}, + Key: SplunkFluentBitSecretTokenKey, + }, + }, + }, + ) + } + } + return envs +} + +func (c *fluentBitComponent) fluentBitConfigMap() *corev1.ConfigMap { + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: c.fluentBitConfConfigMapName(), Namespace: LogCollectorNamespace}, + Data: map[string]string{"fluent-bit.yaml": c.renderFluentBitConf()}, + } +} + +type fluentBitConfig struct { + Service map[string]interface{} `json:"service"` + Parsers []map[string]interface{} `json:"parsers,omitempty"` + Pipeline fluentBitPipeline `json:"pipeline"` + Plugins []map[string]interface{} `json:"plugins,omitempty"` +} + +type fluentBitPipeline struct { + Inputs []map[string]interface{} `json:"inputs"` + Filters []map[string]interface{} `json:"filters,omitempty"` + Outputs []map[string]interface{} `json:"outputs"` +} + +// logInput is one tail input: the canonical tag and the file the producing +// component writes. +type logInput struct { + tag, path, parser string +} + +// linuxLogInputs are the log files the Linux daemonset tails. The paths are +// the producers' output paths, matching the authoritative defaults the fluentd +// image carried (fluentd/Dockerfile ENV *_LOG_FILE) — felix, BIRD, the +// apiserver audit policy, intrusion detection, compliance and the runtime +// security agent all keep writing to the same locations. +var linuxLogInputs = []logInput{ + {"flows", "/var/log/calico/flowlogs/flows.log", "json"}, + {"dns", "/var/log/calico/dnslogs/dns.log", "json"}, + {"l7", "/var/log/calico/l7logs/l7.log", "json"}, + {"waf", "/var/log/calico/waf/waf.log", "json"}, + {"runtime", "/var/log/calico/runtime-security/report.log", "json"}, + {"audit.tsee", "/var/log/calico/audit/tsee-audit.log", "json"}, + {"audit.kube", "/var/log/calico/audit/kube-audit.log", "json"}, + {"bird", "/var/log/calico/bird/current", "bird_regex"}, + {"bird6", "/var/log/calico/bird6/current", "bird_regex"}, + {"ids.events", "/var/log/calico/ids/events.log", "json"}, + {"compliance.reports", "/var/log/calico/compliance/compliance.*.reports.log", "json"}, + {"policy_activity", "/var/log/calico/policy/policy_activity.log", "json"}, +} + +// windowsLogInputs match the fluentd Windows variant +// (fluentd/fluent_sources.conf.windows), which tails only flows and the audit +// logs. +var windowsLogInputs = []logInput{ + {"flows", "/var/log/calico/flowlogs/flows.log", "json"}, + {"audit.tsee", "/var/log/calico/audit/tsee-audit.log", "json"}, + {"audit.kube", "/var/log/calico/audit/kube-audit.log", "json"}, +} + +func (c *fluentBitComponent) logInputs() []logInput { + if c.cfg.OSType == rmeta.OSTypeWindows { + return windowsLogInputs + } + return linuxLogInputs +} + +// logDirsCSV lists the tailed log directories (comma-separated) for the +// pos-migrator init container to pre-create: glob tail inputs (compliance) log +// a scan error on every refresh while their parent directory is missing, e.g. +// on clusters where the producing feature isn't enabled yet. Deriving the list +// from logInputs keeps a single source of truth for the tailed paths. +func (c *fluentBitComponent) logDirsCSV() string { + var dirs []string + seen := map[string]bool{} + for _, in := range c.logInputs() { + dir := c.path(in.path[:strings.LastIndex(in.path, "/")]) + if !seen[dir] { + seen[dir] = true + dirs = append(dirs, dir) + } + } + return strings.Join(dirs, ",") +} + +// linseedMatchRegex builds the match for the linseed output: every tailed tag +// except ids.events and compliance.reports — those are deliberately not +// Linseed-bound (IDS events use a different ingestion path; compliance reports +// are S3-only), and an unknown tag makes out_linseed drop the chunk with an +// error. The non_cluster_* tags are produced by the voltron-facing http input. +func (c *fluentBitComponent) linseedMatchRegex() string { + var tags []string + for _, in := range c.logInputs() { + if in.tag == "ids.events" || in.tag == "compliance.reports" { + continue + } + tags = append(tags, strings.ReplaceAll(in.tag, ".", `\.`)) + } + if c.cfg.NonClusterHost != nil && c.cfg.OSType == rmeta.OSTypeLinux { + tags = append(tags, "non_cluster_flows", "non_cluster_dns", "non_cluster_policy_activity") + } + return fmt.Sprintf("^(%s)$", strings.Join(tags, "|")) +} + +func (c *fluentBitComponent) renderFluentBitConf() string { + linseedEndpoint := relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, LinseedNamespace(c.cfg.Tenant), c.cfg.ManagedCluster, true) + caPath := c.trustedBundlePath() + keyPath := c.keyPath() + certPath := c.certPath() + tokenPath := c.path(GetLinseedTokenPath(c.cfg.ManagedCluster)) + + cfg := fluentBitConfig{ + Service: map[string]interface{}{ + "flush": 5, + "log_level": "info", + "http_server": true, + "http_port": FluentBitMetricsPort, + // Enable the /api/v1/health endpoint that the liveness/readiness/ + // startup probes hit (without this it returns 404 and pods never + // become Ready). + "health_check": true, + // Load the custom out_linseed (and in_eks) Go plugins shipped in the + // image. Without this the `linseed` output is an unknown plugin. + "plugins_file": c.pluginsFilePath(), + // Filesystem buffering under the same hostPath-backed state dir as + // the tail offset DBs, so buffered-but-unsent chunks survive pod + // restarts (fluentd buffered to disk for up to 72h). + "storage.path": c.path("/var/log/calico/calico-fluent-bit/storage"), + }, + // Parsers referenced by the tail inputs. Defined inline so the config is + // self-contained and does not depend on the image's parsers.conf. + Parsers: []map[string]interface{}{ + {"name": "json", "format": "json"}, + {"name": "bird_regex", "format": "regex", "regex": `^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})\.\d{5} bird: (?.*)`}, + }, + } + + for _, in := range c.logInputs() { + cfg.Pipeline.Inputs = append(cfg.Pipeline.Inputs, map[string]interface{}{ + "name": "tail", + "path": c.path(in.path), + "tag": in.tag, + // Persist read offsets in SQLite under /var/log/calico/calico-fluent-bit + // — the same directory the host rpm/deb package uses, and a subdir of the + // already-mounted var-log-calico volume — so the tail resumes across + // restarts instead of re-shipping from the head. The pos-migrator init + // container seeds these DBs from the legacy fluentd .pos files at cutover; + // read_from_head only applies to files with no prior offset (first install). + "db": c.path(fmt.Sprintf("/var/log/calico/calico-fluent-bit/in_tail_%s.db", in.tag)), + "parser": in.parser, + "read_from_head": true, + "storage.type": "filesystem", + }) + } + + if c.cfg.NonClusterHost != nil && c.cfg.OSType == rmeta.OSTypeLinux { + cfg.Pipeline.Inputs = append(cfg.Pipeline.Inputs, map[string]interface{}{ + "name": "http", + "listen": "0.0.0.0", + "port": FluentBitInputPort, + "tls": "on", + "tls.ca_file": caPath, + "tls.crt_file": certPath, + "tls.key_file": keyPath, + // Require a Tigera-CA-signed client certificate, like fluentd's http + // source did (client_cert_auth true) — voltron presents its internal + // client certificate on this hop. + "tls.verify_client_cert": "on", + "storage.type": "filesystem", + }) + } + + // Per-log-type transforms (host injection, flows @timestamp, audit name + // derivation, BIRD ip_version + noise drop, etc.) are implemented in the + // record_transformer.lua filter shipped in the image, keyed by tag. + cfg.Pipeline.Filters = append(cfg.Pipeline.Filters, map[string]interface{}{ + "name": "lua", + "match": "*", + "script": c.luaScriptPath(), + "call": "record_transformer", + }) + + // User-provided flow/dns filters (from the fluent-bit-filters ConfigMap) are + // inlined into the pipeline, replacing fluentd's config-include mechanism. + // Each ConfigMap key holds a YAML list of fluent-bit filter entries. + c.addUserFilters(&cfg) + + // NOTE: linseed is a Go proxy output plugin that performs its own HTTPS + // (it builds an http.Client from the CA pool + client keypair). fluent-bit + // reserves the native `tls.*` property namespace for outputs that use its + // built-in TLS layer and rejects those keys on proxy plugins + // ("linseed.0 does not support TLS"). So the CA/cert/key are passed with + // non-tls key names that the plugin reads itself; it always verifies the + // server certificate (skip_verify defaults to false). + linseedOutput := map[string]interface{}{ + "name": "linseed", + "match_regex": c.linseedMatchRegex(), + "endpoint": linseedEndpoint, + "ca_file": caPath, + "cert_file": certPath, + "key_file": keyPath, + // Retry failed chunks until they send instead of dropping them after + // the default single retry; the filesystem storage bounds what can + // accumulate during a Linseed outage. + "retry_limit": "no_limits", + "storage.total_limit_size": "500M", + } + if c.cfg.ManagedCluster { + linseedOutput["token_path"] = tokenPath + } + if c.cfg.Tenant != nil && c.cfg.ExternalElastic { + linseedOutput["tenant_id"] = c.cfg.Tenant.Spec.ID + } + cfg.Pipeline.Outputs = append(cfg.Pipeline.Outputs, linseedOutput) + + // Additional stores are Linux-only, matching the fluentd Windows variant + // (Linseed only). + if c.cfg.LogCollector.Spec.AdditionalStores != nil && c.cfg.OSType == rmeta.OSTypeLinux { + c.addS3Outputs(&cfg) + c.addSyslogOutputs(&cfg) + c.addSplunkOutputs(&cfg) + } + + out, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return fmt.Sprintf("# error rendering config: %v\n", err) + } + return string(out) +} + +// addUserFilters inlines the user-provided filter snippets into the pipeline. +// The fluent-bit-filters ConfigMap keys (flow, dns) each hold a YAML list of +// fluent-bit filter maps; entries without an explicit match are scoped to the +// key's log tag. Invalid YAML is skipped (and logged) rather than breaking the +// whole pipeline. +func (c *fluentBitComponent) addUserFilters(cfg *fluentBitConfig) { + if c.cfg.Filters == nil || c.cfg.OSType != rmeta.OSTypeLinux { + return + } + for _, uf := range []struct{ content, tag string }{ + {c.cfg.Filters.Flow, "flows"}, + {c.cfg.Filters.DNS, "dns"}, + } { + if uf.content == "" { + continue + } + var filters []map[string]interface{} + if err := yaml.Unmarshal([]byte(uf.content), &filters); err != nil { + log.Error(err, "skipping invalid user filter content", "tag", uf.tag) + continue + } + for _, f := range filters { + if _, ok := f["match"]; !ok { + if _, ok := f["match_regex"]; !ok { + f["match"] = uf.tag + } + } + cfg.Pipeline.Filters = append(cfg.Pipeline.Filters, f) + } + } +} + +func (c *fluentBitComponent) addS3Outputs(cfg *fluentBitConfig) { + s3 := c.cfg.LogCollector.Spec.AdditionalStores.S3 + if s3 == nil { + return + } + // The log types fluentd archived to S3 (fluentd/outputs/out-s3-*.conf): + // BGP, IDS events and policy activity were never S3-archived. + tags := []string{"flows", "dns", "l7", "waf", "runtime", "audit.tsee", "audit.kube", "compliance.reports"} + if s3.HostScope != nil && *s3.HostScope == operatorv1.HostScopeNonClusterOnly { + // Matches fluentd's behavior: FORWARD_NON_CLUSTER_LOGS_TO_S3 only wired + // S3 into the non-cluster flows path (ee_entrypoint.sh), not DNS or + // policy activity. The tag is the one the http input derives from + // voltron's /non-cluster-flows route. + tags = []string{"non_cluster_flows"} + } + for _, tag := range tags { + cfg.Pipeline.Outputs = append(cfg.Pipeline.Outputs, map[string]interface{}{ + "name": "s3", + "match": tag, + "bucket": s3.BucketName, + "region": s3.Region, + "s3_key_format": fmt.Sprintf("%s/%s/%%Y%%m%%d_$INDEX.gz", s3.BucketPath, tag), + "total_file_size": "10M", + "upload_timeout": fluentBitDefaultFlush, + "retry_limit": "no_limits", + "storage.total_limit_size": "500M", + }) + } +} + +func (c *fluentBitComponent) addSyslogOutputs(cfg *fluentBitConfig) { + syslog := c.cfg.LogCollector.Spec.AdditionalStores.Syslog + if syslog == nil { + return + } + proto, host, port, _ := url.ParseEndpoint(syslog.Endpoint) + mode := proto + var syslogTags []string + for _, t := range syslog.LogTypes { + switch t { + case operatorv1.SyslogLogAudit: + syslogTags = append(syslogTags, "audit.tsee", "audit.kube") + case operatorv1.SyslogLogDNS: + syslogTags = append(syslogTags, "dns") + case operatorv1.SyslogLogFlows: + syslogTags = append(syslogTags, "flows") + case operatorv1.SyslogLogIDSEvents: + syslogTags = append(syslogTags, "ids.events") + } + } + for _, tag := range syslogTags { + out := map[string]interface{}{ + "name": "syslog", + "match": tag, + "host": host, + "port": port, + "mode": mode, + "syslog_format": "rfc5424", + "syslog_hostname_key": "host", + "syslog_appname_preset": "tigera_secure", + "syslog_severity_preset": "info", + // The whole record is shipped as one JSON MSG, preserving fluentd + // remote_syslog's ` @type json` wire format: the per-output + // lua processor below packs the record into the `log` key, so other + // outputs still see the unpacked record. + "syslog_message_key": "log", + "processors": map[string]interface{}{ + "logs": []map[string]interface{}{{ + "name": "lua", + "script": c.luaScriptPath(), + "call": "syslog_pack", + }}, + }, + "retry_limit": "no_limits", + "storage.total_limit_size": "500M", + } + if syslog.Encryption == operatorv1.EncryptionTLS { + out["mode"] = "tls" + // `mode tls` only selects the framing; the tls property is what + // actually enables TLS on the upstream connection. + out["tls"] = "on" + out["tls.verify"] = "on" + if c.cfg.UseSyslogCertificate { + // The user-provided syslog CA is part of the trusted bundle + // (fluentd pointed SYSLOG_CA_FILE at the same bundle). + out["tls.ca_file"] = c.trustedBundlePath() + } + } + if syslog.PacketSize != nil { + out["syslog_maxsize"] = *syslog.PacketSize + } + cfg.Pipeline.Outputs = append(cfg.Pipeline.Outputs, out) + } +} + +func (c *fluentBitComponent) addSplunkOutputs(cfg *fluentBitConfig) { + splunk := c.cfg.LogCollector.Spec.AdditionalStores.Splunk + if splunk == nil { + return + } + proto, host, port, _ := url.ParseEndpoint(splunk.Endpoint) + // The log types fluentd forwarded to Splunk HEC + // (fluentd/outputs/out-splunk-{flow,dns,l7,audit}.conf). + for _, tag := range []string{"flows", "dns", "l7", "audit.tsee", "audit.kube"} { + out := map[string]interface{}{ + "name": "splunk", + "match": tag, + "host": host, + "port": port, + "splunk_token": "${SPLUNK_HEC_TOKEN}", + "retry_limit": "no_limits", + "storage.total_limit_size": "500M", + } + // Honor the endpoint scheme like fluentd's SPLUNK_PROTOCOL did: an + // http:// HEC endpoint stays plaintext, https:// gets verified TLS + // against the trusted bundle (which carries any private CA). + if proto != "http" { + out["tls"] = "on" + out["tls.verify"] = "on" + out["tls.ca_file"] = c.trustedBundlePath() + } + cfg.Pipeline.Outputs = append(cfg.Pipeline.Outputs, out) + } +} + +func (c *fluentBitComponent) trustedBundlePath() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return certificatemanagement.TrustedCertBundleMountPathWindows + } + return c.cfg.TrustedBundle.MountPath() +} + +func (c *fluentBitComponent) keyPath() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return fmt.Sprintf("c:/%s/%s", c.cfg.FluentBitKeyPair.GetName(), corev1.TLSPrivateKeyKey) + } + return c.cfg.FluentBitKeyPair.VolumeMountKeyFilePath() +} + +func (c *fluentBitComponent) certPath() string { + if c.cfg.OSType == rmeta.OSTypeWindows { + return fmt.Sprintf("c:/%s/%s", c.cfg.FluentBitKeyPair.GetName(), corev1.TLSCertKey) + } + return c.cfg.FluentBitKeyPair.VolumeMountCertificateFilePath() +} + +// The startup probe uses the same action as the liveness probe, but with +// a higher failure threshold and double the timeout to account for slow +// networks. +func (c *fluentBitComponent) startup() *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: c.healthProbeHandler(), + TimeoutSeconds: c.probeTimeout, + PeriodSeconds: c.probePeriod, + FailureThreshold: 10, + } +} + +func (c *fluentBitComponent) liveness() *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: c.healthProbeHandler(), + TimeoutSeconds: c.probeTimeout, + PeriodSeconds: c.probePeriod, + } +} + +func (c *fluentBitComponent) readiness() *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: c.healthProbeHandler(), + TimeoutSeconds: c.probeTimeout, + PeriodSeconds: c.probePeriod, + } +} + +func (c *fluentBitComponent) volumes() []corev1.Volume { + dirOrCreate := corev1.HostPathDirectoryOrCreate + + volumes := []corev1.Volume{ + { + Name: "var-log-calico", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: c.volumeHostPath(), + Type: &dirOrCreate, + }, + }, + }, + { + Name: "fluent-bit-conf", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: c.fluentBitConfConfigMapName(), + }, + }, + }, + }, + } + if c.cfg.FluentBitKeyPair != nil { + volumes = append(volumes, c.cfg.FluentBitKeyPair.Volume()) + } + if c.cfg.ManagedCluster { + volumes = append(volumes, + corev1.Volume{ + Name: LinseedTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + // Per-OS: the token controller provisions a secret per + // ServiceAccount (calico-fluent-bit / + // calico-fluent-bit-windows). + SecretName: fmt.Sprintf(LinseedTokenSecret, c.fluentBitNodeName()), + Items: []corev1.KeyToPath{{Key: LinseedTokenKey, Path: LinseedTokenSubPath}}, + }, + }, + }) + } + volumes = append(volumes, trustedBundleVolume(c.cfg.TrustedBundle)) + + return volumes +} + +func (c *fluentBitComponent) fluentBitClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: c.fluentBitName(), + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: c.fluentBitName(), + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: c.fluentBitNodeName(), + Namespace: LogCollectorNamespace, + }, + }, + } +} + +func (c *fluentBitComponent) fluentBitClusterRole() *rbacv1.ClusterRole { + role := &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: c.fluentBitName(), + }, + Rules: []rbacv1.PolicyRule{ + { + // Add write access to Linseed APIs. + APIGroups: []string{"linseed.tigera.io"}, + Resources: []string{ + "flowlogs", + "kube_auditlogs", + "ee_auditlogs", + "dnslogs", + "l7logs", + "events", + "bgplogs", + "waflogs", + "runtimereports", + "policyactivity", + }, + Verbs: []string{"create"}, + }, + }, + } + + if c.cfg.Installation.KubernetesProvider.IsOpenShift() { + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{securitycontextconstraints.Privileged}, + }) + } + return role +} + +func (c *fluentBitComponent) eksLogForwarderServiceAccount() *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: EKSLogForwarderName, Namespace: LogCollectorNamespace}, + } +} + +func (c *fluentBitComponent) eksLogForwarderSecret() *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EksLogForwarderSecret, + Namespace: LogCollectorNamespace, + }, + Data: map[string][]byte{ + EksLogForwarderAwsId: c.cfg.EKSConfig.AwsId, + EksLogForwarderAwsKey: c.cfg.EKSConfig.AwsKey, + }, + } +} + +// renderEKSFluentBitConf renders the fluent-bit config for the eks-log-forwarder +// Deployment: a single in_eks input (which polls CloudWatch, applies the EKS +// audit filtering itself, and resumes from the last Linseed-ingested timestamp) +// feeding the linseed output. The in_eks plugin reads its CloudWatch/Linseed +// settings from the Deployment's env vars rather than plugin properties. +func (c *fluentBitComponent) renderEKSFluentBitConf() string { + cfg := fluentBitConfig{ + Service: map[string]interface{}{ + "flush": 5, + "log_level": "info", + "http_server": true, + "http_port": FluentBitMetricsPort, + "health_check": true, + "plugins_file": c.pluginsFilePath(), + }, + } + cfg.Pipeline.Inputs = append(cfg.Pipeline.Inputs, map[string]interface{}{ + "name": "in_eks", + "tag": "audit.kube", + }) + linseedOutput := map[string]interface{}{ + "name": "linseed", + "match": "audit.kube", + "endpoint": relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, LinseedNamespace(c.cfg.Tenant), c.cfg.ManagedCluster, true), + "ca_file": c.trustedBundlePath(), + "cert_file": c.cfg.EKSLogForwarderKeyPair.VolumeMountCertificateFilePath(), + "key_file": c.cfg.EKSLogForwarderKeyPair.VolumeMountKeyFilePath(), + "retry_limit": "no_limits", + } + if c.cfg.ManagedCluster { + linseedOutput["token_path"] = c.path(GetLinseedTokenPath(c.cfg.ManagedCluster)) + } + if c.cfg.Tenant != nil && c.cfg.ExternalElastic { + linseedOutput["tenant_id"] = c.cfg.Tenant.Spec.ID + } + cfg.Pipeline.Outputs = append(cfg.Pipeline.Outputs, linseedOutput) + + out, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return fmt.Sprintf("# error rendering config: %v\n", err) + } + return string(out) +} + +func (c *fluentBitComponent) eksConfigMap() *corev1.ConfigMap { + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: EKSLogForwarderConfConfigMapName, Namespace: LogCollectorNamespace}, + Data: map[string]string{"fluent-bit.yaml": c.renderEKSFluentBitConf()}, + } +} + +func (c *fluentBitComponent) eksLogForwarderDeployment() *appsv1.Deployment { + annots := map[string]string{ + eksCloudwatchLogCredentialHashAnnotation: rmeta.AnnotationHash(c.cfg.EKSConfig), + configHashAnnotation: rmeta.AnnotationHash(c.renderEKSFluentBitConf()), + } + + envVars := []corev1.EnvVar{ + {Name: "LOG_LEVEL", Value: "info"}, + // CloudWatch config, credentials — consumed by the in_eks input plugin + // (fluent-bit/plugins/in_eks/pkg/config) and the AWS SDK credential chain. + {Name: "EKS_CLOUDWATCH_LOG_GROUP", Value: c.cfg.EKSConfig.GroupName}, + {Name: "EKS_CLOUDWATCH_LOG_STREAM_PREFIX", Value: c.cfg.EKSConfig.StreamPrefix}, + {Name: "EKS_CLOUDWATCH_POLL_INTERVAL", Value: fmt.Sprintf("%ds", c.cfg.EKSConfig.FetchInterval)}, + {Name: "AWS_REGION", Value: c.cfg.EKSConfig.AwsRegion}, + {Name: "AWS_ACCESS_KEY_ID", ValueFrom: secret.GetEnvVarSource(EksLogForwarderSecret, EksLogForwarderAwsId, false)}, + {Name: "AWS_SECRET_ACCESS_KEY", ValueFrom: secret.GetEnvVarSource(EksLogForwarderSecret, EksLogForwarderAwsKey, false)}, + // Linseed connection for the plugin's resume-point query (it asks + // Linseed for the last ingested audit timestamp on startup). + // Determine the namespace in which Linseed is running. For managed and standalone clusters, this is always the elasticsearch + // namespace. For multi-tenant management clusters, this may vary. + {Name: "LINSEED_ENDPOINT", Value: relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, LinseedNamespace(c.cfg.Tenant), c.cfg.ManagedCluster, true)}, + {Name: "LINSEED_CA_PATH", Value: c.trustedBundlePath()}, + {Name: "TLS_CRT_PATH", Value: c.cfg.EKSLogForwarderKeyPair.VolumeMountCertificateFilePath()}, + {Name: "TLS_KEY_PATH", Value: c.cfg.EKSLogForwarderKeyPair.VolumeMountKeyFilePath()}, + {Name: "LINSEED_TOKEN", Value: c.path(GetLinseedTokenPath(c.cfg.ManagedCluster))}, + } + if c.cfg.Tenant != nil && c.cfg.ExternalElastic { + envVars = append(envVars, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.Tenant.Spec.ID}) + } + + var eksLogForwarderReplicas int32 = 1 + + tolerations := c.cfg.Installation.ControlPlaneTolerations + if c.cfg.Installation.KubernetesProvider.IsGKE() { + tolerations = append(tolerations, rmeta.TolerateGKEARM64NoSchedule) + } + + d := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EKSLogForwarderName, + Namespace: LogCollectorNamespace, + Labels: map[string]string{ + "k8s-app": EKSLogForwarderName, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &eksLogForwarderReplicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k8s-app": EKSLogForwarderName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: EKSLogForwarderName, + Namespace: LogCollectorNamespace, + Labels: map[string]string{ + "k8s-app": EKSLogForwarderName, + }, + Annotations: annots, + }, + Spec: corev1.PodSpec{ + Tolerations: tolerations, + ServiceAccountName: EKSLogForwarderName, + ImagePullSecrets: secret.GetReferenceList(c.cfg.PullSecrets), + Containers: []corev1.Container{{ + Name: EKSLogForwarderName, + Image: c.image, + Command: []string{c.binPath()}, + Args: []string{"-c", c.configPath()}, + Env: envVars, + SecurityContext: c.securityContext(false), + VolumeMounts: c.eksLogForwarderVolumeMounts(), + StartupProbe: c.startup(), + LivenessProbe: c.liveness(), + ReadinessProbe: c.readiness(), + }}, + Volumes: c.eksLogForwarderVolumes(), + }, + }, + }, + } + + if c.cfg.LogCollector != nil { + if overrides := c.cfg.LogCollector.Spec.EKSLogForwarderDeployment; overrides != nil { + rcomponents.ApplyDeploymentOverrides(d, overrides) + } + } + + return d +} + +func trustedBundleVolume(bundle certificatemanagement.TrustedBundle) corev1.Volume { + volume := bundle.Volume() + // We mount the bundle under two names; the standard name and the name for the expected elastic cert. + volume.ConfigMap.Items = []corev1.KeyToPath{ + {Key: certificatemanagement.TrustedCertConfigMapKeyName, Path: certificatemanagement.TrustedCertConfigMapKeyName}, + //nolint:staticcheck // Ignore SA1019 deprecated + {Key: certificatemanagement.TrustedCertConfigMapKeyName, Path: certificatemanagement.LegacyTrustedCertConfigMapKeyName}, + {Key: certificatemanagement.TrustedCertConfigMapKeyName, Path: SplunkFluentBitSecretCertificateKey}, + {Key: certificatemanagement.RHELRootCertificateBundleName, Path: certificatemanagement.RHELRootCertificateBundleName}, + } + return volume +} + +func (c *fluentBitComponent) eksLogForwarderVolumeMounts() []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + Name: certificatemanagement.TrustedCertConfigMapName, + MountPath: c.path("/etc/fluent-bit/certs/"), + }, + // Mount only the rendered config file (SubPath) so it does not shadow + // the image's /etc/fluent-bit directory (plugins.conf etc.). + { + Name: "fluent-bit-conf", + MountPath: c.configPath(), + SubPath: "fluent-bit.yaml", + ReadOnly: true, + }, + } + volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) + if c.cfg.EKSLogForwarderKeyPair != nil { + volumeMounts = append(volumeMounts, c.cfg.EKSLogForwarderKeyPair.VolumeMount(c.SupportedOSType())) + } + + if c.cfg.ManagedCluster { + volumeMounts = append(volumeMounts, + corev1.VolumeMount{ + Name: LinseedTokenVolumeName, + MountPath: c.path(LinseedVolumeMountPath), + }) + } + return volumeMounts +} + +func (c *fluentBitComponent) eksLogForwarderVolumes() []corev1.Volume { + volumes := []corev1.Volume{ + trustedBundleVolume(c.cfg.TrustedBundle), + { + Name: "fluent-bit-conf", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: EKSLogForwarderConfConfigMapName, + }, + }, + }, + }, + } + if c.cfg.EKSLogForwarderKeyPair != nil { + volumes = append(volumes, c.cfg.EKSLogForwarderKeyPair.Volume()) + } + + if c.cfg.ManagedCluster { + volumes = append(volumes, + corev1.Volume{ + Name: LinseedTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: fmt.Sprintf(LinseedTokenSecret, EKSLogForwarderName), + Items: []corev1.KeyToPath{{Key: LinseedTokenKey, Path: LinseedTokenSubPath}}, + }, + }, + }) + } + return volumes +} + +func (c *fluentBitComponent) eksLogForwarderClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EKSLogForwarderName, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: EKSLogForwarderName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: EKSLogForwarderName, + Namespace: LogCollectorNamespace, + }, + }, + } +} + +func (c *fluentBitComponent) eksLogForwarderClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + { + // Add read access to Linseed APIs. + APIGroups: []string{"linseed.tigera.io"}, + Resources: []string{ + "auditlogs", + }, + Verbs: []string{"get"}, + }, + { + // Add write access to Linseed APIs to flush eks kube audit logs. + APIGroups: []string{"linseed.tigera.io"}, + Resources: []string{ + "kube_auditlogs", + }, + Verbs: []string{"create"}, + }, + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EKSLogForwarderName, + }, + Rules: rules, + } +} + +func (c *fluentBitComponent) calicoSystemPolicy() *v3.NetworkPolicy { + multiTenant := false + tenantNamespace := "" + if c.cfg.Tenant != nil { + multiTenant = true + tenantNamespace = c.cfg.Tenant.Namespace + } + policyHelper := networkpolicy.Helper(multiTenant, tenantNamespace) + + egressRules := []v3.Rule{} + if c.cfg.ManagedCluster { + egressRules = append(egressRules, v3.Rule{ + Action: v3.Deny, + Protocol: &networkpolicy.TCPProtocol, + Source: v3.EntityRule{}, + Destination: v3.EntityRule{ + NamespaceSelector: fmt.Sprintf("projectcalico.org/name == '%s'", GuardianNamespace), + Selector: networkpolicy.KubernetesAppSelector(GuardianServiceName), + NotPorts: networkpolicy.Ports(8080), + }, + }) + } else { + egressRules = append(egressRules, v3.Rule{ + Action: v3.Deny, + Protocol: &networkpolicy.TCPProtocol, + Source: v3.EntityRule{}, + Destination: v3.EntityRule{ + NamespaceSelector: fmt.Sprintf("projectcalico.org/name == '%s'", ElasticsearchNamespace), + Selector: networkpolicy.KubernetesAppSelector("tigera-secure-es-gateway"), + NotPorts: networkpolicy.Ports(5554), + }, + }) + egressRules = append(egressRules, v3.Rule{ + Action: v3.Deny, + Protocol: &networkpolicy.TCPProtocol, + Source: v3.EntityRule{}, + Destination: v3.EntityRule{ + NamespaceSelector: fmt.Sprintf("projectcalico.org/name == '%s'", ElasticsearchNamespace), + Selector: networkpolicy.KubernetesAppSelector("tigera-linseed"), + NotPorts: networkpolicy.Ports(8444), + }, + }) + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, c.cfg.Installation.KubernetesProvider.IsOpenShift()) + } + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + }) + + ingressRules := []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Source: networkpolicy.PrometheusSourceEntityRule, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(FluentBitMetricsPort), + }, + }, + } + + if c.cfg.NonClusterHost != nil { + ingressRules = append(ingressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Source: policyHelper.ManagerSourceEntityRule(), + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(FluentBitInputPort), + }, + }) + } + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: FluentBitPolicyName, + Namespace: LogCollectorNamespace, + }, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(FluentBitNodeName, fluentBitNodeWindowsName), + ServiceAccountSelector: "", + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: ingressRules, + Egress: egressRules, + }, + } +} diff --git a/pkg/render/fluentd_test.go b/pkg/render/fluentbit_test.go similarity index 53% rename from pkg/render/fluentd_test.go rename to pkg/render/fluentbit_test.go index a4627156fd..3604f46850 100644 --- a/pkg/render/fluentd_test.go +++ b/pkg/render/fluentbit_test.go @@ -41,23 +41,21 @@ import ( ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/render" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" - "github.com/tigera/operator/pkg/render/common/secret" + "github.com/tigera/operator/pkg/render/common/resourcequota" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/testutils" "github.com/tigera/operator/pkg/tls" - "github.com/tigera/operator/pkg/tls/certificatemanagement" "github.com/tigera/operator/test" ) -var _ = Describe("Tigera Secure Fluentd rendering tests", func() { - var cfg *render.FluentdConfiguration +var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() { + var cfg *render.FluentBitConfiguration var cli client.Client - expectedFluentdPolicyForUnmanaged := testutils.GetExpectedPolicyFromFile("testutils/expected_policies/fluentd_unmanaged.json") - expectedFluentdPolicyForUnmanagedOpenshift := testutils.GetExpectedPolicyFromFile("testutils/expected_policies/fluentd_unmanaged_ocp.json") - expectedFluentdPolicyForManaged := testutils.GetExpectedPolicyFromFile("testutils/expected_policies/fluentd_managed.json") + expectedFluentBitPolicyForUnmanaged := testutils.GetExpectedPolicyFromFile("testutils/expected_policies/fluentbit_unmanaged.json") + expectedFluentBitPolicyForUnmanagedOpenshift := testutils.GetExpectedPolicyFromFile("testutils/expected_policies/fluentbit_unmanaged_ocp.json") + expectedFluentBitPolicyForManaged := testutils.GetExpectedPolicyFromFile("testutils/expected_policies/fluentbit_managed.json") BeforeEach(func() { // Initialize a default instance to use. Each test can override this to its @@ -69,18 +67,18 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { certificateManager, err := certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) Expect(err).NotTo(HaveOccurred()) - metricsSecret, err := certificateManager.GetOrCreateKeyPair(cli, render.FluentdPrometheusTLSSecretName, common.OperatorNamespace(), []string{""}) + metricsSecret, err := certificateManager.GetOrCreateKeyPair(cli, render.FluentBitTLSSecretName, common.OperatorNamespace(), []string{""}) Expect(err).NotTo(HaveOccurred()) eksSecret, err := certificateManager.GetOrCreateKeyPair(cli, render.EKSLogForwarderTLSSecretName, common.OperatorNamespace(), []string{""}) Expect(err).NotTo(HaveOccurred()) - cfg = &render.FluentdConfiguration{ + cfg = &render.FluentBitConfiguration{ LogCollector: &operatorv1.LogCollector{}, ClusterDomain: dns.DefaultClusterDomain, OSType: rmeta.OSTypeLinux, Installation: &operatorv1.InstallationSpec{ KubernetesProvider: operatorv1.ProviderNone, }, - FluentdKeyPair: metricsSecret, + FluentBitKeyPair: metricsSecret, EKSLogForwarderKeyPair: eksSecret, TrustedBundle: certificateManager.CreateTrustedBundle(), } @@ -88,13 +86,13 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { It("should render SecurityContextConstrains properly when provider is OpenShift", func() { cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - // tigera-fluentd clusterRole should have openshift securitycontextconstraints PolicyRule - fluentdRole := rtest.GetResource(resources, "tigera-fluentd", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(fluentdRole.Rules).To(ContainElement(rbacv1.PolicyRule{ + // calico-fluent-bit clusterRole should have openshift securitycontextconstraints PolicyRule + fluentBitRole := rtest.GetResource(resources, "calico-fluent-bit", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(fluentBitRole.Rules).To(ContainElement(rbacv1.PolicyRule{ APIGroups: []string{"security.openshift.io"}, Resources: []string{"securitycontextconstraints"}, Verbs: []string{"use"}, @@ -104,14 +102,15 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { It("should render with a default configuration", func() { expectedResources := []client.Object{ - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.PacketCaptureAPIRole, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.PacketCaptureAPIRoleBinding, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, } cfg.PacketCapture = &operatorv1.PacketCaptureAPI{ @@ -121,55 +120,73 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { } // Should render the correct resources. - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() rtest.ExpectResources(resources, expectedResources) - ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + ds := rtest.GetResource(resources, "calico-fluent-bit", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Volumes[0].VolumeSource.HostPath.Path).To(Equal("/var/log/calico")) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) + + // The pos-migrator init container pre-creates the tailed log dirs so + // glob inputs (compliance) don't error while a feature's dir is absent. + initContainers := ds.Spec.Template.Spec.InitContainers + Expect(initContainers).NotTo(BeEmpty()) + Expect(initContainers[0].Name).To(Equal("pos-migrator")) + var logDirs string + for _, env := range initContainers[0].Env { + if env.Name == "LOG_DIRS" { + logDirs = env.Value + } + } + Expect(logDirs).To(ContainSubstring("/var/log/calico/compliance")) + Expect(logDirs).To(ContainSubstring("/var/log/calico/waf")) envs := ds.Spec.Template.Spec.Containers[0].Env - Expect(envs).Should(ContainElements( - corev1.EnvVar{Name: "LINSEED_ENABLED", Value: "true"}, - corev1.EnvVar{Name: "LINSEED_ENDPOINT", Value: "https://tigera-linseed.tigera-elasticsearch.svc"}, - corev1.EnvVar{Name: "LINSEED_CA_PATH", Value: "/etc/pki/tls/certs/tigera-ca-bundle.crt"}, - corev1.EnvVar{Name: "TLS_KEY_PATH", Value: "/tigera-fluentd-prometheus-tls/tls.key"}, - corev1.EnvVar{Name: "TLS_CRT_PATH", Value: "/tigera-fluentd-prometheus-tls/tls.crt"}, - corev1.EnvVar{Name: "FLUENT_UID", Value: "0"}, - corev1.EnvVar{Name: "FLOW_LOG_FILE", Value: "/var/log/calico/flowlogs/flows.log"}, - corev1.EnvVar{Name: "DNS_LOG_FILE", Value: "/var/log/calico/dnslogs/dns.log"}, - corev1.EnvVar{Name: "FLUENTD_ES_SECURE", Value: "true"}, + Expect(envs).Should(ContainElement( corev1.EnvVar{ Name: "NODENAME", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, }, }, - corev1.EnvVar{Name: "LINSEED_TOKEN", Value: "/var/run/secrets/kubernetes.io/serviceaccount/token"}, )) + // Linseed/TLS config is now in the ConfigMap, not env vars. Expect(envs).ShouldNot(ContainElements( - corev1.EnvVar{Name: "ELASTIC_INDEX_SUFFIX", Value: "clusterTestName"}, - corev1.EnvVar{Name: "ELASTIC_SCHEME", Value: "https"}, - corev1.EnvVar{Name: "ELASTIC_HOST", Value: "tigera-secure-es-gateway-http.tigera-elasticsearch.svc"}, - corev1.EnvVar{Name: "ELASTIC_PORT", Value: "9200"}, - corev1.EnvVar{Name: "ELASTIC_USER", ValueFrom: secret.GetEnvVarSource("tigera-eks-log-forwarder-elasticsearch-access", "username", false)}, - corev1.EnvVar{Name: "ELASTIC_PASSWORD", ValueFrom: secret.GetEnvVarSource("tigera-eks-log-forwarder-elasticsearch-access", "password", false)}, - corev1.EnvVar{Name: "ELASTIC_CA", Value: "/etc/pki/tls/certs/tigera-ca-bundle.crt"}, + corev1.EnvVar{Name: "LINSEED_ENABLED", Value: "true"}, + corev1.EnvVar{Name: "LINSEED_ENDPOINT", Value: "https://tigera-linseed.tigera-elasticsearch.svc"}, + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: "/calico-fluent-bit-tls/tls.key"}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: "/calico-fluent-bit-tls/tls.crt"}, )) + // Verify the ConfigMap contains the expected config. + cm := rtest.GetResource(resources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + Expect(cm.Data).To(HaveKey("fluent-bit.yaml")) + fluentBitConf := cm.Data["fluent-bit.yaml"] + Expect(fluentBitConf).To(ContainSubstring("linseed")) + Expect(fluentBitConf).To(ContainSubstring("tigera-linseed")) + container := ds.Spec.Template.Spec.Containers[0] - Expect(container.ReadinessProbe.Exec.Command).To(ConsistOf([]string{"sh", "-c", "/bin/readiness.sh"})) + Expect(container.Command).To(Equal([]string{"/usr/bin/fluent-bit"})) + Expect(container.Args).To(Equal([]string{"-c", "/etc/fluent-bit/fluent-bit.yaml"})) + + Expect(container.ReadinessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.ReadinessProbe.HTTPGet.Path).To(Equal("/api/v1/health")) + Expect(container.ReadinessProbe.HTTPGet.Port).To(Equal(intstr.FromInt(render.FluentBitMetricsPort))) Expect(container.ReadinessProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.ReadinessProbe.PeriodSeconds).To(BeEquivalentTo(60)) - Expect(container.LivenessProbe.Exec.Command).To(ConsistOf([]string{"sh", "-c", "/bin/liveness.sh"})) + Expect(container.LivenessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.LivenessProbe.HTTPGet.Path).To(Equal("/api/v1/health")) + Expect(container.LivenessProbe.HTTPGet.Port).To(Equal(intstr.FromInt(render.FluentBitMetricsPort))) Expect(container.LivenessProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.LivenessProbe.PeriodSeconds).To(BeEquivalentTo(60)) - Expect(container.StartupProbe.Exec.Command).To(ConsistOf([]string{"sh", "-c", "/bin/liveness.sh"})) + Expect(container.StartupProbe.HTTPGet).NotTo(BeNil()) + Expect(container.StartupProbe.HTTPGet.Path).To(Equal("/api/v1/health")) + Expect(container.StartupProbe.HTTPGet.Port).To(Equal(intstr.FromInt(render.FluentBitMetricsPort))) Expect(container.StartupProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.StartupProbe.PeriodSeconds).To(BeEquivalentTo(60)) Expect(container.StartupProbe.FailureThreshold).To(BeEquivalentTo(10)) @@ -213,11 +230,11 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { })) // The metrics service should have the correct configuration. - ms := rtest.GetResource(resources, render.FluentdMetricsService, render.LogCollectorNamespace, "", "v1", "Service").(*corev1.Service) + ms := rtest.GetResource(resources, render.FluentBitMetricsService, render.LogCollectorNamespace, "", "v1", "Service").(*corev1.Service) Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") }) - It("should render fluentd Daemonset with resources requests/limits", func() { + It("should render fluent-bit DaemonSet with resources requests/limits", func() { ca, _ := tls.MakeCA(rmeta.DefaultOperatorCASignerName()) cert, _, _ := ca.Config.GetPEMBytes() // create a valid pem block cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: cert} @@ -225,12 +242,12 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { certificateManager, err := certificatemanager.Create(cli, cfg.Installation, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) Expect(err).NotTo(HaveOccurred()) - metricsSecret, err := certificateManager.GetOrCreateKeyPair(cli, render.FluentdPrometheusTLSSecretName, common.OperatorNamespace(), []string{""}) + metricsSecret, err := certificateManager.GetOrCreateKeyPair(cli, render.FluentBitTLSSecretName, common.OperatorNamespace(), []string{""}) Expect(err).NotTo(HaveOccurred()) - cfg.FluentdKeyPair = metricsSecret + cfg.FluentBitKeyPair = metricsSecret - fluentdResources := corev1.ResourceRequirements{ + fluentBitResources := corev1.ResourceRequirements{ Limits: corev1.ResourceList{ "cpu": resource.MustParse("2"), "memory": resource.MustParse("300Mi"), @@ -245,17 +262,17 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { logCollectorcfg := operatorv1.LogCollector{ Spec: operatorv1.LogCollectorSpec{ - FluentdDaemonSet: &operatorv1.FluentdDaemonSet{ + CalicoFluentBitDaemonSet: &operatorv1.FluentdDaemonSet{ Spec: &operatorv1.FluentdDaemonSetSpec{ Template: &operatorv1.FluentdDaemonSetPodTemplateSpec{ Spec: &operatorv1.FluentdDaemonSetPodSpec{ InitContainers: []operatorv1.FluentdDaemonSetInitContainer{{ - Name: "tigera-fluentd-prometheus-tls-key-cert-provisioner", - Resources: &fluentdResources, + Name: "calico-fluent-bit-tls-key-cert-provisioner", + Resources: &fluentBitResources, }}, Containers: []operatorv1.FluentdDaemonSetContainer{{ - Name: "fluentd", - Resources: &fluentdResources, + Name: "calico-fluent-bit", + Resources: &fluentBitResources, }}, }, }, @@ -265,88 +282,90 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { } cfg.LogCollector = &logCollectorcfg - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() - ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + ds := rtest.GetResource(resources, "calico-fluent-bit", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) - container := test.GetContainer(ds.Spec.Template.Spec.Containers, "fluentd") + container := test.GetContainer(ds.Spec.Template.Spec.Containers, "calico-fluent-bit") Expect(container).NotTo(BeNil()) - Expect(container.Resources).To(Equal(fluentdResources)) + Expect(container.Resources).To(Equal(fluentBitResources)) - Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - initContainer := test.GetContainer(ds.Spec.Template.Spec.InitContainers, "tigera-fluentd-prometheus-tls-key-cert-provisioner") + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + initContainer := test.GetContainer(ds.Spec.Template.Spec.InitContainers, "calico-fluent-bit-tls-key-cert-provisioner") Expect(initContainer).NotTo(BeNil()) - Expect(initContainer.Resources).To(Equal(fluentdResources)) + Expect(initContainer.Resources).To(Equal(fluentBitResources)) }) It("should render with a configuration for a managed cluster", func() { expectedResources := []client.Object{ - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdPolicyName, Namespace: render.LogCollectorNamespace}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdMetricsService, Namespace: render.LogCollectorNamespace}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitPolicyName, Namespace: render.LogCollectorNamespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitMetricsService, Namespace: render.LogCollectorNamespace}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-linseed", Namespace: render.LogCollectorNamespace}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdNodeName, Namespace: render.LogCollectorNamespace}}, - &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdNodeName, Namespace: render.LogCollectorNamespace}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitNodeName, Namespace: render.LogCollectorNamespace}}, + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitNodeName, Namespace: render.LogCollectorNamespace}}, &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-linseed", Namespace: render.LogCollectorNamespace}}, } - expectedDeleteResources := []client.Object{ - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera.allow-fluentd-node", Namespace: render.LogCollectorNamespace}}, - } + expectedDeleteResources := append([]client.Object{ + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera.allow-calico-fluent-bit", Namespace: render.LogCollectorNamespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitInputService, Namespace: render.LogCollectorNamespace}}, + }, legacyFluentdDeleteResources()...) // Should render the correct resources. - managedCfg := &render.FluentdConfiguration{ - LogCollector: cfg.LogCollector, - ClusterDomain: cfg.ClusterDomain, - OSType: cfg.OSType, - Installation: cfg.Installation, - FluentdKeyPair: cfg.FluentdKeyPair, - TrustedBundle: cfg.TrustedBundle, - ManagedCluster: true, + managedCfg := &render.FluentBitConfiguration{ + LogCollector: cfg.LogCollector, + ClusterDomain: cfg.ClusterDomain, + OSType: cfg.OSType, + Installation: cfg.Installation, + FluentBitKeyPair: cfg.FluentBitKeyPair, + TrustedBundle: cfg.TrustedBundle, + ManagedCluster: true, } - component := render.Fluentd(managedCfg) + component := render.FluentBit(managedCfg) createResources, deleteResources := component.Objects() rtest.ExpectResources(createResources, expectedResources) rtest.ExpectResources(deleteResources, expectedDeleteResources) - ds := rtest.GetResource(createResources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + ds := rtest.GetResource(createResources, "calico-fluent-bit", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Volumes[0].VolumeSource.HostPath.Path).To(Equal("/var/log/calico")) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) envs := ds.Spec.Template.Spec.Containers[0].Env - Expect(envs).Should(ContainElements( - corev1.EnvVar{Name: "LINSEED_ENABLED", Value: "true"}, - corev1.EnvVar{Name: "LINSEED_ENDPOINT", Value: "https://tigera-linseed"}, - corev1.EnvVar{Name: "LINSEED_CA_PATH", Value: "/etc/pki/tls/certs/tigera-ca-bundle.crt"}, - corev1.EnvVar{Name: "TLS_KEY_PATH", Value: "/tigera-fluentd-prometheus-tls/tls.key"}, - corev1.EnvVar{Name: "TLS_CRT_PATH", Value: "/tigera-fluentd-prometheus-tls/tls.crt"}, - corev1.EnvVar{Name: "FLUENT_UID", Value: "0"}, - corev1.EnvVar{Name: "FLOW_LOG_FILE", Value: "/var/log/calico/flowlogs/flows.log"}, - corev1.EnvVar{Name: "DNS_LOG_FILE", Value: "/var/log/calico/dnslogs/dns.log"}, - corev1.EnvVar{Name: "FLUENTD_ES_SECURE", Value: "true"}, + Expect(envs).Should(ContainElement( corev1.EnvVar{ Name: "NODENAME", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, }, }, - corev1.EnvVar{Name: "LINSEED_TOKEN", Value: "/var/run/secrets/tigera.io/linseed/token"}, )) + // Verify the ConfigMap contains the linseed config for the managed cluster. + cm := rtest.GetResource(createResources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + Expect(cm.Data).To(HaveKey("fluent-bit.yaml")) + fluentBitConf := cm.Data["fluent-bit.yaml"] + Expect(fluentBitConf).To(ContainSubstring("linseed")) + Expect(fluentBitConf).To(ContainSubstring("tigera-linseed")) + container := ds.Spec.Template.Spec.Containers[0] - Expect(container.ReadinessProbe.Exec.Command).To(ConsistOf([]string{"sh", "-c", "/bin/readiness.sh"})) + Expect(container.ReadinessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.ReadinessProbe.HTTPGet.Path).To(Equal("/api/v1/health")) Expect(container.ReadinessProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.ReadinessProbe.PeriodSeconds).To(BeEquivalentTo(60)) - Expect(container.LivenessProbe.Exec.Command).To(ConsistOf([]string{"sh", "-c", "/bin/liveness.sh"})) + Expect(container.LivenessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.LivenessProbe.HTTPGet.Path).To(Equal("/api/v1/health")) Expect(container.LivenessProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.LivenessProbe.PeriodSeconds).To(BeEquivalentTo(60)) - Expect(container.StartupProbe.Exec.Command).To(ConsistOf([]string{"sh", "-c", "/bin/liveness.sh"})) + Expect(container.StartupProbe.HTTPGet).NotTo(BeNil()) + Expect(container.StartupProbe.HTTPGet.Path).To(Equal("/api/v1/health")) Expect(container.StartupProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.StartupProbe.PeriodSeconds).To(BeEquivalentTo(60)) Expect(container.StartupProbe.FailureThreshold).To(BeEquivalentTo(10)) @@ -377,27 +396,29 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { })) // The metrics service should have the correct configuration. - ms := rtest.GetResource(createResources, render.FluentdMetricsService, render.LogCollectorNamespace, "", "v1", "Service").(*corev1.Service) + ms := rtest.GetResource(createResources, render.FluentBitMetricsService, render.LogCollectorNamespace, "", "v1", "Service").(*corev1.Service) Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") }) It("should render with a configuration for a managed cluster with packet capture", func() { expectedResources := []client.Object{ - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdPolicyName, Namespace: render.LogCollectorNamespace}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdMetricsService, Namespace: render.LogCollectorNamespace}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitPolicyName, Namespace: render.LogCollectorNamespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitMetricsService, Namespace: render.LogCollectorNamespace}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-linseed", Namespace: render.LogCollectorNamespace}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdNodeName, Namespace: render.LogCollectorNamespace}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitNodeName, Namespace: render.LogCollectorNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.PacketCaptureAPIRole, Namespace: render.LogCollectorNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.PacketCaptureAPIRoleBinding, Namespace: render.LogCollectorNamespace}}, - &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdNodeName, Namespace: render.LogCollectorNamespace}}, + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitNodeName, Namespace: render.LogCollectorNamespace}}, &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-linseed", Namespace: render.LogCollectorNamespace}}, } - expectedDeleteResources := []client.Object{ - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera.allow-fluentd-node", Namespace: render.LogCollectorNamespace}}, - } + expectedDeleteResources := append([]client.Object{ + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera.allow-calico-fluent-bit", Namespace: render.LogCollectorNamespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitInputService, Namespace: render.LogCollectorNamespace}}, + }, legacyFluentdDeleteResources()...) pc := &operatorv1.PacketCaptureAPI{ ObjectMeta: metav1.ObjectMeta{ @@ -406,56 +427,49 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { } // Should render the correct resources. - managedCfg := &render.FluentdConfiguration{ - LogCollector: cfg.LogCollector, - ClusterDomain: cfg.ClusterDomain, - OSType: cfg.OSType, - Installation: cfg.Installation, - FluentdKeyPair: cfg.FluentdKeyPair, - TrustedBundle: cfg.TrustedBundle, - ManagedCluster: true, - PacketCapture: pc, + managedCfg := &render.FluentBitConfiguration{ + LogCollector: cfg.LogCollector, + ClusterDomain: cfg.ClusterDomain, + OSType: cfg.OSType, + Installation: cfg.Installation, + FluentBitKeyPair: cfg.FluentBitKeyPair, + TrustedBundle: cfg.TrustedBundle, + ManagedCluster: true, + PacketCapture: pc, } - component := render.Fluentd(managedCfg) + component := render.FluentBit(managedCfg) createResources, deleteResources := component.Objects() rtest.ExpectResources(createResources, expectedResources) rtest.ExpectResources(deleteResources, expectedDeleteResources) - ds := rtest.GetResource(createResources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + ds := rtest.GetResource(createResources, "calico-fluent-bit", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Volumes[0].VolumeSource.HostPath.Path).To(Equal("/var/log/calico")) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) envs := ds.Spec.Template.Spec.Containers[0].Env - Expect(envs).Should(ContainElements( - corev1.EnvVar{Name: "LINSEED_ENABLED", Value: "true"}, - corev1.EnvVar{Name: "LINSEED_ENDPOINT", Value: "https://tigera-linseed"}, - corev1.EnvVar{Name: "LINSEED_CA_PATH", Value: "/etc/pki/tls/certs/tigera-ca-bundle.crt"}, - corev1.EnvVar{Name: "TLS_KEY_PATH", Value: "/tigera-fluentd-prometheus-tls/tls.key"}, - corev1.EnvVar{Name: "TLS_CRT_PATH", Value: "/tigera-fluentd-prometheus-tls/tls.crt"}, - corev1.EnvVar{Name: "FLUENT_UID", Value: "0"}, - corev1.EnvVar{Name: "FLOW_LOG_FILE", Value: "/var/log/calico/flowlogs/flows.log"}, - corev1.EnvVar{Name: "DNS_LOG_FILE", Value: "/var/log/calico/dnslogs/dns.log"}, - corev1.EnvVar{Name: "FLUENTD_ES_SECURE", Value: "true"}, + Expect(envs).Should(ContainElement( corev1.EnvVar{ Name: "NODENAME", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, }, }, - corev1.EnvVar{Name: "LINSEED_TOKEN", Value: "/var/run/secrets/tigera.io/linseed/token"}, )) container := ds.Spec.Template.Spec.Containers[0] - Expect(container.ReadinessProbe.Exec.Command).To(ConsistOf([]string{"sh", "-c", "/bin/readiness.sh"})) + Expect(container.ReadinessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.ReadinessProbe.HTTPGet.Path).To(Equal("/api/v1/health")) Expect(container.ReadinessProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.ReadinessProbe.PeriodSeconds).To(BeEquivalentTo(60)) - Expect(container.LivenessProbe.Exec.Command).To(ConsistOf([]string{"sh", "-c", "/bin/liveness.sh"})) + Expect(container.LivenessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.LivenessProbe.HTTPGet.Path).To(Equal("/api/v1/health")) Expect(container.LivenessProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.LivenessProbe.PeriodSeconds).To(BeEquivalentTo(60)) - Expect(container.StartupProbe.Exec.Command).To(ConsistOf([]string{"sh", "-c", "/bin/liveness.sh"})) + Expect(container.StartupProbe.HTTPGet).NotTo(BeNil()) + Expect(container.StartupProbe.HTTPGet.Path).To(Equal("/api/v1/health")) Expect(container.StartupProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.StartupProbe.PeriodSeconds).To(BeEquivalentTo(60)) Expect(container.StartupProbe.FailureThreshold).To(BeEquivalentTo(10)) @@ -509,7 +523,7 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { })) // The metrics service should have the correct configuration. - ms := rtest.GetResource(createResources, render.FluentdMetricsService, render.LogCollectorNamespace, "", "v1", "Service").(*corev1.Service) + ms := rtest.GetResource(createResources, render.FluentBitMetricsService, render.LogCollectorNamespace, "", "v1", "Service").(*corev1.Service) Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") }) @@ -517,86 +531,77 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { cfg.Installation.KubernetesProvider = operatorv1.ProviderGKE // Should render the correct resources. - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() // Should render resource quota - Expect(rtest.GetResource(resources, "tigera-critical-pods", "tigera-fluentd", "", "v1", "ResourceQuota")).ToNot(BeNil()) + Expect(rtest.GetResource(resources, "tigera-critical-pods", "calico-system", "", "v1", "ResourceQuota")).ToNot(BeNil()) }) It("should render for Windows nodes", func() { expectedResources := []client.Object{ - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdMetricsServiceWindows, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd-windows"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd-windows"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node-windows", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node-windows", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitMetricsServiceWindows, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName + "-windows", Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit-windows"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit-windows"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit-windows", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit-windows", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, } cfg.OSType = rmeta.OSTypeWindows // Should render the correct resources. - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() rtest.ExpectResources(resources, expectedResources) - ds := rtest.GetResource(resources, "fluentd-node-windows", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + ds := rtest.GetResource(resources, "calico-fluent-bit-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Volumes[0].VolumeSource.HostPath.Path).To(Equal("c:/TigeraCalico")) envs := ds.Spec.Template.Spec.Containers[0].Env - expectedEnvs := []corev1.EnvVar{ - {Name: "LINSEED_ENABLED", Value: "true"}, - {Name: "LINSEED_ENDPOINT", Value: "https://tigera-linseed.tigera-elasticsearch.svc.cluster.local"}, - {Name: "LINSEED_CA_PATH", Value: certificatemanagement.TrustedCertBundleMountPathWindows}, - {Name: "TLS_KEY_PATH", Value: "c:/tigera-fluentd-prometheus-tls/tls.key"}, - {Name: "TLS_CRT_PATH", Value: "c:/tigera-fluentd-prometheus-tls/tls.crt"}, - {Name: "FLUENT_UID", Value: "0"}, - {Name: "FLOW_LOG_FILE", Value: "c:/var/log/calico/flowlogs/flows.log"}, - {Name: "DNS_LOG_FILE", Value: "c:/var/log/calico/dnslogs/dns.log"}, - {Name: "FLUENTD_ES_SECURE", Value: "true"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, }, - {Name: "LINSEED_TOKEN", Value: "c:/var/run/secrets/kubernetes.io/serviceaccount/token"}, - } - for _, expected := range expectedEnvs { - Expect(envs).To(ContainElement(expected)) - } - - ds = rtest.GetResource(resources, "fluentd-node-windows", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - envs = ds.Spec.Template.Spec.Containers[0].Env + })) - expectedEnvs = []corev1.EnvVar{ - {Name: "FLUENT_UID", Value: "0"}, - {Name: "FLOW_LOG_FILE", Value: "c:/var/log/calico/flowlogs/flows.log"}, - {Name: "DNS_LOG_FILE", Value: "c:/var/log/calico/dnslogs/dns.log"}, - {Name: "FLUENTD_ES_SECURE", Value: "true"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - } - for _, expected := range expectedEnvs { - Expect(envs).To(ContainElement(expected)) - } + // Verify the ConfigMap contains expected config for Windows paths. The + // Windows component renders its own OS-suffixed ConfigMap so it cannot + // fight the Linux one on mixed clusters. + cm := rtest.GetResource(resources, render.FluentBitConfConfigMapName+"-windows", render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + Expect(cm.Data).To(HaveKey("fluent-bit.yaml")) + fluentBitConf := cm.Data["fluent-bit.yaml"] + Expect(fluentBitConf).To(ContainSubstring("linseed")) + // The Windows image lays everything out under C:\fluent-bit. + Expect(fluentBitConf).To(ContainSubstring(`"plugins_file": "c:/fluent-bit/plugins.conf"`)) + Expect(fluentBitConf).To(ContainSubstring(`"script": "c:/fluent-bit/record_transformer.lua"`)) + // Windows tails only the log types the fluentd Windows variant shipped. + Expect(fluentBitConf).To(ContainSubstring("c:/var/log/calico/flowlogs/flows.log")) + Expect(fluentBitConf).NotTo(ContainSubstring("dnslogs")) + Expect(ds.Spec.Template.Spec.Containers[0].Command).To(Equal([]string{"c:/fluent-bit/fluent-bit.exe"})) + Expect(ds.Spec.Template.Spec.Containers[0].Args).To(Equal([]string{"-c", "c:/etc/fluent-bit/conf/fluent-bit.yaml"})) + // The Windows pos-migrator runs with c:-prefixed dirs. + initContainers := ds.Spec.Template.Spec.InitContainers + Expect(initContainers).To(HaveLen(1)) + Expect(initContainers[0].Command).To(Equal([]string{"c:/fluent-bit/pos-migrator.exe"})) + Expect(initContainers[0].Env).To(ContainElement(corev1.EnvVar{Name: "DB_DIR", Value: "c:/var/log/calico/calico-fluent-bit"})) container := ds.Spec.Template.Spec.Containers[0] - Expect(container.ReadinessProbe.Exec.Command).To(ConsistOf([]string{`c:\ruby\msys64\usr\bin\bash.exe`, `-lc`, `/c/bin/readiness.sh`})) + Expect(container.ReadinessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.ReadinessProbe.HTTPGet.Path).To(Equal("/api/v1/health")) Expect(container.ReadinessProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.ReadinessProbe.PeriodSeconds).To(BeEquivalentTo(60)) - Expect(container.LivenessProbe.Exec.Command).To(ConsistOf([]string{`c:\ruby\msys64\usr\bin\bash.exe`, `-lc`, `/c/bin/liveness.sh`})) + Expect(container.LivenessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.LivenessProbe.HTTPGet.Path).To(Equal("/api/v1/health")) Expect(container.LivenessProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.LivenessProbe.PeriodSeconds).To(BeEquivalentTo(60)) - Expect(container.StartupProbe.Exec.Command).To(ConsistOf([]string{`c:\ruby\msys64\usr\bin\bash.exe`, `-lc`, `/c/bin/liveness.sh`})) + Expect(container.StartupProbe.HTTPGet).NotTo(BeNil()) + Expect(container.StartupProbe.HTTPGet.Path).To(Equal("/api/v1/health")) Expect(container.StartupProbe.TimeoutSeconds).To(BeEquivalentTo(10)) Expect(container.StartupProbe.PeriodSeconds).To(BeEquivalentTo(60)) Expect(container.StartupProbe.FailureThreshold).To(BeEquivalentTo(10)) @@ -618,64 +623,65 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { } expectedResources := []client.Object{ - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "log-collector-s3-credentials", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "log-collector-s3-credentials", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, } // Should render the correct resources. - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() rtest.ExpectResources(resources, expectedResources) - ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + ds := rtest.GetResource(resources, "calico-fluent-bit", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) Expect(ds.Spec.Template.Annotations).To(HaveKey("hash.operator.tigera.io/s3-credentials")) + + // S3 credential env vars are still set for the container to consume. envs := ds.Spec.Template.Spec.Containers[0].Env + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "AWS_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "log-collector-s3-credentials"}, + Key: "key-id", + }, + }, + })) + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "AWS_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "log-collector-s3-credentials"}, + Key: "key-secret", + }, + }, + })) - expectedEnvs := []struct { - name string - val string - secretName string - secretKey string - }{ - {"S3_STORAGE", "true", "", ""}, - {"S3_BUCKET_NAME", "thebucket", "", ""}, - {"AWS_REGION", "anyplace", "", ""}, - {"S3_BUCKET_PATH", "bucketpath", "", ""}, - {"S3_FLUSH_INTERVAL", "5s", "", ""}, - {"AWS_KEY_ID", "", "log-collector-s3-credentials", "key-id"}, - {"AWS_SECRET_KEY", "", "log-collector-s3-credentials", "key-secret"}, - } - for _, expected := range expectedEnvs { - if expected.val != "" { - Expect(envs).To(ContainElement(corev1.EnvVar{Name: expected.name, Value: expected.val})) - } else { - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: expected.name, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: expected.secretName}, - Key: expected.secretKey, - }, - }, - })) - } - } + // S3 output configuration is now in the ConfigMap. + cm := rtest.GetResource(resources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + Expect(cm.Data).To(HaveKey("fluent-bit.yaml")) + fluentBitConf := cm.Data["fluent-bit.yaml"] + Expect(fluentBitConf).To(ContainSubstring("s3")) + Expect(fluentBitConf).To(ContainSubstring("thebucket")) + Expect(fluentBitConf).To(ContainSubstring("anyplace")) + Expect(fluentBitConf).To(ContainSubstring("bucketpath")) }) It("should render with Syslog configuration", func() { expectedResources := []client.Object{ - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, } var ps int32 = 180 @@ -690,53 +696,21 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { }, }, } - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() rtest.ExpectResources(resources, expectedResources) - ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + ds := rtest.GetResource(resources, "calico-fluent-bit", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(3)) - envs := ds.Spec.Template.Spec.Containers[0].Env - - expectedEnvs := []struct { - name string - val string - secretName string - secretKey string - }{ - {"SYSLOG_HOST", "1.2.3.4", "", ""}, - {"SYSLOG_PORT", "80", "", ""}, - {"SYSLOG_PROTOCOL", "tcp", "", ""}, - {"SYSLOG_FLUSH_INTERVAL", "5s", "", ""}, - {"SYSLOG_PACKET_SIZE", "180", "", ""}, - {"SYSLOG_DNS_LOG", "true", "", ""}, - {"SYSLOG_FLOW_LOG", "true", "", ""}, - {"SYSLOG_IDS_EVENT_LOG", "true", "", ""}, - } - for _, expected := range expectedEnvs { - if expected.val != "" { - Expect(envs).To(ContainElement(corev1.EnvVar{Name: expected.name, Value: expected.val})) - } else { - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: expected.name, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: expected.secretName}, - Key: expected.secretKey, - }, - }, - })) - } - } - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "SYSLOG_HOSTNAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - })) + Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(4)) + + // Syslog configuration is now in the ConfigMap. + cm := rtest.GetResource(resources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + Expect(cm.Data).To(HaveKey("fluent-bit.yaml")) + fluentBitConf := cm.Data["fluent-bit.yaml"] + Expect(fluentBitConf).To(ContainSubstring("syslog")) + Expect(fluentBitConf).To(ContainSubstring("1.2.3.4")) + Expect(fluentBitConf).To(ContainSubstring("80")) }) It("should render with Syslog configuration with TLS and user's corporate CA", func() { @@ -754,12 +728,12 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { }, }, } - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() - ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + ds := rtest.GetResource(resources, "calico-fluent-bit", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(4)) var volnames []string for _, vol := range ds.Spec.Template.Spec.Volumes { @@ -767,21 +741,12 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { } Expect(volnames).To(ContainElement("tigera-ca-bundle")) - envs := ds.Spec.Template.Spec.Containers[0].Env - - Expect(envs).To(ContainElements([]corev1.EnvVar{ - {Name: "SYSLOG_HOST", Value: "1.2.3.4", ValueFrom: nil}, - {Name: "SYSLOG_PORT", Value: "80", ValueFrom: nil}, - {Name: "SYSLOG_PROTOCOL", Value: "tcp", ValueFrom: nil}, - {Name: "SYSLOG_FLUSH_INTERVAL", Value: "5s", ValueFrom: nil}, - {Name: "SYSLOG_PACKET_SIZE", Value: "180", ValueFrom: nil}, - {Name: "SYSLOG_DNS_LOG", Value: "true", ValueFrom: nil}, - {Name: "SYSLOG_FLOW_LOG", Value: "true", ValueFrom: nil}, - {Name: "SYSLOG_IDS_EVENT_LOG", Value: "true", ValueFrom: nil}, - {Name: "SYSLOG_TLS", Value: "true", ValueFrom: nil}, - {Name: "SYSLOG_VERIFY_MODE", Value: "1", ValueFrom: nil}, - {Name: "SYSLOG_CA_FILE", Value: cfg.TrustedBundle.MountPath(), ValueFrom: nil}, - })) + // Syslog TLS configuration is in the ConfigMap. + cm := rtest.GetResource(resources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + fluentBitConf := cm.Data["fluent-bit.yaml"] + Expect(fluentBitConf).To(ContainSubstring("syslog")) + Expect(fluentBitConf).To(ContainSubstring("1.2.3.4")) + Expect(fluentBitConf).To(ContainSubstring("tls")) }) It("should render with Syslog configuration with TLS and Internet CA", func() { @@ -799,28 +764,19 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { }, }, } - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() - ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + ds := rtest.GetResource(resources, "calico-fluent-bit", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(3)) - - envs := ds.Spec.Template.Spec.Containers[0].Env - - Expect(envs).To(ContainElements([]corev1.EnvVar{ - {Name: "SYSLOG_HOST", Value: "1.2.3.4", ValueFrom: nil}, - {Name: "SYSLOG_PORT", Value: "80", ValueFrom: nil}, - {Name: "SYSLOG_PROTOCOL", Value: "tcp", ValueFrom: nil}, - {Name: "SYSLOG_FLUSH_INTERVAL", Value: "5s", ValueFrom: nil}, - {Name: "SYSLOG_PACKET_SIZE", Value: "180", ValueFrom: nil}, - {Name: "SYSLOG_DNS_LOG", Value: "true", ValueFrom: nil}, - {Name: "SYSLOG_FLOW_LOG", Value: "true", ValueFrom: nil}, - {Name: "SYSLOG_IDS_EVENT_LOG", Value: "true", ValueFrom: nil}, - {Name: "SYSLOG_TLS", Value: "true", ValueFrom: nil}, - {Name: "SYSLOG_VERIFY_MODE", Value: "1", ValueFrom: nil}, - {Name: "SYSLOG_CA_FILE", Value: render.SysLogPublicCAPath, ValueFrom: nil}, - })) + Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(4)) + + // Syslog TLS configuration is in the ConfigMap. + cm := rtest.GetResource(resources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + fluentBitConf := cm.Data["fluent-bit.yaml"] + Expect(fluentBitConf).To(ContainSubstring("syslog")) + Expect(fluentBitConf).To(ContainSubstring("1.2.3.4")) + Expect(fluentBitConf).To(ContainSubstring("tls")) }) It("should render with splunk configuration", func() { @@ -834,91 +790,83 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { } expectedResources := []client.Object{ - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "logcollector-splunk-credentials", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "logcollector-splunk-credentials", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, } // Should render the correct resources. - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() rtest.ExpectResources(resources, expectedResources) - ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + ds := rtest.GetResource(resources, "calico-fluent-bit", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(4)) + // Splunk HEC token credential env var is still set. envs := ds.Spec.Template.Spec.Containers[0].Env + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "SPLUNK_HEC_TOKEN", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "logcollector-splunk-credentials"}, + Key: "token", + }, + }, + })) - expectedEnvs := []struct { - name string - val string - secretName string - secretKey string - }{ - {"SPLUNK_FLOW_LOG", "true", "", ""}, - {"SPLUNK_AUDIT_LOG", "true", "", ""}, - {"SPLUNK_DNS_LOG", "true", "", ""}, - {"SPLUNK_HEC_HOST", "1.2.3.4", "", ""}, - {"SPLUNK_HEC_PORT", "8088", "", ""}, - {"SPLUNK_PROTOCOL", "https", "", ""}, - {"SPLUNK_FLUSH_INTERVAL", "5s", "", ""}, - {"SPLUNK_HEC_TOKEN", "", "logcollector-splunk-credentials", "token"}, - } - for _, expected := range expectedEnvs { - if expected.val != "" { - Expect(envs).To(ContainElement(corev1.EnvVar{Name: expected.name, Value: expected.val})) - } else { - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: expected.name, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: expected.secretName}, - Key: expected.secretKey, - }, - }, - })) - } - } + // Splunk output configuration is now in the ConfigMap. + cm := rtest.GetResource(resources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + Expect(cm.Data).To(HaveKey("fluent-bit.yaml")) + fluentBitConf := cm.Data["fluent-bit.yaml"] + Expect(fluentBitConf).To(ContainSubstring("splunk")) + Expect(fluentBitConf).To(ContainSubstring("1.2.3.4")) + Expect(fluentBitConf).To(ContainSubstring("8088")) }) It("should render with filter", func() { - cfg.Filters = &render.FluentdFilters{ - Flow: "flow-filter", + cfg.Filters = &render.FluentBitFilters{ + Flow: "- name: grep\n exclude: dest_namespace noisy\n", } expectedResources := []client.Object{ - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-filters", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, } // Should render the correct resources. - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() rtest.ExpectResources(resources, expectedResources) - ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + // User filters are inlined into the rendered config, scoped to the log + // type's tag, and roll the pods via the config hash annotation. + cm := rtest.GetResource(resources, render.FluentBitConfConfigMapName, "calico-system", "", "v1", "ConfigMap").(*corev1.ConfigMap) + conf := cm.Data["fluent-bit.yaml"] + Expect(conf).To(ContainSubstring(`"name": "grep"`)) + Expect(conf).To(ContainSubstring(`"exclude": "dest_namespace noisy"`)) + Expect(conf).To(ContainSubstring(`"match": "flows"`)) + + ds := rtest.GetResource(resources, "calico-fluent-bit", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(ds.Spec.Template.Annotations).To(HaveKey("hash.operator.tigera.io/fluentd-filters")) - envs := ds.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{Name: "FLUENTD_FLOW_FILTERS", Value: "true"})) - Expect(envs).ToNot(ContainElement(corev1.EnvVar{Name: "FLUENTD_DNS_FILTERS", Value: "true"})) + Expect(ds.Spec.Template.Annotations).To(HaveKey("hash.operator.tigera.io/fluent-bit-config")) }) It("should render with EKS Cloudwatch Log", func() { expectedResources := getExpectedResourcesForEKS(false) cfg.EKSConfig = setupEKSCloudwatchLogConfig() - cfg.ESClusterConfig = relasticsearch.NewClusterConfig("clusterTestName", 1, 1, 1) t := corev1.Toleration{ Key: "foo", Operator: corev1.TolerationOpEqual, @@ -928,40 +876,28 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { KubernetesProvider: operatorv1.ProviderEKS, ControlPlaneTolerations: []corev1.Toleration{t}, } - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() Expect(len(resources)).To(Equal(len(expectedResources))) // Should render the correct resources. rtest.ExpectResources(resources, expectedResources) - deploy := rtest.GetResource(resources, "eks-log-forwarder", "tigera-fluentd", "apps", "v1", "Deployment").(*appsv1.Deployment) + deploy := rtest.GetResource(resources, "eks-log-forwarder", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + // The fluentd-era startup init container is gone: the in_eks input + // plugin resolves its own resume point from Linseed. + Expect(deploy.Spec.Template.Spec.InitContainers).To(BeEmpty()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) Expect(deploy.Spec.Template.Annotations).To(HaveKey("hash.operator.tigera.io/eks-cloudwatch-log-credentials")) + Expect(deploy.Spec.Template.Annotations).To(HaveKey("hash.operator.tigera.io/fluent-bit-config")) Expect(deploy.Spec.Template.Spec.Tolerations).To(ContainElement(t)) - Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + + Expect(deploy.Spec.Template.Spec.Containers[0].Command).To(Equal([]string{"/usr/bin/fluent-bit"})) + Expect(deploy.Spec.Template.Spec.Containers[0].Args).To(Equal([]string{"-c", "/etc/fluent-bit/fluent-bit.yaml"})) envs := deploy.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{Name: "K8S_PLATFORM", Value: "eks"})) Expect(envs).To(ContainElement(corev1.EnvVar{Name: "AWS_REGION", Value: cfg.EKSConfig.AwsRegion})) - Expect(*deploy.Spec.Template.Spec.InitContainers[0].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) - Expect(*deploy.Spec.Template.Spec.InitContainers[0].SecurityContext.Privileged).To(BeFalse()) - Expect(*deploy.Spec.Template.Spec.InitContainers[0].SecurityContext.RunAsGroup).To(BeEquivalentTo(0)) - Expect(*deploy.Spec.Template.Spec.InitContainers[0].SecurityContext.RunAsNonRoot).To(BeFalse()) - Expect(*deploy.Spec.Template.Spec.InitContainers[0].SecurityContext.RunAsUser).To(BeEquivalentTo(0)) - Expect(deploy.Spec.Template.Spec.InitContainers[0].SecurityContext.Capabilities).To(Equal( - &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - )) - Expect(deploy.Spec.Template.Spec.InitContainers[0].SecurityContext.SeccompProfile).To(Equal( - &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - })) - Expect(*deploy.Spec.Template.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) Expect(*deploy.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeFalse()) Expect(*deploy.Spec.Template.Spec.Containers[0].SecurityContext.RunAsGroup).To(BeEquivalentTo(0)) @@ -979,13 +915,9 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { expectedEnvVars := []corev1.EnvVar{ {Name: "LOG_LEVEL", Value: "info", ValueFrom: nil}, - {Name: "FLUENT_UID", Value: "0", ValueFrom: nil}, - {Name: "MANAGED_K8S", Value: "true", ValueFrom: nil}, - {Name: "K8S_PLATFORM", Value: "eks", ValueFrom: nil}, - {Name: "FLUENTD_ES_SECURE", Value: "true"}, {Name: "EKS_CLOUDWATCH_LOG_GROUP", Value: "dummy-eks-cluster-cloudwatch-log-group"}, {Name: "EKS_CLOUDWATCH_LOG_STREAM_PREFIX", Value: ""}, - {Name: "EKS_CLOUDWATCH_LOG_FETCH_INTERVAL", Value: "900"}, + {Name: "EKS_CLOUDWATCH_POLL_INTERVAL", Value: "900s"}, {Name: "AWS_REGION", Value: "us-west-1", ValueFrom: nil}, { Name: "AWS_ACCESS_KEY_ID", @@ -1010,7 +942,6 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { }, }, }, - {Name: "LINSEED_ENABLED", Value: "true"}, {Name: "LINSEED_ENDPOINT", Value: "https://tigera-linseed.tigera-elasticsearch.svc"}, {Name: "LINSEED_CA_PATH", Value: "/etc/pki/tls/certs/tigera-ca-bundle.crt"}, {Name: "TLS_CRT_PATH", Value: "/tigera-eks-log-forwarder-tls/tls.crt"}, @@ -1023,12 +954,11 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { It("should render EKS Cloudwatch Log toleration on GKE", func() { cfg.EKSConfig = setupEKSCloudwatchLogConfig() - cfg.ESClusterConfig = relasticsearch.NewClusterConfig("clusterTestName", 1, 1, 1) cfg.Installation.KubernetesProvider = operatorv1.ProviderGKE - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() - deploy := rtest.GetResource(resources, "eks-log-forwarder", "tigera-fluentd", "apps", "v1", "Deployment").(*appsv1.Deployment) + deploy := rtest.GetResource(resources, "eks-log-forwarder", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(deploy).NotTo(BeNil()) Expect(deploy.Spec.Template.Spec.Tolerations).To(ContainElements(corev1.Toleration{ Key: "kubernetes.io/arch", @@ -1040,7 +970,6 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { It("should render with EKS Cloudwatch Log with resources", func() { cfg.EKSConfig = setupEKSCloudwatchLogConfig() - cfg.ESClusterConfig = relasticsearch.NewClusterConfig("clusterTestName", 1, 1, 1) cfg.Installation = &operatorv1.InstallationSpec{ KubernetesProvider: operatorv1.ProviderEKS, } @@ -1076,24 +1005,21 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { } cfg.LogCollector = &logCollectorcfg - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() - deploy := rtest.GetResource(resources, "eks-log-forwarder", "tigera-fluentd", "apps", "v1", "Deployment").(*appsv1.Deployment) + deploy := rtest.GetResource(resources, "eks-log-forwarder", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) container := test.GetContainer(deploy.Spec.Template.Spec.Containers, "eks-log-forwarder") Expect(container).NotTo(BeNil()) Expect(container.Resources).To(Equal(eksResources)) - initContainer := test.GetContainer(deploy.Spec.Template.Spec.InitContainers, "eks-log-forwarder-startup") - Expect(initContainer).NotTo(BeNil()) - Expect(initContainer.Resources).To(Equal(corev1.ResourceRequirements{})) + Expect(deploy.Spec.Template.Spec.InitContainers).To(BeEmpty()) }) It("should render with EKS Cloudwatch Log with multi tenant envvars", func() { expectedResources := getExpectedResourcesForEKS(false) cfg.EKSConfig = setupEKSCloudwatchLogConfig() - cfg.ESClusterConfig = relasticsearch.NewClusterConfig("clusterTestName", 1, 1, 1) t := corev1.Toleration{ Key: "foo", Operator: corev1.TolerationOpEqual, @@ -1112,14 +1038,14 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { tenant.Spec.ID = "test-tenant-id" cfg.Tenant = tenant - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() Expect(len(resources)).To(Equal(len(expectedResources))) // Should render the correct resources. rtest.ExpectResources(resources, expectedResources) - deploy := rtest.GetResource(resources, "eks-log-forwarder", "tigera-fluentd", "apps", "v1", "Deployment").(*appsv1.Deployment) + deploy := rtest.GetResource(resources, "eks-log-forwarder", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) envs := deploy.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElement(corev1.EnvVar{Name: "LINSEED_ENDPOINT", Value: "https://tigera-linseed.tenant-namespace.svc"})) Expect(envs).To(ContainElement(corev1.EnvVar{Name: "TENANT_ID", Value: "test-tenant-id"})) @@ -1133,7 +1059,6 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-linseed", Namespace: render.LogCollectorNamespace}}) cfg.EKSConfig = setupEKSCloudwatchLogConfig() - cfg.ESClusterConfig = relasticsearch.NewClusterConfig("clusterTestName", 1, 1, 1) t := corev1.Toleration{ Key: "foo", Operator: corev1.TolerationOpEqual, @@ -1144,13 +1069,13 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { ControlPlaneTolerations: []corev1.Toleration{t}, } cfg.ManagedCluster = true - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() Expect(len(resources)).To(Equal(len(expectedResources))) rtest.ExpectResources(resources, expectedResources) - deploy := rtest.GetResource(resources, "eks-log-forwarder", "tigera-fluentd", "apps", "v1", "Deployment").(*appsv1.Deployment) + deploy := rtest.GetResource(resources, "eks-log-forwarder", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) envs := deploy.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElement(corev1.EnvVar{Name: "LINSEED_TOKEN", Value: "/var/run/secrets/tigera.io/linseed/token"})) @@ -1159,22 +1084,21 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { }) DescribeTable("should render with a valid configuration for non-cluster host and forwarding enabled", - func(destination render.ForwardingDestination) { + func(destination string) { additionalStoreSpecAllHosts := additionalStoreSpecForDestinationAndScope(destination, operatorv1.HostScopeAll) additionalStoreSpecNonClusterHosts := additionalStoreSpecForDestinationAndScope(destination, operatorv1.HostScopeNonClusterOnly) - clusterLogEnvVarName := "FORWARD_CLUSTER_LOGS_TO_" + strings.ToUpper(string(destination)) - nonClusterLogEnvVarName := "FORWARD_NON_CLUSTER_LOGS_TO_" + strings.ToUpper(string(destination)) By("establishing the base case with no non-cluster hosts or forwarding options") expectedResources := []client.Object{ - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitMetricsService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.PacketCaptureAPIRole, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.PacketCaptureAPIRoleBinding, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, } cfg.PacketCapture = &operatorv1.PacketCaptureAPI{ ObjectMeta: metav1.ObjectMeta{ @@ -1182,12 +1106,14 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { }, } - resources, _ := render.Fluentd(cfg).Objects() + resources, _ := render.FluentBit(cfg).Objects() rtest.ExpectResources(resources, expectedResources) - ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) - envs := ds.Spec.Template.Spec.Containers[0].Env - Expect(forwardingEnvVarCount(envs)).To(Equal(0)) + + // Base case: no additional store outputs in ConfigMap besides linseed. + cm := rtest.GetResource(resources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + baseConf := cm.Data["fluent-bit.yaml"] + Expect(baseConf).To(ContainSubstring("linseed")) + Expect(baseConf).NotTo(ContainSubstring(strings.ToLower(destination))) By("enabling non-cluster hosts and forwarding from all hosts") cfg.NonClusterHost = &operatorv1.NonClusterHost{ @@ -1199,67 +1125,47 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { }, } cfg.LogCollector.Spec.AdditionalStores = additionalStoreSpecAllHosts - expectedResources = append(expectedResources, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdInputService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}) + expectedResources = append(expectedResources, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitInputService, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}) // Should render the correct resources. - resources, _ = render.Fluentd(cfg).Objects() + resources, _ = render.FluentBit(cfg).Objects() rtest.ExpectResources(resources, expectedResources) // Service is rendered as expected. - ms := rtest.GetResource(resources, render.FluentdInputService, render.LogCollectorNamespace, "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.Selector).To(Equal(map[string]string{"k8s-app": render.FluentdNodeName})) + ms := rtest.GetResource(resources, render.FluentBitInputService, render.LogCollectorNamespace, "", "v1", "Service").(*corev1.Service) + Expect(ms.Spec.Selector).To(Equal(map[string]string{"k8s-app": render.FluentBitNodeName})) Expect(ms.Spec.Ports).To(HaveLen(1)) - Expect(ms.Spec.Ports[0].Port).To(BeNumerically("==", render.FluentdInputPort)) - Expect(ms.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt32(render.FluentdInputPort))) + Expect(ms.Spec.Ports[0].Port).To(BeNumerically("==", render.FluentBitInputPort)) + Expect(ms.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt32(render.FluentBitInputPort))) Expect(ms.Spec.Ports[0].Protocol).To(Equal(corev1.ProtocolTCP)) - // Should contain the env vars with all forwarding enabled. - rtest.ExpectResources(resources, expectedResources) - ds = rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) - envs = ds.Spec.Template.Spec.Containers[0].Env - Expect(forwardingEnvVarCount(envs)).To(Equal(2)) - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: clusterLogEnvVarName, - Value: "true", - })) - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: nonClusterLogEnvVarName, - Value: "true", - })) + // ConfigMap should contain the destination output section (all hosts scope). + cm = rtest.GetResource(resources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + allHostsConf := cm.Data["fluent-bit.yaml"] + Expect(allHostsConf).To(ContainSubstring(strings.ToLower(destination))) By("enabling forwarding of only non-cluster logs") cfg.LogCollector.Spec.AdditionalStores = additionalStoreSpecNonClusterHosts - resources, _ = render.Fluentd(cfg).Objects() + resources, _ = render.FluentBit(cfg).Objects() rtest.ExpectResources(resources, expectedResources) - // Should contain the env vars with only non-cluster forwarding enabled. - rtest.ExpectResources(resources, expectedResources) - ds = rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) - envs = ds.Spec.Template.Spec.Containers[0].Env - Expect(forwardingEnvVarCount(envs)).To(Equal(2)) - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: clusterLogEnvVarName, - Value: "false", - })) - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: nonClusterLogEnvVarName, - Value: "true", - })) + // ConfigMap should still contain the destination output section (non-cluster scope). + cm = rtest.GetResource(resources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + nonClusterConf := cm.Data["fluent-bit.yaml"] + Expect(nonClusterConf).To(ContainSubstring(strings.ToLower(destination))) }, - Entry("S3", render.ForwardingDestinationS3), - Entry("Syslog", render.ForwardingDestinationSyslog), - Entry("Splunk", render.ForwardingDestinationSplunk)) + Entry("S3", "S3"), + Entry("Syslog", "Syslog"), + Entry("Splunk", "Splunk")) Context("calico-system rendering", func() { - policyName := types.NamespacedName{Name: "calico-system.allow-fluentd-node", Namespace: "tigera-fluentd"} + policyName := types.NamespacedName{Name: "calico-system.allow-calico-fluent-bit", Namespace: "calico-system"} getExpectedPolicy := func(scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { if scenario.ManagedCluster { - return expectedFluentdPolicyForManaged + return expectedFluentBitPolicyForManaged } else { - return testutils.SelectPolicyByProvider(scenario, expectedFluentdPolicyForUnmanaged, expectedFluentdPolicyForUnmanagedOpenshift) + return testutils.SelectPolicyByProvider(scenario, expectedFluentBitPolicyForUnmanaged, expectedFluentBitPolicyForUnmanagedOpenshift) } } @@ -1272,7 +1178,7 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { } cfg.ManagedCluster = scenario.ManagedCluster - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) resources, _ := component.Objects() policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) @@ -1286,7 +1192,7 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { ) It("should render calico-system policy for the non-cluster-host scenario", func() { - resourcesWithoutNonClusterHosts, _ := render.Fluentd(cfg).Objects() + resourcesWithoutNonClusterHosts, _ := render.FluentBit(cfg).Objects() policyWithoutNonClusterHosts := testutils.GetCalicoSystemPolicyFromResources(policyName, resourcesWithoutNonClusterHosts) cfg.NonClusterHost = &operatorv1.NonClusterHost{ ObjectMeta: metav1.ObjectMeta{ @@ -1296,10 +1202,10 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { Endpoint: "https://1.2.3.4:5678", }, } - resourcesWithNonClusterHosts, _ := render.Fluentd(cfg).Objects() + resourcesWithNonClusterHosts, _ := render.FluentBit(cfg).Objects() policyWithNonClusterHosts := testutils.GetCalicoSystemPolicyFromResources(policyName, resourcesWithNonClusterHosts) - // Validate that we have a single ingress rule added for the fluentd service. + // Validate that we have a single ingress rule added for the fluent-bit service. Expect(policyWithoutNonClusterHosts.Spec.Egress).To(Equal(policyWithNonClusterHosts.Spec.Egress)) Expect(len(policyWithoutNonClusterHosts.Spec.Ingress)).To(Equal(len(policyWithNonClusterHosts.Spec.Ingress) - 1)) Expect(len(policyWithNonClusterHosts.Spec.Ingress)).To(Equal(2)) @@ -1311,7 +1217,7 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { NamespaceSelector: fmt.Sprintf("projectcalico.org/name == '%s'", render.ManagerNamespace), }, Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(render.FluentdInputPort), + Ports: networkpolicy.Ports(render.FluentBitInputPort), }, })) }) @@ -1319,7 +1225,7 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { It("should move DaemonSet to toDelete when LicenseExpired is true", func() { cfg.LicenseExpired = true - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) Expect(component.ResolveImages(nil)).To(BeNil()) toCreate, toDelete := component.Objects() @@ -1333,28 +1239,28 @@ var _ = Describe("Tigera Secure Fluentd rendering tests", func() { // DaemonSet should be in toDelete. found := false for _, obj := range toDelete { - if ds, ok := obj.(*appsv1.DaemonSet); ok && ds.Name == "fluentd-node" { + if ds, ok := obj.(*appsv1.DaemonSet); ok && ds.Name == "calico-fluent-bit" { found = true break } } - Expect(found).To(BeTrue(), "Expected fluentd-node DaemonSet to be in toDelete") + Expect(found).To(BeTrue(), "Expected fluent-bit-node DaemonSet to be in toDelete") }) It("should include DaemonSet in toCreate when LicenseExpired is false", func() { cfg.LicenseExpired = false - component := render.Fluentd(cfg) + component := render.FluentBit(cfg) Expect(component.ResolveImages(nil)).To(BeNil()) toCreate, _ := component.Objects() found := false for _, obj := range toCreate { - if ds, ok := obj.(*appsv1.DaemonSet); ok && ds.Name == "fluentd-node" { + if ds, ok := obj.(*appsv1.DaemonSet); ok && ds.Name == "calico-fluent-bit" { found = true break } } - Expect(found).To(BeTrue(), "Expected fluentd-node DaemonSet to be in toCreate") + Expect(found).To(BeTrue(), "Expected fluent-bit-node DaemonSet to be in toCreate") }) }) @@ -1372,20 +1278,22 @@ func setupEKSCloudwatchLogConfig() *render.EksCloudwatchLogConfig { func getExpectedResourcesForEKS(isManagedcluster bool) []client.Object { expectedResources := []client.Object{ &v3.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{Name: render.FluentdPolicyName, Namespace: render.LogCollectorNamespace}, + ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitPolicyName, Namespace: render.LogCollectorNamespace}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, }, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentdMetricsService, Namespace: render.LogCollectorNamespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitMetricsService, Namespace: render.LogCollectorNamespace}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitConfConfigMapName, Namespace: render.LogCollectorNamespace}}, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder"}}, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder", Namespace: "tigera-fluentd"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: "tigera-fluentd"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder", Namespace: "calico-system"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: "calico-system"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: render.EKSLogForwarderConfConfigMapName, Namespace: render.LogCollectorNamespace}}, &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder", Namespace: render.LogCollectorNamespace}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "tigera-eks-log-forwarder-secret", Namespace: render.LogCollectorNamespace}}, - &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: render.LogCollectorNamespace}}, + &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit", Namespace: render.LogCollectorNamespace}}, } if isManagedcluster { @@ -1395,26 +1303,17 @@ func getExpectedResourcesForEKS(isManagedcluster bool) []client.Object { return expectedResources } -func forwardingEnvVarCount(envVars []corev1.EnvVar) (count int) { - for _, envVar := range envVars { - if strings.HasPrefix(envVar.Name, "FORWARD_") { - count++ - } - } - return count -} - -func additionalStoreSpecForDestinationAndScope(destination render.ForwardingDestination, scope operatorv1.HostScope) *operatorv1.AdditionalLogStoreSpec { +func additionalStoreSpecForDestinationAndScope(destination string, scope operatorv1.HostScope) *operatorv1.AdditionalLogStoreSpec { var spec operatorv1.AdditionalLogStoreSpec switch destination { - case render.ForwardingDestinationS3: + case "S3": spec.S3 = &operatorv1.S3StoreSpec{ Region: "anyplace", BucketName: "thebucket", BucketPath: "bucketpath", HostScope: &scope, } - case render.ForwardingDestinationSyslog: + case "Syslog": var ps int32 = 180 spec.Syslog = &operatorv1.SyslogStoreSpec{ Endpoint: "tcp://1.2.3.4:80", @@ -1426,7 +1325,7 @@ func additionalStoreSpecForDestinationAndScope(destination render.ForwardingDest }, HostScope: &scope, } - case render.ForwardingDestinationSplunk: + case "Splunk": spec.Splunk = &operatorv1.SplunkStoreSpec{ Endpoint: "https://1.2.3.4:8088", HostScope: &scope, @@ -1435,3 +1334,39 @@ func additionalStoreSpecForDestinationAndScope(destination render.ForwardingDest return &spec } + +func legacyFluentdDeleteResources() []client.Object { + ns := "tigera-fluentd" + return []client.Object{ + &appsv1.DaemonSet{TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: ns}}, + &appsv1.DaemonSet{TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node-windows", Namespace: ns}}, + &appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder", Namespace: ns}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-metrics", Namespace: ns}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-http-input", Namespace: ns}}, + &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node", Namespace: ns}}, + &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node-windows", Namespace: ns}}, + &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder", Namespace: ns}}, + &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}}, + &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd-windows"}}, + &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd"}}, + &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd-windows"}}, + &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder"}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-metrics-windows", Namespace: ns}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-linseed", Namespace: ns}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-fluentd-prometheus-tls", Namespace: ns}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: render.EKSLogForwarderTLSSecretName, Namespace: ns}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: render.EksLogForwarderSecret, Namespace: ns}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: render.S3FluentBitSecretName, Namespace: ns}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: render.SplunkFluentBitTokenSecretName, Namespace: ns}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-node-tigera-linseed-token", Namespace: ns}}, + &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "eks-log-forwarder-tigera-linseed-token", Namespace: ns}}, + &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "fluentd-filters", Namespace: ns}}, + &rbacv1.Role{TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: render.PacketCaptureAPIRole, Namespace: ns}}, + &rbacv1.RoleBinding{TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: render.PacketCaptureAPIRoleBinding, Namespace: ns}}, + &corev1.ResourceQuota{TypeMeta: metav1.TypeMeta{Kind: "ResourceQuota", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: resourcequota.TigeraCriticalResourceQuotaName, Namespace: ns}}, + &corev1.Namespace{TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: ns}}, + // The rendered copy of the user filters ConfigMap is superseded by + // inlining the filters into the rendered config. + &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: render.FluentBitFilterConfigMapName, Namespace: render.LogCollectorNamespace}}, + } +} diff --git a/pkg/render/fluentd.go b/pkg/render/fluentd.go deleted file mode 100644 index 9673bcc1e6..0000000000 --- a/pkg/render/fluentd.go +++ /dev/null @@ -1,1372 +0,0 @@ -// Copyright (c) 2019-2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package render - -import ( - "crypto/x509" - "fmt" - "strconv" - "strings" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" - - v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/components" - rcomponents "github.com/tigera/operator/pkg/render/common/components" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" - rmeta "github.com/tigera/operator/pkg/render/common/meta" - "github.com/tigera/operator/pkg/render/common/networkpolicy" - "github.com/tigera/operator/pkg/render/common/resourcequota" - "github.com/tigera/operator/pkg/render/common/secret" - "github.com/tigera/operator/pkg/render/common/securitycontext" - "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" - "github.com/tigera/operator/pkg/tls/certificatemanagement" - "github.com/tigera/operator/pkg/tls/certkeyusage" - "github.com/tigera/operator/pkg/url" -) - -type ForwardingDestination string - -const ( - LogCollectorNamespace = "tigera-fluentd" - FluentdFilterConfigMapName = "fluentd-filters" - FluentdFilterFlowName = "flow" - FluentdFilterDNSName = "dns" - S3FluentdSecretName = "log-collector-s3-credentials" - S3KeyIdName = "key-id" - S3KeySecretName = "key-secret" - - // FluentdPrometheusTLSSecretName is the name of the secret containing the key pair fluentd presents to identify itself. - // Somewhat confusingly, this is named the prometheus TLS key pair because that was the first - // use-case for this credential. However, it is used on all TLS connections served by fluentd. - FluentdPrometheusTLSSecretName = "tigera-fluentd-prometheus-tls" - FluentdMetricsService = "fluentd-metrics" - FluentdMetricsServiceWindows = "fluentd-metrics-windows" - FluentdInputService = "fluentd-http-input" - FluentdMetricsPortName = "fluentd-metrics-port" - FluentdMetricsPort = 9081 - FluentdInputPortName = "fluentd-http-input-port" - FluentdInputPort = 9880 - FluentdPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "allow-fluentd-node" - filterHashAnnotation = "hash.operator.tigera.io/fluentd-filters" - s3CredentialHashAnnotation = "hash.operator.tigera.io/s3-credentials" - splunkCredentialHashAnnotation = "hash.operator.tigera.io/splunk-credentials" - eksCloudwatchLogCredentialHashAnnotation = "hash.operator.tigera.io/eks-cloudwatch-log-credentials" - fluentdDefaultFlush = "5s" - ElasticsearchEksLogForwarderUserSecret = "tigera-eks-log-forwarder-elasticsearch-access" - EksLogForwarderSecret = "tigera-eks-log-forwarder-secret" - EksLogForwarderAwsId = "aws-id" - EksLogForwarderAwsKey = "aws-key" - SplunkFluentdTokenSecretName = "logcollector-splunk-credentials" - SplunkFluentdSecretTokenKey = "token" - SplunkFluentdSecretCertificateKey = "ca.pem" - SysLogPublicCADir = "/etc/pki/tls/certs/" - SysLogPublicCertKey = "ca-bundle.crt" - SysLogPublicCAPath = SysLogPublicCADir + SysLogPublicCertKey - SyslogCAConfigMapName = "syslog-ca" - - // Constants for Linseed token volume mounting in managed clusters. - LinseedTokenVolumeName = "linseed-token" - LinseedTokenKey = "token" - LinseedTokenSubPath = "token" - LinseedTokenSecret = "%s-tigera-linseed-token" - LinseedVolumeMountPath = "/var/run/secrets/tigera.io/linseed/" - LinseedTokenPath = "/var/run/secrets/tigera.io/linseed/token" - - fluentdName = "tigera-fluentd" - fluentdWindowsName = "tigera-fluentd-windows" - - FluentdNodeName = "fluentd-node" - fluentdNodeWindowsName = "fluentd-node-windows" - - EKSLogForwarderName = "eks-log-forwarder" - EKSLogForwarderTLSSecretName = "tigera-eks-log-forwarder-tls" - - PacketCaptureAPIRole = "packetcapture-api-role" - PacketCaptureAPIRoleBinding = "packetcapture-api-role-binding" - - ForwardingDestinationS3 ForwardingDestination = "S3" - ForwardingDestinationSyslog ForwardingDestination = "Syslog" - ForwardingDestinationSplunk ForwardingDestination = "Splunk" -) - -var FluentdSourceEntityRule = v3.EntityRule{ - NamespaceSelector: fmt.Sprintf("name == '%s'", LogCollectorNamespace), - Selector: networkpolicy.KubernetesAppSelector(FluentdNodeName, fluentdNodeWindowsName), -} - -var EKSLogForwarderEntityRule = networkpolicy.CreateSourceEntityRule(LogCollectorNamespace, EKSLogForwarderName) - -// Register secret/certs that need Server and Client Key usage -func init() { - certkeyusage.SetCertKeyUsage(FluentdPrometheusTLSSecretName, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}) - certkeyusage.SetCertKeyUsage(EKSLogForwarderTLSSecretName, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}) -} - -type FluentdFilters struct { - Flow string - DNS string -} - -type S3Credential struct { - KeyId []byte - KeySecret []byte -} - -type SplunkCredential struct { - Token []byte -} - -func Fluentd(cfg *FluentdConfiguration) Component { - return &fluentdComponent{ - cfg: cfg, - probeTimeout: 10, - probePeriod: 60, - } -} - -type EksCloudwatchLogConfig struct { - AwsId []byte - AwsKey []byte - AwsRegion string - GroupName string - StreamPrefix string - FetchInterval int32 -} - -// FluentdConfiguration contains all the config information needed to render the component. -type FluentdConfiguration struct { - LogCollector *operatorv1.LogCollector - S3Credential *S3Credential - SplkCredential *SplunkCredential - Filters *FluentdFilters - // ESClusterConfig is only populated for when EKSConfig - // is also defined - ESClusterConfig *relasticsearch.ClusterConfig - EKSConfig *EksCloudwatchLogConfig - PullSecrets []*corev1.Secret - Installation *operatorv1.InstallationSpec - ClusterDomain string - OSType rmeta.OSType - FluentdKeyPair certificatemanagement.KeyPairInterface - TrustedBundle certificatemanagement.TrustedBundle - ManagedCluster bool - - // Set if running as a multi-tenant management cluster. Configures the management cluster's - // own fluentd daemonset. - Tenant *operatorv1.Tenant - ExternalElastic bool - - // Whether to use User provided certificate or not. - UseSyslogCertificate bool - - // EKSLogForwarderKeyPair contains the certificate presented by EKS LogForwarder when communicating with Linseed - EKSLogForwarderKeyPair certificatemanagement.KeyPairInterface - - PacketCapture *operatorv1.PacketCaptureAPI - - NonClusterHost *operatorv1.NonClusterHost - - // LicenseExpired indicates the license has expired and fluentd DaemonSet should be removed. - LicenseExpired bool -} - -type fluentdComponent struct { - cfg *FluentdConfiguration - image string - probeTimeout int32 - probePeriod int32 -} - -func (c *fluentdComponent) ResolveImages(is *operatorv1.ImageSet) error { - reg := c.cfg.Installation.Registry - path := c.cfg.Installation.ImagePath - prefix := c.cfg.Installation.ImagePrefix - - if c.cfg.OSType == rmeta.OSTypeWindows { - var err error - c.image, err = components.GetReference(components.ComponentFluentdWindows, reg, path, prefix, is) - return err - } - - var err error - c.image, err = components.GetReference(components.ComponentFluentd, reg, path, prefix, is) - if err != nil { - return err - } - return err -} - -func (c *fluentdComponent) SupportedOSType() rmeta.OSType { - return c.cfg.OSType -} - -func (c *fluentdComponent) fluentdName() string { - if c.cfg.OSType == rmeta.OSTypeWindows { - return fluentdWindowsName - } - return fluentdName -} - -func (c *fluentdComponent) fluentdNodeName() string { - if c.cfg.OSType == rmeta.OSTypeWindows { - return fluentdNodeWindowsName - } - return FluentdNodeName -} - -// Use different service names depending on the OS type ("fluentd-metrics" -// vs "fluentd-metrics-windows") in order to help identify which OS daemonset -// we are referring to. -func (c *fluentdComponent) fluentdMetricsServiceName() string { - if c.cfg.OSType == rmeta.OSTypeWindows { - return FluentdMetricsServiceWindows - } - return FluentdMetricsService -} - -func (c *fluentdComponent) readinessCmd() []string { - if c.cfg.OSType == rmeta.OSTypeWindows { - // On Windows, we rely on bash via msys2 installed by the fluentd base image. - return []string{`c:\ruby\msys64\usr\bin\bash.exe`, `-lc`, `/c/bin/readiness.sh`} - } - return []string{"sh", "-c", "/bin/readiness.sh"} -} - -func (c *fluentdComponent) livenessCmd() []string { - if c.cfg.OSType == rmeta.OSTypeWindows { - // On Windows, we rely on bash via msys2 installed by the fluentd base image. - return []string{`c:\ruby\msys64\usr\bin\bash.exe`, `-lc`, `/c/bin/liveness.sh`} - } - return []string{"sh", "-c", "/bin/liveness.sh"} -} - -func (c *fluentdComponent) securityContext(privileged bool) *corev1.SecurityContext { - if c.cfg.OSType == rmeta.OSTypeWindows { - return nil - } - return securitycontext.NewRootContext(privileged) -} - -func (c *fluentdComponent) volumeHostPath() string { - if c.cfg.OSType == rmeta.OSTypeWindows { - return "c:/TigeraCalico" - } - return "/var/log/calico" -} - -func (c *fluentdComponent) path(path string) string { - if c.cfg.OSType == rmeta.OSTypeWindows { - // Use c: path prefix for windows. - return "c:" + path - } - // For linux just leave the path as-is. - return path -} - -func (c *fluentdComponent) Objects() ([]client.Object, []client.Object) { - var objs, toDelete []client.Object - objs = append(objs, c.calicoSystemPolicy()) - objs = append(objs, c.metricsService()) - - // allow-tigera Tier was renamed to calico-system - toDelete = append(toDelete, networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("allow-fluentd-node", LogCollectorNamespace)) - - if c.cfg.Installation.KubernetesProvider.IsGKE() { - // We do this only for GKE as other providers don't (yet?) - // automatically add resource quota that constrains whether - // components that are marked cluster or node critical - // can be scheduled. - objs = append(objs, c.fluentdResourceQuota()) - } - if c.cfg.S3Credential != nil { - objs = append(objs, c.s3CredentialSecret()) - } - if c.cfg.SplkCredential != nil { - objs = append(objs, secret.ToRuntimeObjects(secret.CopyToNamespace(LogCollectorNamespace, c.splunkCredentialSecret()...)...)...) - } - if c.cfg.Filters != nil { - objs = append(objs, c.filtersConfigMap()) - } - if c.cfg.EKSConfig != nil && c.cfg.OSType == rmeta.OSTypeLinux { - objs = append(objs, - c.eksLogForwarderClusterRole(), - c.eksLogForwarderClusterRoleBinding()) - - objs = append(objs, c.eksLogForwarderServiceAccount(), - c.eksLogForwarderSecret(), - c.eksLogForwarderDeployment()) - } - - // Add in the cluster role and binding. - objs = append(objs, - c.fluentdClusterRole(), - c.fluentdClusterRoleBinding(), - ) - if c.cfg.ManagedCluster { - objs = append(objs, c.externalLinseedService()) - objs = append(objs, c.externalLinseedRoleBinding()) - } else { - toDelete = append(toDelete, c.externalLinseedService()) - toDelete = append(toDelete, c.externalLinseedRoleBinding()) - } - - objs = append(objs, c.fluentdServiceAccount()) - if c.cfg.PacketCapture != nil { - objs = append(objs, c.packetCaptureApiRole(), c.packetCaptureApiRoleBinding()) - } - - if c.cfg.LicenseExpired { - toDelete = append(toDelete, c.daemonset()) - } else { - objs = append(objs, c.daemonset()) - } - - if c.cfg.NonClusterHost != nil && c.cfg.OSType == rmeta.OSTypeLinux { - objs = append(objs, c.nonClusterHostInputService()) - } - - return objs, toDelete -} - -func (c *fluentdComponent) nonClusterHostInputService() *corev1.Service { - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: FluentdInputService, - Namespace: LogCollectorNamespace, - Labels: map[string]string{"k8s-app": c.fluentdNodeName()}, - }, - // We do not treat this service as a headless service, as we want to ensure traffic is load-balanced. This is because: - // - We have no guarantee that the client (voltron) will perform load balancing across the returned records. The - // golang dialer implementation appears to prefer the first record returned (see dialSerial in the go SDK) - // - We have no guarantee that the DNS server will perform load-balancing or randomize the order of records returned - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": c.fluentdNodeName()}, - Ports: []corev1.ServicePort{ - { - Name: FluentdInputPortName, - Port: int32(FluentdInputPort), - TargetPort: intstr.FromInt(FluentdInputPort), - Protocol: corev1.ProtocolTCP, - }, - }, - }, - } -} - -func (c *fluentdComponent) externalLinseedRoleBinding() *rbacv1.RoleBinding { - // For managed clusters, we must create a role binding to allow Linseed to manage access token secrets - // in our namespace. - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-linseed", - Namespace: LogCollectorNamespace, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: TigeraLinseedSecretsClusterRole, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: GuardianServiceAccountName, - Namespace: GuardianNamespace, - }, - }, - } -} - -func (c *fluentdComponent) externalLinseedService() *corev1.Service { - // For managed clusters, we must create an external service for fluentd to forward the request to guardian. - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-linseed", - Namespace: LogCollectorNamespace, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeExternalName, - ExternalName: fmt.Sprintf("%s.%s.svc.%s", GuardianServiceName, GuardianNamespace, c.cfg.ClusterDomain), - }, - } -} - -func (c *fluentdComponent) Ready() bool { - return true -} - -func (c *fluentdComponent) fluentdResourceQuota() *corev1.ResourceQuota { - criticalPriorityClasses := []string{NodePriorityClassName} - return resourcequota.ResourceQuotaForPriorityClassScope(resourcequota.TigeraCriticalResourceQuotaName, LogCollectorNamespace, criticalPriorityClasses) -} - -func (c *fluentdComponent) s3CredentialSecret() *corev1.Secret { - if c.cfg.S3Credential == nil { - return nil - } - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: S3FluentdSecretName, - Namespace: LogCollectorNamespace, - }, - Data: map[string][]byte{ - S3KeyIdName: c.cfg.S3Credential.KeyId, - S3KeySecretName: c.cfg.S3Credential.KeySecret, - }, - } -} - -func (c *fluentdComponent) filtersConfigMap() *corev1.ConfigMap { - if c.cfg.Filters == nil { - return nil - } - return &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: FluentdFilterConfigMapName, - Namespace: LogCollectorNamespace, - }, - Data: map[string]string{ - FluentdFilterFlowName: c.cfg.Filters.Flow, - FluentdFilterDNSName: c.cfg.Filters.DNS, - }, - } -} - -func (c *fluentdComponent) splunkCredentialSecret() []*corev1.Secret { - if c.cfg.SplkCredential == nil { - return nil - } - return []*corev1.Secret{ - { - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: SplunkFluentdTokenSecretName, - Namespace: LogCollectorNamespace, - }, - Data: map[string][]byte{ - SplunkFluentdSecretTokenKey: c.cfg.SplkCredential.Token, - }, - }, - } -} - -func (c *fluentdComponent) fluentdServiceAccount() *corev1.ServiceAccount { - return &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: c.fluentdNodeName(), Namespace: LogCollectorNamespace}, - } -} - -// packetCaptureApiRole creates a role in the tigera-fluentd namespace to allow pod/exec -// only from fluentd pods. This is being used by the PacketCapture API and created -// by the operator after the namespace tigera-fluentd is created. -func (c *fluentdComponent) packetCaptureApiRole() *rbacv1.Role { - return &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: PacketCaptureAPIRole, - Namespace: LogCollectorNamespace, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"pods/exec"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - }, - } -} - -// packetCaptureApiRoleBinding creates a role binding within the tigera-fluentd namespace between the pod/exec role -// the service account tigera-manager. This is being used by the PacketCapture API and created -// by the operator after the namespace tigera-fluentd is created -func (c *fluentdComponent) packetCaptureApiRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: PacketCaptureAPIRoleBinding, - Namespace: LogCollectorNamespace, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: PacketCaptureAPIRole, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: PacketCaptureServiceAccountName, - Namespace: PacketCaptureNamespace, - }, - }, - } -} - -// managerDeployment creates a deployment for the Tigera Secure manager component. -func (c *fluentdComponent) daemonset() *appsv1.DaemonSet { - var terminationGracePeriod int64 = 0 - // The rationale for this setting is that while there is no need for fluentd to be available, we want to avoid - // potentially negative consequences of an immediate roll-out on huge clusters. - maxUnavailable := intstr.FromInt(10) - - annots := c.cfg.TrustedBundle.HashAnnotations() - - if c.cfg.FluentdKeyPair != nil { - annots[c.cfg.FluentdKeyPair.HashAnnotationKey()] = c.cfg.FluentdKeyPair.HashAnnotationValue() - } - if c.cfg.S3Credential != nil { - annots[s3CredentialHashAnnotation] = rmeta.AnnotationHash(c.cfg.S3Credential) - } - if c.cfg.SplkCredential != nil { - annots[splunkCredentialHashAnnotation] = rmeta.AnnotationHash(c.cfg.SplkCredential) - } - if c.cfg.Filters != nil { - annots[filterHashAnnotation] = rmeta.AnnotationHash(c.cfg.Filters) - } - var initContainers []corev1.Container - if c.cfg.FluentdKeyPair != nil && c.cfg.FluentdKeyPair.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.FluentdKeyPair.InitContainer(LogCollectorNamespace, c.container().SecurityContext)) - } - - podTemplate := &corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: annots, - }, - Spec: corev1.PodSpec{ - NodeSelector: map[string]string{}, - Tolerations: rmeta.TolerateAll, - ImagePullSecrets: secret.GetReferenceList(c.cfg.PullSecrets), - TerminationGracePeriodSeconds: &terminationGracePeriod, - InitContainers: initContainers, - Containers: []corev1.Container{c.container()}, - Volumes: c.volumes(), - ServiceAccountName: c.fluentdNodeName(), - }, - } - - ds := &appsv1.DaemonSet{ - TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: c.fluentdNodeName(), - Namespace: LogCollectorNamespace, - }, - Spec: appsv1.DaemonSetSpec{ - Template: *podTemplate, - UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ - RollingUpdate: &appsv1.RollingUpdateDaemonSet{ - MaxUnavailable: &maxUnavailable, - }, - }, - }, - } - if c.cfg.LogCollector != nil { - if overrides := c.cfg.LogCollector.Spec.FluentdDaemonSet; overrides != nil { - rcomponents.ApplyDaemonSetOverrides(ds, overrides) - } - } - setNodeCriticalPod(&(ds.Spec.Template)) - return ds -} - -// container creates the fluentd container. -func (c *fluentdComponent) container() corev1.Container { - // Determine environment to pass to the CNI init container. - envs := c.envvars() - volumeMounts := []corev1.VolumeMount{ - {MountPath: c.path("/var/log/calico"), Name: "var-log-calico"}, - {MountPath: c.path("/etc/fluentd/elastic"), Name: certificatemanagement.TrustedCertConfigMapName}, - } - if c.cfg.Filters != nil { - if c.cfg.Filters.Flow != "" { - volumeMounts = append(volumeMounts, - corev1.VolumeMount{ - Name: "fluentd-filters", - MountPath: c.path("/etc/fluentd/flow-filters.conf"), - SubPath: FluentdFilterFlowName, - }) - } - if c.cfg.Filters.DNS != "" { - volumeMounts = append(volumeMounts, - corev1.VolumeMount{ - Name: "fluentd-filters", - MountPath: c.path("/etc/fluentd/dns-filters.conf"), - SubPath: FluentdFilterDNSName, - }) - } - } - - volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) - - if c.cfg.FluentdKeyPair != nil { - volumeMounts = append(volumeMounts, c.cfg.FluentdKeyPair.VolumeMount(c.SupportedOSType())) - } - - if c.cfg.ManagedCluster { - volumeMounts = append(volumeMounts, - corev1.VolumeMount{ - Name: LinseedTokenVolumeName, - MountPath: c.path(LinseedVolumeMountPath), - }) - } - - return corev1.Container{ - Name: "fluentd", - Image: c.image, - Env: envs, - // On OpenShift Fluentd needs privileged access to access logs on host path volume - SecurityContext: c.securityContext(c.cfg.Installation.KubernetesProvider.IsOpenShift()), - VolumeMounts: volumeMounts, - StartupProbe: c.startup(), - LivenessProbe: c.liveness(), - ReadinessProbe: c.readiness(), - Ports: []corev1.ContainerPort{{ - Name: "metrics-port", - ContainerPort: FluentdMetricsPort, - }}, - } -} - -func (c *fluentdComponent) metricsService() *corev1.Service { - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: c.fluentdMetricsServiceName(), - Namespace: LogCollectorNamespace, - Labels: map[string]string{"k8s-app": c.fluentdNodeName()}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": c.fluentdNodeName()}, - // Important: "None" tells Kubernetes that we want a headless service with - // no kube-proxy load balancer. If we omit this then kube-proxy will render - // a huge set of iptables rules for this service since there's an instance - // on every node. - ClusterIP: "None", - Ports: []corev1.ServicePort{ - { - Name: FluentdMetricsPortName, - Port: int32(FluentdMetricsPort), - TargetPort: intstr.FromInt(FluentdMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - }, - }, - } -} - -func (c *fluentdComponent) envvars() []corev1.EnvVar { - envs := []corev1.EnvVar{ - {Name: "LINSEED_ENABLED", Value: "true"}, - // Determine the namespace in which Linseed is running. For managed and standalone clusters, this is always the elasticsearch - // namespace. For multi-tenant management clusters, this may vary. - {Name: "LINSEED_ENDPOINT", Value: relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, LinseedNamespace(c.cfg.Tenant), c.cfg.ManagedCluster, true)}, - {Name: "LINSEED_CA_PATH", Value: c.trustedBundlePath()}, - {Name: "TLS_KEY_PATH", Value: c.keyPath()}, - {Name: "TLS_CRT_PATH", Value: c.certPath()}, - {Name: "FLUENT_UID", Value: "0"}, - {Name: "FLOW_LOG_FILE", Value: c.path("/var/log/calico/flowlogs/flows.log")}, - {Name: "DNS_LOG_FILE", Value: c.path("/var/log/calico/dnslogs/dns.log")}, - {Name: "FLUENTD_ES_SECURE", Value: "true"}, - {Name: "NODENAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}}, - {Name: "LINSEED_TOKEN", Value: c.path(GetLinseedTokenPath(c.cfg.ManagedCluster))}, - } - - if c.cfg.Tenant != nil && c.cfg.ExternalElastic { - envs = append(envs, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.Tenant.Spec.ID}) - } - - if c.cfg.LogCollector.Spec.AdditionalStores != nil { - s3 := c.cfg.LogCollector.Spec.AdditionalStores.S3 - if s3 != nil { - envs = append(envs, - corev1.EnvVar{ - Name: "AWS_KEY_ID", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: S3FluentdSecretName, - }, - Key: S3KeyIdName, - }, - }, - }, - corev1.EnvVar{ - Name: "AWS_SECRET_KEY", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: S3FluentdSecretName, - }, - Key: S3KeySecretName, - }, - }, - }, - corev1.EnvVar{Name: "S3_STORAGE", Value: "true"}, - corev1.EnvVar{Name: "S3_BUCKET_NAME", Value: s3.BucketName}, - corev1.EnvVar{Name: "AWS_REGION", Value: s3.Region}, - corev1.EnvVar{Name: "S3_BUCKET_PATH", Value: s3.BucketPath}, - corev1.EnvVar{Name: "S3_FLUSH_INTERVAL", Value: fluentdDefaultFlush}, - ) - - hostScopeEnvVars := envVarsForHostScope(s3.HostScope, ForwardingDestinationS3) - envs = append(envs, hostScopeEnvVars...) - } - syslog := c.cfg.LogCollector.Spec.AdditionalStores.Syslog - if syslog != nil { - proto, host, port, _ := url.ParseEndpoint(syslog.Endpoint) - envs = append(envs, - corev1.EnvVar{Name: "SYSLOG_HOST", Value: host}, - corev1.EnvVar{Name: "SYSLOG_PORT", Value: port}, - corev1.EnvVar{Name: "SYSLOG_PROTOCOL", Value: proto}, - corev1.EnvVar{Name: "SYSLOG_FLUSH_INTERVAL", Value: fluentdDefaultFlush}, - corev1.EnvVar{ - Name: "SYSLOG_HOSTNAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - ) - if syslog.PacketSize != nil { - envs = append(envs, - corev1.EnvVar{ - Name: "SYSLOG_PACKET_SIZE", - Value: fmt.Sprintf("%d", *syslog.PacketSize), - }, - ) - } - - if syslog.LogTypes != nil { - for _, t := range syslog.LogTypes { - switch t { - case operatorv1.SyslogLogAudit: - envs = append(envs, - corev1.EnvVar{Name: "SYSLOG_AUDIT_EE_LOG", Value: "true"}, - ) - envs = append(envs, - corev1.EnvVar{Name: "SYSLOG_AUDIT_KUBE_LOG", Value: "true"}, - ) - case operatorv1.SyslogLogDNS: - envs = append(envs, - corev1.EnvVar{Name: "SYSLOG_DNS_LOG", Value: "true"}, - ) - case operatorv1.SyslogLogFlows: - envs = append(envs, - corev1.EnvVar{Name: "SYSLOG_FLOW_LOG", Value: "true"}, - ) - case operatorv1.SyslogLogIDSEvents: - envs = append(envs, - corev1.EnvVar{Name: "SYSLOG_IDS_EVENT_LOG", Value: "true"}, - ) - } - } - } - - if syslog.Encryption == operatorv1.EncryptionTLS { - envs = append(envs, - corev1.EnvVar{Name: "SYSLOG_TLS", Value: "true"}, - ) - // By default, we would be using the secure verification mode OpenSSL::SSL::VERIFY_PEER(1) - envs = append(envs, - corev1.EnvVar{Name: "SYSLOG_VERIFY_MODE", Value: "1"}, - ) - if c.cfg.UseSyslogCertificate { - envs = append(envs, - corev1.EnvVar{Name: "SYSLOG_CA_FILE", Value: c.cfg.TrustedBundle.MountPath()}, - ) - } else { - envs = append(envs, - corev1.EnvVar{Name: "SYSLOG_CA_FILE", Value: SysLogPublicCAPath}, - ) - } - } - - hostScopeEnvVars := envVarsForHostScope(syslog.HostScope, ForwardingDestinationSyslog) - envs = append(envs, hostScopeEnvVars...) - } - splunk := c.cfg.LogCollector.Spec.AdditionalStores.Splunk - if splunk != nil { - proto, host, port, _ := url.ParseEndpoint(splunk.Endpoint) - envs = append(envs, - corev1.EnvVar{ - Name: "SPLUNK_HEC_TOKEN", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: SplunkFluentdTokenSecretName, - }, - Key: SplunkFluentdSecretTokenKey, - }, - }, - }, - corev1.EnvVar{Name: "SPLUNK_FLOW_LOG", Value: "true"}, - corev1.EnvVar{Name: "SPLUNK_AUDIT_LOG", Value: "true"}, - corev1.EnvVar{Name: "SPLUNK_DNS_LOG", Value: "true"}, - corev1.EnvVar{Name: "SPLUNK_HEC_HOST", Value: host}, - corev1.EnvVar{Name: "SPLUNK_HEC_PORT", Value: port}, - corev1.EnvVar{Name: "SPLUNK_PROTOCOL", Value: proto}, - corev1.EnvVar{Name: "SPLUNK_FLUSH_INTERVAL", Value: fluentdDefaultFlush}, - ) - - hostScopeEnvVars := envVarsForHostScope(splunk.HostScope, ForwardingDestinationSplunk) - envs = append(envs, hostScopeEnvVars...) - } - } - - if c.cfg.Filters != nil { - if c.cfg.Filters.Flow != "" { - envs = append(envs, - corev1.EnvVar{Name: "FLUENTD_FLOW_FILTERS", Value: "true"}) - } - if c.cfg.Filters.DNS != "" { - envs = append(envs, - corev1.EnvVar{Name: "FLUENTD_DNS_FILTERS", Value: "true"}) - } - } - - envs = append(envs, corev1.EnvVar{Name: "CA_CRT_PATH", Value: c.trustedBundlePath()}) - - return envs -} - -func (c *fluentdComponent) trustedBundlePath() string { - if c.cfg.OSType == rmeta.OSTypeWindows { - return certificatemanagement.TrustedCertBundleMountPathWindows - } - return c.cfg.TrustedBundle.MountPath() -} - -func (c *fluentdComponent) keyPath() string { - if c.cfg.OSType == rmeta.OSTypeWindows { - return fmt.Sprintf("c:/%s/%s", c.cfg.FluentdKeyPair.GetName(), corev1.TLSPrivateKeyKey) - } - return c.cfg.FluentdKeyPair.VolumeMountKeyFilePath() -} - -func (c *fluentdComponent) certPath() string { - if c.cfg.OSType == rmeta.OSTypeWindows { - return fmt.Sprintf("c:/%s/%s", c.cfg.FluentdKeyPair.GetName(), corev1.TLSCertKey) - } - return c.cfg.FluentdKeyPair.VolumeMountCertificateFilePath() -} - -// The startup probe uses the same action as the liveness probe, but with -// a higher failure threshold and double the timeout to account for slow -// networks. -func (c *fluentdComponent) startup() *corev1.Probe { - return &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: c.livenessCmd(), - }, - }, - TimeoutSeconds: c.probeTimeout, - PeriodSeconds: c.probePeriod, - // tolerate more failures for the startup probe - FailureThreshold: 10, - } -} - -func (c *fluentdComponent) liveness() *corev1.Probe { - return &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: c.livenessCmd(), - }, - }, - TimeoutSeconds: c.probeTimeout, - PeriodSeconds: c.probePeriod, - } -} - -func (c *fluentdComponent) readiness() *corev1.Probe { - return &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: c.readinessCmd(), - }, - }, - TimeoutSeconds: c.probeTimeout, - PeriodSeconds: c.probePeriod, - } -} - -func (c *fluentdComponent) volumes() []corev1.Volume { - dirOrCreate := corev1.HostPathDirectoryOrCreate - - volumes := []corev1.Volume{ - { - Name: "var-log-calico", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: c.volumeHostPath(), - Type: &dirOrCreate, - }, - }, - }, - } - if c.cfg.Filters != nil { - volumes = append(volumes, - corev1.Volume{ - Name: "fluentd-filters", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: FluentdFilterConfigMapName, - }, - }, - }, - }) - } - if c.cfg.FluentdKeyPair != nil { - volumes = append(volumes, c.cfg.FluentdKeyPair.Volume()) - } - if c.cfg.ManagedCluster { - volumes = append(volumes, - corev1.Volume{ - Name: LinseedTokenVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf(LinseedTokenSecret, FluentdNodeName), - Items: []corev1.KeyToPath{{Key: LinseedTokenKey, Path: LinseedTokenSubPath}}, - }, - }, - }) - } - volumes = append(volumes, trustedBundleVolume(c.cfg.TrustedBundle)) - - return volumes -} - -func (c *fluentdComponent) fluentdClusterRoleBinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: c.fluentdName(), - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: c.fluentdName(), - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: c.fluentdNodeName(), - Namespace: LogCollectorNamespace, - }, - }, - } -} - -func (c *fluentdComponent) fluentdClusterRole() *rbacv1.ClusterRole { - role := &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: c.fluentdName(), - }, - Rules: []rbacv1.PolicyRule{ - { - // Add write access to Linseed APIs. - APIGroups: []string{"linseed.tigera.io"}, - Resources: []string{ - "flowlogs", - "kube_auditlogs", - "ee_auditlogs", - "dnslogs", - "l7logs", - "events", - "bgplogs", - "waflogs", - "runtimereports", - "policyactivity", - }, - Verbs: []string{"create"}, - }, - }, - } - - if c.cfg.Installation.KubernetesProvider.IsOpenShift() { - role.Rules = append(role.Rules, rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{securitycontextconstraints.Privileged}, - }) - } - return role -} - -func (c *fluentdComponent) eksLogForwarderServiceAccount() *corev1.ServiceAccount { - return &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: EKSLogForwarderName, Namespace: LogCollectorNamespace}, - } -} - -func (c *fluentdComponent) eksLogForwarderSecret() *corev1.Secret { - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: EksLogForwarderSecret, - Namespace: LogCollectorNamespace, - }, - Data: map[string][]byte{ - EksLogForwarderAwsId: c.cfg.EKSConfig.AwsId, - EksLogForwarderAwsKey: c.cfg.EKSConfig.AwsKey, - }, - } -} - -func (c *fluentdComponent) eksLogForwarderDeployment() *appsv1.Deployment { - annots := map[string]string{ - eksCloudwatchLogCredentialHashAnnotation: rmeta.AnnotationHash(c.cfg.EKSConfig), - } - - envVars := []corev1.EnvVar{ - // Meta flags. - {Name: "LOG_LEVEL", Value: "info"}, - {Name: "FLUENT_UID", Value: "0"}, - // Use fluentd for EKS log forwarder. - {Name: "MANAGED_K8S", Value: "true"}, - {Name: "K8S_PLATFORM", Value: "eks"}, - {Name: "FLUENTD_ES_SECURE", Value: "true"}, - // Cloudwatch config, credentials. - {Name: "EKS_CLOUDWATCH_LOG_GROUP", Value: c.cfg.EKSConfig.GroupName}, - {Name: "EKS_CLOUDWATCH_LOG_STREAM_PREFIX", Value: c.cfg.EKSConfig.StreamPrefix}, - {Name: "EKS_CLOUDWATCH_LOG_FETCH_INTERVAL", Value: fmt.Sprintf("%d", c.cfg.EKSConfig.FetchInterval)}, - {Name: "AWS_REGION", Value: c.cfg.EKSConfig.AwsRegion}, - {Name: "AWS_ACCESS_KEY_ID", ValueFrom: secret.GetEnvVarSource(EksLogForwarderSecret, EksLogForwarderAwsId, false)}, - {Name: "AWS_SECRET_ACCESS_KEY", ValueFrom: secret.GetEnvVarSource(EksLogForwarderSecret, EksLogForwarderAwsKey, false)}, - {Name: "LINSEED_ENABLED", Value: "true"}, - // Determine the namespace in which Linseed is running. For managed and standalone clusters, this is always the elasticsearch - // namespace. For multi-tenant management clusters, this may vary. - {Name: "LINSEED_ENDPOINT", Value: relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, LinseedNamespace(c.cfg.Tenant), c.cfg.ManagedCluster, true)}, - {Name: "LINSEED_CA_PATH", Value: c.trustedBundlePath()}, - {Name: "TLS_CRT_PATH", Value: c.cfg.EKSLogForwarderKeyPair.VolumeMountCertificateFilePath()}, - {Name: "TLS_KEY_PATH", Value: c.cfg.EKSLogForwarderKeyPair.VolumeMountKeyFilePath()}, - {Name: "LINSEED_TOKEN", Value: c.path(GetLinseedTokenPath(c.cfg.ManagedCluster))}, - } - if c.cfg.Tenant != nil && c.cfg.ExternalElastic { - envVars = append(envVars, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.Tenant.Spec.ID}) - } - - var eksLogForwarderReplicas int32 = 1 - - tolerations := c.cfg.Installation.ControlPlaneTolerations - if c.cfg.Installation.KubernetesProvider.IsGKE() { - tolerations = append(tolerations, rmeta.TolerateGKEARM64NoSchedule) - } - - d := &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: EKSLogForwarderName, - Namespace: LogCollectorNamespace, - Labels: map[string]string{ - "k8s-app": EKSLogForwarderName, - }, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &eksLogForwarderReplicas, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.RecreateDeploymentStrategyType, - }, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "k8s-app": EKSLogForwarderName, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: EKSLogForwarderName, - Namespace: LogCollectorNamespace, - Labels: map[string]string{ - "k8s-app": EKSLogForwarderName, - }, - Annotations: annots, - }, - Spec: corev1.PodSpec{ - Tolerations: tolerations, - ServiceAccountName: EKSLogForwarderName, - ImagePullSecrets: secret.GetReferenceList(c.cfg.PullSecrets), - InitContainers: []corev1.Container{{ - Name: EKSLogForwarderName + "-startup", - Image: c.image, - Command: []string{c.path("/bin/eks-log-forwarder-startup")}, - Env: envVars, - SecurityContext: c.securityContext(false), - VolumeMounts: c.eksLogForwarderVolumeMounts(), - }}, - Containers: []corev1.Container{{ - Name: EKSLogForwarderName, - Image: c.image, - Env: envVars, - SecurityContext: c.securityContext(false), - VolumeMounts: c.eksLogForwarderVolumeMounts(), - }}, - Volumes: c.eksLogForwarderVolumes(), - }, - }, - }, - } - - if c.cfg.LogCollector != nil { - if overrides := c.cfg.LogCollector.Spec.EKSLogForwarderDeployment; overrides != nil { - rcomponents.ApplyDeploymentOverrides(d, overrides) - } - } - - return d -} - -func trustedBundleVolume(bundle certificatemanagement.TrustedBundle) corev1.Volume { - volume := bundle.Volume() - // We mount the bundle under two names; the standard name and the name for the expected elastic cert. - volume.ConfigMap.Items = []corev1.KeyToPath{ - {Key: certificatemanagement.TrustedCertConfigMapKeyName, Path: certificatemanagement.TrustedCertConfigMapKeyName}, - //nolint:staticcheck // Ignore SA1019 deprecated - {Key: certificatemanagement.TrustedCertConfigMapKeyName, Path: certificatemanagement.LegacyTrustedCertConfigMapKeyName}, - {Key: certificatemanagement.TrustedCertConfigMapKeyName, Path: SplunkFluentdSecretCertificateKey}, - {Key: certificatemanagement.RHELRootCertificateBundleName, Path: certificatemanagement.RHELRootCertificateBundleName}, - } - return volume -} - -func (c *fluentdComponent) eksLogForwarderVolumeMounts() []corev1.VolumeMount { - volumeMounts := []corev1.VolumeMount{ - { - Name: "plugin-statefile-dir", - MountPath: c.path("/fluentd/cloudwatch-logs/"), - }, - { - Name: certificatemanagement.TrustedCertConfigMapName, - MountPath: c.path("/etc/fluentd/elastic/"), - }, - } - volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) - if c.cfg.EKSLogForwarderKeyPair != nil { - volumeMounts = append(volumeMounts, c.cfg.EKSLogForwarderKeyPair.VolumeMount(c.SupportedOSType())) - } - - if c.cfg.ManagedCluster { - volumeMounts = append(volumeMounts, - corev1.VolumeMount{ - Name: LinseedTokenVolumeName, - MountPath: c.path(LinseedVolumeMountPath), - }) - } - return volumeMounts -} - -func (c *fluentdComponent) eksLogForwarderVolumes() []corev1.Volume { - volumes := []corev1.Volume{ - trustedBundleVolume(c.cfg.TrustedBundle), - { - Name: "plugin-statefile-dir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: nil, - }, - }, - } - if c.cfg.EKSLogForwarderKeyPair != nil { - volumes = append(volumes, c.cfg.EKSLogForwarderKeyPair.Volume()) - } - - if c.cfg.ManagedCluster { - volumes = append(volumes, - corev1.Volume{ - Name: LinseedTokenVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf(LinseedTokenSecret, EKSLogForwarderName), - Items: []corev1.KeyToPath{{Key: LinseedTokenKey, Path: LinseedTokenSubPath}}, - }, - }, - }) - } - return volumes -} - -func (c *fluentdComponent) eksLogForwarderClusterRoleBinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: EKSLogForwarderName, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: EKSLogForwarderName, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: EKSLogForwarderName, - Namespace: LogCollectorNamespace, - }, - }, - } -} - -func (c *fluentdComponent) eksLogForwarderClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - { - // Add read access to Linseed APIs. - APIGroups: []string{"linseed.tigera.io"}, - Resources: []string{ - "auditlogs", - }, - Verbs: []string{"get"}, - }, - { - // Add write access to Linseed APIs to flush eks kube audit logs. - APIGroups: []string{"linseed.tigera.io"}, - Resources: []string{ - "kube_auditlogs", - }, - Verbs: []string{"create"}, - }, - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: EKSLogForwarderName, - }, - Rules: rules, - } -} - -func (c *fluentdComponent) calicoSystemPolicy() *v3.NetworkPolicy { - multiTenant := false - tenantNamespace := "" - if c.cfg.Tenant != nil { - multiTenant = true - tenantNamespace = c.cfg.Tenant.Namespace - } - policyHelper := networkpolicy.Helper(multiTenant, tenantNamespace) - - egressRules := []v3.Rule{} - if c.cfg.ManagedCluster { - egressRules = append(egressRules, v3.Rule{ - Action: v3.Deny, - Protocol: &networkpolicy.TCPProtocol, - Source: v3.EntityRule{}, - Destination: v3.EntityRule{ - NamespaceSelector: fmt.Sprintf("projectcalico.org/name == '%s'", GuardianNamespace), - Selector: networkpolicy.KubernetesAppSelector(GuardianServiceName), - NotPorts: networkpolicy.Ports(8080), - }, - }) - } else { - egressRules = append(egressRules, v3.Rule{ - Action: v3.Deny, - Protocol: &networkpolicy.TCPProtocol, - Source: v3.EntityRule{}, - Destination: v3.EntityRule{ - NamespaceSelector: fmt.Sprintf("projectcalico.org/name == '%s'", ElasticsearchNamespace), - Selector: networkpolicy.KubernetesAppSelector("tigera-secure-es-gateway"), - NotPorts: networkpolicy.Ports(5554), - }, - }) - egressRules = append(egressRules, v3.Rule{ - Action: v3.Deny, - Protocol: &networkpolicy.TCPProtocol, - Source: v3.EntityRule{}, - Destination: v3.EntityRule{ - NamespaceSelector: fmt.Sprintf("projectcalico.org/name == '%s'", ElasticsearchNamespace), - Selector: networkpolicy.KubernetesAppSelector("tigera-linseed"), - NotPorts: networkpolicy.Ports(8444), - }, - }) - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, c.cfg.Installation.KubernetesProvider.IsOpenShift()) - } - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - }) - - ingressRules := []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicy.PrometheusSourceEntityRule, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(FluentdMetricsPort), - }, - }, - } - - if c.cfg.NonClusterHost != nil { - ingressRules = append(ingressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: policyHelper.ManagerSourceEntityRule(), - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(FluentdInputPort), - }, - }) - } - - return &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: FluentdPolicyName, - Namespace: LogCollectorNamespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(FluentdNodeName, fluentdNodeWindowsName), - ServiceAccountSelector: "", - Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, - Ingress: ingressRules, - Egress: egressRules, - }, - } -} - -func envVarsForHostScope(hostScope *operatorv1.HostScope, destination ForwardingDestination) []corev1.EnvVar { - var forwardClusterLogs, forwardNonClusterLogs bool - if hostScope == nil || *hostScope != operatorv1.HostScopeNonClusterOnly { - forwardClusterLogs = true - forwardNonClusterLogs = true - } else { - forwardClusterLogs = false - forwardNonClusterLogs = true - } - - return []corev1.EnvVar{ - {Name: fmt.Sprintf("FORWARD_CLUSTER_LOGS_TO_%s", strings.ToUpper(string(destination))), Value: strconv.FormatBool(forwardClusterLogs)}, - {Name: fmt.Sprintf("FORWARD_NON_CLUSTER_LOGS_TO_%s", strings.ToUpper(string(destination))), Value: strconv.FormatBool(forwardNonClusterLogs)}, - } -} diff --git a/pkg/render/guardian.go b/pkg/render/guardian.go index 8fd24d35b6..91f6e32e34 100644 --- a/pkg/render/guardian.go +++ b/pkg/render/guardian.go @@ -702,7 +702,7 @@ func guardianCalicoSystemPolicy(cfg *GuardianConfiguration) (*v3.NetworkPolicy, { Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, - Source: FluentdSourceEntityRule, + Source: FluentBitSourceEntityRule, Destination: guardianIngressDestinationEntityRule, }, { diff --git a/pkg/render/intrusion_detection.go b/pkg/render/intrusion_detection.go index f0aedbad0a..4730cb65b1 100644 --- a/pkg/render/intrusion_detection.go +++ b/pkg/render/intrusion_detection.go @@ -579,7 +579,7 @@ func (c *intrusionDetectionComponent) deploymentPodTemplate() *corev1.PodTemplat c.cfg.IntrusionDetectionCertSecret.Volume(), } // If syslog forwarding is enabled then set the necessary hostpath volume to write - // logs for Fluentd to access. + // logs for Fluent Bit to access. if c.cfg.SyslogForwardingIsEnabled { dirOrCreate := corev1.HostPathDirectoryOrCreate volumes = append(volumes, corev1.Volume{ @@ -741,7 +741,7 @@ func (c *intrusionDetectionComponent) intrusionDetectionControllerContainer() co sc := securitycontext.NewNonRootContext() // If syslog forwarding is enabled then set the necessary ENV var and volume mount to - // write logs for Fluentd. + // write logs for Fluent Bit. volumeMounts := c.cfg.TrustedCertBundle.VolumeMounts(c.SupportedOSType()) volumeMounts = append(volumeMounts, c.cfg.IntrusionDetectionCertSecret.VolumeMount(c.SupportedOSType())) if c.cfg.SyslogForwardingIsEnabled { diff --git a/pkg/render/logstorage/esgateway/esgateway.go b/pkg/render/logstorage/esgateway/esgateway.go index d8673fae21..8b63adc641 100644 --- a/pkg/render/logstorage/esgateway/esgateway.go +++ b/pkg/render/logstorage/esgateway/esgateway.go @@ -390,7 +390,7 @@ func (e *esGateway) esGatewayCalicoSystemPolicy() *v3.NetworkPolicy { { Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, - Source: render.FluentdSourceEntityRule, + Source: render.FluentBitSourceEntityRule, Destination: esgatewayIngressDestinationEntityRule, }, { diff --git a/pkg/render/logstorage/linseed/linseed.go b/pkg/render/logstorage/linseed/linseed.go index 7f976280f4..90f1362571 100644 --- a/pkg/render/logstorage/linseed/linseed.go +++ b/pkg/render/logstorage/linseed/linseed.go @@ -596,7 +596,7 @@ func (l *linseed) linseedCalicoSystemPolicy() *v3.NetworkPolicy { { Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, - Source: render.FluentdSourceEntityRule, + Source: render.FluentBitSourceEntityRule, Destination: linseedIngressDestinationEntityRule, }, { diff --git a/pkg/render/manager.go b/pkg/render/manager.go index 765918beb3..ab5b3e9b60 100644 --- a/pkg/render/manager.go +++ b/pkg/render/manager.go @@ -1259,7 +1259,7 @@ func (c *managerComponent) managerCalicoSystemNetworkPolicy() *v3.NetworkPolicy Destination: v3.EntityRule{ Services: &v3.ServiceMatch{ Namespace: LogCollectorNamespace, - Name: FluentdInputService, + Name: FluentBitInputService, }, }, }) @@ -1400,7 +1400,7 @@ func managerClusterWideTigeraLayer() *v3.UISettings { "tigera-dpi", "tigera-eck-operator", "tigera-elasticsearch", - "tigera-fluentd", + "calico-fluent-bit", "tigera-intrusion-detection", "tigera-kibana", "tigera-manager", diff --git a/pkg/render/manager_test.go b/pkg/render/manager_test.go index 618b815ca7..15543fd54a 100644 --- a/pkg/render/manager_test.go +++ b/pkg/render/manager_test.go @@ -1237,7 +1237,7 @@ var _ = Describe("Tigera Secure Manager rendering tests", func() { policyWithNonClusterHosts := testutils.GetCalicoSystemPolicyFromResources(policyName, resourcesWithNonClusterHosts) policyWithoutNonClusterHosts := testutils.GetCalicoSystemPolicyFromResources(policyName, resourcesWithoutNonClusterHosts) - // Validate that we have a single egress rule added for the fluentd service. + // Validate that we have a single egress rule added for the fluent-bit service. Expect(policyWithoutNonClusterHosts.Spec.Ingress).To(Equal(policyWithNonClusterHosts.Spec.Ingress)) Expect(len(policyWithoutNonClusterHosts.Spec.Egress)).To(Equal(len(policyWithNonClusterHosts.Spec.Egress) - 1)) Expect(len(policyWithNonClusterHosts.Spec.Egress)).To(Equal(11)) @@ -1247,7 +1247,7 @@ var _ = Describe("Tigera Secure Manager rendering tests", func() { Destination: v3.EntityRule{ Services: &v3.ServiceMatch{ Namespace: render.LogCollectorNamespace, - Name: render.FluentdInputService, + Name: render.FluentBitInputService, }, }, })) diff --git a/pkg/render/monitor/monitor.go b/pkg/render/monitor/monitor.go index f9a59de790..361b1c4ed6 100644 --- a/pkg/render/monitor/monitor.go +++ b/pkg/render/monitor/monitor.go @@ -84,7 +84,7 @@ const ( MeshAlertmanagerPolicyName = AlertmanagerPolicyName + "-mesh" ElasticsearchMetrics = "elasticsearch-metrics" - FluentdMetrics = "fluentd-metrics" + FluentBitMetrics = "calico-fluent-bit-metrics" calicoNodePrometheusServiceName = "calico-node-prometheus" tigeraPrometheusServiceHealthEndpoint = "/health" @@ -279,7 +279,7 @@ func (mc *monitorComponent) Objects() ([]client.Object, []client.Object) { serviceMonitors := []client.Object{ mc.serviceMonitorCalicoNode(), mc.serviceMonitorElasticsearch(), - mc.serviceMonitorFluentd(), + mc.serviceMonitorFluentBit(), mc.serviceMonitorQueryServer(), mc.serviceMonitorCalicoKubeControllers(), } @@ -324,8 +324,11 @@ func (mc *monitorComponent) Objects() ([]client.Object, []client.Object) { } toDelete = append(toDelete, - // Remove the pod monitor that existed prior to v1.25. - &monitoringv1.PodMonitor{ObjectMeta: metav1.ObjectMeta{Name: FluentdMetrics, Namespace: common.TigeraPrometheusNamespace}}, + // Remove the pod monitor that existed prior to v1.25 and the + // fluentd-era monitors replaced by serviceMonitorFluentBit. + &monitoringv1.PodMonitor{ObjectMeta: metav1.ObjectMeta{Name: FluentBitMetrics, Namespace: common.TigeraPrometheusNamespace}}, + &monitoringv1.PodMonitor{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-metrics", Namespace: common.TigeraPrometheusNamespace}}, + &monitoringv1.ServiceMonitor{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-metrics", Namespace: common.TigeraPrometheusNamespace}}, // Remove the tigera-prometheus-api deployment that was part of release-v1.23, but has been removed since. &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "tigera-prometheus-api", Namespace: common.TigeraPrometheusNamespace}}, ) @@ -1121,14 +1124,14 @@ func (mc *monitorComponent) serviceMonitorElasticsearch() *monitoringv1.ServiceM } } -// serviceMonitorFluentd creates a service monitor to make Prometheus watch Fluentd. Previously, a pod monitor was used. +// serviceMonitorFluentBit creates a service monitor to make Prometheus watch Fluent Bit. Previously, a pod monitor was used. // However, the pod monitor does not have all the tls configuration options that we need, namely reading them from the // file system, as opposed to getting them from watching kubernetes secrets. -func (mc *monitorComponent) serviceMonitorFluentd() *monitoringv1.ServiceMonitor { +func (mc *monitorComponent) serviceMonitorFluentBit() *monitoringv1.ServiceMonitor { return &monitoringv1.ServiceMonitor{ TypeMeta: metav1.TypeMeta{Kind: monitoringv1.ServiceMonitorsKind, APIVersion: MonitoringAPIVersion}, ObjectMeta: metav1.ObjectMeta{ - Name: render.FluentdMetricsService, + Name: render.FluentBitMetricsService, Namespace: common.TigeraPrometheusNamespace, }, Spec: monitoringv1.ServiceMonitorSpec{ @@ -1137,7 +1140,7 @@ func (mc *monitorComponent) serviceMonitorFluentd() *monitoringv1.ServiceMonitor { Key: "k8s-app", Operator: metav1.LabelSelectorOpIn, - Values: []string{"fluentd-node", "fluentd-node-windows"}, + Values: []string{"calico-fluent-bit", "calico-fluent-bit-windows"}, }, }, }, @@ -1146,19 +1149,14 @@ func (mc *monitorComponent) serviceMonitorFluentd() *monitoringv1.ServiceMonitor { HonorLabels: true, Interval: "5s", - Port: render.FluentdMetricsPortName, + Port: render.FluentBitMetricsPortName, + Path: "/api/v2/metrics/prometheus", ScrapeTimeout: "5s", - HTTPConfigWithProxyAndTLSFiles: monitoringv1.HTTPConfigWithProxyAndTLSFiles{ - HTTPConfigWithTLSFiles: monitoringv1.HTTPConfigWithTLSFiles{ - TLSConfig: mc.tlsConfig(render.FluentdPrometheusTLSSecretName), - }, - }, - RelabelConfigs: []monitoringv1.RelabelConfig{ - { - TargetLabel: "__scheme__", - Replacement: ptr.To("https"), - }, - }, + // fluent-bit's built-in monitoring server (:2020) is plain + // HTTP — it has no TLS support, unlike fluentd's Ruby + // prometheus exporter which terminated mTLS on :9081. Access + // to the port is restricted by the allow-calico-fluent-bit + // NetworkPolicy (prometheus ingress only). }, }, }, diff --git a/pkg/render/monitor/monitor_test.go b/pkg/render/monitor/monitor_test.go index 48809ede67..f52170a36e 100644 --- a/pkg/render/monitor/monitor_test.go +++ b/pkg/render/monitor/monitor_test.go @@ -120,7 +120,7 @@ var _ = Describe("monitor rendering tests", func() { expectedResources := expectedBaseResources() rtest.ExpectResources(toCreate, expectedResources) - Expect(toDelete).To(HaveLen(5)) + Expect(toDelete).To(HaveLen(7)) // Check the namespace. namespace := rtest.GetResource(toCreate, "tigera-prometheus", "", "", "v1", "Namespace").(*corev1.Namespace) @@ -177,7 +177,7 @@ var _ = Describe("monitor rendering tests", func() { component := monitor.Monitor(cfg) Expect(component.ResolveImages(nil)).NotTo(HaveOccurred()) toCreate, toDelete := component.Objects() - Expect(toDelete).To(HaveLen(5)) + Expect(toDelete).To(HaveLen(7)) // Prometheus prometheusObj, ok := rtest.GetResource(toCreate, monitor.CalicoNodePrometheus, common.TigeraPrometheusNamespace, "monitoring.coreos.com", "v1", monitoringv1.PrometheusesKind).(*monitoringv1.Prometheus) @@ -457,7 +457,7 @@ var _ = Describe("monitor rendering tests", func() { Expect(prometheusServiceObj.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(9095))) // PodMonitor - servicemonitorObj, ok := rtest.GetResource(toCreate, monitor.FluentdMetrics, common.TigeraPrometheusNamespace, "monitoring.coreos.com", "v1", monitoringv1.ServiceMonitorsKind).(*monitoringv1.ServiceMonitor) + servicemonitorObj, ok := rtest.GetResource(toCreate, monitor.FluentBitMetrics, common.TigeraPrometheusNamespace, "monitoring.coreos.com", "v1", monitoringv1.ServiceMonitorsKind).(*monitoringv1.ServiceMonitor) Expect(ok).To(BeTrue()) Expect(servicemonitorObj.Spec.Selector.MatchLabels).To(HaveLen(0)) Expect(servicemonitorObj.Spec.Selector.MatchExpressions).To(HaveLen(1)) @@ -465,15 +465,15 @@ var _ = Describe("monitor rendering tests", func() { { Key: "k8s-app", Operator: metav1.LabelSelectorOpIn, - Values: []string{"fluentd-node", "fluentd-node-windows"}, + Values: []string{"calico-fluent-bit", "calico-fluent-bit-windows"}, }, })) Expect(servicemonitorObj.Spec.NamespaceSelector.MatchNames).To(HaveLen(1)) - Expect(servicemonitorObj.Spec.NamespaceSelector.MatchNames[0]).To(Equal("tigera-fluentd")) + Expect(servicemonitorObj.Spec.NamespaceSelector.MatchNames[0]).To(Equal("calico-system")) Expect(servicemonitorObj.Spec.Endpoints).To(HaveLen(1)) Expect(servicemonitorObj.Spec.Endpoints[0].HonorLabels).To(BeTrue()) Expect(servicemonitorObj.Spec.Endpoints[0].Interval).To(BeEquivalentTo("5s")) - Expect(servicemonitorObj.Spec.Endpoints[0].Port).To(Equal("fluentd-metrics-port")) + Expect(servicemonitorObj.Spec.Endpoints[0].Port).To(Equal("fluent-bit-metrics-port")) Expect(servicemonitorObj.Spec.Endpoints[0].ScrapeTimeout).To(BeEquivalentTo("5s")) // PrometheusRule @@ -530,7 +530,7 @@ var _ = Describe("monitor rendering tests", func() { Expect(servicemonitorObj.Spec.Endpoints[0].ScrapeTimeout).To(BeEquivalentTo("5s")) Expect(*servicemonitorObj.Spec.Endpoints[0].RelabelConfigs[0].Replacement).To(Equal("https")) - servicemonitorObj, ok = rtest.GetResource(toCreate, "fluentd-metrics", common.TigeraPrometheusNamespace, "monitoring.coreos.com", "v1", monitoringv1.ServiceMonitorsKind).(*monitoringv1.ServiceMonitor) + servicemonitorObj, ok = rtest.GetResource(toCreate, "calico-fluent-bit-metrics", common.TigeraPrometheusNamespace, "monitoring.coreos.com", "v1", monitoringv1.ServiceMonitorsKind).(*monitoringv1.ServiceMonitor) Expect(ok).To(BeTrue()) Expect(servicemonitorObj.Spec.Selector.MatchLabels).To(HaveLen(0)) Expect(servicemonitorObj.Spec.Selector.MatchExpressions).To(HaveLen(1)) @@ -538,17 +538,20 @@ var _ = Describe("monitor rendering tests", func() { { Key: "k8s-app", Operator: metav1.LabelSelectorOpIn, - Values: []string{"fluentd-node", "fluentd-node-windows"}, + Values: []string{"calico-fluent-bit", "calico-fluent-bit-windows"}, }, })) Expect(servicemonitorObj.Spec.NamespaceSelector.MatchNames).To(HaveLen(1)) - Expect(servicemonitorObj.Spec.NamespaceSelector.MatchNames[0]).To(Equal("tigera-fluentd")) + Expect(servicemonitorObj.Spec.NamespaceSelector.MatchNames[0]).To(Equal("calico-system")) Expect(servicemonitorObj.Spec.Endpoints).To(HaveLen(1)) Expect(servicemonitorObj.Spec.Endpoints[0].HonorLabels).To(BeTrue()) Expect(servicemonitorObj.Spec.Endpoints[0].Interval).To(BeEquivalentTo("5s")) - Expect(servicemonitorObj.Spec.Endpoints[0].Port).To(Equal("fluentd-metrics-port")) + Expect(servicemonitorObj.Spec.Endpoints[0].Port).To(Equal("fluent-bit-metrics-port")) Expect(servicemonitorObj.Spec.Endpoints[0].ScrapeTimeout).To(BeEquivalentTo("5s")) - Expect(*servicemonitorObj.Spec.Endpoints[0].RelabelConfigs[0].Replacement).To(Equal("https")) + // fluent-bit's monitoring server is plain HTTP (no TLS support), unlike + // fluentd's mTLS prometheus exporter. + Expect(servicemonitorObj.Spec.Endpoints[0].RelabelConfigs).To(BeEmpty()) + Expect(servicemonitorObj.Spec.Endpoints[0].TLSConfig).To(BeNil()) servicemonitorObj, ok = rtest.GetResource(toCreate, "calico-api", common.TigeraPrometheusNamespace, "monitoring.coreos.com", "v1", monitoringv1.ServiceMonitorsKind).(*monitoringv1.ServiceMonitor) Expect(ok).To(BeTrue()) @@ -679,7 +682,7 @@ var _ = Describe("monitor rendering tests", func() { expectedResources := expectedBaseResources() rtest.ExpectResources(toCreate, expectedResources) - Expect(toDelete).To(HaveLen(5)) + Expect(toDelete).To(HaveLen(7)) // Prometheus prometheusObj, ok := rtest.GetResource(toCreate, monitor.CalicoNodePrometheus, common.TigeraPrometheusNamespace, "monitoring.coreos.com", "v1", monitoringv1.PrometheusesKind).(*monitoringv1.Prometheus) @@ -869,7 +872,7 @@ var _ = Describe("monitor rendering tests", func() { ) rtest.ExpectResources(toCreate, expectedResources) - Expect(toDelete).To(HaveLen(5)) + Expect(toDelete).To(HaveLen(7)) }) It("Should render external prometheus resources with service monitor and custom token", func() { @@ -895,7 +898,7 @@ var _ = Describe("monitor rendering tests", func() { ) rtest.ExpectResources(toCreate, expectedResources) - Expect(toDelete).To(HaveLen(5)) + Expect(toDelete).To(HaveLen(7)) }) It("Should render external prometheus resources without service monitor", func() { @@ -911,7 +914,7 @@ var _ = Describe("monitor rendering tests", func() { ) rtest.ExpectResources(toCreate, expectedResources) - Expect(toDelete).To(HaveLen(5)) + Expect(toDelete).To(HaveLen(7)) }) It("Should render typha service monitor if typha metrics are enabled", func() { @@ -925,7 +928,7 @@ var _ = Describe("monitor rendering tests", func() { ) rtest.ExpectResources(toCreate, expectedResources) - Expect(toDelete).To(HaveLen(4)) + Expect(toDelete).To(HaveLen(6)) sm := rtest.GetResource(toCreate, "calico-typha-metrics", "tigera-prometheus", "monitoring.coreos.com", "v1", "ServiceMonitor").(*monitoringv1.ServiceMonitor) Expect(sm).To(Equal(&monitoringv1.ServiceMonitor{ TypeMeta: metav1.TypeMeta{Kind: monitoringv1.ServiceMonitorsKind, APIVersion: "monitoring.coreos.com/v1"}, @@ -1002,7 +1005,7 @@ var _ = Describe("monitor rendering tests", func() { serviceMonitorNames := []string{ monitor.CalicoNodeMonitor, monitor.ElasticsearchMetrics, - monitor.FluentdMetrics, + monitor.FluentBitMetrics, "calico-api", "calico-kube-controllers-metrics", } @@ -1027,7 +1030,7 @@ var _ = Describe("monitor rendering tests", func() { serviceMonitorNames := []string{ monitor.CalicoNodeMonitor, monitor.ElasticsearchMetrics, - monitor.FluentdMetrics, + monitor.FluentBitMetrics, "calico-api", "calico-kube-controllers-metrics", } @@ -1064,8 +1067,8 @@ var _ = Describe("monitor rendering tests", func() { serviceMonitor := sm.(*monitoringv1.ServiceMonitor) Expect(serviceMonitor.Spec.Endpoints[0].Port).To(Equal(monitor.OperatorMetricsPortName)) - // Neither should be in toDelete (only PodMonitor, Deployment, typhaServiceMonitor). - Expect(toDelete).To(HaveLen(3)) + // Neither should be in toDelete (only the legacy monitors, Deployment, typhaServiceMonitor). + Expect(toDelete).To(HaveLen(5)) }) It("Should include operator alert rules in PrometheusRule when OperatorMetricsEnabled is true", func() { @@ -1188,7 +1191,7 @@ func expectedBaseResources() []client.Object { &monitoringv1.PrometheusRule{ObjectMeta: metav1.ObjectMeta{Name: monitor.TigeraPrometheusRule, Namespace: common.TigeraPrometheusNamespace}, TypeMeta: metav1.TypeMeta{Kind: "PrometheusRule", APIVersion: "monitoring.coreos.com/v1"}}, &monitoringv1.ServiceMonitor{ObjectMeta: metav1.ObjectMeta{Name: "calico-node-monitor", Namespace: common.TigeraPrometheusNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceMonitor", APIVersion: "monitoring.coreos.com/v1"}}, &monitoringv1.ServiceMonitor{ObjectMeta: metav1.ObjectMeta{Name: "elasticsearch-metrics", Namespace: common.TigeraPrometheusNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceMonitor", APIVersion: "monitoring.coreos.com/v1"}}, - &monitoringv1.ServiceMonitor{ObjectMeta: metav1.ObjectMeta{Name: "fluentd-metrics", Namespace: common.TigeraPrometheusNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceMonitor", APIVersion: "monitoring.coreos.com/v1"}}, + &monitoringv1.ServiceMonitor{ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit-metrics", Namespace: common.TigeraPrometheusNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceMonitor", APIVersion: "monitoring.coreos.com/v1"}}, &monitoringv1.ServiceMonitor{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: common.TigeraPrometheusNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceMonitor", APIVersion: "monitoring.coreos.com/v1"}}, &monitoringv1.ServiceMonitor{ObjectMeta: metav1.ObjectMeta{Name: "calico-kube-controllers-metrics", Namespace: common.TigeraPrometheusNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceMonitor", APIVersion: "monitoring.coreos.com/v1"}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.TigeraOperatorSecrets, Namespace: common.TigeraPrometheusNamespace}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, diff --git a/pkg/render/testutils/expected_policies/dns.json b/pkg/render/testutils/expected_policies/dns.json index 1a6562ee4b..56aea31484 100644 --- a/pkg/render/testutils/expected_policies/dns.json +++ b/pkg/render/testutils/expected_policies/dns.json @@ -12,7 +12,7 @@ { "action": "Allow", "source": { - "selector": "projectcalico.org/namespace in {'calico-system','tigera-compliance','tigera-dex','tigera-elasticsearch','tigera-fluentd','tigera-intrusion-detection','tigera-kibana','tigera-eck-operator','tigera-packetcapture','tigera-prometheus','tigera-skraper'}", + "selector": "projectcalico.org/namespace in {'calico-system','tigera-compliance','tigera-dex','tigera-elasticsearch','tigera-intrusion-detection','tigera-kibana','tigera-eck-operator','tigera-packetcapture','tigera-prometheus','tigera-skraper'}", "namespaceSelector": "all()" }, "destination": {} diff --git a/pkg/render/testutils/expected_policies/dns_ocp.json b/pkg/render/testutils/expected_policies/dns_ocp.json index eb05c51a24..f26fab6166 100644 --- a/pkg/render/testutils/expected_policies/dns_ocp.json +++ b/pkg/render/testutils/expected_policies/dns_ocp.json @@ -12,7 +12,7 @@ { "action":"Allow", "source":{ - "selector":"projectcalico.org/namespace in {'calico-system','tigera-compliance','tigera-dex','tigera-elasticsearch','tigera-fluentd','tigera-intrusion-detection','tigera-kibana','tigera-eck-operator','tigera-packetcapture','tigera-prometheus','tigera-skraper'}", + "selector":"projectcalico.org/namespace in {'calico-system','tigera-compliance','tigera-dex','tigera-elasticsearch','tigera-intrusion-detection','tigera-kibana','tigera-eck-operator','tigera-packetcapture','tigera-prometheus','tigera-skraper'}", "namespaceSelector":"all()" }, "destination":{} diff --git a/pkg/render/testutils/expected_policies/es-gateway.json b/pkg/render/testutils/expected_policies/es-gateway.json index 8d7b439c8a..84a847b013 100644 --- a/pkg/render/testutils/expected_policies/es-gateway.json +++ b/pkg/render/testutils/expected_policies/es-gateway.json @@ -18,8 +18,8 @@ "action": "Allow", "protocol": "TCP", "source": { - "selector": "k8s-app == 'fluentd-node' || k8s-app == 'fluentd-node-windows'", - "namespaceSelector": "name == 'tigera-fluentd'" + "selector": "k8s-app == 'calico-fluent-bit' || k8s-app == 'calico-fluent-bit-windows'", + "namespaceSelector": "name == 'calico-system'" }, "destination": { "ports": [ @@ -32,7 +32,7 @@ "protocol": "TCP", "source": { "selector": "k8s-app == 'eks-log-forwarder'", - "namespaceSelector": "projectcalico.org/name == 'tigera-fluentd'" + "namespaceSelector": "projectcalico.org/name == 'calico-system'" }, "destination": { "ports": [ diff --git a/pkg/render/testutils/expected_policies/es-gateway_ocp.json b/pkg/render/testutils/expected_policies/es-gateway_ocp.json index 10a40d03c5..70abdefa00 100644 --- a/pkg/render/testutils/expected_policies/es-gateway_ocp.json +++ b/pkg/render/testutils/expected_policies/es-gateway_ocp.json @@ -18,8 +18,8 @@ "action": "Allow", "protocol": "TCP", "source": { - "selector": "k8s-app == 'fluentd-node' || k8s-app == 'fluentd-node-windows'", - "namespaceSelector": "name == 'tigera-fluentd'" + "selector": "k8s-app == 'calico-fluent-bit' || k8s-app == 'calico-fluent-bit-windows'", + "namespaceSelector": "name == 'calico-system'" }, "destination": { "ports": [ @@ -32,7 +32,7 @@ "protocol": "TCP", "source": { "selector": "k8s-app == 'eks-log-forwarder'", - "namespaceSelector": "projectcalico.org/name == 'tigera-fluentd'" + "namespaceSelector": "projectcalico.org/name == 'calico-system'" }, "destination": { "ports": [ diff --git a/pkg/render/testutils/expected_policies/fluentd_managed.json b/pkg/render/testutils/expected_policies/fluentbit_managed.json similarity index 81% rename from pkg/render/testutils/expected_policies/fluentd_managed.json rename to pkg/render/testutils/expected_policies/fluentbit_managed.json index eaab9c5b1c..aae1dac0e5 100644 --- a/pkg/render/testutils/expected_policies/fluentd_managed.json +++ b/pkg/render/testutils/expected_policies/fluentbit_managed.json @@ -2,13 +2,13 @@ "apiVersion": "projectcalico.org/v3", "kind": "NetworkPolicy", "metadata": { - "name": "calico-system.allow-fluentd-node", - "namespace": "tigera-fluentd" + "name": "calico-system.allow-calico-fluent-bit", + "namespace": "calico-system" }, "spec": { "tier": "calico-system", "order": 1, - "selector": "k8s-app == 'fluentd-node' || k8s-app == 'fluentd-node-windows'", + "selector": "k8s-app == 'calico-fluent-bit' || k8s-app == 'calico-fluent-bit-windows'", "types": [ "Ingress", "Egress" @@ -23,7 +23,7 @@ }, "destination": { "ports": [ - "9081" + "2020" ] } } diff --git a/pkg/render/testutils/expected_policies/fluentd_unmanaged.json b/pkg/render/testutils/expected_policies/fluentbit_unmanaged.json similarity index 88% rename from pkg/render/testutils/expected_policies/fluentd_unmanaged.json rename to pkg/render/testutils/expected_policies/fluentbit_unmanaged.json index 39ef72b426..290d673e1b 100644 --- a/pkg/render/testutils/expected_policies/fluentd_unmanaged.json +++ b/pkg/render/testutils/expected_policies/fluentbit_unmanaged.json @@ -2,13 +2,13 @@ "apiVersion": "projectcalico.org/v3", "kind": "NetworkPolicy", "metadata": { - "name": "calico-system.allow-fluentd-node", - "namespace": "tigera-fluentd" + "name": "calico-system.allow-calico-fluent-bit", + "namespace": "calico-system" }, "spec": { "tier": "calico-system", "order": 1, - "selector": "k8s-app == 'fluentd-node' || k8s-app == 'fluentd-node-windows'", + "selector": "k8s-app == 'calico-fluent-bit' || k8s-app == 'calico-fluent-bit-windows'", "serviceAccountSelector": "", "types": [ "Ingress", @@ -24,7 +24,7 @@ }, "destination": { "ports": [ - "9081" + "2020" ] } } diff --git a/pkg/render/testutils/expected_policies/fluentd_unmanaged_ocp.json b/pkg/render/testutils/expected_policies/fluentbit_unmanaged_ocp.json similarity index 90% rename from pkg/render/testutils/expected_policies/fluentd_unmanaged_ocp.json rename to pkg/render/testutils/expected_policies/fluentbit_unmanaged_ocp.json index 6fb8be3a12..bf6d782cd2 100644 --- a/pkg/render/testutils/expected_policies/fluentd_unmanaged_ocp.json +++ b/pkg/render/testutils/expected_policies/fluentbit_unmanaged_ocp.json @@ -2,13 +2,13 @@ "apiVersion": "projectcalico.org/v3", "kind": "NetworkPolicy", "metadata": { - "name": "calico-system.allow-fluentd-node", - "namespace": "tigera-fluentd" + "name": "calico-system.allow-calico-fluent-bit", + "namespace": "calico-system" }, "spec": { "tier": "calico-system", "order": 1, - "selector": "k8s-app == 'fluentd-node' || k8s-app == 'fluentd-node-windows'", + "selector": "k8s-app == 'calico-fluent-bit' || k8s-app == 'calico-fluent-bit-windows'", "serviceAccountSelector": "", "types": [ "Ingress", @@ -24,7 +24,7 @@ }, "destination": { "ports": [ - "9081" + "2020" ] } } diff --git a/pkg/render/testutils/expected_policies/guardian.json b/pkg/render/testutils/expected_policies/guardian.json index 05cc65475e..8c80cd1049 100644 --- a/pkg/render/testutils/expected_policies/guardian.json +++ b/pkg/render/testutils/expected_policies/guardian.json @@ -23,8 +23,8 @@ }, "protocol": "TCP", "source": { - "namespaceSelector": "name == 'tigera-fluentd'", - "selector": "k8s-app == 'fluentd-node' || k8s-app == 'fluentd-node-windows'" + "namespaceSelector": "name == 'calico-system'", + "selector": "k8s-app == 'calico-fluent-bit' || k8s-app == 'calico-fluent-bit-windows'" } }, { diff --git a/pkg/render/testutils/expected_policies/guardian_ocp.json b/pkg/render/testutils/expected_policies/guardian_ocp.json index 542351d5f6..c866003892 100644 --- a/pkg/render/testutils/expected_policies/guardian_ocp.json +++ b/pkg/render/testutils/expected_policies/guardian_ocp.json @@ -23,8 +23,8 @@ }, "protocol": "TCP", "source": { - "namespaceSelector": "name == 'tigera-fluentd'", - "selector": "k8s-app == 'fluentd-node' || k8s-app == 'fluentd-node-windows'" + "namespaceSelector": "name == 'calico-system'", + "selector": "k8s-app == 'calico-fluent-bit' || k8s-app == 'calico-fluent-bit-windows'" } }, { diff --git a/pkg/render/testutils/expected_policies/linseed.json b/pkg/render/testutils/expected_policies/linseed.json index 6d75c7caa3..0cf23d1bc5 100644 --- a/pkg/render/testutils/expected_policies/linseed.json +++ b/pkg/render/testutils/expected_policies/linseed.json @@ -18,8 +18,8 @@ "action": "Allow", "protocol": "TCP", "source": { - "selector": "k8s-app == 'fluentd-node' || k8s-app == 'fluentd-node-windows'", - "namespaceSelector": "name == 'tigera-fluentd'" + "selector": "k8s-app == 'calico-fluent-bit' || k8s-app == 'calico-fluent-bit-windows'", + "namespaceSelector": "name == 'calico-system'" }, "destination": { "ports": [ @@ -32,7 +32,7 @@ "protocol": "TCP", "source": { "selector": "k8s-app == 'eks-log-forwarder'", - "namespaceSelector": "projectcalico.org/name == 'tigera-fluentd'" + "namespaceSelector": "projectcalico.org/name == 'calico-system'" }, "destination": { "ports": [ diff --git a/pkg/render/testutils/expected_policies/linseed_dpi_enabled.json b/pkg/render/testutils/expected_policies/linseed_dpi_enabled.json index c7ab55cfd1..023474b338 100644 --- a/pkg/render/testutils/expected_policies/linseed_dpi_enabled.json +++ b/pkg/render/testutils/expected_policies/linseed_dpi_enabled.json @@ -18,8 +18,8 @@ "action": "Allow", "protocol": "TCP", "source": { - "selector": "k8s-app == 'fluentd-node' || k8s-app == 'fluentd-node-windows'", - "namespaceSelector": "name == 'tigera-fluentd'" + "selector": "k8s-app == 'calico-fluent-bit' || k8s-app == 'calico-fluent-bit-windows'", + "namespaceSelector": "name == 'calico-system'" }, "destination": { "ports": [ @@ -32,7 +32,7 @@ "protocol": "TCP", "source": { "selector": "k8s-app == 'eks-log-forwarder'", - "namespaceSelector": "projectcalico.org/name == 'tigera-fluentd'" + "namespaceSelector": "projectcalico.org/name == 'calico-system'" }, "destination": { "ports": [ diff --git a/pkg/render/testutils/expected_policies/linseed_ocp.json b/pkg/render/testutils/expected_policies/linseed_ocp.json index f28838a0ed..241f2fe126 100644 --- a/pkg/render/testutils/expected_policies/linseed_ocp.json +++ b/pkg/render/testutils/expected_policies/linseed_ocp.json @@ -18,8 +18,8 @@ "action": "Allow", "protocol": "TCP", "source": { - "selector": "k8s-app == 'fluentd-node' || k8s-app == 'fluentd-node-windows'", - "namespaceSelector": "name == 'tigera-fluentd'" + "selector": "k8s-app == 'calico-fluent-bit' || k8s-app == 'calico-fluent-bit-windows'", + "namespaceSelector": "name == 'calico-system'" }, "destination": { "ports": [ @@ -32,7 +32,7 @@ "protocol": "TCP", "source": { "selector": "k8s-app == 'eks-log-forwarder'", - "namespaceSelector": "projectcalico.org/name == 'tigera-fluentd'" + "namespaceSelector": "projectcalico.org/name == 'calico-system'" }, "destination": { "ports": [ diff --git a/pkg/render/testutils/expected_policies/linseed_ocp_dpi_enabled.json b/pkg/render/testutils/expected_policies/linseed_ocp_dpi_enabled.json index 97ae80ffba..82478056c1 100644 --- a/pkg/render/testutils/expected_policies/linseed_ocp_dpi_enabled.json +++ b/pkg/render/testutils/expected_policies/linseed_ocp_dpi_enabled.json @@ -18,8 +18,8 @@ "action": "Allow", "protocol": "TCP", "source": { - "selector": "k8s-app == 'fluentd-node' || k8s-app == 'fluentd-node-windows'", - "namespaceSelector": "name == 'tigera-fluentd'" + "selector": "k8s-app == 'calico-fluent-bit' || k8s-app == 'calico-fluent-bit-windows'", + "namespaceSelector": "name == 'calico-system'" }, "destination": { "ports": [ @@ -32,7 +32,7 @@ "protocol": "TCP", "source": { "selector": "k8s-app == 'eks-log-forwarder'", - "namespaceSelector": "projectcalico.org/name == 'tigera-fluentd'" + "namespaceSelector": "projectcalico.org/name == 'calico-system'" }, "destination": { "ports": [ diff --git a/pkg/render/testutils/expected_policies/node_local_dns_dual.json b/pkg/render/testutils/expected_policies/node_local_dns_dual.json index aa1e962285..71371ed0e2 100644 --- a/pkg/render/testutils/expected_policies/node_local_dns_dual.json +++ b/pkg/render/testutils/expected_policies/node_local_dns_dual.json @@ -7,7 +7,7 @@ "spec": { "tier":"calico-system", "order":10, - "selector": "projectcalico.org/namespace in {'calico-system','tigera-compliance','tigera-dex','tigera-elasticsearch','tigera-fluentd','tigera-intrusion-detection','tigera-kibana','tigera-eck-operator','tigera-packetcapture','tigera-prometheus','tigera-skraper'}", + "selector": "projectcalico.org/namespace in {'calico-system','tigera-compliance','tigera-dex','tigera-elasticsearch','tigera-intrusion-detection','tigera-kibana','tigera-eck-operator','tigera-packetcapture','tigera-prometheus','tigera-skraper'}", "egress":[ { "action":"Allow", diff --git a/pkg/render/testutils/expected_policies/node_local_dns_ipv4.json b/pkg/render/testutils/expected_policies/node_local_dns_ipv4.json index fe3c073a88..bf482c16eb 100644 --- a/pkg/render/testutils/expected_policies/node_local_dns_ipv4.json +++ b/pkg/render/testutils/expected_policies/node_local_dns_ipv4.json @@ -7,7 +7,7 @@ "spec": { "tier":"calico-system", "order":10, - "selector": "projectcalico.org/namespace in {'calico-system','tigera-compliance','tigera-dex','tigera-elasticsearch','tigera-fluentd','tigera-intrusion-detection','tigera-kibana','tigera-eck-operator','tigera-packetcapture','tigera-prometheus','tigera-skraper'}", + "selector": "projectcalico.org/namespace in {'calico-system','tigera-compliance','tigera-dex','tigera-elasticsearch','tigera-intrusion-detection','tigera-kibana','tigera-eck-operator','tigera-packetcapture','tigera-prometheus','tigera-skraper'}", "egress":[ { "action":"Allow", diff --git a/pkg/render/testutils/expected_policies/node_local_dns_ipv6.json b/pkg/render/testutils/expected_policies/node_local_dns_ipv6.json index 8a2b2d376d..e5e3ecd1a4 100644 --- a/pkg/render/testutils/expected_policies/node_local_dns_ipv6.json +++ b/pkg/render/testutils/expected_policies/node_local_dns_ipv6.json @@ -7,7 +7,7 @@ "spec": { "tier":"calico-system", "order":10, - "selector": "projectcalico.org/namespace in {'calico-system','tigera-compliance','tigera-dex','tigera-elasticsearch','tigera-fluentd','tigera-intrusion-detection','tigera-kibana','tigera-eck-operator','tigera-packetcapture','tigera-prometheus','tigera-skraper'}", + "selector": "projectcalico.org/namespace in {'calico-system','tigera-compliance','tigera-dex','tigera-elasticsearch','tigera-intrusion-detection','tigera-kibana','tigera-eck-operator','tigera-packetcapture','tigera-prometheus','tigera-skraper'}", "egress":[ { "action":"Allow", diff --git a/pkg/render/tiers/tiers_test.go b/pkg/render/tiers/tiers_test.go index dbf936bf4c..ed0e135a32 100644 --- a/pkg/render/tiers/tiers_test.go +++ b/pkg/render/tiers/tiers_test.go @@ -71,7 +71,6 @@ var _ = Describe("Tiers rendering tests", func() { render.ComplianceNamespace, render.DexNamespace, render.ElasticsearchNamespace, - render.LogCollectorNamespace, render.IntrusionDetectionNamespace, kibana.Namespace, eck.OperatorNamespace, From c81ad06d3ea0718c458845153eaa31059b42f52b Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Thu, 11 Jun 2026 00:23:27 -0700 Subject: [PATCH 2/2] Ship fluent-bit logs through the built-in http output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the out_linseed Go proxy output with one built-in http output block per Linseed-bound tag, for the Linux and Windows daemonsets and the eks-log-forwarder. The http output is C compiled into fluent-bit: `format json_lines` with the date key disabled produces exactly the NDJSON body Linseed's bulk APIs expect, native tls.* carries the mTLS client keypair (with hostname verification enabled — fluent-bit's tls.verify alone only checks the chain), and bearer_token_file (re-read per request) carries the ServiceAccount or managed-cluster token. Multi-tenant clusters send the x-tenant-id header. The Windows image runs no Go code at all, so the Windows config no longer loads a plugins file; the Linux configs keep it only for the in_eks EKS CloudWatch input, which stays a Go plugin and now feeds the http output instead of out_linseed. The optional EksCloudwatchLog streamPrefix/fetchInterval settings are omitted from the environment when unset as defense in depth (the logcollector controller defaults them before render, but an empty prefix or zero interval reaching the plugin would override its own defaults with settings that match every stream / disable polling). Per-tag filesystem retry backlogs replace the single shared cap: flows keeps 500M, low-volume tags get 100M each. Co-Authored-By: Claude Fable 5 --- pkg/render/fluentbit.go | 201 ++++++++++++++++++++++++----------- pkg/render/fluentbit_test.go | 96 +++++++++++++++-- 2 files changed, 223 insertions(+), 74 deletions(-) diff --git a/pkg/render/fluentbit.go b/pkg/render/fluentbit.go index deee776246..e6e76b492f 100644 --- a/pkg/render/fluentbit.go +++ b/pkg/render/fluentbit.go @@ -18,6 +18,7 @@ import ( "crypto/x509" "encoding/json" "fmt" + "strconv" "strings" "sigs.k8s.io/yaml" @@ -282,10 +283,10 @@ func (c *fluentBitComponent) binPath() string { return "/usr/bin/fluent-bit" } +// pluginsFilePath is the loader config for the in_eks Go plugin shipped in +// the Linux image. The Windows image carries no Go plugins, so the Windows +// configs never reference it. func (c *fluentBitComponent) pluginsFilePath() string { - if c.cfg.OSType == rmeta.OSTypeWindows { - return "c:/fluent-bit/plugins.conf" - } return "/etc/fluent-bit/plugins.conf" } @@ -735,7 +736,7 @@ func (c *fluentBitComponent) container() corev1.Container { } else { // Mount only the rendered config as a single file (SubPath) so it does // not shadow the image's /etc/fluent-bit directory, which ships - // plugins.conf, record_transformer.lua and the loaded plugins. + // plugins.conf, record_transformer.lua and the in_eks plugin. volumeMounts = append(volumeMounts, corev1.VolumeMount{MountPath: "/etc/fluent-bit/fluent-bit.yaml", Name: "fluent-bit-conf", SubPath: "fluent-bit.yaml", ReadOnly: true}) } @@ -931,31 +932,120 @@ func (c *fluentBitComponent) logDirsCSV() string { return strings.Join(dirs, ",") } -// linseedMatchRegex builds the match for the linseed output: every tailed tag -// except ids.events and compliance.reports — those are deliberately not -// Linseed-bound (IDS events use a different ingestion path; compliance reports -// are S3-only), and an unknown tag makes out_linseed drop the chunk with an -// error. The non_cluster_* tags are produced by the voltron-facing http input. -func (c *fluentBitComponent) linseedMatchRegex() string { +// linseedTags lists the tags shipped to Linseed: every tailed tag except +// ids.events and compliance.reports — those are deliberately not +// Linseed-bound (IDS events use a different ingestion path; compliance +// reports are S3-only). The non_cluster_* tags are produced by the +// voltron-facing http input relaying non-cluster host posts; hosts ship +// flow, DNS and policy activity logs. +func (c *fluentBitComponent) linseedTags() []string { var tags []string for _, in := range c.logInputs() { if in.tag == "ids.events" || in.tag == "compliance.reports" { continue } - tags = append(tags, strings.ReplaceAll(in.tag, ".", `\.`)) + tags = append(tags, in.tag) } if c.cfg.NonClusterHost != nil && c.cfg.OSType == rmeta.OSTypeLinux { tags = append(tags, "non_cluster_flows", "non_cluster_dns", "non_cluster_policy_activity") } - return fmt.Sprintf("^(%s)$", strings.Join(tags, "|")) + return tags +} + +// linseedBulkURI maps a tag to its Linseed bulk-ingestion URI. Voltron-relayed +// non_cluster_* tags post to the same path as their base tag. +func linseedBulkURI(tag string) string { + tag = strings.TrimPrefix(tag, "non_cluster_") + switch tag { + case "runtime": + return "/api/v1/runtime/reports/bulk" + case "audit.tsee": + return "/api/v1/audit/logs/ee/bulk" + case "audit.kube": + return "/api/v1/audit/logs/kube/bulk" + case "bird", "bird6": + return "/api/v1/bgp/logs/bulk" + default: + // flows, dns, l7, waf, policy_activity + return fmt.Sprintf("/api/v1/%s/logs/bulk", tag) + } +} + +// splitEndpoint splits an https:// endpoint into the host and port fields +// fluent-bit's native net layer expects (the port defaults to 443). Plain +// string handling: pkg/url's ParseEndpoint rejects endpoints without an +// explicit port, and Linseed endpoints usually carry none. +func splitEndpoint(endpoint string) (string, int) { + host := strings.TrimPrefix(endpoint, "https://") + host = strings.TrimSuffix(host, "/") + port := 443 + if i := strings.LastIndex(host, ":"); i >= 0 { + if n, err := strconv.Atoi(host[i+1:]); err == nil { + host, port = host[:i], n + } + } + return host, port +} + +// linseedHTTPOutput renders one built-in http output block shipping a tag's +// chunks to its Linseed bulk endpoint. The http output is plain C compiled +// into fluent-bit — no Go proxy plugin is involved — and `format json_lines` +// with the date key disabled produces exactly the NDJSON body Linseed's bulk +// APIs expect. The bearer token file is re-read on every request (a Tigera +// patch carried by the fluent-bit base build), so kubelet-rotated +// ServiceAccount tokens and operator-refreshed managed-cluster tokens are +// picked up without a restart. certPath/keyPath are the mTLS client keypair; +// storageLimit, when non-empty, caps this output's filesystem retry backlog. +func (c *fluentBitComponent) linseedHTTPOutput(tag, certPath, keyPath, storageLimit string) map[string]interface{} { + host, port := splitEndpoint(relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, LinseedNamespace(c.cfg.Tenant), c.cfg.ManagedCluster, true)) + out := map[string]interface{}{ + "name": "http", + "match": tag, + "host": host, + "port": port, + "uri": linseedBulkURI(tag), + "format": "json_lines", + // One record per line, nothing else: Linseed parses each line as the + // log document itself, so no synthetic date field is added. + "json_date_key": false, + "tls": "on", + "tls.verify": "on", + // tls.verify only checks the chain; hostname/SAN verification is a + // separate knob that defaults off in fluent-bit. The Go plugin this + // replaces verified hostnames (crypto/tls default), so keep parity. + "tls.verify_hostname": "on", + "tls.ca_file": c.trustedBundlePath(), + "tls.crt_file": certPath, + "tls.key_file": keyPath, + "bearer_token_file": c.path(GetLinseedTokenPath(c.cfg.ManagedCluster)), + // Retry failed chunks until they send instead of dropping them after + // the default single retry; the filesystem storage bounds what can + // accumulate during a Linseed outage. + "retry_limit": "no_limits", + } + if storageLimit != "" { + out["storage.total_limit_size"] = storageLimit + } + if c.cfg.Tenant != nil && c.cfg.ExternalElastic { + out["header"] = fmt.Sprintf("x-tenant-id %s", c.cfg.Tenant.Spec.ID) + } + return out +} + +// linseedStorageLimit sizes a tag's filesystem retry backlog: flow logs are +// the dominant volume and keep the budget the single shared output used to +// have; everything else is low-volume. +func linseedStorageLimit(tag string) string { + if tag == "flows" || tag == "non_cluster_flows" { + return "500M" + } + return "100M" } func (c *fluentBitComponent) renderFluentBitConf() string { - linseedEndpoint := relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, LinseedNamespace(c.cfg.Tenant), c.cfg.ManagedCluster, true) caPath := c.trustedBundlePath() keyPath := c.keyPath() certPath := c.certPath() - tokenPath := c.path(GetLinseedTokenPath(c.cfg.ManagedCluster)) cfg := fluentBitConfig{ Service: map[string]interface{}{ @@ -967,9 +1057,6 @@ func (c *fluentBitComponent) renderFluentBitConf() string { // startup probes hit (without this it returns 404 and pods never // become Ready). "health_check": true, - // Load the custom out_linseed (and in_eks) Go plugins shipped in the - // image. Without this the `linseed` output is an unknown plugin. - "plugins_file": c.pluginsFilePath(), // Filesystem buffering under the same hostPath-backed state dir as // the tail offset DBs, so buffered-but-unsent chunks survive pod // restarts (fluentd buffered to disk for up to 72h). @@ -1033,33 +1120,14 @@ func (c *fluentBitComponent) renderFluentBitConf() string { // Each ConfigMap key holds a YAML list of fluent-bit filter entries. c.addUserFilters(&cfg) - // NOTE: linseed is a Go proxy output plugin that performs its own HTTPS - // (it builds an http.Client from the CA pool + client keypair). fluent-bit - // reserves the native `tls.*` property namespace for outputs that use its - // built-in TLS layer and rejects those keys on proxy plugins - // ("linseed.0 does not support TLS"). So the CA/cert/key are passed with - // non-tls key names that the plugin reads itself; it always verifies the - // server certificate (skip_verify defaults to false). - linseedOutput := map[string]interface{}{ - "name": "linseed", - "match_regex": c.linseedMatchRegex(), - "endpoint": linseedEndpoint, - "ca_file": caPath, - "cert_file": certPath, - "key_file": keyPath, - // Retry failed chunks until they send instead of dropping them after - // the default single retry; the filesystem storage bounds what can - // accumulate during a Linseed outage. - "retry_limit": "no_limits", - "storage.total_limit_size": "500M", - } - if c.cfg.ManagedCluster { - linseedOutput["token_path"] = tokenPath - } - if c.cfg.Tenant != nil && c.cfg.ExternalElastic { - linseedOutput["tenant_id"] = c.cfg.Tenant.Spec.ID + // One built-in http output per Linseed-bound tag: chunks are per-tag, so + // an exact match per block routes every record to its bulk endpoint. The + // per-tag split replaces the single out_linseed Go proxy output — the C + // http output keeps the container free of Go proxy plugins. + for _, tag := range c.linseedTags() { + cfg.Pipeline.Outputs = append(cfg.Pipeline.Outputs, + c.linseedHTTPOutput(tag, certPath, keyPath, linseedStorageLimit(tag))) } - cfg.Pipeline.Outputs = append(cfg.Pipeline.Outputs, linseedOutput) // Additional stores are Linux-only, matching the fluentd Windows variant // (Linseed only). @@ -1410,10 +1478,11 @@ func (c *fluentBitComponent) eksLogForwarderSecret() *corev1.Secret { } // renderEKSFluentBitConf renders the fluent-bit config for the eks-log-forwarder -// Deployment: a single in_eks input (which polls CloudWatch, applies the EKS -// audit filtering itself, and resumes from the last Linseed-ingested timestamp) -// feeding the linseed output. The in_eks plugin reads its CloudWatch/Linseed -// settings from the Deployment's env vars rather than plugin properties. +// Deployment: a single in_eks input (the Go plugin polls CloudWatch, applies +// the EKS audit shaping itself, and resumes from the last Linseed-ingested +// timestamp on every process start) feeding the built-in http output that +// ships to Linseed. The in_eks plugin reads its CloudWatch/Linseed settings +// from the Deployment's env vars rather than plugin properties. func (c *fluentBitComponent) renderEKSFluentBitConf() string { cfg := fluentBitConfig{ Service: map[string]interface{}{ @@ -1422,6 +1491,8 @@ func (c *fluentBitComponent) renderEKSFluentBitConf() string { "http_server": true, "http_port": FluentBitMetricsPort, "health_check": true, + // Load the custom in_eks Go plugin shipped in the image. Without + // this the `in_eks` input is an unknown plugin. "plugins_file": c.pluginsFilePath(), }, } @@ -1429,22 +1500,12 @@ func (c *fluentBitComponent) renderEKSFluentBitConf() string { "name": "in_eks", "tag": "audit.kube", }) - linseedOutput := map[string]interface{}{ - "name": "linseed", - "match": "audit.kube", - "endpoint": relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, LinseedNamespace(c.cfg.Tenant), c.cfg.ManagedCluster, true), - "ca_file": c.trustedBundlePath(), - "cert_file": c.cfg.EKSLogForwarderKeyPair.VolumeMountCertificateFilePath(), - "key_file": c.cfg.EKSLogForwarderKeyPair.VolumeMountKeyFilePath(), - "retry_limit": "no_limits", - } - if c.cfg.ManagedCluster { - linseedOutput["token_path"] = c.path(GetLinseedTokenPath(c.cfg.ManagedCluster)) - } - if c.cfg.Tenant != nil && c.cfg.ExternalElastic { - linseedOutput["tenant_id"] = c.cfg.Tenant.Spec.ID - } - cfg.Pipeline.Outputs = append(cfg.Pipeline.Outputs, linseedOutput) + cfg.Pipeline.Outputs = append(cfg.Pipeline.Outputs, c.linseedHTTPOutput( + "audit.kube", + c.cfg.EKSLogForwarderKeyPair.VolumeMountCertificateFilePath(), + c.cfg.EKSLogForwarderKeyPair.VolumeMountKeyFilePath(), + // No filesystem storage on this Deployment, so no backlog cap applies. + "")) out, err := json.MarshalIndent(cfg, "", " ") if err != nil { @@ -1472,8 +1533,6 @@ func (c *fluentBitComponent) eksLogForwarderDeployment() *appsv1.Deployment { // CloudWatch config, credentials — consumed by the in_eks input plugin // (fluent-bit/plugins/in_eks/pkg/config) and the AWS SDK credential chain. {Name: "EKS_CLOUDWATCH_LOG_GROUP", Value: c.cfg.EKSConfig.GroupName}, - {Name: "EKS_CLOUDWATCH_LOG_STREAM_PREFIX", Value: c.cfg.EKSConfig.StreamPrefix}, - {Name: "EKS_CLOUDWATCH_POLL_INTERVAL", Value: fmt.Sprintf("%ds", c.cfg.EKSConfig.FetchInterval)}, {Name: "AWS_REGION", Value: c.cfg.EKSConfig.AwsRegion}, {Name: "AWS_ACCESS_KEY_ID", ValueFrom: secret.GetEnvVarSource(EksLogForwarderSecret, EksLogForwarderAwsId, false)}, {Name: "AWS_SECRET_ACCESS_KEY", ValueFrom: secret.GetEnvVarSource(EksLogForwarderSecret, EksLogForwarderAwsKey, false)}, @@ -1487,6 +1546,18 @@ func (c *fluentBitComponent) eksLogForwarderDeployment() *appsv1.Deployment { {Name: "TLS_KEY_PATH", Value: c.cfg.EKSLogForwarderKeyPair.VolumeMountKeyFilePath()}, {Name: "LINSEED_TOKEN", Value: c.path(GetLinseedTokenPath(c.cfg.ManagedCluster))}, } + // The logcollector controller defaults these before render + // (getEksCloudwatchLogConfig: prefix kube-apiserver-audit-, interval + // 60), so in practice both env vars are always set. The guards are + // defense in depth for other callers: rendering an empty prefix or a + // zero interval would override the plugin's own defaults with a broken + // setting (an empty prefix matches every stream in the group). + if c.cfg.EKSConfig.StreamPrefix != "" { + envVars = append(envVars, corev1.EnvVar{Name: "EKS_CLOUDWATCH_LOG_STREAM_PREFIX", Value: c.cfg.EKSConfig.StreamPrefix}) + } + if c.cfg.EKSConfig.FetchInterval > 0 { + envVars = append(envVars, corev1.EnvVar{Name: "EKS_CLOUDWATCH_POLL_INTERVAL", Value: fmt.Sprintf("%ds", c.cfg.EKSConfig.FetchInterval)}) + } if c.cfg.Tenant != nil && c.cfg.ExternalElastic { envVars = append(envVars, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.Tenant.Spec.ID}) } diff --git a/pkg/render/fluentbit_test.go b/pkg/render/fluentbit_test.go index 3604f46850..7bd27d2a84 100644 --- a/pkg/render/fluentbit_test.go +++ b/pkg/render/fluentbit_test.go @@ -164,8 +164,26 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() { cm := rtest.GetResource(resources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) Expect(cm.Data).To(HaveKey("fluent-bit.yaml")) fluentBitConf := cm.Data["fluent-bit.yaml"] - Expect(fluentBitConf).To(ContainSubstring("linseed")) - Expect(fluentBitConf).To(ContainSubstring("tigera-linseed")) + // Linseed shipping uses the built-in http output (no Go proxy + // plugins): one block per tag, NDJSON body, mTLS plus a bearer token + // re-read from file on every request. + Expect(fluentBitConf).To(ContainSubstring(`"name": "http"`)) + Expect(fluentBitConf).To(ContainSubstring(`"host": "tigera-linseed.tigera-elasticsearch.svc"`)) + Expect(fluentBitConf).To(ContainSubstring(`"uri": "/api/v1/flows/logs/bulk"`)) + Expect(fluentBitConf).To(ContainSubstring(`"uri": "/api/v1/dns/logs/bulk"`)) + Expect(fluentBitConf).To(ContainSubstring(`"uri": "/api/v1/audit/logs/ee/bulk"`)) + Expect(fluentBitConf).To(ContainSubstring(`"uri": "/api/v1/audit/logs/kube/bulk"`)) + Expect(fluentBitConf).To(ContainSubstring(`"uri": "/api/v1/bgp/logs/bulk"`)) + Expect(fluentBitConf).To(ContainSubstring(`"format": "json_lines"`)) + Expect(fluentBitConf).To(ContainSubstring(`"json_date_key": false`)) + Expect(fluentBitConf).To(ContainSubstring(`"bearer_token_file": "/var/run/secrets/kubernetes.io/serviceaccount/token"`)) + // Per-tag filesystem retry caps: flows is the dominant volume and + // keeps the budget the single shared output used to have. + Expect(fluentBitConf).To(ContainSubstring(`"storage.total_limit_size": "500M"`)) + Expect(fluentBitConf).To(ContainSubstring(`"storage.total_limit_size": "100M"`)) + // No Go proxy plugins are loaded. + Expect(fluentBitConf).NotTo(ContainSubstring("plugins_file")) + Expect(fluentBitConf).NotTo(ContainSubstring(`"name": "linseed"`)) container := ds.Spec.Template.Spec.Containers[0] @@ -349,8 +367,12 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() { cm := rtest.GetResource(createResources, render.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) Expect(cm.Data).To(HaveKey("fluent-bit.yaml")) fluentBitConf := cm.Data["fluent-bit.yaml"] - Expect(fluentBitConf).To(ContainSubstring("linseed")) - Expect(fluentBitConf).To(ContainSubstring("tigera-linseed")) + Expect(fluentBitConf).To(ContainSubstring(`"name": "http"`)) + // Managed clusters post to the external tigera-linseed service (which + // redirects to Guardian) with the operator-provisioned token, not the + // pod's ServiceAccount token. + Expect(fluentBitConf).To(ContainSubstring(`"host": "tigera-linseed"`)) + Expect(fluentBitConf).To(ContainSubstring(`"bearer_token_file": "/var/run/secrets/tigera.io/linseed/token"`)) container := ds.Spec.Template.Spec.Containers[0] @@ -573,9 +595,15 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() { cm := rtest.GetResource(resources, render.FluentBitConfConfigMapName+"-windows", render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) Expect(cm.Data).To(HaveKey("fluent-bit.yaml")) fluentBitConf := cm.Data["fluent-bit.yaml"] - Expect(fluentBitConf).To(ContainSubstring("linseed")) + // Windows ships through the built-in http output too — the image + // carries no plugin DLLs and no Go runtime at all. + Expect(fluentBitConf).To(ContainSubstring(`"name": "http"`)) + Expect(fluentBitConf).To(ContainSubstring(`"uri": "/api/v1/flows/logs/bulk"`)) + Expect(fluentBitConf).To(ContainSubstring(`"uri": "/api/v1/audit/logs/ee/bulk"`)) + Expect(fluentBitConf).To(ContainSubstring(`"uri": "/api/v1/audit/logs/kube/bulk"`)) + Expect(fluentBitConf).To(ContainSubstring(`"bearer_token_file": "c:/var/run/secrets/kubernetes.io/serviceaccount/token"`)) + Expect(fluentBitConf).NotTo(ContainSubstring("plugins_file")) // The Windows image lays everything out under C:\fluent-bit. - Expect(fluentBitConf).To(ContainSubstring(`"plugins_file": "c:/fluent-bit/plugins.conf"`)) Expect(fluentBitConf).To(ContainSubstring(`"script": "c:/fluent-bit/record_transformer.lua"`)) // Windows tails only the log types the fluentd Windows variant shipped. Expect(fluentBitConf).To(ContainSubstring("c:/var/log/calico/flowlogs/flows.log")) @@ -885,7 +913,7 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() { deploy := rtest.GetResource(resources, "eks-log-forwarder", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) // The fluentd-era startup init container is gone: the in_eks input - // plugin resolves its own resume point from Linseed. + // plugin resolves its own resume point from Linseed on every start. Expect(deploy.Spec.Template.Spec.InitContainers).To(BeEmpty()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) Expect(deploy.Spec.Template.Annotations).To(HaveKey("hash.operator.tigera.io/eks-cloudwatch-log-credentials")) @@ -916,8 +944,6 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() { expectedEnvVars := []corev1.EnvVar{ {Name: "LOG_LEVEL", Value: "info", ValueFrom: nil}, {Name: "EKS_CLOUDWATCH_LOG_GROUP", Value: "dummy-eks-cluster-cloudwatch-log-group"}, - {Name: "EKS_CLOUDWATCH_LOG_STREAM_PREFIX", Value: ""}, - {Name: "EKS_CLOUDWATCH_POLL_INTERVAL", Value: "900s"}, {Name: "AWS_REGION", Value: "us-west-1", ValueFrom: nil}, { Name: "AWS_ACCESS_KEY_ID", @@ -947,9 +973,43 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() { {Name: "TLS_CRT_PATH", Value: "/tigera-eks-log-forwarder-tls/tls.crt"}, {Name: "TLS_KEY_PATH", Value: "/tigera-eks-log-forwarder-tls/tls.key"}, {Name: "LINSEED_TOKEN", Value: "/var/run/secrets/kubernetes.io/serviceaccount/token"}, + // streamPrefix is unset in this render config (the controller + // would have defaulted it), so the env var is omitted and the + // plugin's kube-apiserver-audit- default applies; fetchInterval + // is set (900) so it is rendered. + {Name: "EKS_CLOUDWATCH_POLL_INTERVAL", Value: "900s"}, } Expect(envs).To(Equal(expectedEnvVars)) + + // The rendered config wires the in_eks input into the Linseed http + // output. + cm := rtest.GetResource(resources, render.EKSLogForwarderConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + eksConf := cm.Data["fluent-bit.yaml"] + Expect(eksConf).To(ContainSubstring(`"name": "in_eks"`)) + Expect(eksConf).To(ContainSubstring(`"plugins_file": "/etc/fluent-bit/plugins.conf"`)) + Expect(eksConf).To(ContainSubstring(`"name": "http"`)) + Expect(eksConf).To(ContainSubstring(`"uri": "/api/v1/audit/logs/kube/bulk"`)) + Expect(eksConf).To(ContainSubstring(`"tls.verify_hostname": "on"`)) + }) + + It("should omit unset EKS CloudWatch settings so plugin defaults apply", func() { + cfg.EKSConfig = setupEKSCloudwatchLogConfig() + cfg.EKSConfig.FetchInterval = 0 + cfg.Installation = &operatorv1.InstallationSpec{ + KubernetesProvider: operatorv1.ProviderEKS, + } + + resources, _ := render.FluentBit(cfg).Objects() + deploy := rtest.GetResource(resources, "eks-log-forwarder", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) + // The controller defaults these before render in production; this + // pins the render-level defense in depth — an empty prefix or "0s" + // interval must never reach the plugin, which would override its + // envconfig defaults with broken settings. + for _, env := range deploy.Spec.Template.Spec.Containers[0].Env { + Expect(env.Name).NotTo(Equal("EKS_CLOUDWATCH_LOG_STREAM_PREFIX")) + Expect(env.Name).NotTo(Equal("EKS_CLOUDWATCH_POLL_INTERVAL")) + } }) It("should render EKS Cloudwatch Log toleration on GKE", func() { @@ -1050,6 +1110,10 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() { Expect(envs).To(ContainElement(corev1.EnvVar{Name: "LINSEED_ENDPOINT", Value: "https://tigera-linseed.tenant-namespace.svc"})) Expect(envs).To(ContainElement(corev1.EnvVar{Name: "TENANT_ID", Value: "test-tenant-id"})) Expect(envs).To(ContainElement(corev1.EnvVar{Name: "LINSEED_TOKEN", Value: "/var/run/secrets/kubernetes.io/serviceaccount/token"})) + + // The tenant header rides on the http output config. + cm := rtest.GetResource(resources, render.EKSLogForwarderConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + Expect(cm.Data["fluent-bit.yaml"]).To(ContainSubstring(`"header": "x-tenant-id test-tenant-id"`)) }) It("should render with EKS Cloudwatch Log for managed cluster with linseed token volume", func() { @@ -1079,8 +1143,14 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() { envs := deploy.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElement(corev1.EnvVar{Name: "LINSEED_TOKEN", Value: "/var/run/secrets/tigera.io/linseed/token"})) + // The container mounts the operator-provisioned token for both the + // in_eks resume-point query and the http output's bearer_token_file, + // which re-reads it on every request. volumeMounts := deploy.Spec.Template.Spec.Containers[0].VolumeMounts Expect(volumeMounts).To(ContainElement(corev1.VolumeMount{Name: "linseed-token", MountPath: "/var/run/secrets/tigera.io/linseed/"})) + + cm := rtest.GetResource(resources, render.EKSLogForwarderConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap) + Expect(cm.Data["fluent-bit.yaml"]).To(ContainSubstring(`"bearer_token_file": "/var/run/secrets/tigera.io/linseed/token"`)) }) DescribeTable("should render with a valid configuration for non-cluster host and forwarding enabled", @@ -1144,6 +1214,14 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() { allHostsConf := cm.Data["fluent-bit.yaml"] Expect(allHostsConf).To(ContainSubstring(strings.ToLower(destination))) + // The voltron-relayed non-cluster host tags each get their own http + // output posting to the base tag's bulk URI; a tag without a + // matching output would be silently dropped by the router. + Expect(allHostsConf).To(ContainSubstring(`"match": "non_cluster_flows"`)) + Expect(allHostsConf).To(ContainSubstring(`"match": "non_cluster_dns"`)) + Expect(allHostsConf).To(ContainSubstring(`"match": "non_cluster_policy_activity"`)) + Expect(allHostsConf).To(ContainSubstring(`"uri": "/api/v1/policy_activity/logs/bulk"`)) + By("enabling forwarding of only non-cluster logs") cfg.LogCollector.Spec.AdditionalStores = additionalStoreSpecNonClusterHosts resources, _ = render.FluentBit(cfg).Objects()