diff --git a/api/go.mod b/api/go.mod index 5bc90b576..e927be46b 100644 --- a/api/go.mod +++ b/api/go.mod @@ -5,7 +5,7 @@ go 1.26.3 require ( github.com/onsi/gomega v1.42.1 github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce - github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 + github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260813160234-fdcb3ee3699d github.com/rhobs/observability-operator v1.0.0 k8s.io/api v0.33.13 k8s.io/apimachinery v0.33.13 diff --git a/api/go.sum b/api/go.sum index 4b166f360..19acd32db 100644 --- a/api/go.sum +++ b/api/go.sum @@ -78,8 +78,8 @@ github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce h1:XLbNNe5V6mxfJF+ETXlwgOuMAwVpidXNyBU5HD35AAQ= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce/go.mod h1:XCImRHgk9LhAWyGEYihqOPubYCNwosqe/toIylOaFAc= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 h1:9YpUAOFXCC94SwoWBRrhFyCVbbwnHTce6/j3CqjxaGM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:LY+Q6ms3A6E8DAKCpmvQaUxO5O2CKYm0XMcmTZ6PCcM= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260813160234-fdcb3ee3699d h1:+DIuFNo6jIF0/9bHYz3C6OnnMQIe3C5bbFv2bAJLLo8= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260813160234-fdcb3ee3699d/go.mod h1:LY+Q6ms3A6E8DAKCpmvQaUxO5O2CKYm0XMcmTZ6PCcM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 58e5f75b7..ff1d4bfbd 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -26,12 +26,6 @@ import ( ) const ( - // CloudKittyUserID - Kolla's cloudkitty UID comes from the 'cloudkitty-user' in - // https://github.com/openstack/kolla/blob/master/kolla/common/users.py - CloudKittyUserID = 42408 - // CloudKittyGroupID - Kolla's cloudkitty GID - CloudKittyGroupID = 42408 - // CloudKittyAPIContainerImage - default fall-back image for CloudKitty API CloudKittyAPIContainerImage = "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api:current" // CloudKittyProcContainerImage - default fall-back image for CloudKitty Processor diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7e478e220..85de36420 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -324,8 +324,7 @@ rules: - apiGroups: - security.openshift.io resourceNames: - - anyuid - - privileged + - nonroot-v2 resources: - securitycontextconstraints verbs: diff --git a/go.mod b/go.mod index 302ae5030..74c1b3352 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,8 @@ require ( github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260805073337-c70c88f59cab github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260806140515-16161b7b6b53 github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260806140515-16161b7b6b53 - github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 + github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260813160234-fdcb3ee3699d + github.com/openstack-k8s-operators/lib-common/modules/users v0.0.0-20260813160234-fdcb3ee3699d github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701 github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260806101626-af8d24ac578b github.com/openstack-k8s-operators/telemetry-operator/api v0.0.0-00010101000000-000000000000 @@ -62,8 +63,8 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift/api v3.9.0+incompatible // indirect - github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260725150835-623a52fe0391 // indirect - github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260725150835-623a52fe0391 // indirect + github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260813160234-fdcb3ee3699d // indirect + github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260813160234-fdcb3ee3699d // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect diff --git a/go.sum b/go.sum index c10ed2d50..e7742dfe3 100644 --- a/go.sum +++ b/go.sum @@ -117,12 +117,14 @@ github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.202608061 github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:TkZgZE92VUKHHNKXwuV8b7tT9toaxsZkcGffRafKCSA= github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260806140515-16161b7b6b53 h1:nyN+1pyCXOr+b6D6r4WdoJlrel7kRS4gKY/JYdY2Wdo= github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:FH/zwNFlfKkJaw4lpLa2gkUtxKT9lp38vO6Lg11AiTU= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53 h1:9YpUAOFXCC94SwoWBRrhFyCVbbwnHTce6/j3CqjxaGM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260806140515-16161b7b6b53/go.mod h1:LY+Q6ms3A6E8DAKCpmvQaUxO5O2CKYm0XMcmTZ6PCcM= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260725150835-623a52fe0391 h1:cRyVdm28fshWzWkE5xgRYAzlNX4SreIluOL8s2dpm2U= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260725150835-623a52fe0391/go.mod h1:Q5wcK4PnlDJ55H+M38CFNLzL53OHYiwq81mR/gKtJXs= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260725150835-623a52fe0391 h1:AJEo8hauBmqrT9v//s2S5LbEb5BDKa8cQNFG5lAznXI= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260725150835-623a52fe0391/go.mod h1:fJ9CCRKpo8mQODfv4UDJ3tjUTbyCznYslXQwCkDaGg0= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260813160234-fdcb3ee3699d h1:+DIuFNo6jIF0/9bHYz3C6OnnMQIe3C5bbFv2bAJLLo8= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260813160234-fdcb3ee3699d/go.mod h1:LY+Q6ms3A6E8DAKCpmvQaUxO5O2CKYm0XMcmTZ6PCcM= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260813160234-fdcb3ee3699d h1:btd0N5D9BwZPfXmUCcN5hPPi/v2RlIjsaNwZkbLdtx4= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260813160234-fdcb3ee3699d/go.mod h1:Q5wcK4PnlDJ55H+M38CFNLzL53OHYiwq81mR/gKtJXs= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260813160234-fdcb3ee3699d h1:/FKQGhZ7sE0yXfK1hjmLGhDU18t5Ew3Ex3iUh8ZR81Y= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260813160234-fdcb3ee3699d/go.mod h1:fJ9CCRKpo8mQODfv4UDJ3tjUTbyCznYslXQwCkDaGg0= +github.com/openstack-k8s-operators/lib-common/modules/users v0.0.0-20260813160234-fdcb3ee3699d h1:Tj/j5+ihxeQgfn0j7i65/U1NMPwNoZOEVdG3ZcTprn0= +github.com/openstack-k8s-operators/lib-common/modules/users v0.0.0-20260813160234-fdcb3ee3699d/go.mod h1:jmVVRfwuFQ3fHLIOAkUrtLgOwEL6j45POzS3PBJuwyM= github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701 h1:rYJayTvo7sE3F1NsjLqbRzd/F9RjymfpCGvqYZrj/q8= github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701/go.mod h1:e/G0+fuofGEMmE1v6RWNVn9cImJosyQriG3j4NABZwQ= github.com/openstack-k8s-operators/ovn-operator/api v0.6.1-0.20260806101626-af8d24ac578b h1:F2VJiWumzHrI2+BuqbJJUo8/+5za+hJxLR14/Z7PCsU= diff --git a/internal/autoscaling/aodh_statefulset.go b/internal/autoscaling/aodh_statefulset.go index ee377ddc4..b6f5d39f9 100644 --- a/internal/autoscaling/aodh_statefulset.go +++ b/internal/autoscaling/aodh_statefulset.go @@ -22,9 +22,11 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/annotations" "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/pod" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" + "github.com/openstack-k8s-operators/lib-common/modules/users" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -37,11 +39,6 @@ import ( telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" ) -const ( - // ServiceCommand - - ServiceCommand = "/usr/local/bin/kolla_start" -) - // AodhStatefulSet func func AodhStatefulSet( instance *telemetryv1.Autoscaling, @@ -49,25 +46,21 @@ func AodhStatefulSet( labels map[string]string, topology *topologyv1.Topology, memcached *memcachedv1.Memcached, + customConfigKeys []string, ) (*appsv1.StatefulSet, error) { - aodhUser := int64(AodhUserID) + // TODO might need tuning livenessProbe := &corev1.Probe{ - // TODO might need tuning TimeoutSeconds: 30, PeriodSeconds: 30, InitialDelaySeconds: 5, } readinessProbe := &corev1.Probe{ - // TODO might need tuning TimeoutSeconds: 30, PeriodSeconds: 30, InitialDelaySeconds: 5, } - args := []string{"-c"} - args = append(args, ServiceCommand) - livenessProbe.HTTPGet = &corev1.HTTPGetAction{ Path: "/", Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(AodhAPIPort)}, @@ -84,10 +77,10 @@ func AodhStatefulSet( // create Volume and VolumeMounts volumes := getVolumes(instance) - apiVolumeMounts := getVolumeMounts(instance, "aodh-api") - evaluatorVolumeMounts := getVolumeMounts(instance, "aodh-evaluator") - notifierVolumeMounts := getVolumeMounts(instance, "aodh-notifier") - listenerVolumeMounts := getVolumeMounts(instance, "aodh-listener") + apiVolumeMounts := getAPIVolumeMounts(customConfigKeys) + evaluatorVolumeMounts := getEvaluatorVolumeMounts(customConfigKeys) + notifierVolumeMounts := getWorkerVolumeMounts(customConfigKeys) + listenerVolumeMounts := getWorkerVolumeMounts(customConfigKeys) // add openstack CA cert if defined if instance.Spec.Aodh.TLS.CaBundleSecretName != "" { @@ -108,6 +101,9 @@ func AodhStatefulSet( if memcached.GetMemcachedMTLSSecret() != "" { volumes = append(volumes, memcached.CreateMTLSVolume()) apiVolumeMounts = append(apiVolumeMounts, memcached.CreateMTLSVolumeMounts(nil, nil)...) + evaluatorVolumeMounts = append(evaluatorVolumeMounts, memcached.CreateMTLSVolumeMounts(nil, nil)...) + notifierVolumeMounts = append(notifierVolumeMounts, memcached.CreateMTLSVolumeMounts(nil, nil)...) + listenerVolumeMounts = append(listenerVolumeMounts, memcached.CreateMTLSVolumeMounts(nil, nil)...) } for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { @@ -124,91 +120,90 @@ func AodhStatefulSet( if err != nil { return nil, err } + certMount := fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String()) + keyMount := fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String()) + svc.CertMount = &certMount + svc.KeyMount = &keyMount volumes = append(volumes, svc.CreateVolume(endpt.String())) apiVolumeMounts = append(apiVolumeMounts, svc.CreateVolumeMounts(endpt.String())...) } } envVarsAodh := map[string]env.Setter{} - envVarsAodh["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") envVarsAodh["CONFIG_HASH"] = env.SetValue(configHash) var replicas int32 = 1 apiContainer := corev1.Container{ ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "/bin/bash", - }, - Args: args, - Image: instance.Spec.Aodh.APIImage, - Name: "aodh-api", - Env: env.MergeEnvs([]corev1.EnvVar{}, envVarsAodh), - VolumeMounts: apiVolumeMounts, + Command: []string{"/usr/sbin/httpd"}, + Args: []string{"-DFOREGROUND"}, + Image: instance.Spec.Aodh.APIImage, + Name: "aodh-api", + SecurityContext: pod.RestrictiveSecurityContext(users.AodhUID, users.AodhGID), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVarsAodh), + VolumeMounts: apiVolumeMounts, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, } evaluatorContainer := corev1.Container{ ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "/bin/bash", - }, - Args: args, - Image: instance.Spec.Aodh.EvaluatorImage, - Name: "aodh-evaluator", - Env: env.MergeEnvs([]corev1.EnvVar{}, envVarsAodh), - VolumeMounts: evaluatorVolumeMounts, + Command: []string{"/usr/bin/aodh-evaluator"}, + Args: []string{"--logfile", "/dev/stdout"}, + Image: instance.Spec.Aodh.EvaluatorImage, + Name: "aodh-evaluator", + SecurityContext: pod.RestrictiveSecurityContext(users.AodhUID, users.AodhGID), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVarsAodh), + VolumeMounts: evaluatorVolumeMounts, } notifierContainer := corev1.Container{ ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "/bin/bash", - }, - Args: args, - Image: instance.Spec.Aodh.NotifierImage, - Name: "aodh-notifier", - Env: env.MergeEnvs([]corev1.EnvVar{}, envVarsAodh), - VolumeMounts: notifierVolumeMounts, + Command: []string{"/usr/bin/aodh-notifier"}, + Args: []string{"--logfile", "/dev/stdout"}, + Image: instance.Spec.Aodh.NotifierImage, + Name: "aodh-notifier", + SecurityContext: pod.RestrictiveSecurityContext(users.AodhUID, users.AodhGID), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVarsAodh), + VolumeMounts: notifierVolumeMounts, } listenerContainer := corev1.Container{ ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "/bin/bash", - }, - Args: args, - Image: instance.Spec.Aodh.ListenerImage, - Name: "aodh-listener", - Env: env.MergeEnvs([]corev1.EnvVar{}, envVarsAodh), - VolumeMounts: listenerVolumeMounts, + Command: []string{"/usr/bin/aodh-listener"}, + Args: []string{"--logfile", "/dev/stdout"}, + Image: instance.Spec.Aodh.ListenerImage, + Name: "aodh-listener", + SecurityContext: pod.RestrictiveSecurityContext(users.AodhUID, users.AodhGID), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVarsAodh), + VolumeMounts: listenerVolumeMounts, } - pod := corev1.PodTemplateSpec{ + podSpec := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: ServiceName, Namespace: instance.Namespace, Labels: labels, }, Spec: corev1.PodSpec{ - ServiceAccountName: instance.RbacResourceName(), + ServiceAccountName: instance.RbacResourceName(), + AutomountServiceAccountToken: ptr.To(false), + SecurityContext: pod.RestrictivePodSecurityContext(users.AodhUID, users.AodhGID, users.ApacheGID), Containers: []corev1.Container{ apiContainer, evaluatorContainer, notifierContainer, listenerContainer, }, - SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: &aodhUser, - RunAsNonRoot: ptr.To(true), - }, }, } if instance.Spec.Aodh.NodeSelector != nil { - pod.Spec.NodeSelector = *instance.Spec.Aodh.NodeSelector + podSpec.Spec.NodeSelector = *instance.Spec.Aodh.NodeSelector } if topology != nil { - topology.ApplyTo(&pod) + topology.ApplyTo(&podSpec) } statefulset := &appsv1.StatefulSet{ @@ -223,7 +218,7 @@ func AodhStatefulSet( Selector: &metav1.LabelSelector{ MatchLabels: labels, }, - Template: pod, + Template: podSpec, }, } diff --git a/internal/autoscaling/const.go b/internal/autoscaling/const.go index 75ec25ed9..572248969 100644 --- a/internal/autoscaling/const.go +++ b/internal/autoscaling/const.go @@ -47,9 +47,6 @@ const ( // ACConsumerFinalizer is added to AC secrets that autoscaling (Aodh) is actively consuming ACConsumerFinalizer = "openstack.org/autoscaling-ac-consumer" - - // AodhUserID - - AodhUserID = 42402 ) // PrometheusReplicas - diff --git a/internal/autoscaling/dbsync.go b/internal/autoscaling/dbsync.go index 2642f4c91..b2d28b509 100644 --- a/internal/autoscaling/dbsync.go +++ b/internal/autoscaling/dbsync.go @@ -18,6 +18,8 @@ package autoscaling import ( "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/pod" + "github.com/openstack-k8s-operators/lib-common/modules/users" autoscalingv1beta1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -25,28 +27,18 @@ import ( "k8s.io/utils/ptr" ) -const ( - dbSyncCommand = "/usr/local/bin/kolla_start" -) - // DbSyncJob func -func DbSyncJob(instance *autoscalingv1beta1.Autoscaling, labels map[string]string) *batchv1.Job { - args := []string{"-c"} - args = append(args, dbSyncCommand) - +func DbSyncJob(instance *autoscalingv1beta1.Autoscaling, labels map[string]string, customConfigKeys []string) *batchv1.Job { // create Volume and VolumeMounts volumes := getVolumes(instance) - volumeMounts := getVolumeMounts(instance, "aodh-dbsync") + volumeMounts := getWorkerVolumeMounts(customConfigKeys) // add CA cert if defined if instance.Spec.Aodh.TLS.CaBundleSecretName != "" { volumes = append(volumes, instance.Spec.Aodh.TLS.CreateVolume()) volumeMounts = append(volumeMounts, instance.Spec.Aodh.TLS.CreateVolumeMounts(nil)...) } - runAsUser := int64(AodhUserID) envVars := map[string]env.Setter{} - envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") - envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") aodhPassword := []corev1.EnvVar{ { Name: "AodhPassword", @@ -70,22 +62,18 @@ func DbSyncJob(instance *autoscalingv1beta1.Autoscaling, labels map[string]strin Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyOnFailure, - ServiceAccountName: instance.RbacResourceName(), + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + AutomountServiceAccountToken: ptr.To(false), + SecurityContext: pod.RestrictivePodSecurityContext(users.AodhUID, users.AodhGID), Containers: []corev1.Container{ { - Name: ServiceName + "-db-sync", - Command: []string{ - "/bin/bash", - }, - Args: args, - Image: instance.Spec.Aodh.APIImage, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsUser, - RunAsNonRoot: ptr.To(true), - }, - Env: env.MergeEnvs(aodhPassword, envVars), - VolumeMounts: volumeMounts, + Name: ServiceName + "-db-sync", + Command: []string{"/usr/bin/aodh-dbsync"}, + Image: instance.Spec.Aodh.APIImage, + SecurityContext: pod.RestrictiveSecurityContext(users.AodhUID, users.AodhGID), + Env: env.MergeEnvs(aodhPassword, envVars), + VolumeMounts: volumeMounts, }, }, Volumes: volumes, diff --git a/internal/autoscaling/volumes.go b/internal/autoscaling/volumes.go index 50f6fd85e..6512c1f37 100644 --- a/internal/autoscaling/volumes.go +++ b/internal/autoscaling/volumes.go @@ -16,43 +16,32 @@ limitations under the License. package autoscaling import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/volume" telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/internal/utils" corev1 "k8s.io/api/core/v1" ) const ( - scriptVolume = "aodh-scripts" configVolume = "aodh-config-data" - logVolume = "logs" ) -var ( - // scriptMode is the default permissions mode for Scripts volume - scriptMode int32 = 0740 - // configMode is the 640 permissions mode - configMode int32 = 0640 -) +var config0440AccessMode int32 = 0440 // getVolumes - service volumes func getVolumes(instance *telemetryv1.Autoscaling) []corev1.Volume { vols := []corev1.Volume{ { - Name: "scripts", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - DefaultMode: &scriptMode, - SecretName: scriptVolume, - }, - }, - }, { Name: "config-data", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &configMode, + DefaultMode: &config0440AccessMode, SecretName: configVolume, }, }, }, + volume.WritableDirVolume(volume.RunHttpdVolumeName), + volume.WritableDirVolume(volume.VarLogHttpdVolumeName), } if instance.Spec.Aodh.CustomConfigsSecretName != "" { @@ -60,7 +49,7 @@ func getVolumes(instance *telemetryv1.Autoscaling) []corev1.Volume { Name: "custom-config", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &configMode, + DefaultMode: &config0440AccessMode, SecretName: instance.Spec.Aodh.CustomConfigsSecretName, }, }, @@ -69,34 +58,121 @@ func getVolumes(instance *telemetryv1.Autoscaling) []corev1.Volume { return vols } -// getVolumeMounts - general VolumeMounts -func getVolumeMounts(instance *telemetryv1.Autoscaling, serviceName string) []corev1.VolumeMount { - volMounts := []corev1.VolumeMount{ +func customConfigMounts(customConfigKeys []string) []corev1.VolumeMount { + mounts := make([]corev1.VolumeMount, 0, len(customConfigKeys)) + for _, key := range customConfigKeys { + mounts = append(mounts, corev1.VolumeMount{ + Name: "custom-config", + MountPath: "/etc/aodh/" + key, + SubPath: key, + ReadOnly: true, + }) + } + return mounts +} + +// getAPIVolumeMounts - aodh-api (httpd) VolumeMounts +func getAPIVolumeMounts(customConfigKeys []string) []corev1.VolumeMount { + vm := []corev1.VolumeMount{ + { + Name: "config-data", + MountPath: "/etc/aodh/aodh.conf", + SubPath: "aodh.conf", + ReadOnly: true, + }, { - Name: "scripts", - MountPath: "/var/lib/openstack/bin", + Name: "config-data", + MountPath: "/etc/aodh/aodh.conf.d/01-aodh-custom.conf", + SubPath: "custom.conf", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/etc/httpd/conf.d/00wsgi-aodh.conf", + SubPath: "wsgi-aodh.conf", ReadOnly: true, }, { Name: "config-data", - MountPath: "/var/lib/openstack/config", + MountPath: "/etc/httpd/conf/httpd.conf", + SubPath: "httpd.conf", ReadOnly: true, }, { Name: "config-data", - MountPath: "/var/lib/kolla/config_files/config.json", - SubPath: serviceName + "-config.json", + MountPath: "/etc/httpd/conf.d/ssl.conf", + SubPath: "ssl.conf", ReadOnly: true, }, + { + Name: "config-data", + MountPath: "/etc/my.cnf", + SubPath: "my.cnf", + ReadOnly: true, + }, + volume.WritableDirVolumeMount(volume.RunHttpdVolumeName, volume.RunHttpdMountPath), + volume.WritableDirVolumeMount(volume.VarLogHttpdVolumeName, volume.VarLogHttpdMountPath), } - if instance.Spec.Aodh.CustomConfigsSecretName != "" { - volMounts = append(volMounts, corev1.VolumeMount{ - Name: "custom-config", - MountPath: "/var/lib/openstack/custom-config", + // custom-config files override the default file mounted at the same path + return utils.MergeCustomConfigMounts(vm, customConfigMounts(customConfigKeys)) +} + +// getEvaluatorVolumeMounts - aodh-evaluator VolumeMounts +func getEvaluatorVolumeMounts(customConfigKeys []string) []corev1.VolumeMount { + vm := []corev1.VolumeMount{ + { + Name: "config-data", + MountPath: "/etc/aodh/aodh.conf", + SubPath: "aodh.conf", ReadOnly: true, - }) + }, + { + Name: "config-data", + MountPath: "/etc/aodh/aodh.conf.d/01-aodh-custom.conf", + SubPath: "custom.conf", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/etc/openstack/prometheus.yaml", + SubPath: "prometheus.yaml", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/etc/my.cnf", + SubPath: "my.cnf", + ReadOnly: true, + }, + } + // custom-config files override the default file mounted at the same path + return utils.MergeCustomConfigMounts(vm, customConfigMounts(customConfigKeys)) +} + +// getWorkerVolumeMounts - aodh-notifier/listener/dbsync VolumeMounts +func getWorkerVolumeMounts(customConfigKeys []string) []corev1.VolumeMount { + vm := []corev1.VolumeMount{ + { + Name: "config-data", + MountPath: "/etc/aodh/aodh.conf", + SubPath: "aodh.conf", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/etc/aodh/aodh.conf.d/01-aodh-custom.conf", + SubPath: "custom.conf", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/etc/my.cnf", + SubPath: "my.cnf", + ReadOnly: true, + }, } - return volMounts + // custom-config files override the default file mounted at the same path + return utils.MergeCustomConfigMounts(vm, customConfigMounts(customConfigKeys)) } // getCustomPrometheusCaVolume - Volume for CA certificate of user deployed Prometheus diff --git a/internal/ceilometer/const.go b/internal/ceilometer/const.go index 44229bc43..045cfe6ca 100644 --- a/internal/ceilometer/const.go +++ b/internal/ceilometer/const.go @@ -29,14 +29,10 @@ const ( // CeilometerPrometheusPort - CeilometerPrometheusPort int = 3000 - // KollaConfigCentral - - KollaConfigCentral = "/var/lib/config-data/merged/config-central.json" - - // KollaConfigNotification - - KollaConfigNotification = "/var/lib/config-data/merged/config-notification.json" - - // CeilometerUserID - - CeilometerUserID = 42405 + // CentralHCScript is the path to the central health check script + CentralHCScript = "/var/lib/openstack/bin/centralhealth.py" + // NotificationHCScript is the path to the notification health check script + NotificationHCScript = "/var/lib/openstack/bin/notificationhealth.py" // ACConsumerFinalizer is added to AC secrets that Ceilometer is actively consuming ACConsumerFinalizer = "openstack.org/ceilometer-ac-consumer" diff --git a/internal/ceilometer/statefulset.go b/internal/ceilometer/statefulset.go index dab8bcdbd..05b7d8125 100644 --- a/internal/ceilometer/statefulset.go +++ b/internal/ceilometer/statefulset.go @@ -20,8 +20,10 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/annotations" "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/pod" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" + "github.com/openstack-k8s-operators/lib-common/modules/users" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -33,25 +35,15 @@ import ( telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" ) -const ( - // ServiceCommand - - ServiceCommand = "/usr/local/bin/kolla_start" - // CentralHCScript is the path to the central health check script - CentralHCScript = "/var/lib/openstack/bin/centralhealth.py" - // NotificationHCScript is the path to the notification health check script - NotificationHCScript = "/var/lib/openstack/bin/notificationhealth.py" -) - // StatefulSet func func StatefulSet( instance *telemetryv1.Ceilometer, configHash string, labels map[string]string, topology *topologyv1.Topology, + customConfigKeys []string, ) (*appsv1.StatefulSet, error) { - ceilometerUser := int64(CeilometerUserID) - // container probes sgRootEndpointCurl := corev1.HTTPGetAction{ Path: "/", Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(CeilometerPrometheusPort)}, @@ -93,30 +85,25 @@ func StatefulSet( Command: []string{"/usr/bin/python3", NotificationHCScript}, } - args := []string{"-c"} - args = append(args, ServiceCommand) - - envVarsCentral := map[string]env.Setter{} - envVarsCentral["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") - envVarsCentral["CONFIG_HASH"] = env.SetValue(configHash) - - envVarsNotification := map[string]env.Setter{} - envVarsNotification["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") - envVarsNotification["CONFIG_HASH"] = env.SetValue(configHash) + envVars := map[string]env.Setter{} + envVars["CONFIG_HASH"] = env.SetValue(configHash) var replicas int32 = 1 volumes := getVolumes(instance) - centralVolumeMounts := getVolumeMounts(instance, "ceilometer-central") - notificationVolumeMounts := getVolumeMounts(instance, "ceilometer-notification") + centralVolumeMounts := getCentralVolumeMounts(customConfigKeys) + notificationVolumeMounts := getNotificationVolumeMounts(customConfigKeys) httpdVolumeMounts := getHttpdVolumeMounts() + centralVolumeMounts = append(centralVolumeMounts, getHealthCheckVolumeMounts()...) + notificationVolumeMounts = append(notificationVolumeMounts, getHealthCheckVolumeMounts()...) + + // add TLS cert if defined if instance.Spec.TLS.Enabled() { svc, err := instance.Spec.TLS.ToService() if err != nil { return nil, err } - // httpd container is not using kolla, mount the certs to its dst svc.CertMount = ptr.To(fmt.Sprintf("/etc/pki/tls/certs/%s", tls.CertKey)) svc.KeyMount = ptr.To(fmt.Sprintf("/etc/pki/tls/private/%s", tls.PrivateKey)) @@ -138,41 +125,32 @@ func StatefulSet( centralAgentContainer := corev1.Container{ ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "/bin/bash", - }, - Args: args, - Image: instance.Spec.CentralImage, - Name: "ceilometer-central-agent", - Env: env.MergeEnvs([]corev1.EnvVar{}, envVarsCentral), - VolumeMounts: centralVolumeMounts, - LivenessProbe: centralLivenessProbe, + Command: []string{"/usr/bin/ceilometer-polling"}, + Args: []string{"--polling-namespaces", "central", "--logfile", "/dev/stdout"}, + Image: instance.Spec.CentralImage, + Name: "ceilometer-central-agent", + SecurityContext: pod.RestrictiveSecurityContext(users.CeilometerUID, users.CeilometerGID), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: centralVolumeMounts, + LivenessProbe: centralLivenessProbe, } notificationAgentContainer := corev1.Container{ ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "/bin/bash", - }, - Args: args, - Image: instance.Spec.NotificationImage, - Name: "ceilometer-notification-agent", - Env: env.MergeEnvs([]corev1.EnvVar{}, envVarsNotification), - VolumeMounts: notificationVolumeMounts, - LivenessProbe: notificationLivenessProbe, + Command: []string{"/usr/bin/ceilometer-agent-notification"}, + Args: []string{"--logfile", "/dev/stdout"}, + Image: instance.Spec.NotificationImage, + Name: "ceilometer-notification-agent", + SecurityContext: pod.RestrictiveSecurityContext(users.CeilometerUID, users.CeilometerGID), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: notificationVolumeMounts, + LivenessProbe: notificationLivenessProbe, } sgCoreContainer := corev1.Container{ ImagePullPolicy: corev1.PullIfNotPresent, Image: instance.Spec.SgCoreImage, Name: "sg-core", + SecurityContext: pod.RestrictiveSecurityContext(users.CeilometerUID, users.CeilometerGID), VolumeMounts: getSgCoreVolumeMounts(), - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: ptr.To(false), - Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{ - "ALL", - }, - }, - }, } proxyContainer := corev1.Container{ ImagePullPolicy: corev1.PullIfNotPresent, @@ -182,49 +160,38 @@ func StatefulSet( ContainerPort: int32(CeilometerPrometheusPort), Name: "proxy-httpd", }}, - VolumeMounts: httpdVolumeMounts, - ReadinessProbe: sgReadinessProbe, - LivenessProbe: sgLivenessProbe, - Command: []string{"/usr/sbin/httpd"}, - Args: []string{"-DFOREGROUND"}, - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: ptr.To(false), - Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{ - "ALL", - }, - }, - }, + SecurityContext: pod.RestrictiveSecurityContext(users.CeilometerUID, users.CeilometerGID), + VolumeMounts: httpdVolumeMounts, + ReadinessProbe: sgReadinessProbe, + LivenessProbe: sgLivenessProbe, + Command: []string{"/usr/sbin/httpd"}, + Args: []string{"-DFOREGROUND"}, } - pod := corev1.PodTemplateSpec{ + podSpec := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: ServiceName, Namespace: instance.Namespace, Labels: labels, }, Spec: corev1.PodSpec{ - ServiceAccountName: instance.RbacResourceName(), + ServiceAccountName: instance.RbacResourceName(), + AutomountServiceAccountToken: ptr.To(false), + SecurityContext: pod.RestrictivePodSecurityContext(users.CeilometerUID, users.CeilometerGID, users.ApacheGID), Containers: []corev1.Container{ centralAgentContainer, notificationAgentContainer, sgCoreContainer, proxyContainer, }, - SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: &ceilometerUser, - RunAsGroup: &ceilometerUser, - RunAsNonRoot: ptr.To(true), - FSGroup: &ceilometerUser, - }, }, } if instance.Spec.NodeSelector != nil { - pod.Spec.NodeSelector = *instance.Spec.NodeSelector + podSpec.Spec.NodeSelector = *instance.Spec.NodeSelector } if topology != nil { - topology.ApplyTo(&pod) + topology.ApplyTo(&podSpec) } statefulset := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -238,7 +205,7 @@ func StatefulSet( Selector: &metav1.LabelSelector{ MatchLabels: labels, }, - Template: pod, + Template: podSpec, }, } diff --git a/internal/ceilometer/volumes.go b/internal/ceilometer/volumes.go index 3b8c0a8b5..df971feea 100644 --- a/internal/ceilometer/volumes.go +++ b/internal/ceilometer/volumes.go @@ -16,19 +16,20 @@ limitations under the License. package ceilometer import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/volume" telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/internal/utils" corev1 "k8s.io/api/core/v1" ) const ( scriptVolume = "ceilometer-scripts" configVolume = "ceilometer-config-data" - logVolume = "logs" ) var ( - configMode int32 = 0640 - scriptMode int32 = 0740 + config0440AccessMode int32 = 0440 + script0550AccessMode int32 = 0550 ) func getVolumes(instance *telemetryv1.Ceilometer) []corev1.Volume { @@ -37,7 +38,7 @@ func getVolumes(instance *telemetryv1.Ceilometer) []corev1.Volume { Name: "scripts", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &scriptMode, + DefaultMode: &script0550AccessMode, SecretName: scriptVolume, }, }, @@ -45,7 +46,7 @@ func getVolumes(instance *telemetryv1.Ceilometer) []corev1.Volume { Name: "config-data", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &configMode, + DefaultMode: &config0440AccessMode, SecretName: configVolume, }, }, @@ -53,7 +54,7 @@ func getVolumes(instance *telemetryv1.Ceilometer) []corev1.Volume { Name: "sg-core-conf-yaml", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &configMode, + DefaultMode: &config0440AccessMode, Items: []corev1.KeyToPath{{ Key: "sg-core.conf.yaml", Path: "sg-core.conf.yaml", @@ -62,18 +63,8 @@ func getVolumes(instance *telemetryv1.Ceilometer) []corev1.Volume { }, }, }, - { - Name: "run-httpd", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{Medium: ""}, - }, - }, - { - Name: "log-httpd", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{Medium: ""}, - }, - }, + volume.WritableDirVolume(volume.RunHttpdVolumeName), + volume.WritableDirVolume(volume.VarLogHttpdVolumeName), } if instance.Spec.CustomConfigsSecretName != "" { @@ -81,7 +72,7 @@ func getVolumes(instance *telemetryv1.Ceilometer) []corev1.Volume { Name: "custom-config", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &configMode, + DefaultMode: &config0440AccessMode, SecretName: instance.Spec.CustomConfigsSecretName, }, }, @@ -90,37 +81,77 @@ func getVolumes(instance *telemetryv1.Ceilometer) []corev1.Volume { return vols } -// getVolumeMounts - general VolumeMounts -func getVolumeMounts(instance *telemetryv1.Ceilometer, serviceName string) []corev1.VolumeMount { - volMounts := []corev1.VolumeMount{ +func customConfigMounts(customConfigKeys []string) []corev1.VolumeMount { + mounts := make([]corev1.VolumeMount, 0, len(customConfigKeys)) + for _, key := range customConfigKeys { + mounts = append(mounts, corev1.VolumeMount{ + Name: "custom-config", + MountPath: "/etc/ceilometer/" + key, + SubPath: key, + ReadOnly: true, + }) + } + return mounts +} + +// getCentralVolumeMounts - ceilometer-central VolumeMounts +func getCentralVolumeMounts(customConfigKeys []string) []corev1.VolumeMount { + vm := []corev1.VolumeMount{ { - Name: "scripts", - MountPath: "/var/lib/openstack/bin", + Name: "config-data", + MountPath: "/etc/ceilometer/ceilometer.conf", + SubPath: "ceilometer.conf", ReadOnly: true, }, { Name: "config-data", - MountPath: "/var/lib/openstack/config", + MountPath: "/etc/ceilometer/polling.yaml", + SubPath: "polling.yaml.j2", ReadOnly: true, }, { Name: "config-data", - MountPath: "/var/lib/kolla/config_files/config.json", - SubPath: serviceName + "-config.json", + MountPath: "/etc/ceilometer/ceilometer.conf.d/01-ceilometer-custom.conf", + SubPath: "custom.conf", ReadOnly: true, }, } - if instance.Spec.CustomConfigsSecretName != "" { - volMounts = append(volMounts, corev1.VolumeMount{ - Name: "custom-config", - MountPath: "/var/lib/openstack/custom-config", + // custom-config files override the default file mounted at the same path + return utils.MergeCustomConfigMounts(vm, customConfigMounts(customConfigKeys)) +} + +// getNotificationVolumeMounts - ceilometer-notification VolumeMounts +func getNotificationVolumeMounts(customConfigKeys []string) []corev1.VolumeMount { + vm := []corev1.VolumeMount{ + { + Name: "config-data", + MountPath: "/etc/ceilometer/ceilometer.conf", + SubPath: "ceilometer.conf", ReadOnly: true, - }) + }, + { + Name: "config-data", + MountPath: "/etc/ceilometer/pipeline.yaml", + SubPath: "pipeline.yaml", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/etc/ceilometer/event_pipeline.yaml", + SubPath: "event_pipeline.yaml", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/etc/ceilometer/ceilometer.conf.d/01-ceilometer-custom.conf", + SubPath: "custom.conf", + ReadOnly: true, + }, } - return volMounts + // custom-config files override the default file mounted at the same path + return utils.MergeCustomConfigMounts(vm, customConfigMounts(customConfigKeys)) } -// getSgCoreVolumeMounts - VolumeMounts for SGCore container func getSgCoreVolumeMounts() []corev1.VolumeMount { return []corev1.VolumeMount{ { @@ -131,7 +162,6 @@ func getSgCoreVolumeMounts() []corev1.VolumeMount { } } -// getHttpdVolumeMounts - Returns the VolumeMounts used by the httpd sidecar func getHttpdVolumeMounts() []corev1.VolumeMount { return []corev1.VolumeMount{ { @@ -146,15 +176,25 @@ func getHttpdVolumeMounts() []corev1.VolumeMount { SubPath: "ssl.conf", ReadOnly: true, }, + volume.WritableDirVolumeMount(volume.RunHttpdVolumeName, volume.RunHttpdMountPath), + volume.WritableDirVolumeMount(volume.VarLogHttpdVolumeName, volume.VarLogHttpdMountPath), + } +} + +// getHealthCheckVolumeMounts - health check script SubPath mounts for central and notification agents +func getHealthCheckVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ { - Name: "run-httpd", - MountPath: "/run/httpd", - ReadOnly: false, + Name: "scripts", + MountPath: CentralHCScript, + SubPath: "centralhealth.py", + ReadOnly: true, }, { - Name: "log-httpd", - MountPath: "/var/log/httpd", - ReadOnly: false, + Name: "scripts", + MountPath: NotificationHCScript, + SubPath: "notificationhealth.py", + ReadOnly: true, }, } } diff --git a/internal/cloudkitty/const.go b/internal/cloudkitty/const.go index 3fe83b743..a19037fcd 100644 --- a/internal/cloudkitty/const.go +++ b/internal/cloudkitty/const.go @@ -63,9 +63,6 @@ const ( // CaConfigmapKey is the key in the CA configmap CaConfigmapKey = "ca.crt" - // CloudKittyUserID - - CloudKittyUserID = 42406 - // ACConsumerFinalizer is added to AC secrets that CloudKitty is actively consuming ACConsumerFinalizer = "openstack.org/cloudkitty-ac-consumer" ) diff --git a/internal/cloudkitty/dbsync.go b/internal/cloudkitty/dbsync.go index f70bd7fca..e4ef632f1 100644 --- a/internal/cloudkitty/dbsync.go +++ b/internal/cloudkitty/dbsync.go @@ -14,11 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package cloudkitty provides CloudKitty service configuration and management utilities package cloudkitty import ( "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/pod" + "github.com/openstack-k8s-operators/lib-common/modules/users" telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -26,36 +27,32 @@ import ( "k8s.io/utils/ptr" ) -const ( - // DBSyncCommand - - // TODO: Once we work on update/upgrades revisit the command in the - // the cloudkitty-dbsync-config.json file. - // If we stop all services during the update/upgrade then we can keep - // the --bump-versions flag. - // If we are doing rolling upgrades we'll need to use the flag - // conditionally (only for adoption) and do the restart cycle of - // services as described in the upstream rolling upgrades process. - dbSyncCommand = "/usr/local/bin/kolla_start" -) - // DbSyncJob func +// TODO: Once we work on update/upgrades revisit the command. +// +// If we stop all services during the update/upgrade then we can keep +// the --bump-versions flag. +// If we are doing rolling upgrades we'll need to use the flag +// conditionally (only for adoption) and do the restart cycle of +// services as described in the upstream rolling upgrades process. func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string, annotations map[string]string) *batchv1.Job { - args := []string{"-c"} - args = append(args, dbSyncCommand) - // create Volume and VolumeMounts - volumes := GetVolumes(instance.Name) - volumeMounts := GetVolumeMounts("cloudkitty-dbsync") + volumes := GetJobVolumes(instance.Name) + volumeMounts := []corev1.VolumeMount{ + { + Name: "config-data", + MountPath: "/etc/cloudkitty/cloudkitty.conf", + SubPath: DefaultsConfigFileName, + ReadOnly: true, + }, + } // add CA cert if defined if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { volumes = append(volumes, instance.Spec.CloudKittyAPI.TLS.CreateVolume()) volumeMounts = append(volumeMounts, instance.Spec.CloudKittyAPI.TLS.CreateVolumeMounts(nil)...) } - runAsUser := int64(CloudKittyUserID) envVars := map[string]env.Setter{} - envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") - envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") cloudKittyPassword := []corev1.EnvVar{ { Name: "CloudKittyPassword", @@ -82,22 +79,19 @@ func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string, annot Annotations: annotations, }, Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyOnFailure, - ServiceAccountName: instance.RbacResourceName(), + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + AutomountServiceAccountToken: ptr.To(false), + SecurityContext: pod.RestrictivePodSecurityContext(users.CloudkittyUID, users.CloudkittyGID), Containers: []corev1.Container{ { - Name: ServiceName + "-db-sync", - Command: []string{ - "/bin/bash", - }, - Args: args, - Image: instance.Spec.CloudKittyAPI.ContainerImage, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsUser, - RunAsNonRoot: ptr.To(true), - }, - Env: env.MergeEnvs(cloudKittyPassword, envVars), - VolumeMounts: volumeMounts, + Name: ServiceName + "-db-sync", + Command: []string{"/usr/bin/cloudkitty-dbsync"}, + Args: []string{"upgrade"}, + Image: instance.Spec.CloudKittyAPI.ContainerImage, + SecurityContext: pod.RestrictiveSecurityContext(users.CloudkittyUID, users.CloudkittyGID), + Env: env.MergeEnvs(cloudKittyPassword, envVars), + VolumeMounts: volumeMounts, }, }, Volumes: volumes, diff --git a/internal/cloudkitty/storageinit.go b/internal/cloudkitty/storageinit.go index 21273920b..97ae5a6b0 100644 --- a/internal/cloudkitty/storageinit.go +++ b/internal/cloudkitty/storageinit.go @@ -14,11 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package cloudkitty provides CloudKitty service configuration and management utilities package cloudkitty import ( "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/pod" + "github.com/openstack-k8s-operators/lib-common/modules/users" telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -26,35 +27,32 @@ import ( "k8s.io/utils/ptr" ) -const ( - // storageInitCommand - - // TODO: Once we work on update/upgrades revisit the command in the - // the cloudkitty-storageinit-config.json file. - // If we stop all services during the update/upgrade then we can keep - // the --bump-versions flag. - // If we are doing rolling upgrades we'll need to use the flag - // conditionally (only for adoption) and do the restart cycle of - // services as described in the upstream rolling upgrades process. - storageInitCommand = "/usr/local/bin/kolla_start" -) - // StorageInitJob func +// TODO: Once we work on update/upgrades revisit the command. +// +// If we stop all services during the update/upgrade then we can keep +// the --bump-versions flag. +// If we are doing rolling upgrades we'll need to use the flag +// conditionally (only for adoption) and do the restart cycle of +// services as described in the upstream rolling upgrades process. func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string, annotations map[string]string) *batchv1.Job { - args := []string{"-c", storageInitCommand} - // create Volume and VolumeMounts - volumes := GetVolumes(instance.Name) - volumeMounts := GetVolumeMounts("cloudkitty-storageinit") + volumes := GetJobVolumes(instance.Name) + volumeMounts := []corev1.VolumeMount{ + { + Name: "config-data", + MountPath: "/etc/cloudkitty/cloudkitty.conf", + SubPath: DefaultsConfigFileName, + ReadOnly: true, + }, + } // add CA cert if defined if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { volumes = append(volumes, instance.Spec.CloudKittyAPI.TLS.CreateVolume()) volumeMounts = append(volumeMounts, instance.Spec.CloudKittyAPI.TLS.CreateVolumeMounts(nil)...) } - runAsUser := int64(CloudKittyUserID) envVars := map[string]env.Setter{} - envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") - envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") cloudKittyPassword := []corev1.EnvVar{ { Name: "CloudKittyPassword", @@ -81,22 +79,18 @@ func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string, Annotations: annotations, }, Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyOnFailure, - ServiceAccountName: instance.RbacResourceName(), + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + AutomountServiceAccountToken: ptr.To(false), + SecurityContext: pod.RestrictivePodSecurityContext(users.CloudkittyUID, users.CloudkittyGID), Containers: []corev1.Container{ { - Name: ServiceName + "-storageinit", - Command: []string{ - "/bin/bash", - }, - Args: args, - Image: instance.Spec.CloudKittyAPI.ContainerImage, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsUser, - RunAsNonRoot: ptr.To(true), - }, - Env: env.MergeEnvs(cloudKittyPassword, envVars), - VolumeMounts: volumeMounts, + Name: ServiceName + "-storageinit", + Command: []string{"/usr/bin/cloudkitty-storage-init"}, + Image: instance.Spec.CloudKittyAPI.ContainerImage, + SecurityContext: pod.RestrictiveSecurityContext(users.CloudkittyUID, users.CloudkittyGID), + Env: env.MergeEnvs(cloudKittyPassword, envVars), + VolumeMounts: volumeMounts, }, }, Volumes: volumes, diff --git a/internal/cloudkitty/volumes.go b/internal/cloudkitty/volumes.go index 23478a3a0..5dea64422 100644 --- a/internal/cloudkitty/volumes.go +++ b/internal/cloudkitty/volumes.go @@ -1,3 +1,4 @@ +// Package cloudkitty provides CloudKitty service configuration and management utilities package cloudkitty import ( @@ -5,30 +6,39 @@ import ( ) var ( - // scriptMode is the default permissions mode for Scripts volume - scriptMode int32 = 0755 - // configMode is the 640 permissions mode - configMode int32 = 0640 - // certMode is the 400 permissions mode - certMode int32 = 0400 + config0440AccessMode int32 = 0440 + certMode int32 = 0400 ) -// GetVolumes - service volumes -func GetVolumes(name string) []corev1.Volume { +// GetJobVolumes - volumes for the dbsync/storageinit Jobs. They only need the +// config-data secret (cloudkitty.conf); unlike the API/proc pods they do not +// talk to LokiStack, so the loki client cert + gateway CA bundle projected +// "certs" volume from GetVolumes is intentionally omitted. Including it would +// gate the Jobs on the loki cert/ca-bundle resources (non-optional projected +// sources) even though no job container mounts them. The DB (galera) CA is +// added separately by the caller via the TLS CaBundleSecretName volume. +func GetJobVolumes(name string) []corev1.Volume { return []corev1.Volume{ { - Name: "scripts", + Name: "config-data", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &scriptMode, - SecretName: name + "-scripts", + DefaultMode: &config0440AccessMode, + SecretName: name + "-config-data", }, }, - }, { + }, + } +} + +// GetVolumes - service volumes +func GetVolumes(name string) []corev1.Volume { + return []corev1.Volume{ + { Name: "config-data", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &configMode, + DefaultMode: &config0440AccessMode, SecretName: name + "-config-data", }, }, @@ -58,28 +68,21 @@ func GetVolumes(name string) []corev1.Volume { } } -// GetVolumeMounts - general VolumeMounts -func GetVolumeMounts(serviceName string) []corev1.VolumeMount { +// GetVolumeMounts - VolumeMounts shared by api and proc (metrics + loki certs). +// The base cloudkitty.conf is NOT included here — api/proc get it via +// config-data-custom at conf.d/00-cloudkitty.conf; dbsync/storageinit mount +// it directly in their own inline volumeMount lists. +func GetVolumeMounts() []corev1.VolumeMount { return []corev1.VolumeMount{ - { - Name: "scripts", - MountPath: "/var/lib/openstack/bin", - ReadOnly: true, - }, - { - Name: "config-data", - MountPath: "/var/lib/openstack/config", - ReadOnly: true, - }, { Name: "config-data", - MountPath: "/var/lib/kolla/config_files/config.json", - SubPath: serviceName + "-config.json", + MountPath: "/etc/cloudkitty/metrics.yaml", + SubPath: "metrics.yaml", ReadOnly: true, }, { Name: "certs", - MountPath: "/var/lib/openstack/loki-certs", + MountPath: "/etc/cloudkitty/certs", ReadOnly: true, }, } diff --git a/internal/cloudkittyapi/statefulset.go b/internal/cloudkittyapi/statefulset.go index c53263e54..a1bdc7a39 100644 --- a/internal/cloudkittyapi/statefulset.go +++ b/internal/cloudkittyapi/statefulset.go @@ -16,10 +16,14 @@ limitations under the License. package cloudkittyapi import ( + "fmt" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/pod" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/users" telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" "github.com/openstack-k8s-operators/telemetry-operator/internal/cloudkitty" @@ -30,11 +34,6 @@ import ( "k8s.io/utils/ptr" ) -const ( - // ServiceCommand - - ServiceCommand = "/usr/local/bin/kolla_start" -) - // StatefulSet func func StatefulSet( instance *telemetryv1.CloudKittyAPI, @@ -42,23 +41,21 @@ func StatefulSet( labels map[string]string, annotations map[string]string, topology *topologyv1.Topology, + customConfigKeys []string, ) (*appsv1.StatefulSet, error) { - runAsUser := int64(cloudkitty.CloudKittyUserID) + // TODO might need tuning livenessProbe := &corev1.Probe{ - // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 5, InitialDelaySeconds: 30, } readinessProbe := &corev1.Probe{ - // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 5, InitialDelaySeconds: 30, } - args := []string{"-c", ServiceCommand} // // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ // @@ -75,7 +72,7 @@ func StatefulSet( // create Volume and VolumeMounts volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name, instance) - volumeMounts := GetVolumeMounts(instance) + volumeMounts := GetVolumeMounts(customConfigKeys) // add CA cert if defined if instance.Spec.TLS.CaBundleSecretName != "" { @@ -97,13 +94,16 @@ func StatefulSet( if err != nil { return nil, err } + certMount := fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String()) + keyMount := fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String()) + svc.CertMount = &certMount + svc.KeyMount = &keyMount volumes = append(volumes, svc.CreateVolume(endpt.String())) volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(endpt.String())...) } } envVars := map[string]env.Setter{} - envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") envVars["CONFIG_HASH"] = env.SetValue(configHash) statefulset := &appsv1.StatefulSet{ @@ -124,7 +124,9 @@ func StatefulSet( Labels: labels, }, Spec: corev1.PodSpec{ - ServiceAccountName: instance.Spec.ServiceAccount, + ServiceAccountName: instance.Spec.ServiceAccount, + AutomountServiceAccountToken: ptr.To(false), + SecurityContext: pod.RestrictivePodSecurityContext(users.CloudkittyUID, users.CloudkittyGID, users.ApacheGID), Containers: []corev1.Container{ // the first container in a pod is the default selected // by oc log so define the log stream container first. @@ -140,29 +142,27 @@ func StatefulSet( "-c", "/usr/bin/tail -n+1 -F " + LogFile + " 2>/dev/null", }, - Image: instance.Spec.ContainerImage, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: []corev1.VolumeMount{GetLogVolumeMount()}, - Resources: instance.Spec.Resources, + Image: instance.Spec.ContainerImage, + SecurityContext: pod.RestrictiveSecurityContext(users.CloudkittyUID, users.CloudkittyGID), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: []corev1.VolumeMount{GetLogVolumeMount()}, + Resources: instance.Spec.Resources, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, }, { - Name: ComponentName, - Command: []string{ - "/bin/bash", - }, - Args: args, - Image: instance.Spec.ContainerImage, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, - ReadinessProbe: readinessProbe, - LivenessProbe: livenessProbe, + Name: ComponentName, + Command: []string{"/usr/sbin/httpd"}, + Args: []string{"-DFOREGROUND"}, + Image: instance.Spec.ContainerImage, + SecurityContext: pod.RestrictiveSecurityContext(users.CloudkittyUID, users.CloudkittyGID), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, }, }, - SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: &runAsUser, - RunAsNonRoot: ptr.To(true), - }, Volumes: volumes, }, }, @@ -176,9 +176,6 @@ func StatefulSet( if topology != nil { topology.ApplyTo(&statefulset.Spec.Template) } else { - // If possible two pods of the same service should not - // run on the same worker node. If this is not possible - // the get still created on the same worker node. statefulset.Spec.Template.Spec.Affinity = cloudkitty.GetPodAffinity(ComponentName) } diff --git a/internal/cloudkittyapi/volumes.go b/internal/cloudkittyapi/volumes.go index d8798532c..6447ab6dd 100644 --- a/internal/cloudkittyapi/volumes.go +++ b/internal/cloudkittyapi/volumes.go @@ -1,25 +1,23 @@ package cloudkittyapi import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/volume" telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" "github.com/openstack-k8s-operators/telemetry-operator/internal/cloudkitty" + "github.com/openstack-k8s-operators/telemetry-operator/internal/utils" corev1 "k8s.io/api/core/v1" ) -var ( - configMode int32 = 0640 -) +var config0440AccessMode int32 = 0440 // GetVolumes - func GetVolumes(parentName string, name string, instance *telemetryv1.CloudKittyAPI) []corev1.Volume { - var config0644AccessMode int32 = 0644 - volumes := []corev1.Volume{ { Name: "config-data-custom", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &config0644AccessMode, + DefaultMode: &config0440AccessMode, SecretName: name + "-config-data", }, }, @@ -30,6 +28,8 @@ func GetVolumes(parentName string, name string, instance *telemetryv1.CloudKitty EmptyDir: &corev1.EmptyDirVolumeSource{Medium: ""}, }, }, + volume.WritableDirVolume(volume.RunHttpdVolumeName), + volume.WritableDirVolume(volume.VarLogHttpdVolumeName), } if instance.Spec.CustomConfigsSecretName != "" { @@ -37,7 +37,7 @@ func GetVolumes(parentName string, name string, instance *telemetryv1.CloudKitty Name: "custom-config", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &configMode, + DefaultMode: &config0440AccessMode, SecretName: instance.Spec.CustomConfigsSecretName, }, }, @@ -48,25 +48,72 @@ func GetVolumes(parentName string, name string, instance *telemetryv1.CloudKitty } // GetVolumeMounts - CloudKitty API VolumeMounts -func GetVolumeMounts(instance *telemetryv1.CloudKittyAPI) []corev1.VolumeMount { +// customConfigKeys are the keys from the CustomConfigsSecret; each is +// SubPath-mounted into /etc/cloudkitty/ (matching the API contract: +// "files from this secret will get copied into /etc/cloudkitty/"). +func GetVolumeMounts(customConfigKeys []string) []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", - MountPath: "/var/lib/openstack/service-config/", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d/00-cloudkitty.conf", + SubPath: cloudkitty.DefaultsConfigFileName, ReadOnly: true, }, + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d/" + cloudkitty.CustomConfigFileName, + SubPath: cloudkitty.CustomConfigFileName, + ReadOnly: true, + }, + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d/" + cloudkitty.CustomServiceConfigFileName, + SubPath: cloudkitty.CustomServiceConfigFileName, + ReadOnly: true, + }, + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d/" + cloudkitty.CustomServiceConfigSecretsFileName, + SubPath: cloudkitty.CustomServiceConfigSecretsFileName, + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/etc/httpd/conf.d/00wsgi-cloudkitty.conf", + SubPath: "wsgi-cloudkitty.conf", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/etc/httpd/conf/httpd.conf", + SubPath: "httpd.conf", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/etc/httpd/conf.d/ssl.conf", + SubPath: "ssl.conf", + ReadOnly: true, + }, + volume.WritableDirVolumeMount(volume.RunHttpdVolumeName, volume.RunHttpdMountPath), + volume.WritableDirVolumeMount(volume.VarLogHttpdVolumeName, volume.VarLogHttpdMountPath), GetLogVolumeMount(), } - if instance.Spec.CustomConfigsSecretName != "" { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ + base := append(cloudkitty.GetVolumeMounts(), volumeMounts...) + + customConfigMounts := make([]corev1.VolumeMount, 0, len(customConfigKeys)) + for _, key := range customConfigKeys { + customConfigMounts = append(customConfigMounts, corev1.VolumeMount{ Name: "custom-config", - MountPath: "/var/lib/openstack/custom-config", + MountPath: "/etc/cloudkitty/" + key, + SubPath: key, ReadOnly: true, }) } - return append(cloudkitty.GetVolumeMounts(cloudkitty.ServiceName+"-api"), volumeMounts...) + // custom-config files override the default file mounted at the same path + return utils.MergeCustomConfigMounts(base, customConfigMounts) } // GetLogVolumeMount - CloudKitty API LogVolumeMount diff --git a/internal/cloudkittyproc/statefulset.go b/internal/cloudkittyproc/statefulset.go index 132b00560..62b42f726 100644 --- a/internal/cloudkittyproc/statefulset.go +++ b/internal/cloudkittyproc/statefulset.go @@ -18,6 +18,8 @@ package cloudkittyproc import ( topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/pod" + "github.com/openstack-k8s-operators/lib-common/modules/users" telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" "github.com/openstack-k8s-operators/telemetry-operator/internal/cloudkitty" @@ -27,13 +29,6 @@ import ( "k8s.io/utils/ptr" ) -const ( - // ServiceCommand - - ServiceCommand = "/usr/local/bin/kolla_start" - // CloudKittyHCScript is the path to the health check script - CloudKittyHCScript = "/var/lib/openstack/bin/healthcheck.py" -) - // StatefulSet func func StatefulSet( instance *telemetryv1.CloudKittyProc, @@ -41,35 +36,28 @@ func StatefulSet( labels map[string]string, annotations map[string]string, topology *topologyv1.Topology, + customConfigKeys []string, ) *appsv1.StatefulSet { - runAsUser := int64(cloudkitty.CloudKittyUserID) - // TODO until we determine how to properly query for these + // TODO might need tuning livenessProbe := &corev1.Probe{ - // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 5, InitialDelaySeconds: 3, } - args := []string{"-c", ServiceCommand} - //var probeCommand string - - //probeCommand = "/usr/local/bin/kolla_set_configs && /var/lib/openstack/bin/healthcheck.py --config-dir /etc/cloudkitty/cloudkitty.conf.d/" - livenessProbe.Exec = &corev1.ExecAction{ Command: []string{ - "/usr/bin/python3", - CloudKittyHCScript, + "/usr/bin/pgrep", "-f", "cloudkitty-processor", }, } envVars := map[string]env.Setter{} - envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") envVars["CONFIG_HASH"] = env.SetValue(configHash) + // create Volume and VolumeMounts volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name, instance) - volumeMounts := GetVolumeMounts(instance) + volumeMounts := GetVolumeMounts(customConfigKeys) // Add the CA bundle if instance.Spec.TLS.CaBundleSecretName != "" { @@ -94,23 +82,20 @@ func StatefulSet( Labels: labels, }, Spec: corev1.PodSpec{ - ServiceAccountName: instance.Spec.ServiceAccount, + ServiceAccountName: instance.Spec.ServiceAccount, + AutomountServiceAccountToken: ptr.To(false), + SecurityContext: pod.RestrictivePodSecurityContext(users.CloudkittyUID, users.CloudkittyGID), Containers: []corev1.Container{ { - Name: ComponentName, - Command: []string{ - "/bin/bash", - }, - Args: args, - Image: instance.Spec.ContainerImage, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsUser, - RunAsNonRoot: ptr.To(true), - }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, - LivenessProbe: livenessProbe, + Name: ComponentName, + Command: []string{"/usr/bin/cloudkitty-processor"}, + Args: []string{"--logfile", "/dev/stdout"}, + Image: instance.Spec.ContainerImage, + SecurityContext: pod.RestrictiveSecurityContext(users.CloudkittyUID, users.CloudkittyGID), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + LivenessProbe: livenessProbe, }, }, Volumes: volumes, @@ -126,9 +111,6 @@ func StatefulSet( if topology != nil { topology.ApplyTo(&statefulset.Spec.Template) } else { - // If possible two pods of the same service should not - // run on the same worker node. If this is not possible - // the get still created on the same worker node. statefulset.Spec.Template.Spec.Affinity = cloudkitty.GetPodAffinity(ComponentName) } diff --git a/internal/cloudkittyproc/volumes.go b/internal/cloudkittyproc/volumes.go index 9186f8028..d9c07d0bf 100644 --- a/internal/cloudkittyproc/volumes.go +++ b/internal/cloudkittyproc/volumes.go @@ -3,23 +3,20 @@ package cloudkittyproc import ( telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" "github.com/openstack-k8s-operators/telemetry-operator/internal/cloudkitty" + "github.com/openstack-k8s-operators/telemetry-operator/internal/utils" corev1 "k8s.io/api/core/v1" ) -var ( - configMode int32 = 0640 -) +var config0440AccessMode int32 = 0440 // GetVolumes - func GetVolumes(parentName string, name string, instance *telemetryv1.CloudKittyProc) []corev1.Volume { - var config0644AccessMode int32 = 0644 - volumes := []corev1.Volume{ { Name: "config-data-custom", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &config0644AccessMode, + DefaultMode: &config0440AccessMode, SecretName: name + "-config-data", }, }, @@ -31,7 +28,7 @@ func GetVolumes(parentName string, name string, instance *telemetryv1.CloudKitty Name: "custom-config", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - DefaultMode: &configMode, + DefaultMode: &config0440AccessMode, SecretName: instance.Spec.CustomConfigsSecretName, }, }, @@ -42,22 +39,49 @@ func GetVolumes(parentName string, name string, instance *telemetryv1.CloudKitty } // GetVolumeMounts - CloudKitty Proc VolumeMounts -func GetVolumeMounts(instance *telemetryv1.CloudKittyProc) []corev1.VolumeMount { +// customConfigKeys are the keys from the CustomConfigsSecret; each is +// SubPath-mounted into /etc/cloudkitty/ (matching the API contract: +// "files from this secret will get copied into /etc/cloudkitty/"). +func GetVolumeMounts(customConfigKeys []string) []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", - MountPath: "/var/lib/openstack/service-config/", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d/00-cloudkitty.conf", + SubPath: cloudkitty.DefaultsConfigFileName, + ReadOnly: true, + }, + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d/" + cloudkitty.CustomConfigFileName, + SubPath: cloudkitty.CustomConfigFileName, + ReadOnly: true, + }, + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d/" + cloudkitty.CustomServiceConfigFileName, + SubPath: cloudkitty.CustomServiceConfigFileName, + ReadOnly: true, + }, + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d/" + cloudkitty.CustomServiceConfigSecretsFileName, + SubPath: cloudkitty.CustomServiceConfigSecretsFileName, ReadOnly: true, }, } - if instance.Spec.CustomConfigsSecretName != "" { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ + base := append(cloudkitty.GetVolumeMounts(), volumeMounts...) + + customConfigMounts := make([]corev1.VolumeMount, 0, len(customConfigKeys)) + for _, key := range customConfigKeys { + customConfigMounts = append(customConfigMounts, corev1.VolumeMount{ Name: "custom-config", - MountPath: "/var/lib/openstack/custom-config", + MountPath: "/etc/cloudkitty/" + key, + SubPath: key, ReadOnly: true, }) } - return append(cloudkitty.GetVolumeMounts(cloudkitty.ServiceName+"-proc"), volumeMounts...) + // custom-config files override the default file mounted at the same path + return utils.MergeCustomConfigMounts(base, customConfigMounts) } diff --git a/internal/controller/aodh_controller.go b/internal/controller/aodh_controller.go index fd14fe650..27c310a26 100644 --- a/internal/controller/aodh_controller.go +++ b/internal/controller/aodh_controller.go @@ -20,6 +20,7 @@ package controller import ( "context" "fmt" + "sort" "time" corev1 "k8s.io/api/core/v1" @@ -158,8 +159,19 @@ func (r *AutoscalingReconciler) reconcileInitAodh( // // run Aodh db sync + var customConfigKeys []string + if instance.Spec.Aodh.CustomConfigsSecretName != "" { + customConfigsSecret, _, err := secret.GetSecret(ctx, helper, instance.Spec.Aodh.CustomConfigsSecretName, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + for key := range customConfigsSecret.Data { + customConfigKeys = append(customConfigKeys, key) + } + sort.Strings(customConfigKeys) + } dbSyncHash := instance.Status.Hash[telemetryv1.DbSyncHash] - jobDef := autoscaling.DbSyncJob(instance, serviceLabels) + jobDef := autoscaling.DbSyncJob(instance, serviceLabels, customConfigKeys) dbSyncjob := job.NewJob( jobDef, @@ -322,7 +334,18 @@ func (r *AutoscalingReconciler) reconcileNormalAodh( return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err) } - sfsetDef, err := autoscaling.AodhStatefulSet(instance, inputHash, serviceLabels, topology, memcached) + var aodhCustomConfigKeys []string + if instance.Spec.Aodh.CustomConfigsSecretName != "" { + customConfigsSecret, _, err := secret.GetSecret(ctx, helper, instance.Spec.Aodh.CustomConfigsSecretName, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + for key := range customConfigsSecret.Data { + aodhCustomConfigKeys = append(aodhCustomConfigKeys, key) + } + sort.Strings(aodhCustomConfigKeys) + } + sfsetDef, err := autoscaling.AodhStatefulSet(instance, inputHash, serviceLabels, topology, memcached, aodhCustomConfigKeys) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.DeploymentReadyCondition, diff --git a/internal/controller/autoscaling_controller.go b/internal/controller/autoscaling_controller.go index 446d50cfa..fb11f73f6 100644 --- a/internal/controller/autoscaling_controller.go +++ b/internal/controller/autoscaling_controller.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "maps" + "sort" "strconv" "time" @@ -110,7 +111,7 @@ func (r *AutoscalingReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch // service account permissions that are needed to grant permission to the above -// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use // +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update // Reconcile reconciles an Autoscaling @@ -317,7 +318,7 @@ func (r *AutoscalingReconciler) reconcileNormal( rbacRules := []rbacv1.PolicyRule{ { APIGroups: []string{"security.openshift.io"}, - ResourceNames: []string{"anyuid"}, + ResourceNames: []string{"nonroot-v2"}, Resources: []string{"securitycontextconstraints"}, Verbs: []string{"use"}, }, @@ -680,12 +681,17 @@ func (r *AutoscalingReconciler) reconcileNormal( // // check for custom configs secret holding custom configuration files // + var customConfigKeys []string if instance.Spec.Aodh.CustomConfigsSecretName != "" { - _, hash, err := secret.GetSecret(ctx, helper, instance.Spec.Aodh.CustomConfigsSecretName, instance.Namespace) + customConfigsSecret, hash, err := secret.GetSecret(ctx, helper, instance.Spec.Aodh.CustomConfigsSecretName, instance.Namespace) if err != nil { return ctrl.Result{}, err } configMapVars["custom-configs-secret"] = env.SetValue(hash) + for key := range customConfigsSecret.Data { + customConfigKeys = append(customConfigKeys, key) + } + sort.Strings(customConfigKeys) } // run check custom configs secret - end @@ -906,16 +912,6 @@ func (r *AutoscalingReconciler) generateServiceConfig( } cms := []util.Template{ - // ScriptsSecret - { - Name: fmt.Sprintf("%s-scripts", autoscaling.ServiceName), - Namespace: instance.Namespace, - Type: util.TemplateTypeScripts, - InstanceType: instance.Kind, - AdditionalTemplate: map[string]string{"common.sh": "/common/common.sh"}, - Labels: cmLabels, - }, - // Secret { Name: fmt.Sprintf("%s-config-data", autoscaling.ServiceName), Namespace: instance.Namespace, diff --git a/internal/controller/ceilometer_controller.go b/internal/controller/ceilometer_controller.go index 9bfcfb259..1d77f16a6 100644 --- a/internal/controller/ceilometer_controller.go +++ b/internal/controller/ceilometer_controller.go @@ -105,7 +105,7 @@ func (r *CeilometerReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch // service account permissions that are needed to grant permission to the above -// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update @@ -496,7 +496,7 @@ func (r *CeilometerReconciler) reconcileNormal(ctx context.Context, instance *te rbacRules := []rbacv1.PolicyRule{ { APIGroups: []string{"security.openshift.io"}, - ResourceNames: []string{"anyuid"}, + ResourceNames: []string{"nonroot-v2"}, Resources: []string{"securitycontextconstraints"}, Verbs: []string{"use"}, }, @@ -606,14 +606,19 @@ func (r *CeilometerReconciler) reconcileCeilometer( // run check OpenStack secret - end // - // check for custom configs secret secret holding custom configuration files + // check for custom configs secret holding custom configuration files // + var customConfigKeys []string if instance.Spec.CustomConfigsSecretName != "" { - _, hash, err := secret.GetSecret(ctx, helper, instance.Spec.CustomConfigsSecretName, instance.Namespace) + customConfigsSecret, hash, err := secret.GetSecret(ctx, helper, instance.Spec.CustomConfigsSecretName, instance.Namespace) if err != nil { return ctrlResult, err } configMapVars["custom-configs-secret"] = env.SetValue(hash) + for key := range customConfigsSecret.Data { + customConfigKeys = append(customConfigKeys, key) + } + sort.Strings(customConfigKeys) } // run check custom configs secret - end @@ -809,7 +814,7 @@ func (r *CeilometerReconciler) reconcileCeilometer( } // Define a new StatefulSet object - sfsetDef, err := ceilometer.StatefulSet(instance, inputHash, serviceLabels, topology) + sfsetDef, err := ceilometer.StatefulSet(instance, inputHash, serviceLabels, topology, customConfigKeys) if err != nil { return ctrl.Result{}, err } @@ -1387,7 +1392,6 @@ func (r *CeilometerReconciler) generateServiceConfig( Type: util.TemplateTypeScripts, InstanceType: "ceilometercentral", AdditionalTemplate: map[string]string{ - "common.sh": "/common/common.sh", "centralhealth.py": "/ceilometercentral/bin/centralhealth.py", "notificationhealth.py": "/ceilometercentral/bin/notificationhealth.py", }, @@ -1486,12 +1490,11 @@ func (r *CeilometerReconciler) generateComputeServiceConfig( cms := []util.Template{ // CeilometerCompute ScriptsConfigMap { - Name: fmt.Sprintf("%s-scripts", ceilometer.ComputeServiceName), - Namespace: instance.Namespace, - Type: util.TemplateTypeScripts, - InstanceType: "ceilometercompute", - AdditionalTemplate: map[string]string{"common.sh": "/common/common.sh"}, - Labels: cmLabels, + Name: fmt.Sprintf("%s-scripts", ceilometer.ComputeServiceName), + Namespace: instance.Namespace, + Type: util.TemplateTypeScripts, + InstanceType: "ceilometercompute", + Labels: cmLabels, }, // CeilometerCompute ConfigMap { @@ -1505,12 +1508,11 @@ func (r *CeilometerReconciler) generateComputeServiceConfig( }, // CeilometerIpmi ScriptsConfigMap { - Name: fmt.Sprintf("%s-scripts", ceilometer.IpmiServiceName), - Namespace: instance.Namespace, - Type: util.TemplateTypeScripts, - InstanceType: "ceilometeripmi", - AdditionalTemplate: map[string]string{"common.sh": "/common/common.sh"}, - Labels: ipmiLabels, + Name: fmt.Sprintf("%s-scripts", ceilometer.IpmiServiceName), + Namespace: instance.Namespace, + Type: util.TemplateTypeScripts, + InstanceType: "ceilometeripmi", + Labels: ipmiLabels, }, // CeilometerIpmi ConfigMap { diff --git a/internal/controller/cloudkitty_controller.go b/internal/controller/cloudkitty_controller.go index 610fb25da..1f29a6e6a 100644 --- a/internal/controller/cloudkitty_controller.go +++ b/internal/controller/cloudkitty_controller.go @@ -124,7 +124,7 @@ func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch // service account permissions that are needed to grant permission to the above -// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid;privileged,resources=securitycontextconstraints,verbs=use +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use // Reconcile - func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { @@ -842,7 +842,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te rbacRules := []rbacv1.PolicyRule{ { APIGroups: []string{"security.openshift.io"}, - ResourceNames: []string{"anyuid"}, + ResourceNames: []string{"nonroot-v2"}, Resources: []string{"securitycontextconstraints"}, Verbs: []string{"use"}, }, @@ -1191,7 +1191,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te return ctrl.Result{}, nil } -// generateServiceConfigs - create Secret which hold scripts and service configuration +// generateServiceConfigs - create Secret which holds the service configuration func (r *CloudKittyReconciler) generateServiceConfigs( ctx context.Context, h *helper.Helper, @@ -1204,7 +1204,6 @@ func (r *CloudKittyReconciler) generateServiceConfigs( Log := r.GetLogger(ctx) // // create Secret required for cloudkitty input - // - %-scripts holds scripts to e.g. bootstrap the service // - %-config holds minimal cloudkitty config required to get the service up // @@ -1364,13 +1363,6 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["VHosts"] = httpdVhostConfig configTemplates := []util.Template{ - { - Name: fmt.Sprintf("%s-scripts", instance.Name), - Namespace: instance.Namespace, - Type: util.TemplateTypeScripts, - InstanceType: instance.Kind, - Labels: labels, - }, { Name: fmt.Sprintf("%s-config-data", instance.Name), Namespace: instance.Namespace, diff --git a/internal/controller/cloudkittyapi_controller.go b/internal/controller/cloudkittyapi_controller.go index 8304737a1..f072a4492 100644 --- a/internal/controller/cloudkittyapi_controller.go +++ b/internal/controller/cloudkittyapi_controller.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "sort" telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" "github.com/openstack-k8s-operators/telemetry-operator/internal/cloudkitty" @@ -794,7 +795,6 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance parentCloudKittyName := cloudkitty.GetOwningCloudKittyName(instance) secretNames := []string{ instance.Spec.TransportURLSecret, // TransportURLSecret - fmt.Sprintf("%s-scripts", parentCloudKittyName), // ScriptsSecret fmt.Sprintf("%s-config-data", parentCloudKittyName), // ConfigSecret } // Append CustomServiceConfigSecrets that should be checked @@ -1003,13 +1003,17 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance // // check for custom configs secret holding custom configuration files - // + var customConfigKeys []string if instance.Spec.CustomConfigsSecretName != "" { - _, hash, err := secret.GetSecret(ctx, helper, instance.Spec.CustomConfigsSecretName, instance.Namespace) + customConfigsSecret, hash, err := secret.GetSecret(ctx, helper, instance.Spec.CustomConfigsSecretName, instance.Namespace) if err != nil { return ctrl.Result{}, err } configVars["custom-configs-secret"] = env.SetValue(hash) + for key := range customConfigsSecret.Data { + customConfigKeys = append(customConfigKeys, key) + } + sort.Strings(customConfigKeys) } // run check custom configs secret - end @@ -1039,7 +1043,7 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance } // Deploy a statefulset - ssDef, err := cloudkittyapi.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology) + ssDef, err := cloudkittyapi.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology, customConfigKeys) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.DeploymentReadyCondition, diff --git a/internal/controller/cloudkittyproc_controller.go b/internal/controller/cloudkittyproc_controller.go index 4054515f1..15100396e 100644 --- a/internal/controller/cloudkittyproc_controller.go +++ b/internal/controller/cloudkittyproc_controller.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "sort" telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" "github.com/openstack-k8s-operators/telemetry-operator/internal/cloudkitty" @@ -481,7 +482,6 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance parentCloudKittyName := cloudkitty.GetOwningCloudKittyName(instance) secretNames := []string{ instance.Spec.TransportURLSecret, // TransportURLSecret - fmt.Sprintf("%s-scripts", parentCloudKittyName), // ScriptsSecret fmt.Sprintf("%s-config-data", parentCloudKittyName), // ConfigSecret } // Append CustomServiceConfigSecrets that should be checked @@ -586,13 +586,17 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance // // check for custom configs secret holding custom configuration files - // + var customConfigKeys []string if instance.Spec.CustomConfigsSecretName != "" { - _, hash, err := secret.GetSecret(ctx, helper, instance.Spec.CustomConfigsSecretName, instance.Namespace) + customConfigsSecret, hash, err := secret.GetSecret(ctx, helper, instance.Spec.CustomConfigsSecretName, instance.Namespace) if err != nil { return ctrl.Result{}, err } configVars["custom-configs-secret"] = env.SetValue(hash) + for key := range customConfigsSecret.Data { + customConfigKeys = append(customConfigKeys, key) + } + sort.Strings(customConfigKeys) } // run check custom configs secret - end @@ -694,7 +698,7 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance } // Deploy a statefulset - ssDef := cloudkittyproc.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology) + ssDef := cloudkittyproc.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology, customConfigKeys) ss := statefulset.NewStatefulSet(ssDef, cloudkitty.ShortDuration) var ssData appsv1.StatefulSet @@ -850,11 +854,8 @@ func (r *CloudKittyProcReconciler) generateServiceConfigs( Namespace: instance.Namespace, Type: util.TemplateTypeConfig, InstanceType: instance.Kind, - AdditionalTemplate: map[string]string{ - "healthcheck.py": "/cloudkitty/bin/healthcheck.py", - }, - CustomData: customData, - Labels: labels, + CustomData: customData, + Labels: labels, }, } diff --git a/internal/controller/logging_controller.go b/internal/controller/logging_controller.go index c6b137233..27216fbf1 100644 --- a/internal/controller/logging_controller.go +++ b/internal/controller/logging_controller.go @@ -63,6 +63,7 @@ func (r *LoggingReconciler) GetLogger(ctx context.Context) logr.Logger { //+kubebuilder:rbac:groups=telemetry.openstack.org,resources=loggings/finalizers,verbs=update;patch //+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/internal/mysqldexporter/statefulset.go b/internal/mysqldexporter/statefulset.go index fe4ff036f..56e8d86ea 100644 --- a/internal/mysqldexporter/statefulset.go +++ b/internal/mysqldexporter/statefulset.go @@ -108,22 +108,21 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: ptr.To(false), Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{ - "ALL", - }, + Drop: []corev1.Capability{"ALL"}, }, }, VolumeMounts: volumeMounts, } - pod := corev1.PodTemplateSpec{ + podTemplate := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: ServiceName, Namespace: instance.Namespace, Labels: labels, }, Spec: corev1.PodSpec{ - ServiceAccountName: instance.RbacResourceName(), + ServiceAccountName: instance.RbacResourceName(), + AutomountServiceAccountToken: ptr.To(false), Containers: []corev1.Container{ mysqldExporterContainer, }, @@ -138,12 +137,12 @@ func StatefulSet( } if topology != nil { - topology.ApplyTo(&pod) + topology.ApplyTo(&podTemplate) } else { // If possible two pods of the same service should not // run on the same worker node. If this is not possible // the get still created on the same worker node. - pod.Spec.Affinity = affinity.DistributePods( + podTemplate.Spec.Affinity = affinity.DistributePods( common.AppSelector, []string{ ServiceName, @@ -164,7 +163,7 @@ func StatefulSet( Selector: &metav1.LabelSelector{ MatchLabels: labels, }, - Template: pod, + Template: podTemplate, }, } diff --git a/internal/mysqldexporter/volumes.go b/internal/mysqldexporter/volumes.go index 6cd98426c..7c144b3d6 100644 --- a/internal/mysqldexporter/volumes.go +++ b/internal/mysqldexporter/volumes.go @@ -24,7 +24,7 @@ const ( ) var ( - configMode int32 = 0640 + configMode int32 = 0440 ) func getVolumes() []corev1.Volume { diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 0461d0567..af3b0f52c 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -21,6 +21,7 @@ import ( "context" "fmt" + corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -48,6 +49,29 @@ type ConditionalWatchingReconciler struct { Cache cache.Cache } +// MergeCustomConfigMounts returns base with each override applied: an override +// whose MountPath matches an existing base mount replaces it in place, so a +// custom-config file overrides the default file mounted at that path (matching +// the pre-kolla last-write-wins copy behaviour where custom-config/* was copied +// over the rendered defaults). Overrides that don't match a base path are +// appended. This keeps every MountPath unique so the pod spec stays valid. +func MergeCustomConfigMounts(base, overrides []corev1.VolumeMount) []corev1.VolumeMount { + idx := make(map[string]int, len(base)) + for i, m := range base { + idx[m.MountPath] = i + } + out := append([]corev1.VolumeMount(nil), base...) + for _, o := range overrides { + if i, ok := idx[o.MountPath]; ok { + out[i] = o + } else { + idx[o.MountPath] = len(out) + out = append(out, o) + } + } + return out +} + // EnsureDeleted - Delete the object which in turn will clean the sub resources func EnsureDeleted(ctx context.Context, helper *helper.Helper, obj client.Object) (ctrl.Result, error) { key := client.ObjectKeyFromObject(obj) diff --git a/templates/autoscaling/config/aodh-api-config.json b/templates/autoscaling/config/aodh-api-config.json deleted file mode 100644 index faef19c1a..000000000 --- a/templates/autoscaling/config/aodh-api-config.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "command": "/usr/sbin/httpd -DFOREGROUND -E /dev/stdout", - "config_files": [ - { - "source": "/var/lib/openstack/config/aodh.conf", - "dest": "/etc/aodh/aodh.conf", - "owner": "aodh", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/custom-config/*", - "dest": "/etc/aodh/", - "owner": "aodh", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/custom.conf", - "dest": "/etc/aodh/aodh.conf.d/01-aodh-custom.conf", - "owner": "aodh", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/wsgi-aodh.conf", - "dest": "/etc/httpd/conf.d/00wsgi-aodh.conf", - "owner": "root", - "perm": "0644" - }, - { - "source": "/var/lib/openstack/config/httpd.conf", - "dest": "/etc/httpd/conf/httpd.conf", - "owner": "root", - "perm": "0644" - }, - { - "source": "/var/lib/openstack/config/ssl.conf", - "dest": "/etc/httpd/conf.d/ssl.conf", - "owner": "aodh", - "perm": "0644" - }, - { - "source": "/var/lib/config-data/tls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "aodh", - "perm": "0440", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/tls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "aodh", - "perm": "0400", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/openstack/config/my.cnf", - "dest": "/etc/my.cnf", - "owner": "aodh", - "perm": "0644" - }, - { - "source": "/var/lib/config-data/mtls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "aodh:aodh", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/mtls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "aodh:aodh", - "perm": "0640", - "optional": true, - "merge": true - } - ], - "permissions": [ - { - "path": "/etc/httpd/run", - "owner": "aodh:apache", - "recurse": true - } - ] -} diff --git a/templates/autoscaling/config/aodh-dbsync-config.json b/templates/autoscaling/config/aodh-dbsync-config.json deleted file mode 100644 index b76813089..000000000 --- a/templates/autoscaling/config/aodh-dbsync-config.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "command": "/usr/bin/aodh-dbsync", - "config_files": [ - { - "source": "/var/lib/openstack/config/aodh.conf", - "dest": "/etc/aodh/aodh.conf", - "owner": "aodh", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/custom-config/*", - "dest": "/etc/aodh/", - "owner": "aodh", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/custom.conf", - "dest": "/etc/aodh/aodh.conf.d/01-aodh-custom.conf", - "owner": "aodh", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/my.cnf", - "dest": "/etc/my.cnf", - "owner": "aodh", - "perm": "0644" - } - ] -} diff --git a/templates/autoscaling/config/aodh-evaluator-config.json b/templates/autoscaling/config/aodh-evaluator-config.json deleted file mode 100644 index 2d23e5dfb..000000000 --- a/templates/autoscaling/config/aodh-evaluator-config.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "command": "/usr/bin/aodh-evaluator --logfile /dev/stdout", - "config_files": [ - { - "source": "/var/lib/openstack/config/aodh.conf", - "dest": "/etc/aodh/aodh.conf", - "owner": "aodh", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/custom-config/*", - "dest": "/etc/aodh/", - "owner": "aodh", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/custom.conf", - "dest": "/etc/aodh/aodh.conf.d/01-aodh-custom.conf", - "owner": "aodh", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/prometheus.yaml", - "dest": "/etc/openstack/prometheus.yaml", - "owner": "aodh", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/config/my.cnf", - "dest": "/etc/my.cnf", - "owner": "aodh", - "perm": "0644" - }, - { - "source": "/var/lib/config-data/mtls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "aodh:aodh", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/mtls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "aodh:aodh", - "perm": "0640", - "optional": true, - "merge": true - } - ] - } diff --git a/templates/autoscaling/config/aodh-listener-config.json b/templates/autoscaling/config/aodh-listener-config.json deleted file mode 100644 index d729f31f2..000000000 --- a/templates/autoscaling/config/aodh-listener-config.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "command": "/usr/bin/aodh-listener --logfile /dev/stdout", - "config_files": [ - { - "source": "/var/lib/openstack/config/aodh.conf", - "dest": "/etc/aodh/aodh.conf", - "owner": "aodh", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/custom-config/*", - "dest": "/etc/aodh/", - "owner": "aodh", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/custom.conf", - "dest": "/etc/aodh/aodh.conf.d/01-aodh-custom.conf", - "owner": "aodh", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/my.cnf", - "dest": "/etc/my.cnf", - "owner": "aodh", - "perm": "0644" - }, - { - "source": "/var/lib/config-data/mtls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "aodh:aodh", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/mtls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "aodh:aodh", - "perm": "0640", - "optional": true, - "merge": true - } - ] - } diff --git a/templates/autoscaling/config/aodh-notifier-config.json b/templates/autoscaling/config/aodh-notifier-config.json deleted file mode 100644 index 0ec586866..000000000 --- a/templates/autoscaling/config/aodh-notifier-config.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "command": "/usr/bin/aodh-notifier --logfile /dev/stdout", - "config_files": [ - { - "source": "/var/lib/openstack/config/aodh.conf", - "dest": "/etc/aodh/aodh.conf", - "owner": "aodh", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/custom-config/*", - "dest": "/etc/aodh/", - "owner": "aodh", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/custom.conf", - "dest": "/etc/aodh/aodh.conf.d/01-aodh-custom.conf", - "owner": "aodh", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/my.cnf", - "dest": "/etc/my.cnf", - "owner": "aodh", - "perm": "0644" - }, - { - "source": "/var/lib/config-data/mtls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "aodh:aodh", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/mtls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "aodh:aodh", - "perm": "0640", - "optional": true, - "merge": true - } - ] - } diff --git a/templates/autoscaling/config/httpd.conf b/templates/autoscaling/config/httpd.conf index b71c693be..9df3672f9 100644 --- a/templates/autoscaling/config/httpd.conf +++ b/templates/autoscaling/config/httpd.conf @@ -1,11 +1,12 @@ ServerTokens Prod ServerSignature Off TraceEnable Off +PidFile /run/httpd/httpd.pid ServerRoot "/etc/httpd" ServerName "aodh.openstack.svc" -User apache -Group apache +User aodh +Group aodh Listen 8042 @@ -19,6 +20,6 @@ LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-A SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded CustomLog /dev/stdout combined env=!forwarded CustomLog /dev/stdout proxy env=forwarded -ErrorLog /dev/stdout +ErrorLog /dev/stderr Include conf.d/*.conf diff --git a/templates/ceilometercentral/config/ceilometer-central-config.json b/templates/ceilometercentral/config/ceilometer-central-config.json deleted file mode 100644 index 12a43fca1..000000000 --- a/templates/ceilometercentral/config/ceilometer-central-config.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "command": "/usr/bin/ceilometer-polling --polling-namespaces central --logfile /dev/stdout", - "config_files": [ - { - "source": "/var/lib/openstack/config/ceilometer.conf", - "dest": "/etc/ceilometer/ceilometer.conf", - "owner": "ceilometer", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/config/polling.yaml.j2", - "dest": "/etc/ceilometer/polling.yaml", - "owner": "ceilometer", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/custom-config/*", - "dest": "/etc/ceilometer/", - "owner": "ceilometer", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/custom.conf", - "dest": "/etc/ceilometer/ceilometer.conf.d/01-ceilometer-custom.conf", - "owner": "ceilometer", - "perm": "0600", - "optional": true - } - ] -} diff --git a/templates/ceilometercentral/config/ceilometer-notification-config.json b/templates/ceilometercentral/config/ceilometer-notification-config.json deleted file mode 100644 index 2f324a261..000000000 --- a/templates/ceilometercentral/config/ceilometer-notification-config.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "command": "/usr/bin/ceilometer-agent-notification --logfile /dev/stdout", - "config_files": [ - { - "source": "/var/lib/openstack/config/ceilometer.conf", - "dest": "/etc/ceilometer/ceilometer.conf", - "owner": "ceilometer", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/config/pipeline.yaml", - "dest": "/etc/ceilometer/pipeline.yaml", - "owner": "ceilometer", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/config/event_pipeline.yaml", - "dest": "/etc/ceilometer/event_pipeline.yaml", - "owner": "ceilometer", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/custom-config/*", - "dest": "/etc/ceilometer/", - "owner": "ceilometer", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/custom.conf", - "dest": "/etc/ceilometer/ceilometer.conf.d/01-ceilometer-custom.conf", - "owner": "ceilometer", - "perm": "0600", - "optional": true - } - ] -} diff --git a/templates/ceilometercentral/config/httpd.conf b/templates/ceilometercentral/config/httpd.conf index ab37610e5..8298d5b7d 100644 --- a/templates/ceilometercentral/config/httpd.conf +++ b/templates/ceilometercentral/config/httpd.conf @@ -2,7 +2,7 @@ ServerTokens Prod ServerSignature Off TraceEnable Off -PidFile run/httpd.pid +PidFile /run/httpd/httpd.pid ServerRoot "/etc/httpd" ServerName "localhost.localdomain" @@ -22,13 +22,14 @@ LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-A SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded CustomLog /dev/stdout combined env=!forwarded CustomLog /dev/stdout proxy env=forwarded +ErrorLog /dev/stderr # vhost {{ .vhost.ServerName }} configuration ServerName {{ .vhost.ServerName }} ## Logging - ErrorLog /dev/stdout + ErrorLog /dev/stderr ServerSignature Off SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded CustomLog /dev/stdout combined env=!forwarded diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py deleted file mode 100755 index a1110d7a5..000000000 --- a/templates/cloudkitty/bin/healthcheck.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2025 Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import psutil - - -def check_process() -> tuple[int, str]: - # Return 0 if cloudkitty-processor process with given cmdline exists, else 1 with reason. - for proc in psutil.process_iter(attrs=["name", "cmdline"]): - try: - cmdline = proc.info.get("cmdline", []) - if cmdline and any("cloudkitty-processor" in arg for arg in cmdline): - return 0, "" - except (psutil.NoSuchProcess, psutil.AccessDenied): - continue - return 1, "CloudKitty processor process not found" - - -def run_checks() -> tuple[int, str]: - # Run all health checks and return aggregated result - checks = [check_process] - for check in checks: - rc, reason = check() - if rc != 0: - return rc, reason - return 0, "" - - -if __name__ == "__main__": - try: - rc, reason = run_checks() - except Exception as ex: - rc, reason = 2, f"Unknown error: {ex}" - - if rc != 0: - print(reason) - sys.exit(rc) diff --git a/templates/cloudkitty/bin/run-on-host b/templates/cloudkitty/bin/run-on-host deleted file mode 100755 index e7840ace8..000000000 --- a/templates/cloudkitty/bin/run-on-host +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec nsenter -a -t 1 -- `realpath -s $0` "$@" diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json deleted file mode 100644 index f1e4da7d4..000000000 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "command": "/usr/sbin/httpd -DFOREGROUND -E /dev/stdout", - "config_files": [ - { - "source": "/var/lib/openstack/service-config/cloudkitty.conf", - "dest": "/etc/cloudkitty/cloudkitty.conf.d/00-cloudkitty.conf", - "owner": "cloudkitty", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/config/metrics.yaml", - "dest": "/etc/cloudkitty/metrics.yaml", - "owner": "cloudkitty", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/service-config/0*.conf", - "dest": "/etc/cloudkitty/cloudkitty.conf.d/", - "owner": "cloudkitty", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/wsgi-cloudkitty.conf", - "dest": "/etc/httpd/conf.d/00wsgi-cloudkitty.conf", - "owner": "cloudkitty", - "perm": "0644" - }, - { - "source": "/var/lib/openstack/config/httpd.conf", - "dest": "/etc/httpd/conf/httpd.conf", - "owner": "cloudkitty", - "perm": "0644" - }, - { - "source": "/var/lib/openstack/config/ssl.conf", - "dest": "/etc/httpd/conf.d/ssl.conf", - "owner": "cloudkitty", - "perm": "0644" - }, - { - "source": "/var/lib/openstack/custom-config/*", - "dest": "/etc/cloudkitty/", - "owner": "cloudkitty", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/config-data/tls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "cloudkitty", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/tls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "cloudkitty", - "perm": "0600", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/mtls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "cloudkitty:cloudkitty", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/mtls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "cloudkitty:cloudkitty", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/openstack/loki-certs/*", - "dest": "/etc/cloudkitty/certs/", - "owner": "cloudkitty:cloudkitty", - "perm": "0400", - "merge": true - } - ], - "permissions": [ - { - "path": "/var/log/cloudkitty", - "owner": "cloudkitty:apache", - "recurse": true - }, - { - "path": "/etc/httpd/run", - "owner": "cloudkitty:apache", - "recurse": true - } - ] -} diff --git a/templates/cloudkitty/config/cloudkitty-dbsync-config.json b/templates/cloudkitty/config/cloudkitty-dbsync-config.json deleted file mode 100644 index 1eb4f0a69..000000000 --- a/templates/cloudkitty/config/cloudkitty-dbsync-config.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "command": "/usr/bin/cloudkitty-dbsync upgrade", - "config_files": [ - { - "source": "/var/lib/openstack/config/cloudkitty.conf", - "dest": "/etc/cloudkitty/cloudkitty.conf", - "owner": "cloudkitty", - "perm": "0600" - } - ] -} diff --git a/templates/cloudkitty/config/cloudkitty-proc-config.json b/templates/cloudkitty/config/cloudkitty-proc-config.json deleted file mode 100644 index adcca50d3..000000000 --- a/templates/cloudkitty/config/cloudkitty-proc-config.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "command": "/usr/bin/cloudkitty-processor --logfile /dev/stdout", - "config_files": [ - { - "source": "/var/lib/openstack/service-config/cloudkitty.conf", - "dest": "/etc/cloudkitty/cloudkitty.conf.d/00-cloudkitty.conf", - "owner": "cloudkitty", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/service-config/0*.conf", - "dest": "/etc/cloudkitty/cloudkitty.conf.d/", - "owner": "cloudkitty", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/config/metrics.yaml", - "dest": "/etc/cloudkitty/metrics.yaml", - "owner": "cloudkitty", - "perm": "0600" - }, - { - "source": "/var/lib/openstack/custom-config/*", - "dest": "/etc/cloudkitty/", - "owner": "cloudkitty", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/openstack/loki-certs/*", - "dest": "/etc/cloudkitty/certs/", - "owner": "cloudkitty:cloudkitty", - "perm": "0400", - "merge": true - } - ] -} diff --git a/templates/cloudkitty/config/cloudkitty-storageinit-config.json b/templates/cloudkitty/config/cloudkitty-storageinit-config.json deleted file mode 100644 index 3d9561c58..000000000 --- a/templates/cloudkitty/config/cloudkitty-storageinit-config.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "command": "/usr/bin/cloudkitty-storage-init", - "config_files": [ - { - "source": "/var/lib/openstack/config/cloudkitty.conf", - "dest": "/etc/cloudkitty/cloudkitty.conf", - "owner": "cloudkitty", - "perm": "0600" - } - ] -} diff --git a/templates/cloudkitty/config/httpd.conf b/templates/cloudkitty/config/httpd.conf index b844bf4a4..118435a61 100644 --- a/templates/cloudkitty/config/httpd.conf +++ b/templates/cloudkitty/config/httpd.conf @@ -1,11 +1,12 @@ ServerTokens Prod ServerSignature Off TraceEnable Off +PidFile /run/httpd/httpd.pid ServerRoot "/etc/httpd" ServerName "cloudkitty.openstack.svc" -User apache -Group apache +User cloudkitty +Group cloudkitty Listen 8889 @@ -19,6 +20,6 @@ LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-A SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded CustomLog /dev/stdout combined env=!forwarded CustomLog /dev/stdout proxy env=forwarded -ErrorLog /dev/stdout +ErrorLog /dev/stderr Include conf.d/*.conf diff --git a/templates/common/common.sh b/templates/common/common.sh deleted file mode 100755 index 8b9b424fa..000000000 --- a/templates/common/common.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin//bash -# -# Copyright 2022 Red Hat Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -e - -function merge_config_dir { - echo merge config dir $1 - for conf in $(find $1 -type f); do - conf_base=$(basename $conf) - - # If CFG already exist in ../merged and is not a json file, - # we expect for now it can be merged using crudini. - # Else, just copy the full file. - if [[ -f /var/lib/config-data/merged/${conf_base} && ${conf_base} != *.json ]]; then - echo merging ${conf} into /var/lib/config-data/merged/${conf_base} - crudini --merge /var/lib/config-data/merged/${conf_base} < ${conf} - else - echo copy ${conf} to /var/lib/config-data/merged/ - cp -f ${conf} /var/lib/config-data/merged/ - fi - done -} diff --git a/test/kuttl/tests/autoscaling/01-assert.yaml b/test/kuttl/tests/autoscaling/01-assert.yaml index 219d0a893..13fc8a047 100644 --- a/test/kuttl/tests/autoscaling/01-assert.yaml +++ b/test/kuttl/tests/autoscaling/01-assert.yaml @@ -8,30 +8,30 @@ metadata: - kind: StatefulSet name: aodh spec: + automountServiceAccountToken: false containers: - args: - - -c - - /usr/local/bin/kolla_start + - -DFOREGROUND command: - - /bin/bash + - /usr/sbin/httpd name: aodh-api - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-evaluator name: aodh-evaluator - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-notifier name: aodh-notifier - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-listener name: aodh-listener hostname: aodh-0 status: @@ -65,30 +65,30 @@ spec: service: aodh template: spec: + automountServiceAccountToken: false containers: - args: - - -c - - /usr/local/bin/kolla_start + - -DFOREGROUND command: - - /bin/bash + - /usr/sbin/httpd name: aodh-api - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-evaluator name: aodh-evaluator - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-notifier name: aodh-notifier - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-listener name: aodh-listener status: availableReplicas: 1 diff --git a/test/kuttl/tests/autoscaling/02-assert.yaml b/test/kuttl/tests/autoscaling/02-assert.yaml index 4fed38015..41e1849a6 100644 --- a/test/kuttl/tests/autoscaling/02-assert.yaml +++ b/test/kuttl/tests/autoscaling/02-assert.yaml @@ -12,64 +12,87 @@ spec: containers: - name: aodh-api volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/openstack/config + - mountPath: /etc/aodh/aodh.conf name: config-data - - mountPath: /var/lib/kolla/config_files/config.json + subPath: aodh.conf + - mountPath: /etc/aodh/aodh.conf.d/01-aodh-custom.conf name: config-data - subPath: aodh-api-config.json - - mountPath: /var/lib/openstack/custom-config + subPath: custom.conf + - mountPath: /etc/httpd/conf.d/00wsgi-aodh.conf + name: config-data + subPath: wsgi-aodh.conf + - mountPath: /etc/httpd/conf/httpd.conf + name: config-data + subPath: httpd.conf + - mountPath: /etc/httpd/conf.d/ssl.conf + name: config-data + subPath: ssl.conf + - mountPath: /etc/my.cnf + name: config-data + subPath: my.cnf + - mountPath: /run/httpd + name: run-httpd + - mountPath: /var/log/httpd + name: var-log-httpd + - mountPath: /etc/aodh/policy.yaml name: custom-config - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + subPath: policy.yaml - name: aodh-evaluator volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/openstack/config + - mountPath: /etc/aodh/aodh.conf + name: config-data + subPath: aodh.conf + - mountPath: /etc/aodh/aodh.conf.d/01-aodh-custom.conf name: config-data - - mountPath: /var/lib/kolla/config_files/config.json + subPath: custom.conf + - mountPath: /etc/openstack/prometheus.yaml name: config-data - subPath: aodh-evaluator-config.json - - mountPath: /var/lib/openstack/custom-config + subPath: prometheus.yaml + - mountPath: /etc/my.cnf + name: config-data + subPath: my.cnf + - mountPath: /etc/aodh/policy.yaml name: custom-config - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + subPath: policy.yaml - name: aodh-notifier volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/openstack/config + - mountPath: /etc/aodh/aodh.conf + name: config-data + subPath: aodh.conf + - mountPath: /etc/aodh/aodh.conf.d/01-aodh-custom.conf name: config-data - - mountPath: /var/lib/kolla/config_files/config.json + subPath: custom.conf + - mountPath: /etc/my.cnf name: config-data - subPath: aodh-notifier-config.json - - mountPath: /var/lib/openstack/custom-config + subPath: my.cnf + - mountPath: /etc/aodh/policy.yaml name: custom-config - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + subPath: policy.yaml - name: aodh-listener volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/openstack/config + - mountPath: /etc/aodh/aodh.conf name: config-data - - mountPath: /var/lib/kolla/config_files/config.json + subPath: aodh.conf + - mountPath: /etc/aodh/aodh.conf.d/01-aodh-custom.conf name: config-data - subPath: aodh-listener-config.json - - mountPath: /var/lib/openstack/custom-config + subPath: custom.conf + - mountPath: /etc/my.cnf + name: config-data + subPath: my.cnf + - mountPath: /etc/aodh/policy.yaml name: custom-config - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + subPath: policy.yaml volumes: - - name: scripts - secret: - secretName: aodh-scripts - name: config-data secret: secretName: aodh-config-data + - emptyDir: {} + name: run-httpd + - emptyDir: {} + name: var-log-httpd - name: custom-config secret: secretName: custom-config - - projected: - defaultMode: 420 --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert diff --git a/test/kuttl/tests/ceilometer/00-assert.yaml b/test/kuttl/tests/ceilometer/00-assert.yaml index 87fc67127..d5237109f 100644 --- a/test/kuttl/tests/ceilometer/00-assert.yaml +++ b/test/kuttl/tests/ceilometer/00-assert.yaml @@ -8,12 +8,15 @@ metadata: - kind: StatefulSet name: ceilometer spec: + automountServiceAccountToken: false containers: - args: - - -c - - /usr/local/bin/kolla_start + - --polling-namespaces + - central + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/ceilometer-polling livenessProbe: exec: command: @@ -21,10 +24,10 @@ spec: - /var/lib/openstack/bin/centralhealth.py name: ceilometer-central-agent - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/ceilometer-agent-notification livenessProbe: exec: command: @@ -77,18 +80,21 @@ spec: service: ceilometer template: spec: + automountServiceAccountToken: false containers: - args: - - -c - - /usr/local/bin/kolla_start + - --polling-namespaces + - central + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/ceilometer-polling name: ceilometer-central-agent - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/ceilometer-agent-notification name: ceilometer-notification-agent - name: sg-core - name: proxy-httpd diff --git a/test/kuttl/tests/ceilometer/01-assert.yaml b/test/kuttl/tests/ceilometer/01-assert.yaml index f66732017..5aa0afaf8 100644 --- a/test/kuttl/tests/ceilometer/01-assert.yaml +++ b/test/kuttl/tests/ceilometer/01-assert.yaml @@ -11,28 +11,44 @@ spec: containers: - name: ceilometer-central-agent volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/openstack/config - name: config-data - - mountPath: /var/lib/kolla/config_files/config.json + - mountPath: /etc/ceilometer/ceilometer.conf name: config-data - subPath: ceilometer-central-config.json - - mountPath: /var/lib/openstack/custom-config + subPath: ceilometer.conf + - mountPath: /etc/ceilometer/polling.yaml name: custom-config - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + subPath: polling.yaml + - mountPath: /etc/ceilometer/ceilometer.conf.d/01-ceilometer-custom.conf + name: config-data + subPath: custom.conf + - mountPath: /var/lib/openstack/bin/centralhealth.py + name: scripts + subPath: centralhealth.py + - mountPath: /var/lib/openstack/bin/notificationhealth.py + name: scripts + subPath: notificationhealth.py - name: ceilometer-notification-agent volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/openstack/config + - mountPath: /etc/ceilometer/ceilometer.conf + name: config-data + subPath: ceilometer.conf + - mountPath: /etc/ceilometer/pipeline.yaml + name: config-data + subPath: pipeline.yaml + - mountPath: /etc/ceilometer/event_pipeline.yaml name: config-data - - mountPath: /var/lib/kolla/config_files/config.json + subPath: event_pipeline.yaml + - mountPath: /etc/ceilometer/ceilometer.conf.d/01-ceilometer-custom.conf name: config-data - subPath: ceilometer-notification-config.json - - mountPath: /var/lib/openstack/custom-config + subPath: custom.conf + - mountPath: /etc/ceilometer/polling.yaml name: custom-config - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + subPath: polling.yaml + - mountPath: /var/lib/openstack/bin/centralhealth.py + name: scripts + subPath: centralhealth.py + - mountPath: /var/lib/openstack/bin/notificationhealth.py + name: scripts + subPath: notificationhealth.py - name: sg-core - name: proxy-httpd volumes: @@ -51,12 +67,10 @@ spec: - emptyDir: {} name: run-httpd - emptyDir: {} - name: log-httpd + name: var-log-httpd - name: custom-config secret: secretName: custom-config - - projected: - defaultMode: 420 --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert diff --git a/test/kuttl/tests/cloudkitty/01-deploy-dependencies.yaml b/test/kuttl/tests/cloudkitty/01-deploy-dependencies.yaml index 2cbc1ee09..772f80dd7 100644 --- a/test/kuttl/tests/cloudkitty/01-deploy-dependencies.yaml +++ b/test/kuttl/tests/cloudkitty/01-deploy-dependencies.yaml @@ -117,4 +117,4 @@ stringData: access_key_id: minio access_key_secret: minio123 bucketnames: loki - endpoint: http://minio.svc.cluster.local:9000 + endpoint: http://minio:9000 diff --git a/test/kuttl/tests/cloudkitty/03-assert.yaml b/test/kuttl/tests/cloudkitty/03-assert.yaml index b549af186..360965eec 100644 --- a/test/kuttl/tests/cloudkitty/03-assert.yaml +++ b/test/kuttl/tests/cloudkitty/03-assert.yaml @@ -14,26 +14,39 @@ spec: - name: telemetry-kuttl-cloudkitty-api-log - name: cloudkitty-api volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/openstack/config - name: config-data - - mountPath: /var/lib/kolla/config_files/config.json - name: config-data - subPath: cloudkitty-api-config.json - - mountPath: /var/lib/openstack/loki-certs + - mountPath: /etc/cloudkitty/metrics.yaml + name: custom-config + subPath: metrics.yaml + - mountPath: /etc/cloudkitty/certs name: certs - - mountPath: /var/lib/openstack/service-config/ + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/00-cloudkitty.conf + name: config-data-custom + subPath: cloudkitty.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/02-global-custom.conf + name: config-data-custom + subPath: 02-global-custom.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/03-service-custom.conf + name: config-data-custom + subPath: 03-service-custom.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/04-service-custom-secrets.conf name: config-data-custom + subPath: 04-service-custom-secrets.conf + - mountPath: /etc/httpd/conf.d/00wsgi-cloudkitty.conf + name: config-data + subPath: wsgi-cloudkitty.conf + - mountPath: /etc/httpd/conf/httpd.conf + name: config-data + subPath: httpd.conf + - mountPath: /etc/httpd/conf.d/ssl.conf + name: config-data + subPath: ssl.conf + - mountPath: /run/httpd + name: run-httpd + - mountPath: /var/log/httpd + name: var-log-httpd - mountPath: /var/log/cloudkitty name: logs - - mountPath: /var/lib/openstack/custom-config - name: custom-config - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount volumes: - - name: scripts - secret: - secretName: telemetry-kuttl-cloudkitty-scripts - name: config-data secret: secretName: telemetry-kuttl-cloudkitty-config-data @@ -49,11 +62,13 @@ spec: secretName: telemetry-kuttl-cloudkitty-api-config-data - emptyDir: {} name: logs + - emptyDir: {} + name: run-httpd + - emptyDir: {} + name: var-log-httpd - name: custom-config secret: secretName: custom-config - - projected: - defaultMode: 420 --- apiVersion: v1 kind: Pod @@ -69,24 +84,24 @@ spec: containers: - name: cloudkitty-proc volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/openstack/config - name: config-data - - mountPath: /var/lib/kolla/config_files/config.json - name: config-data - subPath: cloudkitty-proc-config.json - - mountPath: /var/lib/openstack/loki-certs + - mountPath: /etc/cloudkitty/metrics.yaml + name: custom-config + subPath: metrics.yaml + - mountPath: /etc/cloudkitty/certs name: certs - - mountPath: /var/lib/openstack/service-config/ + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/00-cloudkitty.conf name: config-data-custom - - mountPath: /var/lib/openstack/custom-config - name: custom-config - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + subPath: cloudkitty.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/02-global-custom.conf + name: config-data-custom + subPath: 02-global-custom.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/03-service-custom.conf + name: config-data-custom + subPath: 03-service-custom.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/04-service-custom-secrets.conf + name: config-data-custom + subPath: 04-service-custom-secrets.conf volumes: - - name: scripts - secret: - secretName: telemetry-kuttl-cloudkitty-scripts - name: config-data secret: secretName: telemetry-kuttl-cloudkitty-config-data @@ -103,8 +118,6 @@ spec: - name: custom-config secret: secretName: custom-config - - projected: - defaultMode: 420 --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert diff --git a/test/kuttl/tests/default/00-cloudkitty-deps.yaml b/test/kuttl/tests/default/00-cloudkitty-deps.yaml index 74a306185..68b7008e8 100644 --- a/test/kuttl/tests/default/00-cloudkitty-deps.yaml +++ b/test/kuttl/tests/default/00-cloudkitty-deps.yaml @@ -100,4 +100,4 @@ stringData: access_key_id: minio access_key_secret: minio123 bucketnames: loki - endpoint: http://minio.svc.cluster.local:9000 + endpoint: http://minio:9000 diff --git a/test/kuttl/tests/default/01-assert.yaml b/test/kuttl/tests/default/01-assert.yaml index 6aab8bf46..83a61f676 100644 --- a/test/kuttl/tests/default/01-assert.yaml +++ b/test/kuttl/tests/default/01-assert.yaml @@ -25,10 +25,12 @@ metadata: spec: containers: - args: - - -c - - /usr/local/bin/kolla_start + - --polling-namespaces + - central + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/ceilometer-polling livenessProbe: exec: command: @@ -36,10 +38,10 @@ spec: - /var/lib/openstack/bin/centralhealth.py name: ceilometer-central-agent - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/ceilometer-agent-notification livenessProbe: exec: command: @@ -94,16 +96,18 @@ spec: spec: containers: - args: - - -c - - /usr/local/bin/kolla_start + - --polling-namespaces + - central + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/ceilometer-polling name: ceilometer-central-agent - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/ceilometer-agent-notification name: ceilometer-notification-agent - name: sg-core - name: proxy-httpd @@ -183,10 +187,6 @@ apiVersion: telemetry.openstack.org/v1beta1 kind: MetricStorage metadata: name: metric-storage -spec: - monitoringStack: - nodeSelector: - kubernetes.io/os: linux status: conditions: - type: Ready @@ -256,28 +256,27 @@ metadata: spec: containers: - args: - - -c - - /usr/local/bin/kolla_start + - -DFOREGROUND command: - - /bin/bash + - /usr/sbin/httpd name: aodh-api - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-evaluator name: aodh-evaluator - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-notifier name: aodh-notifier - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-listener name: aodh-listener hostname: aodh-0 status: @@ -313,28 +312,27 @@ spec: spec: containers: - args: - - -c - - /usr/local/bin/kolla_start + - -DFOREGROUND command: - - /bin/bash + - /usr/sbin/httpd name: aodh-api - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-evaluator name: aodh-evaluator - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-notifier name: aodh-notifier - args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/aodh-listener name: aodh-listener status: availableReplicas: 1 diff --git a/test/kuttl/tests/tls/00-cloudkitty-deps.yaml b/test/kuttl/tests/tls/00-cloudkitty-deps.yaml index 74a306185..68b7008e8 100644 --- a/test/kuttl/tests/tls/00-cloudkitty-deps.yaml +++ b/test/kuttl/tests/tls/00-cloudkitty-deps.yaml @@ -100,4 +100,4 @@ stringData: access_key_id: minio access_key_secret: minio123 bucketnames: loki - endpoint: http://minio.svc.cluster.local:9000 + endpoint: http://minio:9000 diff --git a/test/kuttl/tests/tls/02-assert.yaml b/test/kuttl/tests/tls/02-assert.yaml index b6b814f05..a0b01017b 100644 --- a/test/kuttl/tests/tls/02-assert.yaml +++ b/test/kuttl/tests/tls/02-assert.yaml @@ -11,112 +11,131 @@ spec: containers: - name: aodh-api command: - - /bin/bash + - /usr/sbin/httpd args: - - -c - - /usr/local/bin/kolla_start + - -DFOREGROUND volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts + - mountPath: /etc/aodh/aodh.conf + name: config-data + readOnly: true + subPath: aodh.conf + - mountPath: /etc/aodh/aodh.conf.d/01-aodh-custom.conf + name: config-data + readOnly: true + subPath: custom.conf + - mountPath: /etc/httpd/conf.d/00wsgi-aodh.conf + name: config-data + readOnly: true + subPath: wsgi-aodh.conf + - mountPath: /etc/httpd/conf/httpd.conf + name: config-data readOnly: true - - mountPath: /var/lib/openstack/config + subPath: httpd.conf + - mountPath: /etc/httpd/conf.d/ssl.conf name: config-data readOnly: true - - mountPath: /var/lib/kolla/config_files/config.json + subPath: ssl.conf + - mountPath: /etc/my.cnf name: config-data readOnly: true - subPath: aodh-api-config.json + subPath: my.cnf + - mountPath: /run/httpd + name: run-httpd + - mountPath: /var/log/httpd + name: var-log-httpd - mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem name: combined-ca-bundle readOnly: true subPath: tls-ca-bundle.pem - - mountPath: /var/lib/config-data/tls/certs/internal.crt + - mountPath: /etc/pki/tls/certs/internal.crt name: internal-tls-certs readOnly: true subPath: tls.crt - - mountPath: /var/lib/config-data/tls/private/internal.key + - mountPath: /etc/pki/tls/private/internal.key name: internal-tls-certs readOnly: true subPath: tls.key - - mountPath: /var/lib/config-data/tls/certs/public.crt + - mountPath: /etc/pki/tls/certs/public.crt name: public-tls-certs readOnly: true subPath: tls.crt - - mountPath: /var/lib/config-data/tls/private/public.key + - mountPath: /etc/pki/tls/private/public.key name: public-tls-certs readOnly: true subPath: tls.key - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - readOnly: true - name: aodh-evaluator command: - - /bin/bash + - /usr/bin/aodh-evaluator args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts + - mountPath: /etc/aodh/aodh.conf + name: config-data readOnly: true - - mountPath: /var/lib/openstack/config + subPath: aodh.conf + - mountPath: /etc/aodh/aodh.conf.d/01-aodh-custom.conf name: config-data readOnly: true - - mountPath: /var/lib/kolla/config_files/config.json + subPath: custom.conf + - mountPath: /etc/openstack/prometheus.yaml name: config-data readOnly: true - subPath: aodh-evaluator-config.json + subPath: prometheus.yaml + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem name: combined-ca-bundle readOnly: true subPath: tls-ca-bundle.pem - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - readOnly: true - name: aodh-notifier command: - - /bin/bash + - /usr/bin/aodh-notifier args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts + - mountPath: /etc/aodh/aodh.conf + name: config-data readOnly: true - - mountPath: /var/lib/openstack/config + subPath: aodh.conf + - mountPath: /etc/aodh/aodh.conf.d/01-aodh-custom.conf name: config-data readOnly: true - - mountPath: /var/lib/kolla/config_files/config.json + subPath: custom.conf + - mountPath: /etc/my.cnf name: config-data readOnly: true - subPath: aodh-notifier-config.json + subPath: my.cnf - mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem name: combined-ca-bundle readOnly: true subPath: tls-ca-bundle.pem - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - readOnly: true - name: aodh-listener command: - - /bin/bash + - /usr/bin/aodh-listener args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts + - mountPath: /etc/aodh/aodh.conf + name: config-data readOnly: true - - mountPath: /var/lib/openstack/config + subPath: aodh.conf + - mountPath: /etc/aodh/aodh.conf.d/01-aodh-custom.conf name: config-data readOnly: true - - mountPath: /var/lib/kolla/config_files/config.json + subPath: custom.conf + - mountPath: /etc/my.cnf name: config-data readOnly: true - subPath: aodh-listener-config.json + subPath: my.cnf - mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem name: combined-ca-bundle readOnly: true subPath: tls-ca-bundle.pem - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - readOnly: true --- apiVersion: v1 kind: Pod @@ -131,57 +150,77 @@ spec: containers: - name: ceilometer-central-agent args: - - -c - - /usr/local/bin/kolla_start + - --polling-namespaces + - central + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/ceilometer-polling volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts + - mountPath: /etc/ceilometer/ceilometer.conf + name: config-data readOnly: true - - mountPath: /var/lib/openstack/config + subPath: ceilometer.conf + - mountPath: /etc/ceilometer/polling.yaml name: config-data readOnly: true - - mountPath: /var/lib/kolla/config_files/config.json + subPath: polling.yaml.j2 + - mountPath: /etc/ceilometer/ceilometer.conf.d/01-ceilometer-custom.conf name: config-data readOnly: true - subPath: ceilometer-central-config.json + subPath: custom.conf + - mountPath: /var/lib/openstack/bin/centralhealth.py + name: scripts + readOnly: true + subPath: centralhealth.py + - mountPath: /var/lib/openstack/bin/notificationhealth.py + name: scripts + readOnly: true + subPath: notificationhealth.py - mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem name: combined-ca-bundle readOnly: true subPath: tls-ca-bundle.pem - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - readOnly: true - name: ceilometer-notification-agent args: - - -c - - /usr/local/bin/kolla_start + - --logfile + - /dev/stdout command: - - /bin/bash + - /usr/bin/ceilometer-agent-notification volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts + - mountPath: /etc/ceilometer/ceilometer.conf + name: config-data + readOnly: true + subPath: ceilometer.conf + - mountPath: /etc/ceilometer/pipeline.yaml + name: config-data readOnly: true - - mountPath: /var/lib/openstack/config + subPath: pipeline.yaml + - mountPath: /etc/ceilometer/event_pipeline.yaml name: config-data readOnly: true - - mountPath: /var/lib/kolla/config_files/config.json + subPath: event_pipeline.yaml + - mountPath: /etc/ceilometer/ceilometer.conf.d/01-ceilometer-custom.conf name: config-data readOnly: true - subPath: ceilometer-notification-config.json + subPath: custom.conf + - mountPath: /var/lib/openstack/bin/centralhealth.py + name: scripts + readOnly: true + subPath: centralhealth.py + - mountPath: /var/lib/openstack/bin/notificationhealth.py + name: scripts + readOnly: true + subPath: notificationhealth.py - mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem name: combined-ca-bundle readOnly: true subPath: tls-ca-bundle.pem - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - readOnly: true - name: sg-core volumeMounts: - mountPath: /etc/sg-core.conf.yaml name: sg-core-conf-yaml subPath: sg-core.conf.yaml - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - readOnly: true - name: proxy-httpd args: - -DFOREGROUND @@ -199,7 +238,7 @@ spec: - mountPath: /run/httpd name: run-httpd - mountPath: /var/log/httpd - name: log-httpd + name: var-log-httpd - mountPath: /etc/pki/tls/certs/tls.crt name: ceilometer-tls-certs readOnly: true @@ -212,8 +251,6 @@ spec: name: combined-ca-bundle readOnly: true subPath: tls-ca-bundle.pem - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - readOnly: true --- apiVersion: v1 kind: Pod @@ -485,35 +522,53 @@ spec: - name: telemetry-kuttl-cloudkitty-api-log - name: cloudkitty-api volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/openstack/config - name: config-data - - mountPath: /var/lib/kolla/config_files/config.json + - mountPath: /etc/cloudkitty/metrics.yaml name: config-data - subPath: cloudkitty-api-config.json - - mountPath: /var/lib/openstack/loki-certs + subPath: metrics.yaml + - mountPath: /etc/cloudkitty/certs name: certs - - mountPath: /var/lib/openstack/service-config/ + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/00-cloudkitty.conf + name: config-data-custom + subPath: cloudkitty.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/02-global-custom.conf + name: config-data-custom + subPath: 02-global-custom.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/03-service-custom.conf + name: config-data-custom + subPath: 03-service-custom.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/04-service-custom-secrets.conf name: config-data-custom + subPath: 04-service-custom-secrets.conf + - mountPath: /etc/httpd/conf.d/00wsgi-cloudkitty.conf + name: config-data + subPath: wsgi-cloudkitty.conf + - mountPath: /etc/httpd/conf/httpd.conf + name: config-data + subPath: httpd.conf + - mountPath: /etc/httpd/conf.d/ssl.conf + name: config-data + subPath: ssl.conf + - mountPath: /run/httpd + name: run-httpd + - mountPath: /var/log/httpd + name: var-log-httpd - mountPath: /var/log/cloudkitty name: logs - mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem name: combined-ca-bundle subPath: tls-ca-bundle.pem - - mountPath: /var/lib/config-data/tls/certs/internal.crt + - mountPath: /etc/pki/tls/certs/internal.crt name: internal-tls-certs subPath: tls.crt - - mountPath: /var/lib/config-data/tls/private/internal.key + - mountPath: /etc/pki/tls/private/internal.key name: internal-tls-certs subPath: tls.key - - mountPath: /var/lib/config-data/tls/certs/public.crt + - mountPath: /etc/pki/tls/certs/public.crt name: public-tls-certs subPath: tls.crt - - mountPath: /var/lib/config-data/tls/private/public.key + - mountPath: /etc/pki/tls/private/public.key name: public-tls-certs subPath: tls.key - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount hostname: telemetry-kuttl-cloudkitty-api-0 status: containerStatuses: @@ -538,21 +593,26 @@ spec: containers: - name: cloudkitty-proc volumeMounts: - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/openstack/config - name: config-data - - mountPath: /var/lib/kolla/config_files/config.json + - mountPath: /etc/cloudkitty/metrics.yaml name: config-data - subPath: cloudkitty-proc-config.json - - mountPath: /var/lib/openstack/loki-certs + subPath: metrics.yaml + - mountPath: /etc/cloudkitty/certs name: certs - - mountPath: /var/lib/openstack/service-config/ + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/00-cloudkitty.conf + name: config-data-custom + subPath: cloudkitty.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/02-global-custom.conf + name: config-data-custom + subPath: 02-global-custom.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/03-service-custom.conf + name: config-data-custom + subPath: 03-service-custom.conf + - mountPath: /etc/cloudkitty/cloudkitty.conf.d/04-service-custom-secrets.conf name: config-data-custom + subPath: 04-service-custom-secrets.conf - mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem name: combined-ca-bundle subPath: tls-ca-bundle.pem - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount hostname: telemetry-kuttl-cloudkitty-proc-0 status: containerStatuses: diff --git a/test/kuttl/tests/topology/00-cloudkitty-deps.yaml b/test/kuttl/tests/topology/00-cloudkitty-deps.yaml index 74a306185..68b7008e8 100644 --- a/test/kuttl/tests/topology/00-cloudkitty-deps.yaml +++ b/test/kuttl/tests/topology/00-cloudkitty-deps.yaml @@ -100,4 +100,4 @@ stringData: access_key_id: minio access_key_secret: minio123 bucketnames: loki - endpoint: http://minio.svc.cluster.local:9000 + endpoint: http://minio:9000