feature: Add support to opt-out from zone aware routing via Ingress/RouteGroup

dataclients/kubernetes/clusterstate.go

@@ -79,7 +79,7 @@ func (state *clusterState) GetEndpointsByService(namespace, name, protocol strin
 }
 
 // GetEndpointSlicesByService returns the skipper endpointslices for kubernetes endpointslices.
-func (state *clusterState) GetEndpointSlicesByService(zone, namespace, name, protocol string, servicePort *servicePort) []skipperEndpoint {
+func (state *clusterState) GetEndpointSlicesByService(zone, namespace, name, protocol string, servicePort *servicePort, zoneAwarenessDisabled bool) []skipperEndpoint {
 	epID := endpointID{
 		ResourceID: newResourceID(namespace, name),
 		Protocol:   protocol,
@@ -105,6 +105,10 @@ func (state *clusterState) GetEndpointSlicesByService(zone, namespace, name, pro
 		state.cachedEndpointSlices[epID] = targets
 	}
 
+	if zoneAwarenessDisabled {
+		return targets
+	}
+
 	return filterByZone(zone, targets)
 }
 
@@ -167,7 +171,7 @@ func (state *clusterState) GetEndpointsByTarget(namespace, name, protocol, schem
 }
 
 // GetEndpointSlicesByTarget returns the skipper endpointslices for kubernetes endpointslices.
-func (state *clusterState) GetEndpointSlicesByTarget(zone, namespace, name, protocol, scheme string, target *definitions.BackendPort) []skipperEndpoint {
+func (state *clusterState) GetEndpointSlicesByTarget(zone, namespace, name, protocol, scheme string, target *definitions.BackendPort, zoneAwarenessDisabled bool) []skipperEndpoint {
 	epID := endpointID{
 		ResourceID: newResourceID(namespace, name),
 		Protocol:   protocol,
@@ -192,6 +196,10 @@ func (state *clusterState) GetEndpointSlicesByTarget(zone, namespace, name, prot
 		state.cachedEndpointSlices[epID] = targets
 	}
 
+	if zoneAwarenessDisabled {
+		return targets
+	}
+
 	return filterByZone(zone, targets)
 }
 

dataclients/kubernetes/ingress.go

@@ -24,6 +24,7 @@ const (
 	skipperLoadBalancerAnnotationKey    = "zalando.org/skipper-loadbalancer"
 	skipperBackendProtocolAnnotationKey = "zalando.org/skipper-backend-protocol"
 	pathModeAnnotationKey               = "zalando.org/skipper-ingress-path-mode"
+	trafficZoneAwareAnnotationKey       = "zalando.org/traffic-zone-aware"
 	ingressOriginName                   = "ingress"
 	tlsSecretType                       = "kubernetes.io/tls"
 	tlsSecretDataCrt                    = "tls.crt"

dataclients/kubernetes/ingressv1.go

@@ -103,6 +103,8 @@ func convertPathRuleV1(
 		return nil, err
 	}
 
+	zoneAwarenessDisabled := metadata.Annotations[trafficZoneAwareAnnotationKey] == "false"
+
 	servicePort, err := svc.getServicePortV1(svcPort)
 	if err != nil {
 		// service definition is wrong or no pods
@@ -125,7 +127,7 @@ func convertPathRuleV1(
 		}
 
 		if state.enableEndpointSlices {
-			epSlices = state.GetEndpointSlicesByService(dataclientZone, ns, svcName, protocol, servicePort)
+			epSlices = state.GetEndpointSlicesByService(dataclientZone, ns, svcName, protocol, servicePort, zoneAwarenessDisabled)
 			for _, ep := range epSlices {
 				eps = append(eps, ep.Address)
 			}
@@ -179,6 +181,8 @@ func convertPathRuleV1(
 		r.LBEndpoints = eskip.NewLBEndpoints(eps)
 	}
 
+	r.DisableZoneAwareness = zoneAwarenessDisabled
+
 	setPathV1(pathMode, r, prule.PathType, prule.Path)
 	traffic.apply(r)
 	return r, nil
@@ -386,6 +390,8 @@ func (ing *ingress) convertDefaultBackendV1(
 		return nil, false, err
 	}
 
+	zoneAwarenessDisabled := i.Metadata.Annotations[trafficZoneAwareAnnotationKey] == "false"
+
 	servicePort, err := svc.getServicePortV1(svcPort)
 	if err != nil {
 		ic.logger.Errorf("Failed to find target port %v, %s, for service %s add shuntroute: %v", svc.Spec.Ports, svcPort, svcName, err)
@@ -407,7 +413,7 @@ func (ing *ingress) convertDefaultBackendV1(
 		}
 
 		if state.enableEndpointSlices {
-			epSlices = state.GetEndpointSlicesByService(dataclientZone, ns, svcName, protocol, servicePort)
+			epSlices = state.GetEndpointSlicesByService(dataclientZone, ns, svcName, protocol, servicePort, zoneAwarenessDisabled)
 			for _, ep := range epSlices {
 				eps = append(eps, ep.Address)
 			}
@@ -454,6 +460,8 @@ func (ing *ingress) convertDefaultBackendV1(
 		r.LBEndpoints = eskip.NewLBEndpoints(eps)
 	}
 
+	r.DisableZoneAwareness = zoneAwarenessDisabled
+
 	return r, true, nil
 }
 

dataclients/kubernetes/routegroup.go

@@ -178,6 +178,9 @@ func applyServiceBackend(ctx *routeGroupContext, backend *definitions.SkipperBac
 		protocol = p
 	}
 
+	zoneAwarenessDisabled := ctx.routeGroup.Metadata.Annotations[trafficZoneAwareAnnotationKey] == "false"
+	r.DisableZoneAwareness = zoneAwarenessDisabled
+
 	s, err := getBackendService(ctx, backend)
 	if err != nil {
 		return err
@@ -197,7 +200,8 @@ func applyServiceBackend(ctx *routeGroupContext, backend *definitions.SkipperBac
 			s.Meta.Name,
 			"TCP",
 			protocol,
-			targetPort)
+			targetPort,
+			zoneAwarenessDisabled)
 		for _, epSlice := range epSlices {
 			eps = append(eps, epSlice.Address)
 		}

dataclients/kubernetes/testdata/ingressV1/zone-aware-traffic/default-backend-with-opt-out.eskip

@@ -0,0 +1,3 @@
+kube_default__ingress_default_backend_opt_out______:
+  *
+  -> <roundRobin, "http://10.3.0.10:8080", "http://10.3.0.11:8080", "http://10.3.0.12:8080", "http://10.3.1.10:8080", "http://10.3.1.11:8080">;

dataclients/kubernetes/testdata/ingressV1/zone-aware-traffic/default-backend-with-opt-out.kube

@@ -0,0 +1,2 @@
+enable-kubernetes-endpointslices: true
+topology-zone: eu-central-1a

dataclients/kubernetes/testdata/ingressV1/zone-aware-traffic/default-backend-with-opt-out.yaml

@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: app-default-backend-opt-out
+  name: svc-default-backend-opt-out
+spec:
+  clusterIP: 10.3.191.1
+  ports:
+    - name: web
+      port: 80
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: app-default-backend-opt-out
+  type: ClusterIP
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  labels:
+    app: app-default-backend-opt-out
+  name: ingress-default-backend-opt-out
+  namespace: default
+  annotations:
+    zalando.org/traffic-zone-aware: "false"
+spec:
+  defaultBackend:
+    service:
+      name: svc-default-backend-opt-out
+      port:
+        number: 80
+---
+# 3 endpoints in eu-central-1a, 2 in eu-central-1b; opt-out means all 5 are returned
+apiVersion: v1
+kind: EndpointSlice
+metadata:
+  labels:
+    app: app-default-backend-opt-out
+    kubernetes.io/service-name: svc-default-backend-opt-out
+  name: svc-default-backend-opt-out-eps
+  namespace: default
+endpoints:
+  - addresses:
+    - 10.3.0.10
+    zone: eu-central-1a
+  - addresses:
+    - 10.3.0.11
+    zone: eu-central-1a
+  - addresses:
+    - 10.3.0.12
+    zone: eu-central-1a
+  - addresses:
+    - 10.3.1.10
+    zone: eu-central-1b
+  - addresses:
+    - 10.3.1.11
+    zone: eu-central-1b
+ports:
+  - name: web
+    port: 8080
+    protocol: TCP
+apiVersion: v1

dataclients/kubernetes/testdata/ingressV1/zone-aware-traffic/mixed-zone-threshold-with-opt-out.eskip

@@ -0,0 +1,4 @@
+kube_default__ingress_with_opt_out__with_example_org____svc_with_opt_out: Host("^(with[.]example[.]org[.]?(:[0-9]+)?)$")
+	-> <roundRobin,"http://10.3.0.3:8080", "http://10.3.0.4:8080", "http://10.3.0.5:8080", "http://10.3.1.3:8080", "http://10.3.1.4:8080", "http://10.3.1.5:8080">;
+kube_default__ingress_without_opt_out__without_example_org____svc_without_opt_out: Host("^(without[.]example[.]org[.]?(:[0-9]+)?)$")
+	-> <roundRobin,"http://10.3.0.3:8080", "http://10.3.0.4:8080", "http://10.3.0.5:8080">;

dataclients/kubernetes/testdata/ingressV1/zone-aware-traffic/mixed-zone-threshold-with-opt-out.kube

@@ -0,0 +1,2 @@
+enable-kubernetes-endpointslices: true
+topology-zone: eu-central-1a

dataclients/kubernetes/testdata/ingressV1/zone-aware-traffic/mixed-zone-threshold-with-opt-out.yaml

@@ -0,0 +1,135 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: app-with-opt-out
+  name: svc-with-opt-out
+spec:
+  clusterIP: 10.3.190.1
+  ports:
+    - name: web
+      port: 80
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: app-with-opt-out
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: app-without-opt-out
+  name: svc-without-opt-out
+spec:
+  clusterIP: 10.3.190.2
+  ports:
+    - name: web
+      port: 80
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: app-without-opt-out
+  type: ClusterIP
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  labels:
+    app: app-with-opt-out
+  name: ingress-with-opt-out
+  namespace: default
+  annotations:
+    zalando.org/traffic-zone-aware: "false"
+spec:
+  rules:
+    - host: with.example.org
+      http:
+        paths:
+          - backend:
+              service:
+                name: svc-with-opt-out
+                port:
+                  number: 80
+            pathType: ImplementationSpecific
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  labels:
+    app: app-without-opt-out
+  name: ingress-without-opt-out
+  namespace: default
+spec:
+  rules:
+    - host: without.example.org
+      http:
+        paths:
+          - backend:
+              service:
+                name: svc-without-opt-out
+                port:
+                  number: 80
+            pathType: ImplementationSpecific
+---
+# svc-with-opt-out: 3 endpoints in eu-central-1a — meets minEndpointsByZone, but with opt-out; should return all endpoints
+apiVersion: v1
+kind: EndpointSlice
+metadata:
+  labels:
+    app: app-with-opt-out
+    kubernetes.io/service-name: svc-with-opt-out
+  name: svc-with-opt-out-eps
+  namespace: default
+endpoints:
+  - addresses:
+    - 10.3.0.3
+    zone: eu-central-1a
+  - addresses:
+    - 10.3.0.4
+    zone: eu-central-1a
+  - addresses:
+    - 10.3.0.5
+    zone: eu-central-1a
+  - addresses:
+    - 10.3.1.3
+    zone: eu-central-1b
+  - addresses:
+    - 10.3.1.4
+    zone: eu-central-1b
+  - addresses:
+    - 10.3.1.5
+    zone: eu-central-1b
+ports:
+  - name: web
+    port: 8080
+    protocol: TCP
+apiVersion: v1
+---
+# svc-without-opt-out: only 3 endpoints in eu-central-1a — meets minEndpointsByZone
+apiVersion: v1
+kind: EndpointSlice
+metadata:
+  labels:
+    app: app-without-opt-out
+    kubernetes.io/service-name: svc-without-opt-out
+  name: svc-without-opt-out-eps
+  namespace: default
+endpoints:
+  - addresses:
+    - 10.3.0.3
+    zone: eu-central-1a
+  - addresses:
+    - 10.3.0.4
+    zone: eu-central-1a
+  - addresses:
+    - 10.3.0.5
+    zone: eu-central-1a
+  - addresses:
+    - 10.4.1.4
+    zone: eu-central-1b
+ports:
+  - name: web
+    port: 8080
+    protocol: TCP
+apiVersion: v1

dataclients/kubernetes/testdata/routegroups/zone-aware-traffic/mixed-zone-threshold-with-opt-out.eskip

@@ -0,0 +1,4 @@
+kube_rg__default__ingress_with_opt_out__all__0_0: Host("^(with[.]example[.]org[.]?(:[0-9]+)?)$")
+	-> <roundRobin,"http://10.3.0.3:8080", "http://10.3.0.4:8080", "http://10.3.0.5:8080", "http://10.3.1.3:8080", "http://10.3.1.4:8080", "http://10.3.1.5:8080">;
+kube_rg__default__ingress_without_opt_out__all__0_0: Host("^(without[.]example[.]org[.]?(:[0-9]+)?)$")
+	-> <roundRobin,"http://10.3.0.3:8080", "http://10.3.0.4:8080", "http://10.3.0.5:8080">;

dataclients/kubernetes/testdata/routegroups/zone-aware-traffic/mixed-zone-threshold-with-opt-out.kube

@@ -0,0 +1,2 @@
+enable-kubernetes-endpointslices: true
+topology-zone: eu-central-1a