diff --git a/internal/tools/orchestrator/scheduler/executor/plugins/edas/edas_app_flow.go b/internal/tools/orchestrator/scheduler/executor/plugins/edas/edas_app_flow.go index d7fe5038d65..546b9093841 100644 --- a/internal/tools/orchestrator/scheduler/executor/plugins/edas/edas_app_flow.go +++ b/internal/tools/orchestrator/scheduler/executor/plugins/edas/edas_app_flow.go @@ -115,7 +115,7 @@ func (e *EDAS) updateService(ctx context.Context, sg *apistructs.ServiceGroup, s } else { //查询最新一次的发布单,如果存在运行中则终止 orderList, _ := e.wrapEDASClient.ListRecentChangeOrderInfo(appID) - if len(orderList.ChangeOrder) > 0 && orderList.ChangeOrder[0].Status == 1 { + if orderList != nil && len(orderList.ChangeOrder) > 0 && orderList.ChangeOrder[0].Status == 1 { e.wrapEDASClient.AbortChangeOrder(orderList.ChangeOrder[0].ChangeOrderId) } @@ -140,12 +140,37 @@ func (e *EDAS) updateService(ctx context.Context, sg *apistructs.ServiceGroup, s } func (e *EDAS) resolveServiceSelector(appName, sgID, serviceName string) map[string]string { - var deployment *appsv1.Deployment - if currentDeployment, err := e.wrapEDASClient.GetAppDeployment(appName); err == nil { - deployment = currentDeployment + l := e.l.WithField("func", "resolveServiceSelector") + + for i := 0; i < 3; i++ { + if dep, err := e.wrapEDASClient.GetAppDeployment(appName); err == nil && dep != nil { + selector := resolveServiceSelectorFromDeployment(dep, sgID, serviceName) + l.Infof("resolved selector from edas api, appName: %s, selector: %v", appName, selector) + return selector + } else if err != nil { + l.Warnf("failed to get deployment from edas api, appName: %s, attempt: %d, err: %v", appName, i+1, err) + } + + if i < 2 { + time.Sleep(5 * time.Second) + } + } + + // EDAS API 全部失败,通过 k8s API 最终兜底 + l.Warnf("edas api all retries exhausted, falling back to k8s api, appName: %s", appName) + if dep, err := e.wrapClientSet.GetDeployment(context.Background(), appName); err == nil && dep != nil { + selector := resolveServiceSelectorFromDeployment(dep, sgID, serviceName) + l.Infof("resolved selector from k8s api fallback, appName: %s, selector: %v", appName, selector) + return selector + } else if err != nil { + l.Errorf("failed to get deployment from k8s api, appName: %s, err: %v", appName, err) } - return resolveServiceSelectorFromDeployment(deployment, sgID, serviceName) + l.Errorf("failed to resolve selector from all sources, appName: %s, using default selector", appName) + return map[string]string{ + types.LabelServiceName: serviceName, + types.LabelServiceGroupID: sgID, + } } func resolveServiceSelectorFromDeployment(deployment *appsv1.Deployment, sgID, serviceName string) map[string]string { diff --git a/internal/tools/orchestrator/scheduler/executor/plugins/edas/wrapclient/kubernetes/deployment.go b/internal/tools/orchestrator/scheduler/executor/plugins/edas/wrapclient/kubernetes/deployment.go index c1a67c54f78..8013755ac32 100644 --- a/internal/tools/orchestrator/scheduler/executor/plugins/edas/wrapclient/kubernetes/deployment.go +++ b/internal/tools/orchestrator/scheduler/executor/plugins/edas/wrapclient/kubernetes/deployment.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -90,3 +91,18 @@ func (e *wrapKubernetes) GetK8sDeployList(group string, services *[]apistructs.S l.Debugf("old service list : %+v", services) return nil } + +// GetDeployment get deployment by name via k8s API. +// EDAS may alter the deployment name in k8s, so use list + contains match. +func (e *wrapKubernetes) GetDeployment(ctx context.Context, name string) (*appsv1.Deployment, error) { + deployList, err := e.cs.AppsV1().Deployments(e.namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + for i := range deployList.Items { + if strings.Contains(deployList.Items[i].Name, name) { + return &deployList.Items[i], nil + } + } + return nil, errors.Errorf("deployment contains %q not found", name) +} diff --git a/internal/tools/orchestrator/scheduler/executor/plugins/edas/wrapclient/kubernetes/provider.go b/internal/tools/orchestrator/scheduler/executor/plugins/edas/wrapclient/kubernetes/provider.go index 8cc431bffc1..bf1151237ea 100644 --- a/internal/tools/orchestrator/scheduler/executor/plugins/edas/wrapclient/kubernetes/provider.go +++ b/internal/tools/orchestrator/scheduler/executor/plugins/edas/wrapclient/kubernetes/provider.go @@ -18,6 +18,7 @@ import ( "context" "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" @@ -30,6 +31,7 @@ import ( type Interface interface { GetK8sService(name string) (*corev1.Service, error) GetK8sDeployList(group string, services *[]apistructs.Service) error + GetDeployment(ctx context.Context, name string) (*appsv1.Deployment, error) CreateK8sService(appName string, selectors map[string]string, ports []int) error CreateOrUpdateK8sService(ctx context.Context, appName string, selectors map[string]string, ports []int) error DeleteK8sService(appName string) error