feat(scheduler): correct resource accounting for init containers

pkg/device/common/common.go

@@ -18,6 +18,7 @@ package common
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 )
 
@@ -45,6 +46,7 @@ func GenReason(reasons map[string]int, cards int) string {
 	for r, cnt := range reasons {
 		reason = append(reason, fmt.Sprintf("%d/%d %s", cnt, cards, r))
 	}
+	sort.Strings(reason)
 	return strings.Join(reason, ", ")
 }
 

pkg/scheduler/scheduler.go

@@ -92,8 +92,6 @@ func NewScheduler() *Scheduler {
 	s.nodeManager = newNodeManager()
 	s.podManager = device.NewPodManager()
 	s.quotaManager = device.NewQuotaManager()
-	// Use dummy leader manager when leaderElect is disabled
-	// This ensures IsLeader() always returns true and synced will not be set to false
 	s.leaderManager = leaderelection.NewDummyLeaderManager(true)
 	if config.LeaderElect {
 		callbacks := leaderelection.LeaderCallbacks{
@@ -161,6 +159,22 @@ func (s *Scheduler) onAddPod(obj any) {
 		klog.ErrorS(err, "failed to decode pod devices", "pod", klog.KObj(pod))
 		return
 	}
+
+	numInit := len(pod.Spec.InitContainers)
+	if numInit > 0 {
+		allInitDone := len(pod.Status.InitContainerStatuses) >= numInit
+		for _, cs := range pod.Status.InitContainerStatuses {
+			if cs.State.Terminated == nil || cs.State.Terminated.ExitCode != 0 {
+				allInitDone = false
+				break
+			}
+		}
+		if allInitDone {
+			resourceReqs := device.Resourcereqs(pod)
+			podDev = stripInitContainerAliasSlots(pod, resourceReqs, podDev)
+		}
+	}
+
 	if s.podManager.AddPod(pod, nodeID, podDev) {
 		s.quotaManager.AddUsage(pod, podDev)
 	}
@@ -400,7 +414,14 @@ func (s *Scheduler) register(labelSelector labels.Selector, printedLog map[strin
 					klog.V(5).InfoS("Skipping device cleanup for vendor not present in scheduler cache", "nodeName", val.Name, "deviceVendor", devhandsk)
 					continue
 				}
-				klog.Warning("Device is unhealthy, cleaning up node", "nodeName", val.Name, "deviceVendor", devhandsk)
+				// klog.Warning does plain fmt.Print-style concatenation of its arguments -
+				// klog v2 has no structured WarningS variant. Passing alternating
+				// "key", value pairs to it (as if it were InfoS/ErrorS) produces a garbled,
+				// unstructured log line instead of the intended structured fields. Use
+				// ErrorS (nil error is fine here; this is a detected condition, not a Go
+				// error) to match the structured logging used throughout the rest of this
+				// file.
+				klog.ErrorS(nil, "Device is unhealthy, cleaning up node", "nodeName", val.Name, "deviceVendor", devhandsk)
 				err := devInstance.NodeCleanUp(val.Name)
 				if err != nil {
 					klog.ErrorS(err, "Node cleanup failed", "nodeName", val.Name, "deviceVendor", devhandsk)
@@ -741,22 +762,25 @@ ReleaseNodeLocks:
 func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFilterResult, error) {
 	klog.InfoS("Starting schedule filter process", "pod", args.Pod.Name, "uuid", args.Pod.UID, "namespace", args.Pod.Namespace)
 	resourceReqs := device.Resourcereqs(args.Pod)
-	resourceReqTotal := 0
-	for _, n := range resourceReqs {
-		for _, k := range n {
-			resourceReqTotal += int(k.Nums)
+
+	hasHAMiResource := false
+
+	for _, reqMap := range resourceReqs {
+		if len(reqMap) > 0 {
+			hasHAMiResource = true
+			break
 		}
 	}
-	if resourceReqTotal == 0 {
-		klog.V(1).InfoS("Pod does not request any resources",
-			"pod", args.Pod.Name)
-		s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", fmt.Errorf("does not request any resource"))
+
+	if !hasHAMiResource {
+		klog.V(1).InfoS("Pod does not request any resources", "pod", args.Pod.Name)
 		return &extenderv1.ExtenderFilterResult{
 			NodeNames:   args.NodeNames,
 			FailedNodes: nil,
 			Error:       "",
 		}, nil
 	}
+
 	if pi, ok := s.podManager.TakeAndDeletePod(args.Pod); ok {
 		s.quotaManager.RmUsage(args.Pod, pi.Devices)
 	}
@@ -766,8 +790,7 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi
 		return nil, err
 	}
 	if len(failedNodes) != 0 {
-		klog.V(5).InfoS("Nodes failed during usage retrieval",
-			"nodes", failedNodes)
+		klog.V(5).InfoS("Nodes failed during usage retrieval", "nodes", failedNodes)
 	}
 	nodeScores, err := s.calcScore(nodeUsage, resourceReqs, args.Pod, failedNodes)
 	if err != nil {
@@ -776,8 +799,7 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi
 		return nil, err
 	}
 	if len((*nodeScores).NodeList) == 0 {
-		klog.V(4).InfoS("No available nodes meet the required scores",
-			"pod", args.Pod.Name)
+		klog.V(4).InfoS("No available nodes meet the required scores", "pod", args.Pod.Name)
 		s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", fmt.Errorf("no available node, %d nodes do not meet", len(*args.NodeNames)))
 		return &extenderv1.ExtenderFilterResult{
 			FailedNodes: failedNodes,
@@ -799,16 +821,18 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi
 		val.PatchAnnotations(args.Pod, &annotations, m.Devices)
 	}
 
-	added := s.podManager.AddPod(args.Pod, m.NodeID, m.Devices)
+	usageDevices := stripInitContainerAliasSlots(args.Pod, resourceReqs, m.Devices)
+
+	added := s.podManager.AddPod(args.Pod, m.NodeID, usageDevices)
 	if added {
-		s.quotaManager.AddUsage(args.Pod, m.Devices)
+		s.quotaManager.AddUsage(args.Pod, usageDevices)
 	}
 
 	err = util.PatchPodAnnotations(args.Pod, annotations)
 	if err != nil {
 		s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", err)
 		if added {
-			s.quotaManager.RmUsage(args.Pod, m.Devices)
+			s.quotaManager.RmUsage(args.Pod, usageDevices)
 		}
 		s.podManager.DelPod(args.Pod)
 		return nil, err

pkg/scheduler/scheduler_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"maps"
+	"slices"
 	"strings"
 	"sync/atomic"
 	"testing"
@@ -459,11 +460,11 @@ func Test_Filter(t *testing.T) {
 	}
 
 	tests := []struct {
-		name                      string
-		args                      extenderv1.ExtenderArgs
-		want                      *extenderv1.ExtenderFilterResult
-		wantPodAnnotationDeviceID string
-		wantErr                   error
+		name                       string
+		args                       extenderv1.ExtenderArgs
+		want                       *extenderv1.ExtenderFilterResult
+		wantPodAnnotationDeviceIDs []string
+		wantErr                    error
 	}{
 		{
 			name: "node use binpack gpu use binpack policy",
@@ -500,7 +501,56 @@ func Test_Filter(t *testing.T) {
 			want: &extenderv1.ExtenderFilterResult{
 				NodeNames: &[]string{"node2"},
 			},
-			wantPodAnnotationDeviceID: "device4",
+			wantPodAnnotationDeviceIDs: []string{"device4"},
+		},
+		{
+			name: "pod with init containers fits correctly using max resource logic (Binpack)",
+			args: extenderv1.ExtenderArgs{
+				Pod: &corev1.Pod{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "test-init-containers",
+						UID:  "test-init-uid",
+						Annotations: map[string]string{
+							util.GPUSchedulerPolicyAnnotationKey:  util.GPUSchedulerPolicyBinpack.String(),
+							util.NodeSchedulerPolicyAnnotationKey: util.NodeSchedulerPolicyBinpack.String(),
+						},
+					},
+					Spec: corev1.PodSpec{
+						InitContainers: []corev1.Container{
+							{
+								Name:  "init-1",
+								Image: "busybox",
+								Resources: corev1.ResourceRequirements{
+									Limits: corev1.ResourceList{
+										"hami.io/gpu":      *resource.NewQuantity(1, resource.BinarySI),
+										"hami.io/gpucores": *resource.NewQuantity(20, resource.BinarySI),
+										"hami.io/gpumem":   *resource.NewQuantity(5000, resource.BinarySI),
+									},
+								},
+							},
+						},
+						Containers: []corev1.Container{
+							{
+								Name:  "app-1",
+								Image: "chrstnhntschl/gpu_burn",
+								Resources: corev1.ResourceRequirements{
+									Limits: corev1.ResourceList{
+										"hami.io/gpu":      *resource.NewQuantity(1, resource.BinarySI),
+										"hami.io/gpucores": *resource.NewQuantity(20, resource.BinarySI),
+										"hami.io/gpumem":   *resource.NewQuantity(4000, resource.BinarySI),
+									},
+								},
+							},
+						},
+					},
+				},
+				NodeNames: &[]string{"node1", "node2"},
+			},
+			wantErr: nil,
+			want: &extenderv1.ExtenderFilterResult{
+				NodeNames: &[]string{"node2"},
+			},
+			wantPodAnnotationDeviceIDs: []string{"device3"},
 		},
 		{
 			name: "node use binpack gpu use spread policy",
@@ -537,7 +587,7 @@ func Test_Filter(t *testing.T) {
 			want: &extenderv1.ExtenderFilterResult{
 				NodeNames: &[]string{"node2"},
 			},
-			wantPodAnnotationDeviceID: "device3",
+			wantPodAnnotationDeviceIDs: []string{"device3", "device4"}, // Both are acceptable due to tie
 		},
 		{
 			name: "node use spread gpu use binpack policy",
@@ -574,7 +624,7 @@ func Test_Filter(t *testing.T) {
 			want: &extenderv1.ExtenderFilterResult{
 				NodeNames: &[]string{"node1"},
 			},
-			wantPodAnnotationDeviceID: "device1",
+			wantPodAnnotationDeviceIDs: []string{"device1"},
 		},
 		{
 			name: "node use spread gpu use spread policy",
@@ -611,7 +661,7 @@ func Test_Filter(t *testing.T) {
 			want: &extenderv1.ExtenderFilterResult{
 				NodeNames: &[]string{"node1"},
 			},
-			wantPodAnnotationDeviceID: "device2",
+			wantPodAnnotationDeviceIDs: []string{"device2"},
 		},
 	}
 
@@ -624,7 +674,12 @@ func Test_Filter(t *testing.T) {
 			assert.DeepEqual(t, test.want, got)
 			getPod, _ := client.KubeClient.CoreV1().Pods(test.args.Pod.Namespace).Get(context.Background(), test.args.Pod.Name, metav1.GetOptions{})
 			podDevices, _ := device.DecodePodDevices(device.SupportDevices, getPod.Annotations)
-			assert.DeepEqual(t, test.wantPodAnnotationDeviceID, podDevices["NVIDIA"][0][0].UUID)
+
+			actualUUID := podDevices["NVIDIA"][0][0].UUID
+
+			if !slices.Contains(test.wantPodAnnotationDeviceIDs, actualUUID) {
+				t.Errorf("expected one of %v, got %s", test.wantPodAnnotationDeviceIDs, actualUUID)
+			}
 		})
 	}
 }
@@ -1232,7 +1287,7 @@ func Test_ResourceQuota(t *testing.T) {
 			wantErr: nil,
 			want: &extenderv1.ExtenderFilterResult{
 				FailedNodes: map[string]string{
-					"node1": "NodeUnfitPod",
+					"node1": "1/4 AllocatedCardsInsufficientRequest, 3/4 ResourceQuotaNotFit",
 				},
 			},
 		},
@@ -1320,7 +1375,7 @@ func Test_ResourceQuota(t *testing.T) {
 			wantErr: nil,
 			want: &extenderv1.ExtenderFilterResult{
 				FailedNodes: map[string]string{
-					"node1": "NodeUnfitPod",
+					"node1": "4/4 ResourceQuotaNotFit",
 				},
 			},
 		},