diff --git a/pkg/device/amd/device.go b/pkg/device/amd/device.go index 8c4fa1988..b27e99401 100644 --- a/pkg/device/amd/device.go +++ b/pkg/device/amd/device.go @@ -24,6 +24,7 @@ import ( "github.com/Project-HAMi/HAMi/pkg/device" "github.com/Project-HAMi/HAMi/pkg/device/common" + "github.com/Project-HAMi/HAMi/pkg/util" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -189,6 +190,7 @@ func (amddevice *AMDDevices) Fit(devices []*device.DeviceUsage, request device.C klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k) tmpDevs := make(map[string]device.ContainerDevices) reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) @@ -216,6 +218,11 @@ func (amddevice *AMDDevices) Fit(devices []*device.DeviceUsage, request device.C klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 34d807a33..93737ef1e 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -437,6 +437,7 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD var tmpDevs map[string]device.ContainerDevices tmpDevs = make(map[string]device.ContainerDevices) reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() vnpuMode := "" if pod != nil && pod.Annotations != nil { @@ -509,6 +510,11 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } if k.Coresreq > 100 { klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID) k.Coresreq = 100 diff --git a/pkg/device/awsneuron/device.go b/pkg/device/awsneuron/device.go index 844aba970..f6914bf6b 100644 --- a/pkg/device/awsneuron/device.go +++ b/pkg/device/awsneuron/device.go @@ -369,6 +369,7 @@ func (neuron *AWSNeuronDevices) Fit(devices []*device.DeviceUsage, request devic klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k) tmpDevs := make(map[string]device.ContainerDevices) reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() if k.Nums > 1 { alloc := graphSelect(devices, int(request.Nums)) if len(alloc) == 0 { @@ -425,6 +426,11 @@ func (neuron *AWSNeuronDevices) Fit(devices []*device.DeviceUsage, request devic klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } if countMaskAvailable(dev.Totalcore)-countMaskAvailable(dev.Usedcores) < k.Coresreq { reason[common.CardInsufficientCore]++ diff --git a/pkg/device/biren/device.go b/pkg/device/biren/device.go index 55f29cdd3..010f25fae 100644 --- a/pkg/device/biren/device.go +++ b/pkg/device/biren/device.go @@ -193,6 +193,7 @@ func (br *BirenDevices) Fit(devices []*device.DeviceUsage, request device.Contai klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k) tmpDevs := make(map[string]device.ContainerDevices) reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() for i, dev := range slices.Backward(devices) { klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) @@ -213,6 +214,11 @@ func (br *BirenDevices) Fit(devices []*device.DeviceUsage, request device.Contai klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } if k.Nums > 0 { klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID) k.Nums-- diff --git a/pkg/device/cambricon/device.go b/pkg/device/cambricon/device.go index 509f6765e..ff2d2228d 100644 --- a/pkg/device/cambricon/device.go +++ b/pkg/device/cambricon/device.go @@ -28,6 +28,7 @@ import ( "github.com/Project-HAMi/HAMi/pkg/device" "github.com/Project-HAMi/HAMi/pkg/device/common" + "github.com/Project-HAMi/HAMi/pkg/util" "github.com/Project-HAMi/HAMi/pkg/util/client" corev1 "k8s.io/api/core/v1" @@ -318,6 +319,7 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C var tmpDevs map[string]device.ContainerDevices tmpDevs = make(map[string]device.ContainerDevices) reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) @@ -349,6 +351,11 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } if k.Coresreq > 100 { klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID) k.Coresreq = 100 diff --git a/pkg/device/enflame/device.go b/pkg/device/enflame/device.go index be7c2183e..74d187e4b 100644 --- a/pkg/device/enflame/device.go +++ b/pkg/device/enflame/device.go @@ -29,6 +29,7 @@ import ( "github.com/Project-HAMi/HAMi/pkg/device" "github.com/Project-HAMi/HAMi/pkg/device/common" + "github.com/Project-HAMi/HAMi/pkg/util" corev1 "k8s.io/api/core/v1" "k8s.io/klog/v2" @@ -384,6 +385,7 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k) tmpDevs := make(map[string]device.ContainerDevices) reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() profile, profileMatch := enf.selectProfileByRequest(devices, k) if !profileMatch { reason[common.ModeNotFit]++ @@ -424,6 +426,11 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } if dev.Totalmem-dev.Usedmem < profileMemoryMiB { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", profileMemoryMiB) diff --git a/pkg/device/hygon/device.go b/pkg/device/hygon/device.go index af4c37fc6..f9edfee60 100644 --- a/pkg/device/hygon/device.go +++ b/pkg/device/hygon/device.go @@ -280,6 +280,7 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain var tmpDevs map[string]device.ContainerDevices tmpDevs = make(map[string]device.ContainerDevices) reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) @@ -311,6 +312,11 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } if k.Coresreq > 100 { klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID) k.Coresreq = 100 diff --git a/pkg/device/iluvatar/device.go b/pkg/device/iluvatar/device.go index 7096d8197..cdfe225f9 100644 --- a/pkg/device/iluvatar/device.go +++ b/pkg/device/iluvatar/device.go @@ -264,6 +264,7 @@ func (ilu *IluvatarDevices) Fit(devices []*device.DeviceUsage, request device.Co var tmpDevs map[string]device.ContainerDevices tmpDevs = make(map[string]device.ContainerDevices) reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) @@ -295,6 +296,11 @@ func (ilu *IluvatarDevices) Fit(devices []*device.DeviceUsage, request device.Co klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } if k.Coresreq > 100 { klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID) k.Coresreq = 100 diff --git a/pkg/device/metax/device.go b/pkg/device/metax/device.go index 226dc1c61..d120a413e 100644 --- a/pkg/device/metax/device.go +++ b/pkg/device/metax/device.go @@ -228,6 +228,7 @@ func (mat *MetaxDevices) Fit(devices []*device.DeviceUsage, request device.Conta var tmpDevs map[string]device.ContainerDevices tmpDevs = make(map[string]device.ContainerDevices) reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) @@ -259,6 +260,11 @@ func (mat *MetaxDevices) Fit(devices []*device.DeviceUsage, request device.Conta klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } if k.Coresreq > 100 { klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID) k.Coresreq = 100 diff --git a/pkg/device/metax/sdevice.go b/pkg/device/metax/sdevice.go index 80e13ee2d..e678cdd65 100644 --- a/pkg/device/metax/sdevice.go +++ b/pkg/device/metax/sdevice.go @@ -311,6 +311,7 @@ func (mats *MetaxSDevices) Fit(devices []*device.DeviceUsage, request device.Con // filter device reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() candidateDevices := []*device.DeviceUsage{} for i, v := range slices.Backward(devices) { dev := v @@ -353,6 +354,11 @@ func (mats *MetaxSDevices) Fit(devices []*device.DeviceUsage, request device.Con klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } memreq := int32(0) if request.Memreq > 0 { diff --git a/pkg/device/mthreads/device.go b/pkg/device/mthreads/device.go index bd81dade0..ee28dbb75 100644 --- a/pkg/device/mthreads/device.go +++ b/pkg/device/mthreads/device.go @@ -272,6 +272,7 @@ func (mth *MthreadsDevices) Fit(devices []*device.DeviceUsage, request device.Co var tmpDevs map[string]device.ContainerDevices tmpDevs = make(map[string]device.ContainerDevices) reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) @@ -309,6 +310,11 @@ func (mth *MthreadsDevices) Fit(devices []*device.DeviceUsage, request device.Co klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } if k.Coresreq > 100 { klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID) k.Coresreq = 100 diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index 84285c250..9003fab53 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -754,7 +754,9 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co var tmpDevs map[string]device.ContainerDevices tmpDevs = make(map[string]device.ContainerDevices) reason := make(map[string]int) - needTopology := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyTopology.String() + gpuPolicy := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) + needTopology := gpuPolicy == util.GPUSchedulerPolicyTopology.String() + isMutex := gpuPolicy == util.GPUSchedulerPolicyMutex.String() for i := len(devices) - 1; i >= 0; i-- { dev := devices[i] klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) @@ -790,6 +792,11 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } if k.Coresreq > 100 { klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID) k.Coresreq = 100 diff --git a/pkg/device/vastai/device.go b/pkg/device/vastai/device.go index 3f56efd80..d914f0d78 100644 --- a/pkg/device/vastai/device.go +++ b/pkg/device/vastai/device.go @@ -231,6 +231,7 @@ func (va *VastaiDevices) Fit(devices []*device.DeviceUsage, request device.Conta klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k) tmpDevs := make(map[string]device.ContainerDevices) reason := make(map[string]int) + isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() dieMode := isDieMode(devices) for i, dev := range slices.Backward(devices) { klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) @@ -252,6 +253,11 @@ func (va *VastaiDevices) Fit(devices []*device.DeviceUsage, request device.Conta klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used) continue } + if isMutex && dev.Used > 0 { + reason[common.ExclusiveDeviceAllocateConflict]++ + klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) + continue + } if k.Nums > 0 { klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID) if !dieMode { diff --git a/pkg/scheduler/policy/gpu_policy.go b/pkg/scheduler/policy/gpu_policy.go index e404dde56..6c7f039aa 100644 --- a/pkg/scheduler/policy/gpu_policy.go +++ b/pkg/scheduler/policy/gpu_policy.go @@ -43,17 +43,26 @@ func (l DeviceUsageList) Swap(i, j int) { } func (l DeviceUsageList) Less(i, j int) bool { + si, sj := l.DeviceLists[i].Score, l.DeviceLists[j].Score + ni, nj := l.DeviceLists[i].Device.Numa, l.DeviceLists[j].Device.Numa + if l.Policy == util.GPUSchedulerPolicyMutex.String() { + ui, uj := l.DeviceLists[i].Device.Used, l.DeviceLists[j].Device.Used + if ui != uj { + return ui > uj // busy GPUs first, idle GPUs at tail (picked first by Fit) + } + return ni < nj + } if l.Policy == util.GPUSchedulerPolicyBinpack.String() { - if l.DeviceLists[i].Device.Numa == l.DeviceLists[j].Device.Numa { - return l.DeviceLists[i].Score < l.DeviceLists[j].Score + if si != sj { + return si < sj } - return l.DeviceLists[i].Device.Numa > l.DeviceLists[j].Device.Numa + return ni < nj } // default policy is spread - if l.DeviceLists[i].Device.Numa == l.DeviceLists[j].Device.Numa { - return l.DeviceLists[i].Score > l.DeviceLists[j].Score + if si != sj { + return si > sj } - return l.DeviceLists[i].Device.Numa < l.DeviceLists[j].Device.Numa + return ni < nj } func (l DeviceUsageList) DeepCopy() DeviceUsageList { diff --git a/pkg/scheduler/policy/gpu_policy_test.go b/pkg/scheduler/policy/gpu_policy_test.go index f70eb2f21..2719653b5 100644 --- a/pkg/scheduler/policy/gpu_policy_test.go +++ b/pkg/scheduler/policy/gpu_policy_test.go @@ -137,7 +137,7 @@ func TestDeviceUsageList_Less(t *testing.T) { expectedLess: true, }, { - name: "Binpack policy with different Numa true", + name: "Binpack: score primary, lower score sorts first across NUMA", policy: "binpack", deviceLists: []*DeviceListsScore{ {Device: &device.DeviceUsage{Numa: 1, Used: 10}, Score: 10}, @@ -146,16 +146,34 @@ func TestDeviceUsageList_Less(t *testing.T) { expectedLess: true, }, { - name: "Binpack policy with different Numa false", + name: "Binpack: score primary, higher score sorts last even on lower NUMA", policy: "binpack", deviceLists: []*DeviceListsScore{ {Device: &device.DeviceUsage{Numa: 0, Used: 10}, Score: 10}, {Device: &device.DeviceUsage{Numa: 1, Used: 20}, Score: 20}, }, + expectedLess: true, + }, + { + name: "Binpack: NUMA tiebreaker when scores equal", + policy: "binpack", + deviceLists: []*DeviceListsScore{ + {Device: &device.DeviceUsage{Numa: 0}, Score: 15}, + {Device: &device.DeviceUsage{Numa: 1}, Score: 15}, + }, + expectedLess: true, + }, + { + name: "Binpack: score primary over NUMA, high-NUMA high-score sorts last", + policy: "binpack", + deviceLists: []*DeviceListsScore{ + {Device: &device.DeviceUsage{Numa: 1}, Score: 20}, + {Device: &device.DeviceUsage{Numa: 0}, Score: 10}, + }, expectedLess: false, }, { - name: "Spread policy with same Numa", + name: "Spread: score primary, higher score sorts first", policy: "spread", deviceLists: []*DeviceListsScore{ {Device: &device.DeviceUsage{Numa: 0, Used: 10}, Score: 10}, @@ -164,7 +182,7 @@ func TestDeviceUsageList_Less(t *testing.T) { expectedLess: false, }, { - name: "Spread policy with different Numa false", + name: "Spread: score primary, lower score sorts last across NUMA", policy: "spread", deviceLists: []*DeviceListsScore{ {Device: &device.DeviceUsage{Numa: 1, Used: 10}, Score: 10}, @@ -173,11 +191,47 @@ func TestDeviceUsageList_Less(t *testing.T) { expectedLess: false, }, { - name: "Spread policy with different Numa true", + name: "Spread: score primary over NUMA, high-NUMA high-score sorts first", policy: "spread", deviceLists: []*DeviceListsScore{ - {Device: &device.DeviceUsage{Numa: 0, Used: 10}, Score: 10}, - {Device: &device.DeviceUsage{Numa: 1, Used: 20}, Score: 20}, + {Device: &device.DeviceUsage{Numa: 1}, Score: 20}, + {Device: &device.DeviceUsage{Numa: 0}, Score: 10}, + }, + expectedLess: true, + }, + { + name: "Spread: NUMA tiebreaker when scores equal", + policy: "spread", + deviceLists: []*DeviceListsScore{ + {Device: &device.DeviceUsage{Numa: 0}, Score: 15}, + {Device: &device.DeviceUsage{Numa: 1}, Score: 15}, + }, + expectedLess: true, + }, + { + name: "Mutex: busy GPU sorts before idle GPU", + policy: "mutex", + deviceLists: []*DeviceListsScore{ + {Device: &device.DeviceUsage{Used: 2, Numa: 0}}, + {Device: &device.DeviceUsage{Used: 0, Numa: 0}}, + }, + expectedLess: true, + }, + { + name: "Mutex: idle GPU sorts after busy GPU", + policy: "mutex", + deviceLists: []*DeviceListsScore{ + {Device: &device.DeviceUsage{Used: 0, Numa: 0}}, + {Device: &device.DeviceUsage{Used: 1, Numa: 0}}, + }, + expectedLess: false, + }, + { + name: "Mutex: NUMA tiebreaker when both idle", + policy: "mutex", + deviceLists: []*DeviceListsScore{ + {Device: &device.DeviceUsage{Used: 0, Numa: 0}}, + {Device: &device.DeviceUsage{Used: 0, Numa: 1}}, }, expectedLess: true, }, diff --git a/pkg/util/types.go b/pkg/util/types.go index 780e4dfe1..c1b97651b 100644 --- a/pkg/util/types.go +++ b/pkg/util/types.go @@ -71,6 +71,8 @@ const ( GPUSchedulerPolicySpread SchedulerPolicyName = "spread" // GPUSchedulerPolicyTopology is GPU use topology scheduler. GPUSchedulerPolicyTopology SchedulerPolicyName = "topology-aware" + // GPUSchedulerPolicyMutex is GPU use mutex scheduler, only idle GPUs are allocated. + GPUSchedulerPolicyMutex SchedulerPolicyName = "mutex" ) const (