Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pkg/device/amd/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"github.com/Project-HAMi/HAMi/pkg/device"
"github.com/Project-HAMi/HAMi/pkg/device/common"
"github.com/Project-HAMi/HAMi/pkg/util"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
Expand Down Expand Up @@ -189,6 +190,7 @@ func (amddevice *AMDDevices) Fit(devices []*device.DeviceUsage, request device.C
klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
tmpDevs := make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
for i, v := range slices.Backward(devices) {
dev := v
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
Expand Down Expand Up @@ -216,6 +218,11 @@ func (amddevice *AMDDevices) Fit(devices []*device.DeviceUsage, request device.C
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}

klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)

Expand Down
6 changes: 6 additions & 0 deletions pkg/device/ascend/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,7 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD
var tmpDevs map[string]device.ContainerDevices
tmpDevs = make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()

vnpuMode := ""
if pod != nil && pod.Annotations != nil {
Expand Down Expand Up @@ -509,6 +510,11 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Coresreq > 100 {
klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
k.Coresreq = 100
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/awsneuron/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,7 @@ func (neuron *AWSNeuronDevices) Fit(devices []*device.DeviceUsage, request devic
klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
tmpDevs := make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
if k.Nums > 1 {
alloc := graphSelect(devices, int(request.Nums))
if len(alloc) == 0 {
Expand Down Expand Up @@ -425,6 +426,11 @@ func (neuron *AWSNeuronDevices) Fit(devices []*device.DeviceUsage, request devic
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}

if countMaskAvailable(dev.Totalcore)-countMaskAvailable(dev.Usedcores) < k.Coresreq {
reason[common.CardInsufficientCore]++
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/biren/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ func (br *BirenDevices) Fit(devices []*device.DeviceUsage, request device.Contai
klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
tmpDevs := make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
for i, dev := range slices.Backward(devices) {
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)

Expand All @@ -213,6 +214,11 @@ func (br *BirenDevices) Fit(devices []*device.DeviceUsage, request device.Contai
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Nums > 0 {
klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
k.Nums--
Expand Down
7 changes: 7 additions & 0 deletions pkg/device/cambricon/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (

"github.com/Project-HAMi/HAMi/pkg/device"
"github.com/Project-HAMi/HAMi/pkg/device/common"
"github.com/Project-HAMi/HAMi/pkg/util"
"github.com/Project-HAMi/HAMi/pkg/util/client"

corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -318,6 +319,7 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C
var tmpDevs map[string]device.ContainerDevices
tmpDevs = make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
for i, v := range slices.Backward(devices) {
dev := v
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
Expand Down Expand Up @@ -349,6 +351,11 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Coresreq > 100 {
klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
k.Coresreq = 100
Expand Down
7 changes: 7 additions & 0 deletions pkg/device/enflame/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (

"github.com/Project-HAMi/HAMi/pkg/device"
"github.com/Project-HAMi/HAMi/pkg/device/common"
"github.com/Project-HAMi/HAMi/pkg/util"

corev1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -384,6 +385,7 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con
klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
tmpDevs := make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
profile, profileMatch := enf.selectProfileByRequest(devices, k)
if !profileMatch {
reason[common.ModeNotFit]++
Expand Down Expand Up @@ -424,6 +426,11 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if dev.Totalmem-dev.Usedmem < profileMemoryMiB {
reason[common.CardInsufficientMemory]++
klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", profileMemoryMiB)
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/hygon/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain
var tmpDevs map[string]device.ContainerDevices
tmpDevs = make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
for i, v := range slices.Backward(devices) {
dev := v
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
Expand Down Expand Up @@ -311,6 +312,11 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Coresreq > 100 {
klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
k.Coresreq = 100
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/iluvatar/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ func (ilu *IluvatarDevices) Fit(devices []*device.DeviceUsage, request device.Co
var tmpDevs map[string]device.ContainerDevices
tmpDevs = make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
for i, v := range slices.Backward(devices) {
dev := v
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
Expand Down Expand Up @@ -295,6 +296,11 @@ func (ilu *IluvatarDevices) Fit(devices []*device.DeviceUsage, request device.Co
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Coresreq > 100 {
klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
k.Coresreq = 100
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/metax/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ func (mat *MetaxDevices) Fit(devices []*device.DeviceUsage, request device.Conta
var tmpDevs map[string]device.ContainerDevices
tmpDevs = make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
for i, v := range slices.Backward(devices) {
dev := v
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
Expand Down Expand Up @@ -259,6 +260,11 @@ func (mat *MetaxDevices) Fit(devices []*device.DeviceUsage, request device.Conta
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Coresreq > 100 {
klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
k.Coresreq = 100
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/metax/sdevice.go
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,7 @@ func (mats *MetaxSDevices) Fit(devices []*device.DeviceUsage, request device.Con

// filter device
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
candidateDevices := []*device.DeviceUsage{}
for i, v := range slices.Backward(devices) {
dev := v
Expand Down Expand Up @@ -353,6 +354,11 @@ func (mats *MetaxSDevices) Fit(devices []*device.DeviceUsage, request device.Con
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}

memreq := int32(0)
if request.Memreq > 0 {
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/mthreads/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ func (mth *MthreadsDevices) Fit(devices []*device.DeviceUsage, request device.Co
var tmpDevs map[string]device.ContainerDevices
tmpDevs = make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
for i, v := range slices.Backward(devices) {
dev := v
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
Expand Down Expand Up @@ -309,6 +310,11 @@ func (mth *MthreadsDevices) Fit(devices []*device.DeviceUsage, request device.Co
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Coresreq > 100 {
klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
k.Coresreq = 100
Expand Down
9 changes: 8 additions & 1 deletion pkg/device/nvidia/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -754,7 +754,9 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co
var tmpDevs map[string]device.ContainerDevices
tmpDevs = make(map[string]device.ContainerDevices)
reason := make(map[string]int)
needTopology := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyTopology.String()
gpuPolicy := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod)
needTopology := gpuPolicy == util.GPUSchedulerPolicyTopology.String()
isMutex := gpuPolicy == util.GPUSchedulerPolicyMutex.String()
for i := len(devices) - 1; i >= 0; i-- {
dev := devices[i]
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
Expand Down Expand Up @@ -790,6 +792,11 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Coresreq > 100 {
klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
k.Coresreq = 100
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/vastai/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ func (va *VastaiDevices) Fit(devices []*device.DeviceUsage, request device.Conta
klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
tmpDevs := make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
dieMode := isDieMode(devices)
for i, dev := range slices.Backward(devices) {
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
Expand All @@ -252,6 +253,11 @@ func (va *VastaiDevices) Fit(devices []*device.DeviceUsage, request device.Conta
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Nums > 0 {
klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
if !dieMode {
Expand Down
21 changes: 15 additions & 6 deletions pkg/scheduler/policy/gpu_policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,17 +43,26 @@ func (l DeviceUsageList) Swap(i, j int) {
}

func (l DeviceUsageList) Less(i, j int) bool {
si, sj := l.DeviceLists[i].Score, l.DeviceLists[j].Score
ni, nj := l.DeviceLists[i].Device.Numa, l.DeviceLists[j].Device.Numa
if l.Policy == util.GPUSchedulerPolicyMutex.String() {
ui, uj := l.DeviceLists[i].Device.Used, l.DeviceLists[j].Device.Used
if ui != uj {
return ui > uj // busy GPUs first, idle GPUs at tail (picked first by Fit)
}
return ni < nj
}
if l.Policy == util.GPUSchedulerPolicyBinpack.String() {
if l.DeviceLists[i].Device.Numa == l.DeviceLists[j].Device.Numa {
return l.DeviceLists[i].Score < l.DeviceLists[j].Score
if si != sj {
return si < sj
}
return l.DeviceLists[i].Device.Numa > l.DeviceLists[j].Device.Numa
return ni < nj
}
// default policy is spread
if l.DeviceLists[i].Device.Numa == l.DeviceLists[j].Device.Numa {
return l.DeviceLists[i].Score > l.DeviceLists[j].Score
if si != sj {
return si > sj
}
return l.DeviceLists[i].Device.Numa < l.DeviceLists[j].Device.Numa
return ni < nj
}

func (l DeviceUsageList) DeepCopy() DeviceUsageList {
Expand Down
Loading
Loading