Skip to content

fix(ascend): proportionally split card-level metrics for vnpu container metrics #106

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
peachest wants to merge 6 commits into
base: main
Choose a base branch
from
Open

fix(ascend): proportionally split card-level metrics for vnpu container metrics #106

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
646b6cd
fix(ascend): proportionally split card-level metrics for vnpu contain…
Jun 26, 2026
1a3b61e
fix(ascend): fix compile error, add query warn logs, and keep memory …
Jun 29, 2026
a793689
ci: clean comments
Jun 29, 2026
f71748f
fix: add ascend switch case for taskMemory
Jul 3, 2026
45d3993
fix(ascend): normalize provider from cd.Type to canonical Ascend value
Jul 3, 2026
994264f
fix: node device provider use ==
Jul 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 48 additions & 4 deletions server/internal/exporter/exporter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,33 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error {

s.generateMetricsForMetaxGPU(containers)
for _, device := range deviceInfos {
// === Ascend pre-calculation: card-level metrics + total allocation ===
// ponytail: proportional split — npu-exporter doesn't support vnpu for 910B/A3 yet.
// Card-level metrics are divided across containers by Usedmem ratio.
var ascendCardUtil float32
var ascendCardMemUsedMB float32
var ascendTotalMemoryOnCard int32
var ascendCardQueriesOK bool
if strings.HasPrefix(device.Provider, biz.AscendGPUDevice) {
var cdMemBytes float32
var ascendCardUtilErr, ascendCardMemErr error
ascendCardUtil, ascendCardUtilErr = s.deviceCoreUtil(ctx, device.Provider, device.Id)
cdMemBytes, ascendCardMemErr = s.deviceMemUsed(ctx, device.Provider, device.Id)
ascendCardQueriesOK = ascendCardUtilErr == nil && ascendCardMemErr == nil
if ascendCardQueriesOK && cdMemBytes > 0 {
ascendCardMemUsedMB = cdMemBytes / 1024 / 1024
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
}
for _, c := range containers {
for _, cd := range c.ContainerDevices {
if device.AliasId != "" && !device.MatchAlias(cd.UUID) {
continue
}
if strings.HasPrefix(cd.Type, biz.AscendGPUDevice) {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
ascendTotalMemoryOnCard += cd.Usedmem
}
}
}
}
for _, c := range containers {
var vGPU int32 = 0
var core int32 = 0
Expand All @@ -385,8 +412,15 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error {
s.set(HamiContainerVmemoryAllocated, float64(memory), device.NodeName, provider, device.Type, device.Id, c.PodName, c.Name, c.Namespace, podUIDLabel)
s.set(HamiContainerVcoreAllocated, float64(core), device.NodeName, provider, device.Type, device.Id, c.PodName, c.Name, c.Namespace, podUIDLabel)
// 查询任务在当前设备下的算力利用率
taskCoreUsed, err := s.taskCoreUsed(ctx, provider, c.Namespace, c.PodName, c.Name, c.PodUID, device.Id, device.NodeName, device.Index)
if err == nil {
var taskCoreUsed float32
var taskCoreUsedErr error
if provider == biz.AscendGPUDevice && ascendCardQueriesOK && ascendTotalMemoryOnCard > 0 {
ratio := float32(memory) / float32(ascendTotalMemoryOnCard)
taskCoreUsed = ascendCardUtil * ratio
} else {
taskCoreUsed, taskCoreUsedErr = s.taskCoreUsed(ctx, provider, c.Namespace, c.PodName, c.Name, c.PodUID, device.Id, device.NodeName, device.Index)
}
if taskCoreUsedErr == nil {
used := float64(0)
util := float64(0)
switch provider {
Expand All @@ -396,6 +430,9 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error {
case biz.CambriconGPUDevice:
used = float64(taskCoreUsed) / 100 * float64(core)
util = float64(taskCoreUsed)
case biz.AscendGPUDevice:
used = float64(taskCoreUsed)
util = roundToOneDecimal(100 * float64(taskCoreUsed) / float64(core))
case biz.HygonGPUDevice:
used = float64(taskCoreUsed)
util = roundToOneDecimal(100 * float64(taskCoreUsed) / float64(core))
Expand All @@ -412,8 +449,15 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error {
s.set(HamiContainerCoreUsed, used, device.NodeName, provider, device.Type, device.Id, c.PodName, c.Name, c.Namespace)
s.set(HamiContainerCoreUtil, util, device.NodeName, provider, device.Type, device.Id, c.PodName, c.Name, c.Namespace)
}
taskMemoryUsed, err := s.taskMemoryUsed(ctx, provider, c.Namespace, c.PodName, c.Name, c.PodUID, device.Id, device.NodeName, device.Index)
if err == nil {
var taskMemoryUsed float32
var taskMemoryUsedErr error
if provider == biz.AscendGPUDevice && ascendCardQueriesOK && ascendTotalMemoryOnCard > 0 {
ratio := float32(memory) / float32(ascendTotalMemoryOnCard)
taskMemoryUsed = ascendCardMemUsedMB * ratio
} else {
taskMemoryUsed, taskMemoryUsedErr = s.taskMemoryUsed(ctx, provider, c.Namespace, c.PodName, c.Name, c.PodUID, device.Id, device.NodeName, device.Index)
}
if taskMemoryUsedErr == nil {
switch provider {
case biz.CambriconGPUDevice:
taskMemoryUsed = float32((taskMemoryUsed/100)*float32(memory)) * 1024 * 1024
Expand Down