Add configurable --sysfs-root for NUMA detection and PCI discovery

README.md

@@ -221,6 +221,7 @@ deploying the plugin via `helm`.
 | `--mig-strategy`         | `$MIG_STRATEGY`         | `"none"`        |
 | `--fail-on-init-error`   | `$FAIL_ON_INIT_ERROR`   | `true`          |
 | `--nvidia-driver-root`   | `$NVIDIA_DRIVER_ROOT`   | `"/"`           |
+| `--sysfs-root`           | `$SYSFS_ROOT`           | `"/sys"`        |
 | `--pass-device-specs`    | `$PASS_DEVICE_SPECS`    | `false`         |
 | `--device-list-strategy` | `$DEVICE_LIST_STRATEGY` | `"envvar"`      |
 | `--device-id-strategy`   | `$DEVICE_ID_STRATEGY`   | `"uuid"`        |
@@ -234,6 +235,7 @@ flags:
   migStrategy: "none"
   failOnInitError: true
   nvidiaDriverRoot: "/"
+  sysfsRoot: "/sys"
   plugin:
     passDeviceSpecs: false
     deviceListStrategy: "envvar"

api/config/v1/config.go

@@ -69,6 +69,11 @@ func NewConfig(c *cli.Context, flags []cli.Flag) (*Config, error) {
 		config.Flags.NvidiaDevRoot = config.Flags.NvidiaDriverRoot
 	}
 
+	if config.Flags.SysfsRoot == nil || *config.Flags.SysfsRoot == "" {
+		sysfsRoot := DefaultSysfsRoot
+		config.Flags.SysfsRoot = &sysfsRoot
+	}
+
 	// Preserve the historical MPS behavior unless the config explicitly relaxes it.
 	if config.Sharing.MPS != nil && config.Sharing.MPS.FailRequestsGreaterThanOne == nil {
 		t := true

api/config/v1/consts.go

@@ -53,4 +53,5 @@ const (
 	DefaultCDIAnnotationPrefix = cdiapi.AnnotationPrefix
 	DefaultNvidiaCTKPath       = "/usr/bin/nvidia-ctk"
 	DefaultContainerDriverRoot = "/driver-root"
+	DefaultSysfsRoot           = "/sys"
 )

api/config/v1/flags.go

@@ -62,6 +62,7 @@ type CommandLineFlags struct {
 	MpsRoot                 *string                 `json:"mpsRoot,omitempty"          yaml:"mpsRoot,omitempty"`
 	NvidiaDriverRoot        *string                 `json:"nvidiaDriverRoot,omitempty" yaml:"nvidiaDriverRoot,omitempty"`
 	NvidiaDevRoot           *string                 `json:"nvidiaDevRoot,omitempty"    yaml:"nvidiaDevRoot,omitempty"`
+	SysfsRoot               *string                 `json:"sysfsRoot,omitempty"        yaml:"sysfsRoot,omitempty"`
 	GDRCopyEnabled          *bool                   `json:"gdrcopyEnabled"             yaml:"gdrcopyEnabled"`
 	GDSEnabled              *bool                   `json:"gdsEnabled"                 yaml:"gdsEnabled"`
 	MOFEDEnabled            *bool                   `json:"mofedEnabled"               yaml:"mofedEnabled"`
@@ -129,6 +130,8 @@ func (f *Flags) UpdateFromCLIFlags(c *cli.Context, flags []cli.Flag) {
 				updateFromCLIFlag(&f.NvidiaDriverRoot, c, n)
 			case "dev-root", "nvidia-dev-root":
 				updateFromCLIFlag(&f.NvidiaDevRoot, c, n)
+			case "sysfs-root":
+				updateFromCLIFlag(&f.SysfsRoot, c, n)
 			case "gdrcopy-enabled":
 				updateFromCLIFlag(&f.GDRCopyEnabled, c, n)
 			case "gds-enabled":

cmd/gpu-feature-discovery/main.go

@@ -109,6 +109,12 @@ func main() {
 			Usage:   "the strategy to use to discover devices: 'auto', 'nvml', 'tegra' or 'vfio'",
 			EnvVars: []string{"DEVICE_DISCOVERY_STRATEGY"},
 		},
+		&cli.StringFlag{
+			Name:    "sysfs-root",
+			Value:   spec.DefaultSysfsRoot,
+			Usage:   "the root path for sysfs; used for PCI device discovery",
+			EnvVars: []string{"SYSFS_ROOT"},
+		},
 		&cli.StringFlag{
 			Name:    "driver-root-ctr-path",
 			Aliases: []string{"container-driver-root"},
@@ -191,7 +197,7 @@ func start(c *cli.Context, cfg *Config) error {
 			return fmt.Errorf("failed to create resource manager: %w", err)
 
 		}
-		vgpul := vgpu.NewVGPULib(vgpu.NewNvidiaPCILib())
+		vgpul := vgpu.NewVGPULib(vgpu.NewNvidiaPCILib(*config.Flags.SysfsRoot))
 
 		var clientSets flags.ClientSets
 		if config.Flags.UseNodeFeatureAPI != nil && *config.Flags.UseNodeFeatureAPI {

cmd/nvidia-device-plugin/main.go

@@ -84,6 +84,12 @@ func main() {
 			Usage:   "the root path for the NVIDIA device nodes on the host (typical values are '/' or '/run/nvidia/driver')",
 			EnvVars: []string{"NVIDIA_DEV_ROOT"},
 		},
+		&cli.StringFlag{
+			Name:    "sysfs-root",
+			Value:   spec.DefaultSysfsRoot,
+			Usage:   "the root path for sysfs; used for PCI device NUMA detection",
+			EnvVars: []string{"SYSFS_ROOT"},
+		},
 		&cli.BoolFlag{
 			Name:    "pass-device-specs",
 			Value:   false,

deployments/helm/nvidia-device-plugin/templates/daemonset-device-plugin.yml

@@ -166,6 +166,10 @@ spec:
           - name: NVIDIA_DEV_ROOT
             value: "{{ .Values.nvidiaDevRoot }}"
         {{- end }}
+        {{- if typeIs "string" .Values.sysfsRoot }}
+          - name: SYSFS_ROOT
+            value: {{ .Values.sysfsRoot }}
+        {{- end }}
         {{- if typeIs "string" .Values.cdi.nvidiaHookPath }}
           - name: NVIDIA_CDI_HOOK_PATH
             value: {{ .Values.cdi.nvidiaHookPath }}
@@ -213,6 +217,11 @@ spec:
           - name: driver-root
             mountPath: /driver-root
             readOnly: true
+        {{- end }}
+        {{- if and (typeIs "string" .Values.sysfsRoot) (typeIs "string" .Values.sysfsHostPath) }}
+          - name: sysfs-root
+            mountPath: {{ .Values.sysfsRoot }}
+            readOnly: true
         {{- end }}
           # The MPS /dev/shm is needed to allow for MPS daemon health-checking.
           - name: mps-shm
@@ -249,6 +258,12 @@ spec:
             path: {{ .Values.nvidiaDriverRoot }}
             type: Directory
         {{- end }}
+        {{- if and (typeIs "string" .Values.sysfsRoot) (typeIs "string" .Values.sysfsHostPath) }}
+        - name: sysfs-root
+          hostPath:
+            path: {{ .Values.sysfsHostPath }}
+            type: Directory
+        {{- end }}
         - name: cdi-root
           hostPath:
             path: /var/run/cdi

deployments/helm/nvidia-device-plugin/templates/daemonset-gfd.yml

@@ -178,13 +178,22 @@ spec:
           - name: DEVICE_DISCOVERY_STRATEGY
             value: {{ .Values.deviceDiscoveryStrategy }}
         {{- end }}
+        {{- if typeIs "string" .Values.sysfsRoot }}
+          - name: SYSFS_ROOT
+            value: {{ .Values.sysfsRoot }}
+        {{- end }}
         securityContext:
           {{- include "gpu-feature-discovery.securityContext" . | nindent 10 }}
         volumeMounts:
           - name: output-dir
             mountPath: "/etc/kubernetes/node-feature-discovery/features.d"
           - name: host-sys
             mountPath: "/sys"
+        {{- if and (typeIs "string" .Values.sysfsRoot) (typeIs "string" .Values.sysfsHostPath) }}
+          - name: sysfs-root
+            mountPath: {{ .Values.sysfsRoot }}
+            readOnly: true
+        {{- end }}
         {{- if $options.hasConfigMap }}
           - name: available-configs
             mountPath: /available-configs
@@ -208,6 +217,12 @@ spec:
           hostPath:
             path: {{ clean ( join "/" ( list "/" .Values.nvidiaDriverRoot ) ) | quote }}
             type: Directory
+        {{- if and (typeIs "string" .Values.sysfsRoot) (typeIs "string" .Values.sysfsHostPath) }}
+        - name: sysfs-root
+          hostPath:
+            path: {{ .Values.sysfsHostPath }}
+            type: Directory
+        {{- end }}
       {{- if $options.hasConfigMap }}
         - name: available-configs
           configMap:

deployments/helm/nvidia-device-plugin/values.yaml

@@ -33,6 +33,8 @@ failOnInitError: null
 deviceListStrategy: null
 deviceIDStrategy: null
 nvidiaDriverRoot: null
+sysfsRoot: null
+sysfsHostPath: null
 gdrcopyEnabled: null
 gdsEnabled: null
 mofedEnabled: null

internal/rm/device_map.go

@@ -35,14 +35,20 @@ type deviceMapBuilder struct {
 	replicatedResources *spec.ReplicatedResources
 
 	newGPUDevice func(i int, gpu nvml.Device) (string, deviceInfo)
+	newMigDevice func(i int, j int, mig nvml.Device) (string, nvmlMigDevice)
 }
 
 // DeviceMap stores a set of devices per resource name.
 type DeviceMap map[spec.ResourceName]Devices
 
 // NewDeviceMap creates a device map for the specified NVML library and config.
 func NewDeviceMap(devicelib device.Interface, config *spec.Config, platform info.Platform) (DeviceMap, error) {
-	newGPUDevice := newNvmlGPUDevice
+	sysfsRoot := spec.DefaultSysfsRoot
+	if config.Flags.SysfsRoot != nil && *config.Flags.SysfsRoot != "" {
+		sysfsRoot = *config.Flags.SysfsRoot
+	}
+
+	newGPUDevice := newNvmlGPUDevice(sysfsRoot)
 	if platform == info.PlatformWSL {
 		newGPUDevice = newWslAllGPUsDevice
 	}
@@ -53,6 +59,7 @@ func NewDeviceMap(devicelib device.Interface, config *spec.Config, platform info
 		resources:           &config.Resources,
 		replicatedResources: config.Sharing.ReplicatedResources(),
 		newGPUDevice:        newGPUDevice,
+		newMigDevice:        newMigDevice(sysfsRoot),
 	}
 
 	return b.build()
@@ -143,7 +150,7 @@ func (b *deviceMapBuilder) buildMigDeviceMap() (DeviceMap, error) {
 		}
 		for _, resource := range b.resources.MIGs {
 			if resource.Pattern.Matches(migProfile.String()) {
-				index, info := newMigDevice(i, j, mig)
+				index, info := b.newMigDevice(i, j, mig)
 				return devices.setEntry(resource.Name, index, info)
 			}
 		}

internal/rm/nvml_devices.go

@@ -20,11 +20,13 @@ import (
 	"bytes"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/NVIDIA/go-nvml/pkg/nvml"
 
+	spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
 	"github.com/NVIDIA/k8s-device-plugin/internal/mig"
 )
 
@@ -36,25 +38,32 @@ const (
 // nvmlDevice wraps an nvml.Device with more functions.
 type nvmlDevice struct {
 	nvml.Device
+	sysfsRoot string
 }
 
 // nvmlMigDevice allows for specific functions of nvmlDevice to be overridden.
-type nvmlMigDevice nvmlDevice
+type nvmlMigDevice struct {
+	nvmlDevice
+}
 
 var _ deviceInfo = (*nvmlDevice)(nil)
 var _ deviceInfo = (*nvmlMigDevice)(nil)
 
-func newNvmlGPUDevice(i int, gpu nvml.Device) (string, deviceInfo) {
-	index := fmt.Sprintf("%v", i)
-	return index, nvmlDevice{gpu}
+func newNvmlGPUDevice(sysfsRoot string) func(i int, gpu nvml.Device) (string, deviceInfo) {
+	return func(i int, gpu nvml.Device) (string, deviceInfo) {
+		index := fmt.Sprintf("%v", i)
+		return index, nvmlDevice{Device: gpu, sysfsRoot: sysfsRoot}
+	}
 }
 
 func newWslAllGPUsDevice(_ int, _ nvml.Device) (string, deviceInfo) {
 	return "all", wslAllGPUsDevice{}
 }
 
-func newMigDevice(i int, j int, mig nvml.Device) (string, nvmlMigDevice) {
-	return fmt.Sprintf("%v:%v", i, j), nvmlMigDevice{mig}
+func newMigDevice(sysfsRoot string) func(i int, j int, mig nvml.Device) (string, nvmlMigDevice) {
+	return func(i int, j int, mig nvml.Device) (string, nvmlMigDevice) {
+		return fmt.Sprintf("%v:%v", i, j), nvmlMigDevice{nvmlDevice{Device: mig, sysfsRoot: sysfsRoot}}
+	}
 }
 
 // GetUUID returns the UUID of the device
@@ -68,7 +77,7 @@ func (d nvmlDevice) GetUUID() (string, error) {
 
 // GetUUID returns the UUID of the device
 func (d nvmlMigDevice) GetUUID() (string, error) {
-	return nvmlDevice(d).GetUUID()
+	return d.nvmlDevice.GetUUID()
 }
 
 // GetPaths returns the paths for a GPU device
@@ -97,7 +106,7 @@ func (d nvmlMigDevice) GetComputeCapability() (string, error) {
 	if ret != nvml.SUCCESS {
 		return "", fmt.Errorf("failed to get parent device: %w", ret)
 	}
-	return nvmlDevice{parent}.GetComputeCapability()
+	return nvmlDevice{Device: parent, sysfsRoot: d.sysfsRoot}.GetComputeCapability()
 }
 
 // GetPaths returns the paths for a MIG device
@@ -146,17 +155,9 @@ func (d nvmlMigDevice) GetPaths() ([]string, error) {
 	return devicePaths, nil
 }
 
-// GetNumaNode returns the NUMA node associated with the GPU device
-func (d nvmlDevice) GetNumaNode() (bool, int, error) {
-	info, ret := d.GetPciInfo()
-	if ret != nvml.SUCCESS {
-		return false, 0, fmt.Errorf("error getting PCI Bus Info of device: %v", ret)
-	}
-
-	// Discard leading zeros.
-	busID := strings.ToLower(strings.TrimPrefix(uint8Slice(info.BusId[:]).String(), "0000"))
-
-	b, err := os.ReadFile(fmt.Sprintf("/sys/bus/pci/devices/%s/numa_node", busID))
+func readNumaNodeFromSysfs(sysfsRoot, busID string) (bool, int, error) {
+	path := filepath.Join(sysfsRoot, "bus", "pci", "devices", busID, "numa_node")
+	b, err := os.ReadFile(path)
 	if err != nil {
 		return false, 0, nil
 	}
@@ -173,14 +174,31 @@ func (d nvmlDevice) GetNumaNode() (bool, int, error) {
 	return true, node, nil
 }
 
+// GetNumaNode returns the NUMA node associated with the GPU device
+func (d nvmlDevice) GetNumaNode() (bool, int, error) {
+	info, ret := d.GetPciInfo()
+	if ret != nvml.SUCCESS {
+		return false, 0, fmt.Errorf("error getting PCI Bus Info of device: %v", ret)
+	}
+
+	// Discard leading zeros.
+	busID := strings.ToLower(strings.TrimPrefix(uint8Slice(info.BusId[:]).String(), "0000"))
+
+	sysfsRoot := d.sysfsRoot
+	if sysfsRoot == "" {
+		sysfsRoot = spec.DefaultSysfsRoot
+	}
+	return readNumaNodeFromSysfs(sysfsRoot, busID)
+}
+
 // GetNumaNode for a MIG device is the NUMA node of the parent device.
 func (d nvmlMigDevice) GetNumaNode() (bool, int, error) {
 	parent, ret := d.GetDeviceHandleFromMigDeviceHandle()
 	if ret != nvml.SUCCESS {
 		return false, 0, fmt.Errorf("error getting parent GPU device from MIG device: %v", ret)
 	}
 
-	return nvmlDevice{parent}.GetNumaNode()
+	return nvmlDevice{Device: parent, sysfsRoot: d.sysfsRoot}.GetNumaNode()
 }
 
 // GetTotalMemory returns the total memory available on the device.

internal/rm/nvml_devices_test.go

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rm
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func writeNumaNodeFile(t *testing.T, root, busID, content string) {
+	t.Helper()
+	dir := filepath.Join(root, "bus", "pci", "devices", busID)
+	require.NoError(t, os.MkdirAll(dir, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "numa_node"), []byte(content), 0o644))
+}
+
+func TestReadNumaNodeFromSysfs(t *testing.T) {
+	root := t.TempDir()
+
+	t.Run("valid node", func(t *testing.T) {
+		writeNumaNodeFile(t, root, "0000:03:00.0", "1\n")
+		hasNuma, node, err := readNumaNodeFromSysfs(root, "0000:03:00.0")
+		require.NoError(t, err)
+		require.True(t, hasNuma)
+		require.Equal(t, 1, node)
+	})
+
+	t.Run("missing file returns no numa", func(t *testing.T) {
+		hasNuma, node, err := readNumaNodeFromSysfs(root, "0000:99:00.0")
+		require.NoError(t, err)
+		require.False(t, hasNuma)
+		require.Equal(t, 0, node)
+	})
+
+	t.Run("negative one returns no numa", func(t *testing.T) {
+		writeNumaNodeFile(t, root, "0000:04:00.0", "-1\n")
+		hasNuma, node, err := readNumaNodeFromSysfs(root, "0000:04:00.0")
+		require.NoError(t, err)
+		require.False(t, hasNuma)
+		require.Equal(t, 0, node)
+	})
+
+	t.Run("invalid content returns error", func(t *testing.T) {
+		writeNumaNodeFile(t, root, "0000:05:00.0", "not-a-number\n")
+		_, _, err := readNumaNodeFromSysfs(root, "0000:05:00.0")
+		require.Error(t, err)
+	})
+}