diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md
new file mode 100644
index 000000000..80c8688c6
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug-report.md
@@ -0,0 +1,33 @@
+---
+name: Bug Report
+about: Report a bug encountered while using HAMi.
+labels: kind/bug
+
+---
+
+
+
+**What happened**:
+
+**What you expected to happen**:
+
+**How to reproduce it (as minimally and precisely as possible)**:
+
+**Anything else we need to know?**:
+
+- The output of `nvidia-smi -a` on your host
+- Your docker or containerd configuration file (e.g: `/etc/docker/daemon.json`)
+- The hami-device-plugin container logs
+- The hami-scheduler container logs
+- The kubelet logs on the node (e.g: `sudo journalctl -r -u kubelet`)
+- Any relevant kernel output lines from `dmesg`
+
+**Environment**:
+- HAMi version:
+- nvidia driver or other AI device driver version:
+- Docker version from `docker version`
+- Docker command, image and tag used
+- Kernel version from `uname -a`
+- Others:
+
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
new file mode 100644
index 000000000..f7ee71277
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1,4 @@
+contact_links:
+ - name: FAQ
+ url: https://github.com/Project-HAMi/HAMi/issues/646
+ about: Frequently asked questions and common solutions.
\ No newline at end of file
diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md
new file mode 100644
index 000000000..e8a277834
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/enhancement.md
@@ -0,0 +1,22 @@
+---
+name: Enhancement Request
+about: Suggest an enhancement to the project
+labels: kind/feature
+
+---
+
+
+**What would you like to be added**:
+
+**What type of PR is this?**
+
+/kind feature
+
+**What this PR does / why we need it**:
+
+**Which issue(s) this PR fixes**:
+Fixes #
+
+**Special notes for your reviewer**:
+
+**Does this PR introduce a user-facing change?**:
\ No newline at end of file
diff --git a/.github/ISSUE_TEMPLATE/good-first.md b/.github/ISSUE_TEMPLATE/good-first.md
new file mode 100644
index 000000000..9ed778e48
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/good-first.md
@@ -0,0 +1,31 @@
+---
+name: Good First Issue
+about: Publish a good first issue
+labels: good first issue
+
+---
+
+
+
+**Task description**:
+
+**Solution**:
+
+**Who can join or take the task**:
+
+The good first issue is intended for `first-time contributors` to get started on his/her contributor journey.
+
+After a contributor has successfully completed 1-2 good first issue's,
+they should be ready to move on to `help wanted` items, saving the remaining `good first issue` for other new contributors.
+
+**How to join or take the task**:
+
+Just reply on the issue with the message `/assign` in a separate line.
+
+Then, the issue will be assigned to you.
+
+**How to ask for help**:
+
+If you need help or have questions, please feel free to ask on this issue.
+The issue author or other members of the community will guide you through the contribution process.
\ No newline at end of file
diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md
new file mode 100644
index 000000000..209838853
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/question.md
@@ -0,0 +1,15 @@
+---
+name: Question
+about: Question relating to HAMi.
+labels: kind/question
+
+---
+
+**Please provide an in-depth description of the question you have**:
+
+**What do you think about this question?**:
+
+**Environment**:
+- HAMi version:
+- Kubernetes version:
+- Others:
\ No newline at end of file
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 000000000..b2d955148
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,22 @@
+**What type of PR is this?**
+
+
+
+**What this PR does / why we need it**:
+
+**Which issue(s) this PR fixes**:
+Fixes #
+
+**Special notes for your reviewer**:
+
+**Does this PR introduce a user-facing change?**:
\ No newline at end of file
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 000000000..c6044996b
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,21 @@
+
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+
+version: 2
+updates:
+ - package-ecosystem: "gomod"
+ directory: "/"
+ schedule:
+ interval: "daily"
+ - package-ecosystem: "docker"
+ directory: "/docker"
+ schedule:
+ interval: "daily"
+ - package-ecosystem: "github-actions"
+ directory: "/"
+ schedule:
+ interval: "daily"
diff --git a/.github/labeler.yml b/.github/labeler.yml
new file mode 100644
index 000000000..96756010b
--- /dev/null
+++ b/.github/labeler.yml
@@ -0,0 +1,10 @@
+"kind/bug":
+ - '^[Ff]ix(\(.*\))?:?.*'
+"kind/cleanup":
+ - '^[Cc]hore(\(.*\))?:?.*'
+"kind/documentation":
+ - '^[Dd]ocs?(\(.*\))?:?.*'
+"kind/enhancement":
+ - '^[Rr]efactor(\(.*\))?:?.*'
+"kind/feature":
+ - '^[Ff]eat(\(.*\))?:?.*'
diff --git a/.github/release.yml b/.github/release.yml
new file mode 100644
index 000000000..2582674ae
--- /dev/null
+++ b/.github/release.yml
@@ -0,0 +1,34 @@
+# .github/release.yml
+changelog:
+ exclude:
+ labels:
+ - ignore-for-release
+ - github-actions
+ authors:
+ - dependabot[bot]
+ categories:
+ - title: ✨ New Features
+ labels:
+ - feature
+ - design
+ - enhancement
+ - kind/feature
+ - kind/design
+ - kind/enhancement
+ - title: 🐛 Bug Fixes
+ labels:
+ - bug
+ - kind/bug
+ - title: 📚 Documentation
+ labels:
+ - documentation
+ - kind/documentation
+ - title: ⬆️ Dependencies
+ labels:
+ - dependencies
+ - title: 💥 Breaking Changes
+ labels:
+ - breaking-change
+ - title: 🔨 Other Changes
+ labels:
+ - "*"
diff --git a/.github/workflows/auto-label-pr.yaml b/.github/workflows/auto-label-pr.yaml
new file mode 100644
index 000000000..6f5d0ca0c
--- /dev/null
+++ b/.github/workflows/auto-label-pr.yaml
@@ -0,0 +1,22 @@
+name: "PR Labeler"
+on:
+ pull_request_target:
+ types: [opened, edited]
+
+permissions:
+ issues: write
+ pull-requests: write
+ contents: read
+
+jobs:
+ labeling:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: github/issue-labeler@v3.4
+ with:
+ configuration-path: .github/labeler.yml
+ enable-versioned-regex: 0
+ sync-labels: 1
+ include-title: 1
+ include-body: 0
+ repo-token: ${{ github.token }}
diff --git a/.github/workflows/auto-release.yaml b/.github/workflows/auto-release.yaml
new file mode 100644
index 000000000..d87700027
--- /dev/null
+++ b/.github/workflows/auto-release.yaml
@@ -0,0 +1,152 @@
+name: 1.Release
+
+# it is trigger by tag event:
+# 1 build the release image, push images to ghcr.io, and build image like: ghcr.io/xxxx:v1.0.0
+# 2 package the chart package, update index.yaml and commit to '/charts' of branch 'github_pages' ( PR with label pr/release/robot_update_githubpage )
+# 3 create changelog file, commit to '/changelogs' of branch 'github_pages' for githubPage ( PR with label pr/release/robot_update_githubpage )
+# 4 commit '/docs' to '/docs' of branch 'github_pages'
+# 5 create a release , attached with the chart and changelog
+
+on:
+ release:
+ types:
+ - prereleased
+ push:
+ tags:
+ - v[0-9]+.[0-9]+.[0-9]+
+ - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
+ workflow_dispatch:
+ inputs:
+ tag:
+ description: 'Tag'
+ required: true
+ default: v1.0.0
+permissions: write-all
+
+jobs:
+ pre-release:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ outputs:
+ tag: ${{ env.RUN_TAG }}
+ steps:
+ - name: Check Version
+ run: |
+ TagVersion="${{ env.RUN_TAG }}"
+ RecordVersion=` cat VERSION | tr -d ' ' | tr -d '\n' `
+ if [ "$RecordVersion" != "$TagVersion" ] ; then
+ echo "error, version $RecordVersion of '/VERSION' is different with Tag $TagVersion "
+ exit 1
+ fi
+ #no need to check chart version, which will auto update to /VERSION by CI
+ # generate release notes for the new release branch
+ - name: Generate pre release notes
+ if: startsWith(github.ref, 'refs/tags/')
+ uses: softprops/action-gh-release@v2
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ prerelease: true
+ generate_release_notes: true
+
+ ensure-tag:
+ runs-on: ubuntu-latest
+ needs: pre-release
+ outputs:
+ tag: ${{ env.RUN_TAG }}
+ steps:
+ - name: Free disk space
+ # https://github.com/actions/virtual-environments/issues/709
+ run: |
+ echo "=========original CI disk space"
+ df -h
+ sudo rm -rf "/usr/local/share/boost"
+ sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+ echo "=========after clean up, the left CI disk space"
+ df -h
+ - name: Get Ref
+ id: get_ref
+ run: |
+ if ${{ github.event_name == 'workflow_dispatch' }} ; then
+ echo "call by self workflow_dispatch"
+ echo "RUN_TAG=${{ github.event.inputs.tag }}" >> $GITHUB_ENV
+ YBranchName=` grep -Eo "v[0-9]+\.[0-9]+" <<< "${{ github.event.inputs.tag }}" `
+ elif ${{ github.event_name == 'push' }} ; then
+ echo "call by push tag"
+ echo "RUN_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
+ YBranchName=` grep -Eo "v[0-9]+\.[0-9]+" <<< "${GITHUB_REF##*/}" `
+ else
+ echo "unexpected event: ${{ github.event_name }}"
+ exit 1
+ fi
+ echo "YBranchName=${YBranchName}"
+ if [ -n "$YBranchName" ] ; then
+ echo "RUN_YBranchName=${YBranchName}" >> $GITHUB_ENV
+ else
+ echo "error, failed to find y branch"
+ exit 1
+ fi
+ - name: Checkout
+ uses: actions/checkout@v5
+ with:
+ ref: ${{ env.RUN_TAG }}
+ # if branch exists, the action will no fail, and it output created=false
+ - name: release Y branch
+ uses: peterjgrainger/action-create-branch@v3.0.0
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ branch: 'release-${{ env.RUN_YBranchName }}'
+ sha: '${{ github.sha }}'
+
+
+ release-image-hamicore:
+ needs: ensure-tag
+ uses: ./.github/workflows/call-release-image-hamicore.yaml
+ with:
+ ref: ${{ needs.ensure-tag.outputs.tag }}
+ secrets: inherit
+
+ release-image:
+ needs: [ensure-tag,release-image-hamicore]
+ uses: ./.github/workflows/call-release-image.yaml
+ with:
+ ref: ${{ needs.ensure-tag.outputs.tag }}
+ secrets: inherit
+
+ release-chart:
+ needs: [ensure-tag]
+ uses: ./.github/workflows/call-release-helm.yaml
+ with:
+ ref: ${{ needs.ensure-tag.outputs.tag }}
+ submit: true
+ secrets: inherit
+
+ # generate changelog and update to github releases pages
+ release-notes:
+ needs: [release-chart,release-image]
+ uses: ./.github/workflows/call-release-notes.yaml
+ with:
+ ref: ${{ needs.ensure-tag.outputs.tag }}
+
+ release-website:
+ needs: [release-notes]
+ uses: ./.github/workflows/call-release-website.yaml
+ with:
+ ref: ${{ needs.ensure-tag.outputs.tag }}
+ # excute a full e2e test when hami release
+ release-e2e:
+ needs: [release-notes]
+ uses: ./.github/workflows/call-e2e.yaml
+ with:
+ ref: ${{ needs.ensure-tag.outputs.tag }}
+ type: "release"
+
+ # excute a compatibility test when hami release
+ release-e2e-upgrade:
+ needs: [release-notes]
+ uses: ./.github/workflows/call-e2e-upgrade.yaml
+ with:
+ ref: ${{ needs.ensure-tag.outputs.tag }}
+
\ No newline at end of file
diff --git a/.github/workflows/call-e2e-upgrade.yaml b/.github/workflows/call-e2e-upgrade.yaml
new file mode 100644
index 000000000..528e37397
--- /dev/null
+++ b/.github/workflows/call-e2e-upgrade.yaml
@@ -0,0 +1,18 @@
+name: Call e2e upgrade test
+
+on:
+ workflow_call:
+ inputs:
+ ref:
+ required: true
+ type: string
+permissions: write-all
+
+jobs:
+ upgrade-e2e:
+ runs-on: ubuntu-latest
+ steps:
+ - name: e2e upgrade test
+ # https://github.com/actions/virtual-environments/issues/709
+ run: |
+ echo "Need to add e2e upgrade test"
diff --git a/.github/workflows/call-e2e.yaml b/.github/workflows/call-e2e.yaml
new file mode 100644
index 000000000..4f084e9b4
--- /dev/null
+++ b/.github/workflows/call-e2e.yaml
@@ -0,0 +1,77 @@
+name: Call e2e test
+
+on:
+ workflow_call:
+ inputs:
+ ref:
+ description: 'Reference id to run tests'
+ required: true
+ type: string
+ type:
+ description: 'E2E type'
+ required: true
+ type: string
+ default: pullrequest
+
+jobs:
+ e2e-test:
+ strategy:
+ matrix:
+ include:
+ - device: nvidia
+ type: tesla-p4
+# - device: nvidia
+# type: rtx-4090
+# - device: huawei
+# type: ascend-910b
+ runs-on: [ "${{ matrix.device }}", "${{ matrix.type }}" ]
+ environment: ${{ matrix.device }}
+ env:
+ E2E_TYPE: ${{ inputs.type }}
+ HAMI_VERSION: ${{ inputs.ref }}
+ steps:
+ - name: checkout code
+ uses: actions/checkout@v5
+
+ - name: install Go
+ uses: actions/setup-go@v5
+ with:
+ go-version: "1.21"
+
+ - name: setup e2e env
+ run: |
+ make e2e-env-setup
+
+ - name: download hami helm
+ if: inputs.type == 'pullrequest'
+ uses: actions/download-artifact@v5
+ with:
+ name: chart_package_artifact
+ path: charts/
+
+ - name: download hami image
+ if: inputs.type == 'pullrequest'
+ uses: actions/download-artifact@v5
+ with:
+ name: hami-image
+ path: ./image
+
+ - name: load e2e image
+ if: inputs.type == 'pullrequest'
+ run: |
+ echo "Loading Docker image from image.tar..."
+ if [ -z "${VSPHERE_GPU_VM_IP}" ]; then
+ echo "Error: VSPHERE_GPU_VM_IP is not defined!"
+ exit 1
+ fi
+ scp ./image/image.tar root@$VSPHERE_GPU_VM_IP:/home/
+ ssh root@$VSPHERE_GPU_VM_IP "nerdctl load -i /home/image.tar"
+ ssh root@$VSPHERE_GPU_VM_IP "nerdctl image ls | grep hami"
+
+ - name: deploy hami helm
+ run: |
+ make helm-deploy
+
+ - name: e2e test
+ run: |
+ make e2e-test
diff --git a/.github/workflows/call-release-helm.yaml b/.github/workflows/call-release-helm.yaml
new file mode 100644
index 000000000..8a29305b3
--- /dev/null
+++ b/.github/workflows/call-release-helm.yaml
@@ -0,0 +1,139 @@
+name: Call Release Helm Charts
+
+# must set branch 'github_pages' as github page
+# this workflow will create the tgz from "/charts/*" of branch main,
+# and deploy to "/charts" of branch "github_pages"
+# and on branch "github_pages", update '/index.yaml' for '/charts/*.tgz'
+
+env:
+ HELM_VERSION: v3.8.1
+ MERGE_BRANCH: gh-pages
+on:
+ workflow_call:
+ inputs:
+ ref:
+ required: true
+ type: string
+ submit:
+ required: true
+ type: string
+ outputs:
+ artifact:
+ description: "name of artifact"
+ value: chart_package_artifact
+ workflow_dispatch:
+ inputs:
+ ref:
+ description: 'tag, sha, branch'
+ required: true
+ default: v1.0.0
+
+permissions: write-all
+
+jobs:
+ get_ref:
+ runs-on: ubuntu-latest
+ outputs:
+ ref: ${{ env.REF }}
+ submit: ${{ env.SUBMIT }}
+ steps:
+ - name: Get Original Ref
+ id: get_original_ref
+ run: |
+ if ${{ inputs.ref != '' }} ; then
+ echo "call by workflow_call"
+ echo "REF=${{ inputs.ref }}" >> $GITHUB_ENV
+ echo "SUBMIT=${{ inputs.submit }}" >> $GITHUB_ENV
+ elif ${{ github.event_name == 'workflow_dispatch' }} ; then
+ echo "call by self workflow_dispatch"
+ echo "REF=${{ inputs.ref }}" >> $GITHUB_ENV
+ echo "SUBMIT=true" >> $GITHUB_ENV
+ else
+ echo "unexpected event: ${{ github.event_name }}"
+ exit 1
+ fi
+
+ # packages tgz from /charts of original branch, deploy to /charts of target branch
+ package_chart:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v5
+ with:
+ fetch-depth: 0
+ ref: ${{ needs.get_ref.outputs.ref }}
+ - name: Configure Git
+ run: |
+ git config user.name "$GITHUB_ACTOR"
+ git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+ - name: Install Helm
+ uses: azure/setup-helm@v4
+ with:
+ version: ${{ env.HELM_VERSION }}
+
+ - name: Lint helm chart
+ run: |
+ make lint_chart
+ - name: Package Chart
+ continue-on-error: false
+ run: |
+ cd charts
+ make clean
+ make
+ if ! ls *.tgz &>/dev/null ; then
+ echo "failed to generate chart"
+ exit 1
+ fi
+ cd ..
+ mkdir -p tmp
+ mv charts/*.tgz tmp
+ - name: Upload Artifact
+ uses: actions/upload-artifact@v4.5.0
+ with:
+ name: chart_package_artifact
+ path: tmp/*
+ retention-days: 1
+ if-no-files-found: error
+
+ # update /index.yaml in the target branch
+ update_githubpage:
+ runs-on: ubuntu-latest
+ needs: [package_chart, get_ref]
+ if: ${{ needs.get_ref.outputs.submit == 'true' }}
+ steps:
+ - name: Get Base Chart URL
+ id: get_base_url
+ run: |
+ name=${{ github.repository }}
+ proj=${name#*/}
+ url=https://${{ github.repository_owner }}.github.io/${proj}
+ echo "URL=${url}" >> $GITHUB_ENV
+
+ - name: Checkout Code
+ uses: actions/checkout@v5
+ with:
+ ref: ${{ env.MERGE_BRANCH }}
+ persist-credentials: "true"
+
+ - name: Download Artifact
+ uses: actions/download-artifact@v5
+ with:
+ name: chart_package_artifact
+ path: charts/
+
+ - name: Update Chart Yaml
+ run: |
+ helm repo index ./charts --url ${{ env.URL }}/charts
+ mv ./charts/index.yaml ./index.yaml
+ - name: update helm release
+ id: push_directory
+ uses: cpina/github-action-push-to-another-repository@v1.7.2
+ env:
+ API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
+ with:
+ source-directory: .
+ destination-github-username: ${{ github.repository_owner }}
+ destination-repository-name: hami
+ user-email: xiaozhang0210@hotmail.com
+ commit-message: sync ORIGIN_COMMIT from $GITHUB_REF
+ target-branch: ${{ env.MERGE_BRANCH }}
diff --git a/.github/workflows/call-release-image-hamicore.yaml b/.github/workflows/call-release-image-hamicore.yaml
new file mode 100644
index 000000000..543c8abd7
--- /dev/null
+++ b/.github/workflows/call-release-image-hamicore.yaml
@@ -0,0 +1,116 @@
+name: Call Release Image
+
+env:
+ REGISTRY: docker.io
+ IMAGE_REPO: ${{ secrets.IMAGE_REPO || 'projecthami/hami' }}
+ IMAGE_REPO_HAMICORE: ${{ secrets.IMAGE_REPO || 'projecthami/hamicore' }}
+ IMAGE_ROOT_PATH: docker
+ BUILD_PLATFORM: linux/arm64,linux/amd64
+ REGISTER_USER: ${{ github.actor }}
+ REGISTER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+on:
+ workflow_call:
+ inputs:
+ ref:
+ required: true
+ type: string
+ suffix:
+ required: false
+ type: string
+ tagoverride:
+ required: false
+ type: string
+permissions: write-all
+
+jobs:
+ docker-build:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@master
+ - name: Free disk space
+ # https://github.com/actions/virtual-environments/issues/709
+ run: |
+ echo "=========original CI disk space"
+ df -h
+ sudo rm -rf "/usr/local/share/boost"
+ sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+ echo "=========after clean up, the left CI disk space"
+ df -h
+ - name: Getting Image Ref
+ id: ref
+ run: |
+ if ${{ inputs.ref != '' }} ; then
+ echo "call by workflow_call"
+ image_ref=${{ inputs.ref }}
+ image_suffix=${{ inputs.suffix }}
+ elif ${{ github.event_name == 'workflow_dispatch' }} ; then
+ echo "call by workflow_dispatch"
+ image_ref=${{ github.event.inputs.ref }}
+ image_suffix=${{ github.event.inputs.suffix }}
+ else
+ echo "unexpected event: ${{ github.event_name }}"
+ exit 1
+ fi
+ echo "ref=${image_ref}" >> $GITHUB_ENV
+ [ -n "${image_suffix}" ] && echo "suffix=-${image_suffix}" >> $GITHUB_ENV
+ if ${{ inputs.tagoverride != '' }} ; then
+ echo "imagetag=${{ inputs.tagoverride }}" >> $GITHUB_ENV
+ else
+ # it is invalid if the ref is branch name "xx/xx/xx"
+ echo "imagetag=${image_ref##*/}" >> $GITHUB_ENV
+ fi
+ exit 0
+
+
+ - name: Checkout code
+ uses: actions/checkout@v5
+ with:
+ persist-credentials: false
+ ref: ${{ steps.ref.outputs.ref }}
+
+ - name: Checkout submodule
+ uses: Mushus/checkout-submodule@v1.0.1
+ with:
+ basePath: # optional, default is .
+ submodulePath: libvgpu
+
+ - name: Lint Dockerfile
+ run: |
+ make lint_dockerfile
+
+ - name: Docker Login
+ uses: docker/login-action@v3.5.0
+ with:
+ username: ${{ secrets.DOCKERHUB_TOKEN }}
+ password: ${{ secrets.DOCKERHUB_PASSWD }}
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ with:
+ driver-opts: image=moby/buildkit:master
+
+ - name: Extract metadata (tags, labels) for Docker
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_REPO_HAMICORE }}
+
+ - name: Build & Pushing hami-core image
+ uses: docker/build-push-action@v6.18.0
+ with:
+ context: .
+ file: ${{ env.IMAGE_ROOT_PATH }}/Dockerfile.hamicore
+ labels: ${{ steps.meta.outputs.labels }}
+ platforms: ${{ env.BUILD_PLATFORM }}
+ build-args: |
+ VERSION=${{ env.ref }}
+ GOLANG_IMAGE=golang:1.22.5-bullseye
+ NVIDIA_IMAGE=nvidia/cuda:12.2.0-devel-ubuntu20.04
+ DEST_DIR=/usr/local
+ tags: ${{ steps.meta.outputs.tags }}
+ push: true
+ github-token: ${{ env.REGISTER_PASSWORD }}
diff --git a/.github/workflows/call-release-image.yaml b/.github/workflows/call-release-image.yaml
new file mode 100644
index 000000000..a0cfd135f
--- /dev/null
+++ b/.github/workflows/call-release-image.yaml
@@ -0,0 +1,139 @@
+name: Call Release Image
+
+env:
+ REGISTRY: docker.io
+ IMAGE_REPO: ${{ secrets.IMAGE_REPO || 'projecthami/hami' }}
+ IMAGE_REPO_HAMICORE: ${{ secrets.IMAGE_REPO || 'projecthami/hamicore' }}
+ IMAGE_ROOT_PATH: docker
+ BUILD_PLATFORM: linux/arm64,linux/amd64
+ REGISTER_USER: ${{ github.actor }}
+ REGISTER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+on:
+ workflow_call:
+ inputs:
+ ref:
+ required: true
+ type: string
+ suffix:
+ required: false
+ type: string
+ tagoverride:
+ required: false
+ type: string
+permissions: write-all
+
+jobs:
+ docker-build:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@master
+ - name: Free disk space
+ # https://github.com/actions/virtual-environments/issues/709
+ run: |
+ echo "=========original CI disk space"
+ df -h
+ sudo rm -rf "/usr/local/share/boost"
+ sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+ echo "=========after clean up, the left CI disk space"
+ df -h
+ - name: Getting Image Ref
+ id: ref
+ run: |
+ if ${{ inputs.ref != '' }} ; then
+ echo "call by workflow_call"
+ image_ref=${{ inputs.ref }}
+ image_suffix=${{ inputs.suffix }}
+ elif ${{ github.event_name == 'workflow_dispatch' }} ; then
+ echo "call by workflow_dispatch"
+ image_ref=${{ github.event.inputs.ref }}
+ image_suffix=${{ github.event.inputs.suffix }}
+ else
+ echo "unexpected event: ${{ github.event_name }}"
+ exit 1
+ fi
+ echo "ref=${image_ref}" >> $GITHUB_ENV
+ [ -n "${image_suffix}" ] && echo "suffix=-${image_suffix}" >> $GITHUB_ENV
+ if ${{ inputs.tagoverride != '' }} ; then
+ echo "imagetag=${{ inputs.tagoverride }}" >> $GITHUB_ENV
+ else
+ # it is invalid if the ref is branch name "xx/xx/xx"
+ echo "imagetag=${image_ref##*/}" >> $GITHUB_ENV
+ fi
+ exit 0
+
+
+ - name: Checkout code
+ uses: actions/checkout@v5
+ with:
+ persist-credentials: false
+ ref: ${{ steps.ref.outputs.ref }}
+
+ - name: Checkout submodule
+ uses: Mushus/checkout-submodule@v1.0.1
+ with:
+ basePath: # optional, default is .
+ submodulePath: libvgpu
+
+ - name: Lint Dockerfile
+ run: |
+ make lint_dockerfile
+
+ - name: Docker Login
+ uses: docker/login-action@v3.5.0
+ with:
+ username: ${{ secrets.DOCKERHUB_TOKEN }}
+ password: ${{ secrets.DOCKERHUB_PASSWD }}
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ with:
+ driver-opts: image=moby/buildkit:master
+
+ - name: Extract metadata (tags, labels) for Docker
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_REPO_HAMICORE }}
+
+ #- name: Build & Pushing hami image
+ # uses: docker/build-push-action@v6.18.0
+ # with:
+ # context: .
+ # file: ${{ env.IMAGE_ROOT_PATH }}/Dockerfile
+ # labels: ${{ steps.meta.outputs.labels }}
+ # platforms: ${{ env.BUILD_PLATFORM }}
+ # build-args: |
+ # VERSION=${{ env.ref }}
+ # GOLANG_IMAGE=golang:1.22.5-bullseye
+ # NVIDIA_IMAGE=nvidia/cuda:12.2.0-devel-ubuntu20.04
+ # DEST_DIR=/usr/local
+ # tags: ${{ steps.meta.outputs.tags }}
+ # push: true
+ # github-token: ${{ env.REGISTER_PASSWORD }}
+
+ - name: Extract metadata (tags, labels) for Docker
+ id: meta1
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_REPO }}
+
+ - name: Build & Pushing hami image
+ uses: docker/build-push-action@v6.18.0
+ with:
+ context: .
+ file: ${{ env.IMAGE_ROOT_PATH }}/Dockerfile.hamimaster
+ labels: ${{ steps.meta1.outputs.labels }}
+ platforms: ${{ env.BUILD_PLATFORM }}
+ build-args: |
+ VERSION=${{ env.ref }}
+ GOLANG_IMAGE=golang:1.22.5-bullseye
+ HAMICORE_IMAGE=${{ steps.meta.outputs.tags }}
+ DEST_DIR=/usr/local
+ tags: ${{ steps.meta1.outputs.tags }}
+ push: true
+ github-token: ${{ env.REGISTER_PASSWORD }}
+
diff --git a/.github/workflows/call-release-notes.yaml b/.github/workflows/call-release-notes.yaml
new file mode 100644
index 000000000..966399527
--- /dev/null
+++ b/.github/workflows/call-release-notes.yaml
@@ -0,0 +1,76 @@
+name: save release notes
+
+
+on:
+ workflow_call:
+ inputs:
+ ref:
+ required: true
+ type: string
+
+permissions: write-all
+
+jobs:
+ generate-release-notes:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Download Artifact
+ uses: actions/download-artifact@v5
+ with:
+ name: chart_package_artifact
+ path: charts/
+ # generate release notes for the new release branch
+ - name: upload chart tgz to release
+ if: startsWith(github.ref, 'refs/tags/')
+ uses: softprops/action-gh-release@v2
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ files: "charts/*.tgz"
+ - name: Checkout
+ uses: actions/checkout@v5
+ with:
+ fetch-depth: 0
+ ref: master
+
+ - name: Create release notes markdown
+ run: |
+ echo "something need to fix https://github.com/wawa0210/HAMi/actions/runs/11008166155/job/30565367256"
+ # 1. get pre-release id
+ # pre_release_id=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${GITHUB_REPOSITORY}/releases | yq '.[] | select(.prerelease == true) | .id')
+ # echo "pre_release_id: ${pre_release_id}"
+
+ # 2. update release status from pre-release to latest-release
+ # curl -L \
+ # -X PATCH \
+ # -H "Accept: application/vnd.github+json" \
+ # -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+ # -H "X-GitHub-Api-Version: 2022-11-28" \
+ # https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/${pre_release_id} \
+ # -d '{"draft":false,"prerelease":false,"make_latest":"true"}'
+
+ # 3. generate release notes content for new release
+ # timeout need to fix, ref https://github.com/wawa0210/HAMi/actions/runs/11008166155/job/30565367256
+ # latest_tag=${GITHUB_REF#refs/*/}
+ # curl -o result.json \
+ # -H "Accept: application/vnd.github+json" \
+ # -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+ # https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${latest_tag}
+
+ # cat result.json | yq '.body' > docs/CHANGELOG/${latest_tag}.md
+
+ # rm -rf result.json
+
+
+ # - name: Push release notes
+ # id: push_directory
+ # uses: cpina/github-action-push-to-another-repository@v1.7.2
+ # env:
+ # API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
+ # with:
+ # source-directory: .
+ # destination-github-username: ${{ github.repository_owner }}
+ # destination-repository-name: hami
+ # user-email: xiaozhang0210@hotmail.com
+ # commit-message: upgrade ORIGIN_COMMIT from $GITHUB_REF
+ # target-branch: master
diff --git a/.github/workflows/call-release-website.yaml b/.github/workflows/call-release-website.yaml
new file mode 100644
index 000000000..518c4c2fb
--- /dev/null
+++ b/.github/workflows/call-release-website.yaml
@@ -0,0 +1,18 @@
+name: Call Release webiste
+
+on:
+ workflow_call:
+ inputs:
+ ref:
+ required: true
+ type: string
+permissions: write-all
+
+jobs:
+ build-website:
+ runs-on: ubuntu-latest
+ steps:
+ - name: release hami website
+ # https://github.com/actions/virtual-environments/issues/709
+ run: |
+ echo "Need to publish hami website"
diff --git a/.github/workflows/ci-image-scanning.yaml b/.github/workflows/ci-image-scanning.yaml
new file mode 100644
index 000000000..0f8b59700
--- /dev/null
+++ b/.github/workflows/ci-image-scanning.yaml
@@ -0,0 +1,76 @@
+name: Trivy Scan
+on:
+ schedule:
+ - cron: "0 0 * * *"
+jobs:
+ trivy-scan:
+ permissions:
+ contents: read # for actions/checkout to fetch code
+ security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+ actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v5
+ - name: Free disk space
+ # https://github.com/actions/virtual-environments/issues/709
+ run: |
+ echo "=========original CI disk space"
+ df -h
+ sudo rm -rf "/usr/local/share/boost"
+ sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+ echo "=========after clean up, the left CI disk space"
+ df -h
+ - name: Checkout submodule
+ uses: Mushus/checkout-submodule@v1.0.1
+ with:
+ basePath: # optional, default is .
+ submodulePath: libvgpu
+ - name: Get branch name
+ uses: nelonoel/branch-name@v1.0.1
+ - name: Docker Login
+ uses: docker/login-action@v3.5.0
+ with:
+ username: ${{ secrets.DOCKERHUB_TOKEN }}
+ password: ${{ secrets.DOCKERHUB_PASSWD }}
+ - name: Set up Docker Buildx
+ id: buildx
+ uses: docker/setup-buildx-action@v3
+ - name: install Go
+ uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ - name: Generating image tag
+ id: runtime-tag
+ run: |
+ echo tag="$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+ - run: make tidy
+ - run: SHORT_VERSION=${{ steps.runtime-tag.outputs.tag }} bash ./hack/build.sh
+ - name: Run Trivy vulnerability scanner (table output)
+ # Prevent running from the forked repository that doesn't need to upload coverage.
+ # In addition, running on the forked repository would fail as missing the necessary secret.
+ if: ${{ github.repository == 'Project-HAMi/HAMi' }}
+ uses: aquasecurity/trivy-action@0.32.0
+ with:
+ image-ref: "projecthami/hami:${{ steps.runtime-tag.outputs.tag }}"
+ format: "table"
+ ignore-unfixed: true
+ severity: "CRITICAL"
+ vuln-type: "os,library"
+ trivyignores: .trivyignore
+ - name: Run Trivy vulnerability scanner (SARIF)
+ uses: aquasecurity/trivy-action@0.32.0
+ with:
+ image-ref: "projecthami/hami:${{ steps.runtime-tag.outputs.tag }}"
+ format: "sarif"
+ output: "trivy-results.sarif"
+ ignore-unfixed: true
+ severity: "CRITICAL"
+ vuln-type: "os,library"
+ trivyignores: .trivyignore
+ if: always() && github.repository == 'Project-HAMi/HAMi'
+ - name: Upload Trivy scan results to GitHub Security tab
+ uses: github/codeql-action/upload-sarif@v3
+ with:
+ sarif_file: "trivy-results.sarif"
+ if: always() && github.repository == 'Project-HAMi/HAMi'
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
new file mode 100644
index 000000000..363788f19
--- /dev/null
+++ b/.github/workflows/ci.yaml
@@ -0,0 +1,253 @@
+name: CI Workflow
+on:
+ # Run this workflow every time a new commit pushed to upstream/fork repository.
+ # Run workflow on fork repository will help contributors find and resolve issues before sending a PR.
+ pull_request:
+ push:
+ # Exclude branches created by Dependabot to avoid triggering current workflow
+ # for PRs initiated by Dependabot.
+ branches-ignore:
+ - 'dependabot/**'
+permissions:
+ contents: read # for actions/checkout to fetch code
+
+env:
+ REGISTRY: docker.io
+ IMAGE_REPO: ${{ secrets.IMAGE_REPO || 'projecthami/hami' }}
+ IMAGE_REPO_HAMICORE: ${{ secrets.IMAGE_REPO || 'projecthami/hamicore' }}
+ IMAGE_ROOT_PATH: docker
+ BUILD_PLATFORM: linux/arm64,linux/amd64
+ REGISTER_USER: ${{ github.actor }}
+ REGISTER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+ lint:
+ name: lint
+ runs-on: ubuntu-22.04
+ steps:
+ - name: checkout code
+ uses: actions/checkout@v5
+ - name: install Go
+ uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ - name: verify license
+ run: hack/verify-license.sh
+ - name: go tidy
+ run: make tidy
+ - name: golangci-lint
+ uses: golangci/golangci-lint-action@v7
+ with:
+ version: v2.0
+ - name: import alias
+ run: hack/verify-import-aliases.sh
+
+ get_info:
+ name: get_info
+ runs-on: ubuntu-22.04
+ outputs:
+ version: ${{ steps.parse_version.outputs.version }}
+ e2e_run: ${{ steps.parse_code.outputs.e2e_run }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v5
+ with:
+ fetch-depth: 3
+
+ - name: Parse the version
+ id: parse_version
+ run: |
+ value=$(make -f Makefile.defs print-version)
+ suffix="$(git rev-parse --short HEAD)"
+ tag="${value}-${suffix}"
+ echo "Generated tag: ${tag}"
+ echo "version=${tag}" >> $GITHUB_OUTPUT
+
+ # Check the code diff, ignore the docs and examples change
+ - name: Parse the code diff
+ id: parse_code
+ run: |
+ pr_commit=${{ github.event.pull_request.base.sha }}
+ base_commit=${{ github.sha }}
+ echo "PR commit: ${pr_commit}"
+ echo "Git log: $(git log ${pr_commit})"
+ echo "Base commit: ${base_commit}"
+ echo "Git log: $(git log ${base_commit})"
+ changed_files=$(git diff --name-only ${pr_commit} ${base_commit} -- . ':(exclude)docs/*' ':(exclude)examples/*' ':(exclude)*.md')
+ echo "Changed files: ${changed_files}"
+ if [ -n "$changed_files" ]; then
+ echo "Will run e2e-test"
+ echo "e2e_run=true" >> $GITHUB_OUTPUT
+ else
+ echo "Won't run e2e-test"
+ echo "e2e_run=false" >> $GITHUB_OUTPUT
+ fi
+
+ test:
+ name: Unit test
+ needs: [ lint, get_info ] # rely on lint & get_info successful completion
+ runs-on: ubuntu-22.04
+ if: needs.get_info.outputs.e2e_run == 'true' || github.ref == 'refs/heads/master'
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v5
+ - name: Checkout submodule
+ uses: Mushus/checkout-submodule@v1.0.1
+ with:
+ basePath: # optional, default is .
+ submodulePath: libvgpu
+ - name: Install Go
+ uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ - run: make tidy
+ - run: make test
+ - name: Upload coverage to Codecov
+ # Prevent running from the forked repository that doesn't need to upload coverage.
+ # In addition, running on the forked repository would fail as missing the necessary secret.
+ if: ${{ github.repository == 'Project-HAMi/HAMi' }}
+ uses: codecov/codecov-action@v5
+ with:
+ # Even though token upload token is not required for public repos,
+ # but adding a token might increase successful uploads as per:
+ # https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954
+ token: ${{secrets.CODECOV_TOKEN}}
+ files: ./_output/coverage/coverage.out
+ flags: unittests
+ fail_ci_if_error: false
+ verbose: true
+
+ build:
+ name: Compile
+ needs: [ test, get_info ]
+ runs-on: ubuntu-22.04
+ if: needs.get_info.outputs.e2e_run == 'true'
+ steps:
+ - uses: actions/checkout@master
+ - name: Free disk space
+ # https://github.com/actions/virtual-environments/issues/709
+ run: |
+ echo "=========original CI disk space"
+ df -h
+ sudo rm -rf "/usr/local/share/boost"
+ sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+ echo "=========after clean up, the left CI disk space"
+ df -h
+
+ - name: Checkout code
+ uses: actions/checkout@v5
+ with:
+ fetch-depth: 0
+
+ - name: Checkout submodule
+ uses: Mushus/checkout-submodule@v1.0.1
+ with:
+ basePath: # optional, default is .
+ submodulePath: libvgpu
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ with:
+ driver-opts: image=moby/buildkit:master
+
+ - name: Extract metadata (tags, labels) for Docker
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_REPO }}
+
+ - name: Build & Pushing hami image
+ uses: docker/build-push-action@v6.18.0
+ with:
+ context: .
+ file: ${{ env.IMAGE_ROOT_PATH }}/Dockerfile
+ labels: ${{ needs.get_info.outputs.version }}
+ build-args: |
+ VERSION=${{ needs.get_info.outputs.version }}
+ GOLANG_IMAGE=golang:1.24.6-bullseye
+ NVIDIA_IMAGE=nvidia/cuda:12.2.0-devel-ubuntu20.04
+ DEST_DIR=/usr/local
+ tags: ${{ env.REGISTRY }}/${{ env.IMAGE_REPO }}:${{ needs.get_info.outputs.version }}
+ push: false
+ load: true
+ github-token: ${{ env.REGISTER_PASSWORD }}
+
+ - name: List images
+ run: |
+ docker images
+
+ - name: Save Docker image to image.tar
+ run: |
+ docker save ${{ env.REGISTRY }}/${{ env.IMAGE_REPO }}:${{ needs.get_info.outputs.version }} -o image.tar
+
+ - name: Upload image.tar as artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: hami-image
+ path: image.tar
+ retention-days: 5
+ if-no-files-found: error
+
+ # packages tgz from /charts of original branch, deploy to /charts of target branch
+ package_chart:
+ runs-on: ubuntu-22.04
+ needs: [ test, get_info ]
+ if: needs.get_info.outputs.e2e_run == 'true'
+ env:
+ HELM_VERSION: v3.8.1
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v5
+ with:
+ fetch-depth: 0
+ ref: ${{ needs.get_ref.outputs.ref }}
+
+ - name: Configure Git
+ run: |
+ git config user.name "$GITHUB_ACTOR"
+ git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+ - name: Install Helm
+ uses: azure/setup-helm@v4
+ with:
+ version: ${{ env.HELM_VERSION }}
+
+ - name: Lint helm chart
+ run: |
+ make lint_chart
+
+ - name: Package Chart
+ continue-on-error: false
+ env:
+ VERSION: ${{ needs.get_info.outputs.version }}
+ run: |
+ cd charts
+ make clean
+ make
+ if ! ls *.tgz &>/dev/null ; then
+ echo "failed to generate chart"
+ exit 1
+ fi
+ cd ..
+ mkdir -p tmp
+ mv charts/*.tgz tmp
+
+ - name: Upload Artifact
+ uses: actions/upload-artifact@v4.4.3
+ with:
+ name: chart_package_artifact
+ path: tmp/*
+ retention-days: 5
+ if-no-files-found: error
+
+ # execute a full e2e test when hami code merge
+ e2e_test:
+ uses: ./.github/workflows/call-e2e.yaml
+ needs: [ package_chart, get_info, build ]
+ if: needs.get_info.outputs.e2e_run == 'true'
+ with:
+ ref: ${{ needs.get_info.outputs.version }}
+ type: "pullrequest"
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
new file mode 100644
index 000000000..98b669df6
--- /dev/null
+++ b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,65 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL"
+
+on:
+ workflow_dispatch:
+ push:
+ branches: ["master","dev"]
+ paths-ignore:
+ - "**/*.json"
+ - "**/*.md"
+ - "**/*.txt"
+ - "**/*.yml"
+ schedule:
+ - cron: "0 4 * * 6"
+
+permissions:
+ security-events: write
+ # required to fetch internal or private CodeQL packs
+ packages: read
+
+ # only required for workflows in private repositories
+ actions: read
+ contents: read
+
+jobs:
+ analyze:
+ name: Analyze
+ runs-on: ubuntu-latest
+ if: github.repository == 'Project-HAMi/HAMi'
+
+ strategy:
+ fail-fast: false
+ matrix:
+ language: ["go"]
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v5
+ - name: Checkout submodule
+ uses: Mushus/checkout-submodule@v1.0.1
+ with:
+ basePath: # optional, default is .
+ submodulePath: libvgpu
+ - if: matrix.language == 'go'
+ name: Set go version
+ uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+
+ # Initializes the CodeQL tools for scanning.
+ - name: Initialize CodeQL
+ uses: github/codeql-action/init@v3
+ with:
+ languages: ${{ matrix.language }}
+ # If you wish to specify custom queries, you can do so here or in a config file.
+ # By default, queries listed here will override any specified in a config file.
+ # Prefix the list here with "+" to use these queries and those in the config file.
+ # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+ - name: Perform CodeQL Analysis
+ uses: github/codeql-action/analyze@v3
diff --git a/.github/workflows/issue-translate.yaml b/.github/workflows/issue-translate.yaml
new file mode 100644
index 000000000..0c344929d
--- /dev/null
+++ b/.github/workflows/issue-translate.yaml
@@ -0,0 +1,15 @@
+name: 'issue-translate'
+on:
+ issue_comment:
+ types: [created]
+ issues:
+ types: [opened]
+
+jobs:
+ translate: # make sure the action works on a clean machine without building
+ runs-on: ubuntu-latest
+ steps:
+ - uses: tomsun28/issues-translate-action@v2.7
+ with:
+ IS_MODIFY_TITLE: true
+ BOT_GITHUB_TOKEN: ${{ secrets.TRANSLATE_ROBOT_TOKEN }}
diff --git a/.github/workflows/lint-chart.yaml b/.github/workflows/lint-chart.yaml
new file mode 100644
index 000000000..de7f2416a
--- /dev/null
+++ b/.github/workflows/lint-chart.yaml
@@ -0,0 +1,31 @@
+name: Chart Lint
+
+on:
+ push:
+ # Exclude branches created by Dependabot to avoid triggering current workflow
+ # for PRs initiated by Dependabot.
+ branches-ignore:
+ - 'dependabot/**'
+ pull_request:
+ paths:
+ - "charts/**"
+
+jobs:
+ chart-lint-test:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v5
+ with:
+ fetch-depth: 0
+
+ - name: Set up Helm
+ uses: azure/setup-helm@v4
+ with:
+ version: v3.7.1
+ - name: Lint Chart
+ run: |
+ make lint_chart
+ - name: Check chart version
+ run: bash ./hack/verify-chart-version.sh
+
diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml
new file mode 100644
index 000000000..00440c2ae
--- /dev/null
+++ b/.github/workflows/stale.yaml
@@ -0,0 +1,23 @@
+name: 'Close stale issues and PRs'
+on:
+ schedule:
+ - cron: '0 0 * * 2'
+
+jobs:
+ stale:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/stale@v9
+ with:
+ days-before-issue-stale: 30
+ stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.'
+ days-before-issue-close: 5
+ close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.'
+ days-before-pr-stale: 45
+ stale-pr-message: 'This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days.'
+ days-before-pr-close: 10
+ close-pr-message: 'This PR was closed because it has been stalled for 10 days with no activity.'
+ exempt-issue-labels: pinned,wip,security,notice
+ exempt-draft-pr: true
+ debug-only: true
+ enable-statistics: true
diff --git a/.github/workflows/test-self-hosted.yaml b/.github/workflows/test-self-hosted.yaml
new file mode 100644
index 000000000..cec1c35e7
--- /dev/null
+++ b/.github/workflows/test-self-hosted.yaml
@@ -0,0 +1,21 @@
+name: Test self-hosted-runner
+
+on:
+ push:
+ # Exclude branches created by Dependabot to avoid triggering current workflow
+ # for PRs initiated by Dependabot.
+ branches-ignore:
+ - 'dependabot/**'
+ pull_request:
+ paths:
+ - "charts/**"
+
+jobs:
+ e2e:
+ runs-on: self-hosted
+ steps:
+ - name: e2e test
+ # https://github.com/actions/virtual-environments/issues/709
+ run: |
+ echo "Need to add e2e test"
+
diff --git a/.gitignore b/.gitignore
index e660fd93d..1de20116f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,15 @@
bin/
+run_device_plugin.sh
+run_scheduler.sh
+device_plugin.sh
+libvgpu/build
+updateso.sh
+libvgpu.so
+.idea
+.vscode
+vendor
+license
+vgpuvalidator
+_output/
+coverage.out
+.DS_Store
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
deleted file mode 100644
index 8765cac77..000000000
--- a/.gitlab-ci.yml
+++ /dev/null
@@ -1,87 +0,0 @@
-stages:
- - build_image
- - deploy
-
-variables:
- IMAGE_NAME: k8s-vgpu
-
-.build_image:
- stage: build_image
- image: '${DIND_IMAGE}'
- script:
- - IMAGE_FULL_NAME=${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}
- - >
- docker build -t ${IMAGE_FULL_NAME}
- --build-arg GOLANG_IMAGE=${GOLANG_IMAGE}
- --build-arg NVIDIA_IMAGE=${NVIDIA_IMAGE}
- --build-arg VERSION=${VERSION}
- --build-arg GOPROXY=${GOPROXY} -f ./docker/Dockerfile .
- - docker push ${IMAGE_FULL_NAME}
-
-build_dev_image:
- extends: .build_image
- variables:
- IMAGE_TAG: ${CI_COMMIT_SHA}
- VERSION: ${CI_COMMIT_SHA}
- only:
- - master
-
-build_release_image:
- extends: .build_image
- variables:
- IMAGE_TAG: ${CI_COMMIT_TAG}
- VERSION: ${CI_COMMIT_TAG}-${CI_COMMIT_SHA}
- only:
- - tags
-
-.deploy:
- stage: deploy
- image: '${HELM_IMAGE}'
- variables:
- RELEASE_NAME: vgpu
- RELEASE_NAMESPACE: vgpu
- EXTRA_ARGS: ''
- script:
- - IMAGE_FULL_NAME=${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}
- - >
- helm upgrade --install ${RELEASE_NAME} ./deployments/4pd-vgpu
- -n ${RELEASE_NAMESPACE}
- --set scheduler.extender.image=${IMAGE_FULL_NAME}
- --set devicePlugin.image=${IMAGE_FULL_NAME}
- ${EXTRA_ARGS}
-
-deploy_develop:
- extends: .deploy
- variables:
- IMAGE_TAG: ${CI_COMMIT_SHA}
- environment:
- name: vgpu-develop
- only:
- - master
- tags:
- - deploy-test
-
-deploy_pre_product:
- extends: .deploy
- variables:
- IMAGE_TAG: ${CI_COMMIT_TAG}
- EXTRA_ARGS: "--wait --timeout=30m"
- environment:
- name: vgpu-develop
- only:
- - tags
- tags:
- - deploy-test
-
-deploy_product:
- extends: .deploy
- variables:
- IMAGE_TAG: ${CI_COMMIT_TAG}
- environment:
- name: vgpu-product
- only:
- - tags
- tags:
- - deploy-product
- when: manual
-
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 000000000..0f8525526
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,4 @@
+[submodule "libvgpu"]
+ path = libvgpu
+ url = https://github.com/Project-HAMi/HAMi-core.git
+ branch = main
diff --git a/.golangci.yaml b/.golangci.yaml
new file mode 100644
index 000000000..f9a91cfe7
--- /dev/null
+++ b/.golangci.yaml
@@ -0,0 +1,88 @@
+version: "2"
+run:
+ concurrency: 4
+ modules-download-mode: readonly
+output:
+ formats:
+ text:
+ path: stdout
+ print-linter-name: true
+ print-issued-lines: true
+ colors: true
+linters:
+ default: none
+ enable:
+ - asciicheck
+ - forcetypeassert
+ - godot
+ - misspell
+ - staticcheck
+ settings:
+ dupl:
+ threshold: 800
+ errcheck:
+ check-type-assertions: true
+ check-blank: true
+ errorlint:
+ errorf: true
+ asserts: true
+ comparison: true
+ goconst:
+ min-len: 3
+ min-occurrences: 3
+ gocritic:
+ disabled-checks:
+ - commentedOutCode
+ - whyNoLint
+ enabled-tags:
+ - diagnostic
+ - experimental
+ - opinionated
+ - performance
+ - style
+ settings:
+ hugeParam:
+ sizeThreshold: 80
+ rangeExprCopy:
+ sizeThreshold: 512
+ rangeValCopy:
+ sizeThreshold: 128
+ gocyclo:
+ min-complexity: 20
+ godot:
+ scope: declarations
+ capital: false
+ nestif:
+ min-complexity: 20
+ exclusions:
+ generated: lax
+ presets:
+ - comments
+ - common-false-positives
+ - legacy
+ - std-error-handling
+ paths:
+ - third_party$
+ - builtin$
+ - examples$
+ - pkg/device-plugin
+issues:
+ uniq-by-line: true
+formatters:
+ enable:
+ - gofmt
+ - goimports
+ settings:
+ gofmt:
+ simplify: true
+ gofumpt:
+ extra-rules: true
+ goimports:
+ local-prefixes:
+ - github.com/Project-HAMi/HAMi
+ exclusions:
+ generated: lax
+ paths:
+ - third_party$
+ - builtin$
+ - examples$
diff --git a/.trivyignore b/.trivyignore
new file mode 100644
index 000000000..e69de29bb
diff --git a/AUTHORS.md b/AUTHORS.md
new file mode 100644
index 000000000..280e651b6
--- /dev/null
+++ b/AUTHORS.md
@@ -0,0 +1,24 @@
+# Contributors
+
+- Please check [HAMi Community Membership](https://github.com/Project-HAMi/community/blob/main/community-membership.md) to find how to be a contributor.
+- Here is the full list of the [MAINTAINERS](./MAINTAINERS.md).
+
+The following people, in alphabetical order, have either authored or signed off on commits in the HAMi repository:
+
+
+| Contributor | Email |
+|-----------------|-----------|
+| [archlitchi](https://github.com/archlitchi) | archlitchi@gmail.com|
+| [atttx123](https://github.com/atttx123) | - |
+| [chaunceyjiang](https://github.com/chaunceyjiang) | chaunceyjiang@gmail.com|
+| [CoderTH](https://github.com/CoderTH) | - |
+| [fyp711](https://github.com/fyp711) | - |
+| [gsakun](https://github.com/gsakun) | - |
+| [lengrongfu](https://github.com/lengrongfu) | - |
+| [ouyangluwei](https://github.com/ouyangluwei163) | ouyangluwei@riseunion.io |
+| peizhaoyou | peizhaoyou@4paradigm.com |
+| [wawa0210](https://github.com/wawa0210) | xiaozhang0210@hotmail.com |
+| [whybeyoung](https://github.com/whybeyoung) | - |
+| [yangshiqi](https://github.com/yangshiqi) | yangshiqi@riseunion.io |
+| [yinyu](https://github.com/Nimbus318) | nimbus-nimo@proton.me |
+| zhengbingxian | - |
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 000000000..cd21797f5
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,236 @@
+# CHANGELOG
+
+## v1.0.1
+
+**Add MIG support:"mixed strategy"**
+
+**Add support for kubernetes v1.22+**
+
+## v1.0.1.1
+
+**Bugs fixed**
+
+a pod can be scheduled to a node where its core usage is 100 - Fixed
+
+cudevshr.cache can't be modified with non-root users - Fixed
+
+## v1.0.1.2
+
+**Add custom resource name**
+
+A task with cores=100 will allocate all device memory(virtual device memory excluded)
+
+## v1.0.1.3
+
+**nvidia.com/gpucores will limit the GPU utilization inside container**
+Prior than v1.0.1.3, nvidia.com/gpucores will not limit utilization inside container, we have fixed it in v1.0.1.3
+
+## v1.0.1.4
+
+**Add nvidia.com/gpumem-percentage resoure name**
+This resource indicates the device memory percentage of GPU, can not be used with "nvidia.com/gpumem". If you want an exclusive GPU, specify both the "nvidia.com/gpucores" and "nvidia.com/gpumem-percentage" to 100
+
+**Add GPU type specification**
+You can set "nvidia.com/use-gputype" annotation to specify which type of GPU to use. "nvidia.com/nouse-gputype" annotation to specify which type of GPU to avoid.
+
+## v1.0.1.5
+
+Fix an monitor "desc not found" error
+
+Add "devicePlugin.sockPath" parameter to set the location of vgpu.sock
+
+## v1.1.0.0
+
+**Major Update: Device Memory will be counted more accurately**
+serveral device memory usage, including cuda context, modules, parameters, reserved addresses will be counted in v1.1.0.0
+
+**Update to be compatable with CUDA 11.6 and Driver 500+**
+
+**Rework monitor strategy**
+Monitor will mmap control file into address space instead of reading it in each query.
+
+## v1.1.1.0
+
+**Fix segmentation fault when invoking cuMallocAsync**
+
+**Core Utilization Oversubscribe and priority-base scheduling**
+Currently we have two priority, 0 for high and 1 for low. The core utilization of high priority task won't be limited to resourceCores unless sharing GPU node with other high priority tasks.
+The core utilization of low priority task won't be limited to resourceCores if no other tasks sharing its GPU.
+See exmaple.yaml for more details
+
+**Add Container Core Utilization policy**
+See details in docs/config.md(docs/config_cn.md)
+
+## v2.2
+
+**Update device memory counting mechanism to compat with CUDA 11.3+ task**
+sometimes vgpu-scheduler won't able to collect device memory usage when running cuda11.3+ compiled tasks in v1.x version of vgpu-scheduler. We solve this problem by reworking device memory counting mechanism.
+
+**Use node annotation instead of grpc to communicate between scheduler and device-plugin**
+In v1.x version of vgpu-scheduler, we use grpc to communicate between scheduler and device-plugin, but we reimplement this communication in v2.x by using node annotation, to make it more stable and readable.
+
+**modified nvidia-container-runtime is no longer needed**
+We remove self-modified nvidia-container-runtime in v1.x, because we now use node lock to track pod and container information. so this nvidia-container-runtime is no longer needed.
+
+## v2.2.7
+
+**BUG fix**
+
+fix tasks with "gpumem-percentage" not working properly
+
+fix dead lock when a process die with its lock not released
+
+**Adjust certain logs**
+
+**update go modules to more recent version in order to support k8s v1.25**
+
+## v2.2.8
+
+**BUG fix**
+
+fix vGPUmonitor not working properly with containerd
+
+fix installation error on k8s v1.25+
+
+## v2.2.9
+
+**BUG fix**
+
+fix non-root user in container can't access /tmp/vgpulock, result in "unified lock error"
+
+**Rework device registration**
+
+device registration used to be done in gRpc between device-plugin and scheduler. However, in some cluster, this communication may be blocked by firewall or selinux configuration. So, we reimplement device registration mechanism by using node annotations:
+A-device-plugin will put its usable device and its status in "Node-A-device-register" annotation
+scheduler will read from this annotation and acknowledge this registration. So, gRpc will no longer be used.
+
+**Optimization in code**
+
+Put nvidia-device-plugin related code in a seperate directory "nvidiadevice"
+
+**Libvgpu log adjusting**
+
+Downgrade the following API from LOG:WARN to LOG:INFO
+cuFuncSetCacheConfig, cuFuncSetCacheConfig ,cuModuleGetTexRef, cuModuleGetSurfRef
+
+## v2.2.10
+
+**BUG fix**
+
+fix process can't initialize properly in driver 440
+
+fix cuCtxCreate failed in some tensorRT task
+
+fix env CUDA_VISIBLE_DEVICES not working properly sometimes.
+
+## v2.2.11
+
+**BUG fix**
+
+fix initialization failed with non-root uers
+
+fix core limitation not working properly on A30
+
+## v2.2.12
+
+Downgrade core control output from LOG:WARN to LOG:DEBUG
+
+## v2.2.13
+
+Adjust default memory to 0, which means use 100% device memory
+
+Move cache file directory from /tmp/vgpu/containers to /usr/local/vgpu/containers
+
+## v2.2.14
+
+Fix device memory calculation error after container crashloop
+
+Fix env cuda_oversubscribe not set properly when MemoryScaling < 1
+
+Fix MemoryScaling not working when set < 1
+
+## v2.2.15
+
+Move shared-memory from from /tmp/xxx.cache to /usr/local/vgpu/xxx.cache inside container
+
+Add Deviceidx to scheduler monitor apis(31993)
+
+## v2.2.16
+
+Fix crash during initlization in vGPUmonitor
+
+# v2.3
+
+## v2.3.0
+
+Fix oom can't be triggered when loading module
+
+Update device-plugin version to v0.14.0
+
+## v2.3.1
+
+Fix a bug where a cuda process can't be launched properly
+
+## v2.3.2
+
+Remove node selector for scheduler
+
+Fix an issue where mlu device-plugin can't be launched properly
+
+Major rework on devices-related code
+
+Add support for hygon DCU device
+
+## v2.3.3
+
+Fix an issue where pod pending on nodes with multi-architect devices.
+
+## v2.3.4
+
+Fix an issue where 31993 port can't list all GPU nodes
+
+Add a switch on cuda_control by set env "CUDA_DISABLE_ENV=true" in container
+
+## v2.3.6
+
+Fix initialization error when using ray
+
+## v2.3.7
+
+Fix error when "N/A" is shown in command "nvidia-smi topo -m"
+Fix core utilization not working on some cases
+Adjust some documents
+
+## v2.3.8
+
+Fix device-plugin launch error on driver version < 500
+
+support manual config MutatingWebhookConfiguration failurePolicy
+
+add metrics bind address flag for scheduler
+
+Improved log messages
+
+fix: loss of metrics after vdeivce restart
+
+bugfix: device-plugin monitor serves too slowly in big cluster
+
+## v2.3.9
+
+Add support for iluvatar GPU devices
+
+Fix issue on "get_host_pid" func in HAMi-core
+
+Regular devices API, make it easier to add new devices
+
+## v2.3.10
+
+Fix issue where device-plugin failed to start
+
+## v2.3.11
+
+Add support for Ascend910B device
+
+Add "NVIDIA_VISIBLE_DEVICES=none" to none-gpu tasks
+
+
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 000000000..63314a55d
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,3 @@
+# HAMi Community Code of Conduct
+
+Please refer to our [HAMi Community Code of Conduct](https://github.com/Project-HAMi/community/blob/main/CODE-OF-CONDUCT.md).
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 000000000..14fc2888a
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,101 @@
+# Contributing
+
+Welcome to HAMi!
+
+- [Contributing](#contributing)
+- [Before you get started](#before-you-get-started)
+ - [Code of Conduct](#code-of-conduct)
+ - [Community Expectations](#community-expectations)
+- [Getting started](#getting-started)
+- [Your First Contribution](#your-first-contribution)
+ - [Find something to work on](#find-something-to-work-on)
+ - [Find a good first topic](#find-a-good-first-topic)
+ - [Work on an issue](#work-on-an-issue)
+ - [File an Issue](#file-an-issue)
+- [Contributor Workflow](#contributor-workflow)
+ - [Creating Pull Requests](#creating-pull-requests)
+ - [Code Review](#code-review)
+
+# Before you get started
+
+## Code of Conduct
+
+Please make sure to read and observe our [Code of Conduct](/CODE_OF_CONDUCT.md).
+
+## Community Expectations
+
+HAMi is a community project driven by its community which strives to promote a healthy, friendly and productive environment.
+
+# Getting started
+
+- Fork the repository on GitHub
+- Make your changes in your forked repository
+- Submit a Pull Request (PR)
+
+# Your First Contribution
+
+We will help you contribute in different areas such as filing issues, developing features, fixing critical bugs and
+getting your work reviewed and merged.
+
+If you have questions about the development process,
+feel free to [file an issue](https://github.com/Project-HAMi/HAMi/issues/new/choose).
+
+## Find something to work on
+
+We are always in need of help, whether it's fixing documentation, reporting bugs, or writing code.
+Look for places where best coding practices aren't followed, code refactoring is needed, or tests are missing.
+Here's how you can get started.
+
+### Find a good first topic
+
+There are [multiple repositories](https://github.com/Project-HAMi/) within the HAMi organization.
+Each repository has beginner-friendly issues marked as "good first issues".
+For example, [Project-HAMi/HAMi](https://github.com/Project-HAMi/HAMi) has
+[help wanted](https://github.com/Project-HAMi/HAMi/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22) and
+[good first issue](https://github.com/Project-HAMi/HAMi/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
+labels for issues that should not require deep knowledge of the system.
+We can help new contributors who wish to work on such issues.
+
+Another good way to contribute is to find documentation improvements, such as fixing missing or broken links.
+Please see [Contributing](#contributing) below for the workflow.
+
+#### Work on an issue
+
+When you are willing to take on an issue, simply reply to the issue and a maintainer will assign it to you.
+
+### File an Issue
+
+While we encourage everyone to contribute code, we also appreciate when someone reports an issue.
+Issues should be filed under the appropriate HAMi sub-repository.
+
+*Example:* A HAMi issue should be opened in [Project-HAMi/HAMi](https://github.com/Project-HAMi/HAMi/issues).
+
+Please follow the provided submission guidelines when opening an issue.
+
+# Contributor Workflow
+
+Please never hesitate to ask questions or submit a pull request.
+
+This is a rough outline of what a contributor's workflow looks like:
+
+- Create a topic branch from where you want to base the contribution (usually master)
+- Make commits of logical units
+- Push changes in your topic branch to your personal fork of the repository
+- Submit a pull request to [Project-HAMi/HAMi](https://github.com/Project-HAMi/HAMi)
+
+## Creating Pull Requests
+
+Pull requests are often called simply "PRs".
+HAMi generally follows the standard [GitHub pull request](https://help.github.com/articles/about-pull-requests/) process.
+To submit a proposed change, please develop the code/fix and add new test cases.
+Before submitting a pull request, run these local verifications to predict whether continuous integration will pass or fail:
+
+* Run and pass `make verify`
+
+## Code Review
+
+To make it easier for your PR to receive reviews, consider that reviewers will need you to:
+
+* Follow [good coding guidelines](https://github.com/golang/go/wiki/CodeReviewComments)
+* Write [good commit messages](https://chris.beams.io/posts/git-commit/)
+* Break large changes into a logical series of smaller patches which individually make easily understandable changes, and in aggregate solve a broader issue
diff --git a/DEPENDENCY.md b/DEPENDENCY.md
new file mode 100644
index 000000000..7fa4c122a
--- /dev/null
+++ b/DEPENDENCY.md
@@ -0,0 +1,46 @@
+# Environment Dependencies Policy
+
+## Purpose
+
+This policy establishes guidelines for managing third-party packages in the HAMi repository. Its goal is to ensure that all dependencies are secure, up-to-date, and necessary for the project’s functionality.
+
+## Scope
+
+This policy applies to all maintainers of the HAMi repository and governs all third-party packages incorporated into the project.
+
+## Policy
+
+Maintainers must adhere to the following when incorporating third-party packages:
+
+- **Necessity:** Include only those packages that are essential to the project’s functionality.
+- **Latest Stable Versions:** Use the latest stable releases whenever possible.
+- **Security:** Avoid packages with known security vulnerabilities.
+- **Version Pinning:** Lock all dependencies to specific versions to maintain consistency.
+- **Dependency Management:** Utilize an appropriate dependency management tool (e.g., Go modules, npm, pip) to handle third-party packages.
+- **Testing:** Ensure that any new dependency passes all automated tests before integration.
+
+## Procedure
+
+When adding a new third-party package, maintainers should:
+
+1. **Assess Need:** Determine whether the package is truly necessary for the project.
+2. **Conduct Research:** Review the package’s maintenance status and reputation within the community.
+3. **Select Version:** Opt for the latest stable version that meets the project’s requirements.
+4. **Pin the Version:** Explicitly pin the dependency to the chosen version within the repository.
+5. **Update Documentation:** Revise the project documentation to include details about the new dependency.
+
+## Archive/Deprecation
+
+If a third-party package becomes deprecated or discontinued, maintainers must promptly identify and integrate a suitable alternative while updating the documentation accordingly.
+
+## Enforcement
+
+Compliance with this policy is monitored by the HAMi maintainers. All dependency-related changes are subject to peer review to ensure adherence to these guidelines.
+
+## Exceptions
+
+Exceptions to this policy may be granted by the HAMi project lead on a case-by-case basis. Any exceptions must be documented with a clear rationale.
+
+## Credits
+
+This policy has been adapted and optimized based on guidelines from the [Kubescape Community](https://github.com/kubescape/kubescape/blob/master/docs/environment-dependencies-policy.md).
\ No newline at end of file
diff --git a/HAMi.jpg b/HAMi.jpg
new file mode 100644
index 000000000..53ecf9432
Binary files /dev/null and b/HAMi.jpg differ
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
new file mode 100644
index 000000000..13cf36221
--- /dev/null
+++ b/MAINTAINERS.md
@@ -0,0 +1,13 @@
+# Maintainers
+
+- Please check [HAMi Community Membership](https://github.com/Project-HAMi/community/blob/main/community-membership.md) to find how to level up through the project.
+- Please see [Contributors](./AUTHORS.md) for the full list of contributors to the project.
+
+## HAMi Committers
+
+| Maintainer | Email | Employer |
+|---------------------------------------------------|-----------|-----------|
+| [Li Mengxuan](https://github.com/archlitchi) | archlitchi@gmail.com | [dynamia.ai](https://www.dynamia.ai/) |
+| [Xiao Zhang](https://github.com/wawa0210) | xiaozhang0210@hotmail.com | [dynamia.ai](https://www.dynamia.ai/) |
+| [Wang Leibo](https://github.com/william-wang) | wang.platform@gmail.com | [HuaweiCloud](https://www.huaweicloud.com/) |
+| [Yin Yu](https://github.com/Nimbus318) | nimbus-nimo@proton.me | Independent Developer |
diff --git a/Makefile b/Makefile
index 81fd816a0..148b9ed8b 100644
--- a/Makefile
+++ b/Makefile
@@ -1,19 +1,89 @@
-GO=go
-GO111MODULE=on
-CMDS=scheduler device-plugin
-OUTPUT_DIR=bin
-
-VERSION ?= unknown
+##### Global variables #####
+include version.mk Makefile.defs
all: build
-build: $(CMDS)
+docker:
+ docker build \
+ --build-arg GOLANG_IMAGE=${GOLANG_IMAGE} \
+ --build-arg TARGET_ARCH=${TARGET_ARCH} \
+ --build-arg NVIDIA_IMAGE=${NVIDIA_IMAGE} \
+ --build-arg DEST_DIR=${DEST_DIR} \
+ --build-arg VERSION=${VERSION} \
+ --build-arg GOPROXY=https://goproxy.cn,direct \
+ . -f=docker/Dockerfile -t ${IMG_TAG}
+
+dockerwithlib:
+ docker build \
+ --no-cache \
+ --build-arg GOLANG_IMAGE=${GOLANG_IMAGE} \
+ --build-arg TARGET_ARCH=${TARGET_ARCH} \
+ --build-arg NVIDIA_IMAGE=${NVIDIA_IMAGE} \
+ --build-arg DEST_DIR=${DEST_DIR} \
+ --build-arg VERSION=${VERSION} \
+ --build-arg GOPROXY=https://goproxy.cn,direct \
+ . -f=docker/Dockerfile.withlib -t ${IMG_TAG}
+
+tidy:
+ $(GO) mod tidy
+
+proto:
+ $(GO) get github.com/gogo/protobuf/protoc-gen-gofast@v1.3.2
+ protoc --gofast_out=plugins=grpc:. ./pkg/api/*.proto
+
+build: $(CMDS) $(DEVICES)
$(CMDS):
- $(GO) build -ldflags '-s -w -X 4pd.io/k8s-vgpu/pkg/version.version=$(VERSION)' -o ${OUTPUT_DIR}/$@ ./cmd/$@
+ $(GO) build -ldflags '-s -w -X github.com/Project-HAMi/HAMi/pkg/version.version=$(VERSION)' -o ${OUTPUT_DIR}/$@ ./cmd/$@
+
+$(DEVICES):
+ $(GO) build -ldflags '-s -w -X github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/info.version=$(VERSION)' -o ${OUTPUT_DIR}/$@-device-plugin ./cmd/device-plugin/$@
clean:
$(GO) clean -r -x ./cmd/...
-rm -rf $(OUTPUT_DIR)
-.PHONY: all build clean $(CMDS)
+.PHONY: all build docker clean test $(CMDS)
+
+test:
+ mkdir -p ./_output/coverage/
+ bash hack/unit-test.sh
+
+lint:
+ bash hack/verify-staticcheck.sh
+
+.PHONY: verify
+verify:
+ hack/verify-all.sh
+
+.PHONY: lint_dockerfile
+lint_dockerfile:
+ @ docker run --rm \
+ -v $(ROOT_DIR)/.trivyignore:/.trivyignore \
+ -v /tmp/trivy:/root/trivy.cache/ \
+ -v $(ROOT_DIR):/tmp/src \
+ aquasec/trivy:$(TRIVY_VERSION) config --exit-code 1 --severity $(LINT_TRIVY_SEVERITY_LEVEL) /tmp/src/docker ; \
+ (($$?==0)) || { echo "error, failed to check dockerfile trivy" && exit 1 ; } ; \
+ echo "dockerfile trivy check: pass"
+
+.PHONY: lint_chart
+lint_chart:
+ @ docker run --rm \
+ -v $(ROOT_DIR)/.trivyignore:/.trivyignore \
+ -v /tmp/trivy:/root/trivy.cache/ \
+ -v $(ROOT_DIR):/tmp/src \
+ aquasec/trivy:$(TRIVY_VERSION) config --exit-code 1 --severity $(LINT_TRIVY_SEVERITY_LEVEL) /tmp/src/charts ; \
+ (($$?==0)) || { echo "error, failed to check chart trivy" && exit 1 ; } ; \
+ echo "chart trivy check: pass"
+
+.PHONY: e2e-env-setup
+e2e-env-setup:
+ ./hack/e2e-test-setup.sh
+
+.PHONY: helm-deploy
+helm-deploy:
+ ./hack/deploy-helm.sh "${E2E_TYPE}" "${KUBE_CONF}" "${HAMI_VERSION}"
+
+.PHONY: e2e-test
+e2e-test:
+ ./hack/e2e-test.sh "${E2E_TYPE}" "${KUBE_CONF}"
diff --git a/Makefile.defs b/Makefile.defs
new file mode 100644
index 000000000..ab0248161
--- /dev/null
+++ b/Makefile.defs
@@ -0,0 +1,28 @@
+
+SHELL := /bin/bash
+.SHELLFLAGS := -eu -o pipefail -c
+
+ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+
+INSTALL = install
+
+PREFIX?=/usr
+BINDIR?=$(PREFIX)/bin
+TARGETARCH ?= amd64
+
+DESTDIR_BIN ?= $(ROOT_DIR)/output/$(TARGETARCH)/bin
+DESTDIR_BASH_COMPLETION ?= $(ROOT_DIR)/output/$(TARGETARCH)/bash-completion
+
+VERSION?=""
+ifeq ($(VERSION), "")
+ VERSION=$(shell cat $(dir $(lastword $(MAKEFILE_LIST)))/VERSION)
+endif
+
+ECHO_GEN=echo " GEN $(RELATIVE_DIR)/"
+
+LINT_TRIVY_SEVERITY_LEVEL ?= CRITICAL
+TRIVY_VERSION=0.36.0
+
+.PHONY: print-version
+print-version:
+ @echo $(VERSION)
diff --git a/NOTICE.txt b/NOTICE.txt
new file mode 100644
index 000000000..3099570f8
--- /dev/null
+++ b/NOTICE.txt
@@ -0,0 +1,12 @@
+HAMi(https://project-hami.io/)
+Copyright HAMi Contributors
+
+This product includes software developed by
+NVIDIA CORPORATION (https://www.nvidia.com).
+Copyright (c) NVIDIA CORPORATION. All rights reserved.
+
+This product includes software developed by
+The HAMi Authors.
+Copyright 2024 The HAMi Authors.
+
+Both are licensed under the Apache License, Version 2.0.
diff --git a/OWNERS b/OWNERS
new file mode 100644
index 000000000..66757d7a3
--- /dev/null
+++ b/OWNERS
@@ -0,0 +1,10 @@
+reviewers:
+ - archlitchi
+ - chaunceyjiang
+ - lengrongfu
+ - ouyangluwei163
+ - shouren
+ - wawa0210
+approvers:
+ - archlitchi
+ - wawa0210
diff --git a/README.md b/README.md
new file mode 100644
index 000000000..85f4ea21f
--- /dev/null
+++ b/README.md
@@ -0,0 +1,193 @@
+English version | [中文版](README_cn.md)
+
+
+
+[](/LICENSE)
+[](https://github.com/Project-HAMi/HAMi/actions/workflows/ci.yaml)
+[](https://github.com/Project-HAMi/HAMi/releases/latest)
+[](https://www.bestpractices.dev/en/projects/9416)
+[](https://goreportcard.com/report/github.com/Project-HAMi/HAMi)
+[](https://codecov.io/gh/Project-HAMi/HAMi)
+[](https://app.fossa.com/projects/git%2Bgithub.com%2FProject-HAMi%2FHAMi?ref=badge_shield)
+[](https://hub.docker.com/r/projecthami/hami)
+[](https://cloud-native.slack.com/archives/C07T10BU4R2)
+[](https://github.com/Project-HAMi/HAMi/discussions)
+[](http://project-hami.io)
+[](https://github.com/Project-HAMi/HAMi#contact)
+
+## Project-HAMi: Heterogeneous AI Computing Virtualization Middleware
+
+## Introduction
+
+HAMi, formerly known as 'k8s-vGPU-scheduler', is a Heterogeneous device management middleware for Kubernetes. It can manage different types of heterogeneous devices (like GPU, NPU, etc.), share heterogeneous devices among pods, make better scheduling decisions based on topology of devices and scheduling policies.
+
+It aims to remove the gap between different Heterogeneous devices, and provide a unified interface for users to manage with no changes to their applications. As of December 2024, HAMi has been widely used not only in Internet, public cloud and private cloud, but also broadly adopted in various vertical industries including finance, securities, energy, telecommunications, education, and manufacturing. More than 50 companies or institutions are not only end users but also active contributors.
+
+
+
+HAMi is a sandbox and [landscape](https://landscape.cncf.io/?item=orchestration-management--scheduling-orchestration--hami) project of
+[Cloud Native Computing Foundation](https://cncf.io/)(CNCF),
+[CNAI Landscape project](https://landscape.cncf.io/?group=cnai&item=cnai--general-orchestration--hami).
+
+
+## Device virtualization
+
+HAMi provides device virtualization for several heterogeneous devices including GPU, by supporting device sharing and device resource isolation. For the list of devices supporting device virtualization, see [supported devices](#supported-devices)
+
+### Device sharing
+
+- Allows partial device allocation by specifying device core usage.
+- Allows partial device allocation by specifying device memory.
+- Imposes a hard limit on streaming multiprocessors.
+- Requires zero changes to existing programs.
+- Support [dynamic-mig](docs/dynamic-mig-support.md) feature, [example](examples/nvidia/dynamic_mig_example.yaml)
+
+
+
+### Device Resources Isolation
+
+A simple demonstration of device isolation:
+A task with the following resources will see 3000M device memory inside container:
+
+```yaml
+ resources:
+ limits:
+ nvidia.com/gpu: 1 # declare how many physical GPUs the pod needs
+ nvidia.com/gpumem: 3000 # identifies 3G GPU memory each physical GPU allocates to the pod
+```
+
+
+
+> Note:
+1. **After installing HAMi, the value of `nvidia.com/gpu` registered on the node defaults to the number of vGPUs.**
+2. **When requesting resources in a pod, `nvidia.com/gpu` refers to the number of physical GPUs required by the current pod.**
+
+### Supported devices
+
+[NVIDIA GPU](https://github.com/Project-HAMi/HAMi#preparing-your-gpu-nodes)
+[Cambricon MLU](docs/cambricon-mlu-support.md)
+[HYGON DCU](docs/hygon-dcu-support.md)
+[Iluvatar CoreX GPU](docs/iluvatar-gpu-support.md)
+[Moore Threads GPU](docs/mthreads-support.md)
+[HUAWEI Ascend NPU](https://github.com/Project-HAMi/ascend-device-plugin/blob/main/README.md)
+[MetaX GPU](docs/metax-support.md)
+
+## Architect
+
+
+
+HAMi consists of several components, including a unified mutatingwebhook, a unified scheduler extender, different device-plugins and different in-container virtualization technics for each heterogeneous AI devices.
+
+## Quick Start
+
+### Choose your orchestrator
+
+[](#prerequisites)
+[](docs/how-to-use-volcano-vgpu.md)
+
+### Prerequisites
+
+The list of prerequisites for running the NVIDIA device plugin is described below:
+
+- NVIDIA drivers >= 440
+- nvidia-docker version > 2.0
+- default runtime configured as nvidia for containerd/docker/cri-o container runtime
+- Kubernetes version >= 1.18
+- glibc >= 2.17 & glibc < 2.30
+- kernel version >= 3.10
+- helm > 3.0
+
+### Install
+
+First, Label your GPU nodes for scheduling with HAMi by adding the label "gpu=on". Without this label, the nodes cannot be managed by our scheduler.
+
+```
+kubectl label nodes {nodeid} gpu=on
+```
+
+Add our repo in helm
+
+```
+helm repo add hami-charts https://project-hami.github.io/HAMi/
+```
+
+Use the following command for deployment:
+
+```
+helm install hami hami-charts/hami -n kube-system
+```
+
+Customize your installation by adjusting the [configs](docs/config.md).
+
+Verify your installation using the following command:
+
+```
+kubectl get pods -n kube-system
+```
+
+If both `hami-device-plugin` (formerly known as `vgpu-device-plugin`) and `hami-scheduler` (formerly known as `vgpu-scheduler`) pods are in the *Running* state, your installation is successful. You can try examples [here](examples/nvidia/default_use.yaml)
+
+### WebUI
+
+[HAMi-WebUI](https://github.com/Project-HAMi/HAMi-WebUI) is available after HAMi v2.4
+
+For installation guide, click [here](https://github.com/Project-HAMi/HAMi-WebUI/blob/main/docs/installation/helm/index.md)
+
+### Monitor
+
+Monitoring is automatically enabled after installation. Obtain an overview of cluster information by visiting the following URL:
+
+```
+http://{scheduler ip}:{monitorPort}/metrics
+```
+
+The default monitorPort is 31993; other values can be set using `--set devicePlugin.service.httpPort` during installation.
+
+Grafana dashboard [example](docs/dashboard.md)
+
+> **Note** The status of a node won't be collected before you submit a task
+
+## Notes
+
+- If you don't request vGPUs when using the device plugin with NVIDIA images all the GPUs on the machine may be exposed inside your container
+- Currently, A100 MIG can be supported in only "none" and "mixed" modes.
+- Tasks with the "nodeName" field cannot be scheduled at the moment; please use "nodeSelector" instead.
+
+## RoadMap, Governance & Contributing
+
+The project is governed by a group of [Maintainers](./MAINTAINERS.md) and [Contributors](./AUTHORS.md). How they are selected and govern is outlined in our [Governance Document](https://github.com/Project-HAMi/community/blob/main/governance.md).
+
+If you're interested in being a contributor and want to get involved in developing the HAMi code, please see [CONTRIBUTING](CONTRIBUTING.md) for details on submitting patches and the contribution workflow.
+
+See [RoadMap](docs/develop/roadmap.md) to see anything you interested.
+
+## Meeting & Contact
+
+The HAMi community is committed to fostering an open and welcoming environment, with several ways to engage with other users and developers.
+
+If you have any questions, please feel free to reach out to us through the following channels:
+
+- Regular Community Meeting: Friday at 16:00 UTC+8 (Chinese)(weekly). [Convert to your timezone](https://www.thetimezoneconverter.com/?t=14%3A30&tz=GMT%2B8&).
+ - [Meeting Notes and Agenda](https://docs.google.com/document/d/1YC6hco03_oXbF9IOUPJ29VWEddmITIKIfSmBX8JtGBw/edit#heading=h.g61sgp7w0d0c)
+ - [Meeting Link](https://meeting.tencent.com/dm/Ntiwq1BICD1P)
+- Email: refer to the [MAINTAINERS.md](MAINTAINERS.md) to find the email addresses of all maintainers. Feel free to contact them via email to report any issues or ask questions.
+- [mailing list](https://groups.google.com/forum/#!forum/hami-project)
+- [slack](https://cloud-native.slack.com/archives/C07T10BU4R2) | [Join](https://slack.cncf.io/)
+
+## Talks and References
+
+| | Link |
+|------------------|-------------------------------------------------------------------------------------------------------------------------|
+| CHINA CLOUD COMPUTING INFRASTRUCTURE DEVELOPER CONFERENCE (Beijing 2024) | [Unlocking heterogeneous AI infrastructure on k8s clusters](https://live.csdn.net/room/csdnnews/3zwDP09S) Starting from 03:06:15 |
+| KubeDay(Japan 2024) | [Unlocking Heterogeneous AI Infrastructure K8s Cluster:Leveraging the Power of HAMi](https://www.youtube.com/watch?v=owoaSb4nZwg) |
+| KubeCon & AI_dev Open Source GenAI & ML Summit(China 2024) | [Is Your GPU Really Working Efficiently in the Data Center?N Ways to Improve GPU Usage](https://www.youtube.com/watch?v=ApkyK3zLF5Q) |
+| KubeCon & AI_dev Open Source GenAI & ML Summit(China 2024) | [Unlocking Heterogeneous AI Infrastructure K8s Cluster](https://www.youtube.com/watch?v=kcGXnp_QShs) |
+| KubeCon(EU 2024)| [Cloud Native Batch Computing with Volcano: Updates and Future](https://youtu.be/fVYKk6xSOsw) |
+
+## License
+
+HAMi is under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details.
+
+## Star History
+
+[](https://star-history.com/#Project-HAMi/HAMi&Date)
diff --git a/README_cn.md b/README_cn.md
new file mode 100644
index 000000000..89f6758b4
--- /dev/null
+++ b/README_cn.md
@@ -0,0 +1,191 @@
+[English version](README.md) | 中文版
+
+
+
+[](/LICENSE)
+[](https://github.com/Project-HAMi/HAMi/actions/workflows/ci.yaml)
+[](https://github.com/Project-HAMi/HAMi/releases/latest)
+[](https://www.bestpractices.dev/en/projects/9416)
+[](https://goreportcard.com/report/github.com/Project-HAMi/HAMi)
+[](https://codecov.io/gh/Project-HAMi/HAMi)
+[](https://app.fossa.com/projects/git%2Bgithub.com%2FProject-HAMi%2FHAMi?ref=badge_shield)
+[](https://hub.docker.com/r/projecthami/hami)
+[](https://cloud-native.slack.com/archives/C07T10BU4R2)
+[](https://github.com/Project-HAMi/HAMi/discussions)
+[](http://project-hami.io)
+[](https://github.com/Project-HAMi/HAMi#contact)
+
+## Project-HAMi：异构 AI 计算虚拟化中间件
+
+## 简介
+
+HAMi（前身为 'k8s-vGPU-scheduler'）是一个面向 Kubernetes 的异构设备管理中间件。它可以管理不同类型的异构设备（如 GPU、NPU、MLU、DCU 等），实现异构设备在 Pod 之间的共享，并基于设备拓扑和调度策略做出更优的调度决策。
+
+HAMi 旨在消除不同异构设备之间的差异，为用户提供统一的管理接口，无需对应用程序进行任何修改。截至 2024 年 12 月，HAMi 除了在互联网、公有云、私有云等领域外，在金融、证券、能源、运营商、教育、制造业等垂直领域，也得到了广泛采纳。超过 40 家企业和机构不仅是最终用户，同时也是活跃的贡献者。
+
+
+
+HAMi 是 [Cloud Native Computing Foundation](https://cncf.io/)(CNCF) 基金会的沙箱项目和 [landscape](https://landscape.cncf.io/?item=orchestration-management--scheduling-orchestration--hami) 项目，同时也是 [CNAI Landscape 项目](https://landscape.cncf.io/?group=cnai&item=cnai--general-orchestration--hami)。
+
+## 虚拟化能力
+
+HAMi 可为多种异构设备提供虚拟化功能，支持设备共享和资源隔离。关于支持设备虚拟化的设备列表，请参见[支持的设备](#支持的设备)。
+
+### 设备共享能力
+
+- 通过设置核心使用率（百分比），进行设备的部分分配
+- 通过设置显存（单位：MB），进行设备的部分分配
+- 对流式多处理器进行硬限制
+- 无需对现有程序进行任何修改
+- 支持[动态MIG切片](docs/dynamic-mig-support_cn.md)能力，[样例](examples/nvidia/dynamic_mig_example.yaml)
+
+
+
+### 设备资源隔离能力
+
+HAMi支持设备资源的硬隔离
+一个以 NVIDIA GPU 为例硬隔离的简单展示：
+```yaml
+ resources:
+ limits:
+ nvidia.com/gpu: 1 # 请求 1 个虚拟 GPU
+ nvidia.com/gpumem: 3000 # 每个虚拟 GPU 包含 3000M 设备内存
+```
+
+在容器内将看到 3G 设备内存
+
+
+
+> 注意：
+1. **安装HAMi后，节点上注册的 `nvidia.com/gpu` 值默认为vGPU数量**
+2. **pod中申请资源时，`nvidia.com/gpu` 指当前pod需要的物理GPU数量**
+
+### 支持的设备
+
+[NVIDIA GPU](https://github.com/Project-HAMi/HAMi#preparing-your-gpu-nodes)
+[寒武纪 MLU](docs/cambricon-mlu-support.md)
+[海光 DCU](docs/hygon-dcu-support.md)
+[天数智芯 GPU](docs/iluvatar-gpu-support.md)
+[摩尔线程 GPU](docs/mthreads-support.md)
+[昇腾 NPU](https://github.com/Project-HAMi/ascend-device-plugin/blob/main/README.md)
+[沐曦 GPU](docs/metax-support.md)
+
+## 架构
+
+
+
+HAMi 由多个组件组成，包括统一的 mutatingwebhook、统一的调度器扩展器、不同的设备插件以及针对每种异构 AI 设备的容器内虚拟化技术。
+
+## 快速开始
+
+### 选择你的调度器
+
+[](#前置条件)
+[](docs/how-to-use-volcano-vgpu.md)
+
+### 前置条件
+
+运行 NVIDIA 设备插件的前置条件如下：
+
+- NVIDIA 驱动 >= 440
+- nvidia-docker 版本 > 2.0
+- containerd/docker/cri-o 容器运行时的默认运行时配置为 nvidia
+- Kubernetes 版本 >= 1.18
+- glibc >= 2.17 & glibc < 2.30
+- 内核版本 >= 3.10
+- helm > 3.0
+
+### 安装
+
+首先，通过添加标签 "gpu=on" 来标记你的 GPU 节点以进行 HAMi 调度。没有此标签的节点将无法被我们的调度器管理。
+
+```
+kubectl label nodes {nodeid} gpu=on
+```
+
+在 helm 中添加我们的仓库
+
+```
+helm repo add hami-charts https://project-hami.github.io/HAMi/
+```
+
+使用以下命令进行部署：
+
+```
+helm install hami hami-charts/hami -n kube-system
+```
+
+通过调整[配置](docs/config.md)来自定义你的安装。
+
+使用以下命令验证你的安装：
+
+```
+kubectl get pods -n kube-system
+```
+
+如果 `vgpu-device-plugin` 和 `vgpu-scheduler` pod 都处于 *Running* 状态，则安装成功。你可以在[这里](examples/nvidia/default_use.yaml)尝试示例。
+
+### Web 界面
+
+[HAMi-WebUI](https://github.com/Project-HAMi/HAMi-WebUI) 从 HAMi v2.4 版本开始可用
+
+安装指南请点击[这里](https://github.com/Project-HAMi/HAMi-WebUI/blob/main/docs/installation/helm/index.md)
+
+### 监控
+
+安装后自动启用监控。通过访问以下 URL 获取集群信息概览：
+
+```
+http://{scheduler ip}:{monitorPort}/metrics
+```
+
+默认 monitorPort 为 31993；可以在安装时使用 `--set devicePlugin.service.httpPort` 设置其他值。
+
+Grafana 仪表板[示例](docs/dashboard.md)
+
+> **注意** 在提交任务之前不会收集节点状态
+
+## 注意事项
+
+- 如果在使用带有 NVIDIA 镜像的设备插件时不请求虚拟 GPU，机器上的所有 GPU 可能会在容器内暴露
+- 目前，A100 MIG 仅支持 "none" 和 "mixed" 模式
+- 带有 "nodeName" 字段的任务目前无法调度；请使用 "nodeSelector" 代替
+
+## 社区治理
+
+本项目由一组 [维护者](./MAINTAINERS.md) 和 [贡献者](./AUTHORS.md) 管理。他们的选择和管理方式在我们的[治理文档](https://github.com/Project-HAMi/community/blob/main/governance.md)中有详细说明。
+
+如果你有兴趣成为贡献者并希望参与 HAMi 代码开发，请查看 [CONTRIBUTING](CONTRIBUTING.md) 了解提交补丁和贡献工作流程的详细信息。
+
+查看[路线图](docs/develop/roadmap.md)了解你感兴趣的内容。
+
+## 会议与联系方式
+
+HAMi 社区致力于营造开放和友好的环境，提供多种方式与其他用户和开发者互动。
+
+如果你有任何问题，请随时通过以下渠道与我们联系：
+
+- 常规社区会议：每周五 16:00（UTC+8）（中文）。[转换为你的时区](https://www.thetimezoneconverter.com/?t=14%3A30&tz=GMT%2B8&)。
+ - [会议记录和议程](https://docs.google.com/document/d/1YC6hco03_oXbF9IOUPJ29VWEddmITIKIfSmBX8JtGBw/edit#heading=h.g61sgp7w0d0c)
+ - [会议链接](https://meeting.tencent.com/dm/Ntiwq1BICD1P)
+- 电子邮件：请参考 [MAINTAINERS.md](MAINTAINERS.md) 查找所有维护者的电子邮件地址。如有任何问题或需要报告问题，请随时通过电子邮件联系他们。
+- [邮件列表](https://groups.google.com/forum/#!forum/hami-project)
+- [Slack](https://cloud-native.slack.com/archives/C07T10BU4R2) | [加入](https://slack.cncf.io/)
+
+## 演讲和参考资料
+
+| | 链接 |
+|------------------|-------------------------------------------------------------------------------------------------------------------------|
+| 中国云计算基础架构开发者大会 (Beijing 2024) | [在 Kubernetes 集群式解锁异构 AI 基础设施](https://live.csdn.net/room/csdnnews/3zwDP09S) Starting from 03:06:15 |
+| KubeDay(Japan 2024) | [Unlocking Heterogeneous AI Infrastructure K8s Cluster:Leveraging the Power of HAMi](https://www.youtube.com/watch?v=owoaSb4nZwg) |
+| KubeCon & AI_dev Open Source GenAI & ML Summit(China 2024) | [Is Your GPU Really Working Efficiently in the Data Center?N Ways to Improve GPU Usage](https://www.youtube.com/watch?v=ApkyK3zLF5Q) |
+| KubeCon & AI_dev Open Source GenAI & ML Summit(China 2024) | [Unlocking Heterogeneous AI Infrastructure K8s Cluster](https://www.youtube.com/watch?v=kcGXnp_QShs) |
+| KubeCon(EU 2024)| [Cloud Native Batch Computing with Volcano: Updates and Future](https://youtu.be/fVYKk6xSOsw) |
+
+## 许可证
+
+HAMi 采用 Apache 2.0 许可证。详情请参见 [LICENSE](LICENSE) 文件。
+
+## Star 趋势
+
+[](https://star-history.com/#Project-HAMi/HAMi&Date)
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 000000000..caf353223
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,40 @@
+# Security Policy
+
+## Supported Versions
+
+The following table outlines which versions of HAMi receive security updates:
+
+| Version | Supported |
+|---------|--------------------|
+| 2.5.x | ✅ Security fixes |
+| 2.4.x | ✅ Security fixes |
+| before 2.4.0 | ❌ No longer supported |
+
+## Reporting a Vulnerability
+
+If you discover a security vulnerability in HAMi, we strongly encourage you to report it responsibly. Please **do not** disclose security vulnerabilities publicly without following our responsible disclosure process.
+
+### How to Report
+- **GitHub Security Advisories**: [submit a private vulnerability report via GitHub](https://github.com/Project-HAMi/HAMi/security/advisories/new).
+- **Bug Bounty**: Currently, HAMi does not offer a public bug bounty program.
+
+### Information to Include
+When reporting a security issue, please include:
+- A clear and concise description of the vulnerability.
+- Steps to reproduce the issue.
+- Any potential attack scenarios or security impact.
+- Suggested mitigations or fixes, if available.
+
+## Response Process
+
+We follow a structured process to handle security reports:
+
+Response times could be affected by weekends, holidays, breaks or time zone differences. That said, the maintainers will endeavour to reply as soon as possible, ideally within 5 working days.
+
+
+## Third-Party Dependencies
+
+HAMi relies on third-party libraries and containers. We monitor dependencies and promptly apply security patches.
+
+
+Thank you for helping us make HAMi more secure! 🔒
\ No newline at end of file
diff --git a/VERSION b/VERSION
new file mode 100644
index 000000000..ca136a9df
--- /dev/null
+++ b/VERSION
@@ -0,0 +1 @@
+v2.6.1
\ No newline at end of file
diff --git a/benchmarks/README.md b/benchmarks/README.md
new file mode 100644
index 000000000..2ef076818
--- /dev/null
+++ b/benchmarks/README.md
@@ -0,0 +1,25 @@
+# Benchmarking the vGPU scheduler
+
+## Prerequisites
+
+### how to build the benchmark image
+
+```bash
+cd HAMi/benchmarks/ai-benchmark
+
+sh build.sh
+```
+
+## How to install the official nvidia device plugin
+
+Please refer to [Quick Start](https://github.com/NVIDIA/k8s-device-plugin?tab=readme-ov-file#quick-start) in the official nvidia device plugin repository.
+
+## Run the benchmark
+
+```bash
+cd HAMi/benchmarks/deployments
+
+kubectl apply -f job-on-hami.yml
+
+kubectl apply -f job-on-nvidia-device-plugin.yml
+```
\ No newline at end of file
diff --git a/benchmarks/ai-benchmark/Dockerfile b/benchmarks/ai-benchmark/Dockerfile
new file mode 100644
index 000000000..feb112122
--- /dev/null
+++ b/benchmarks/ai-benchmark/Dockerfile
@@ -0,0 +1,18 @@
+# This Dockerfile is used to build a Docker image for running the AI Benchmark.
+# It is based on the tensorflow/tensorflow:latest-gpu image.
+
+FROM tensorflow/tensorflow:2.20.0rc0-gpu
+
+# Set the working directory to /ai-benchmark
+WORKDIR ai-benchmark
+
+# Update the package list and install git and apt-utils
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends apt-utils git && \
+ rm -rf /var/lib/apt/lists/* && \
+ pip install --no-cache-dir --upgrade pip && \
+ git clone https://github.com/Project-HAMi/ai-benchmark . && \
+ pip install --no-cache-dir -r requirements.txt
+
+# Set the default command to run when the container starts
+CMD ["python", "./main.py"]
\ No newline at end of file
diff --git a/benchmarks/ai-benchmark/build.sh b/benchmarks/ai-benchmark/build.sh
new file mode 100644
index 000000000..917770845
--- /dev/null
+++ b/benchmarks/ai-benchmark/build.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+IMAGE="vgpu-benchmark"
+TAG="v0.0.1"
+PLATFORM="linux/amd64"
+
+docker buildx build --push \
+ --platform $PLATFORM \
+ --no-cache \
+ -t "$IMAGE:$TAG" \
+ -f Dockerfile .
\ No newline at end of file
diff --git a/benchmarks/deployments/job-on-hami.yml b/benchmarks/deployments/job-on-hami.yml
new file mode 100644
index 000000000..210a9586a
--- /dev/null
+++ b/benchmarks/deployments/job-on-hami.yml
@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: ai-benchmark-on-hami
+spec:
+ template:
+ metadata:
+ name: ai-benchmark-on-hami
+ spec:
+ containers:
+ - name: ai-benchmark-on-hami
+ image: 4pdosc/ai-benchmark:2.4.1-gpu
+ resources:
+ requests:
+ nvidia.com/gpu: 1
+ nvidia.com/gpumem-percentage: 50
+ limits:
+ nvidia.com/gpu: 1
+ nvidia.com/gpumem-percentage: 50
+ restartPolicy: Never
\ No newline at end of file
diff --git a/benchmarks/deployments/job-on-nvidia-device-plugin.yml b/benchmarks/deployments/job-on-nvidia-device-plugin.yml
new file mode 100644
index 000000000..63cc68e73
--- /dev/null
+++ b/benchmarks/deployments/job-on-nvidia-device-plugin.yml
@@ -0,0 +1,18 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: ai-benchmark-on-official
+spec:
+ template:
+ metadata:
+ name: ai-benchmark-on-official
+ spec:
+ containers:
+ - name: ai-benchmark-on-official
+ image: 4pdosc/ai-benchmark:2.4.1-gpu
+ resources:
+ requests:
+ nvidia.com/gpu: 1
+ limits:
+ nvidia.com/gpu: 1
+ restartPolicy: Never
\ No newline at end of file
diff --git a/charts/Makefile b/charts/Makefile
new file mode 100644
index 000000000..7ed4187dc
--- /dev/null
+++ b/charts/Makefile
@@ -0,0 +1,34 @@
+# get VERSION
+.DEFAULT_GOAL := all
+include ../Makefile.defs
+
+VERSION_REGEX := '[vV]*[0-9]\+\.[0-9]\+\.[0-9]\+.*'
+CHART_FILE := "./hami/Chart.yaml"
+VALUES_FILE := "./hami/values.yaml"
+
+.PHONY: all lint update-versions
+all: update-versions lint package
+
+#update version in chart
+update-versions:
+ $(ECHO_GEN) " Updating Chart version to $(VERSION)"
+ echo "VERSION=$(VERSION)"
+ echo "VERSION_MAJOR=$(VERSION_MAJOR)"
+ echo "GIT_VERSION=$(GIT_VERSION)"
+ echo "FULL_BUILD_VERSION=$(FULL_BUILD_VERSION)"
+ @# Update chart versions to point to the current version.
+ hami_version="$(VERSION)"; \
+ chart_version=` echo $(VERSION) | tr -d 'v' ` ; \
+ sed -i 's/version: "*'$(VERSION_REGEX)'"*/version: '$$chart_version'/g' $(CHART_FILE); \
+ sed -i 's/appVersion: "*'$(VERSION_REGEX)'"*/appVersion: "'$$chart_version'"/g' $(CHART_FILE); \
+ sed -i 's/version: "*'$(VERSION_REGEX)'"*/version: "'$$hami_version'"/g' $(VALUES_FILE)
+
+lint: update-versions
+ helm lint --with-subcharts --values ./hami/values.yaml ./hami --debug
+
+package: lint
+ helm package ./hami --debug
+
+clean:
+ rm -f *.tgz
+
diff --git a/charts/hami/Chart.yaml b/charts/hami/Chart.yaml
new file mode 100644
index 000000000..6386fd9a8
--- /dev/null
+++ b/charts/hami/Chart.yaml
@@ -0,0 +1,40 @@
+apiVersion: v2
+name: hami
+description: Heterogeneous AI Computing Virtualization Middleware
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 2.6.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "2.6.1"
+
+kubeVersion: ">= 1.18.0-0"
+
+keywords:
+ - vgpu
+ - gpu
+
+sources:
+ - "https://github.com/Project-HAMi/HAMi"
+
+maintainers:
+ - name: limengxuan
+ email: archlitchi@gmail.com
+ - name: zhangxiao
+ email: xiaozhang0210@hotmail.com
+
diff --git a/charts/hami/README.md b/charts/hami/README.md
new file mode 100644
index 000000000..c149b42ec
--- /dev/null
+++ b/charts/hami/README.md
@@ -0,0 +1,227 @@
+# HAMi Helm Chart Values Documentation
+
+This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart.
+
+## Global Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `global.imageRegistry` | Global Docker image registry | `""` |
+| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` |
+| `global.imageTag` | Image tag | `"v2.6.1"` |
+| `global.gpuHookPath` | GPU Hook path | `/usr/local` |
+| `global.labels` | Global labels | `{}` |
+| `global.annotations` | Global annotations | `{}` |
+| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` |
+| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` |
+| `nameOverride` | Name override | `""` |
+| `fullnameOverride` | Full name override | `""` |
+| `namespaceOverride` | Namespace override | `""` |
+
+## Resource Name Configuration
+
+### NVIDIA GPU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `resourceName` | GPU resource name | `"nvidia.com/gpu"` |
+| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` |
+| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` |
+| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` |
+| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` |
+
+### Cambricon MLU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` |
+| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` |
+| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` |
+
+### Hygon DCU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` |
+| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` |
+| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` |
+
+### Iluvatar GPU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `iluvatarResourceName` | GPU resource name | `"iluvatar.ai/vgpu"` |
+| `iluvatarResourceMem` | GPU memory resource name | `"iluvatar.ai/vcuda-memory"` |
+| `iluvatarResourceCore` | GPU core resource name | `"iluvatar.ai/vcuda-core"` |
+
+### Metax GPU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` |
+| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` |
+| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` |
+| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` |
+
+### Kunlunxin XPU Resources
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` |
+
+## Scheduler Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `schedulerName` | Scheduler name | `"hami-scheduler"` |
+| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` |
+| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` |
+| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` |
+| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` |
+| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` |
+| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` |
+| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` |
+| `scheduler.leaderElect` | Whether to enable leader election | `true` |
+| `scheduler.replicas` | Number of replicas | `1` |
+
+### Kube Scheduler Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` |
+| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` |
+| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` |
+| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` |
+| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` |
+| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` |
+| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` |
+| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` |
+
+### Extender Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` |
+| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` |
+| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` |
+| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` |
+| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` |
+| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` |
+
+### Admission Webhook Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` |
+| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` |
+| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` |
+| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` |
+| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` |
+| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` |
+| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` |
+
+### TLS Certificate Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` |
+| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` |
+| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` |
+| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` |
+| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` |
+| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` |
+| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` |
+| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` |
+| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` |
+
+### Scheduler Service Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `scheduler.service.type` | Service type | `NodePort` |
+| `scheduler.service.httpPort` | HTTP port | `443` |
+| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` |
+| `scheduler.service.monitorPort` | Monitor port | `31993` |
+| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` |
+
+## Device Plugin Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` |
+| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` |
+| `devicePlugin.image.tag` | Device plugin image tag | `""` |
+| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` |
+| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` |
+
+### Monitor Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` |
+| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` |
+| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` |
+| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` |
+| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` |
+| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` |
+
+### Device Plugin Other Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` |
+| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` |
+| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` |
+| `devicePlugin.runtimeClassName` | Runtime class name | `""` |
+| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` |
+| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` |
+| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` |
+| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` |
+| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` |
+
+### Device Plugin Service Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devicePlugin.service.type` | Service type | `NodePort` |
+| `devicePlugin.service.httpPort` | HTTP port | `31992` |
+
+### Device Plugin Deployment Configuration
+
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` |
+| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` |
+| `devicePlugin.nvidianodeSelector` | NVIDIA node selector | `{"gpu": "on"}` |
+| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` |
+| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` |
+
+## Device Configuration
+
+### AWS Neuron
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` |
+
+### Kunlunxin
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devices.kunlun.enabled` | Whether to enable | `true` |
+| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` |
+
+### Mthreads
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devices.mthreads.enabled` | Whether to enable | `true` |
+| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` |
+
+### NVIDIA
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` |
+| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` |
+
+### Huawei Ascend
+| Parameter | Description | Default Value |
+|-----------|-------------|---------------|
+| `devices.ascend.enabled` | Whether to enable | `false` |
+| `devices.ascend.image` | Image | `""` |
+| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` |
+| `devices.ascend.extraArgs` | Extra arguments | `[]` |
+| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` |
+| `devices.ascend.tolerations` | Tolerations | `[]` |
+| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` |
diff --git a/deployments/4pd-vgpu/templates/NOTES.txt b/charts/hami/templates/NOTES.txt
similarity index 100%
rename from deployments/4pd-vgpu/templates/NOTES.txt
rename to charts/hami/templates/NOTES.txt
diff --git a/charts/hami/templates/_commons.tpl b/charts/hami/templates/_commons.tpl
new file mode 100644
index 000000000..78a262c90
--- /dev/null
+++ b/charts/hami/templates/_commons.tpl
@@ -0,0 +1,49 @@
+{{/*
+Return the proper image name
+{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }}
+*/}}
+{{- define "common.images.image" -}}
+{{- $registryName := .imageRoot.registry -}}
+{{- $repositoryName := .imageRoot.repository -}}
+{{- $tag := .imageRoot.tag | toString -}}
+{{- if .global }}
+ {{- if .global.imageRegistry }}
+ {{- $registryName = .global.imageRegistry -}}
+ {{- end -}}
+{{- end -}}
+{{- if .tag }}
+ {{- $tag = .tag | toString -}}
+{{- end -}}
+{{- if $registryName }}
+{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
+{{- else -}}
+{{- printf "%s:%s" $repositoryName $tag -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead)
+{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }}
+*/}}
+{{- define "common.images.pullSecrets" -}}
+ {{- $pullSecrets := list }}
+
+ {{- if .global }}
+ {{- range .global.imagePullSecrets -}}
+ {{- $pullSecrets = append $pullSecrets . -}}
+ {{- end -}}
+ {{- end -}}
+
+ {{- range .images -}}
+ {{- range .pullSecrets -}}
+ {{- $pullSecrets = append $pullSecrets . -}}
+ {{- end -}}
+ {{- end -}}
+
+ {{- if (not (empty $pullSecrets)) }}
+imagePullSecrets:
+ {{- range $pullSecrets }}
+ - name: {{ . }}
+ {{- end }}
+ {{- end }}
+{{- end -}}
\ No newline at end of file
diff --git a/charts/hami/templates/_helpers.tpl b/charts/hami/templates/_helpers.tpl
new file mode 100644
index 000000000..663c5e74e
--- /dev/null
+++ b/charts/hami/templates/_helpers.tpl
@@ -0,0 +1,152 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "hami-vgpu.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "hami-vgpu.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Allow the release namespace to be overridden for multi-namespace deployments in combined charts
+*/}}
+{{- define "hami-vgpu.namespace" -}}
+ {{- if .Values.namespaceOverride -}}
+ {{- .Values.namespaceOverride -}}
+ {{- else -}}
+ {{- .Release.Namespace -}}
+ {{- end -}}
+{{- end -}}
+
+{{/*
+The app name for Scheduler
+*/}}
+{{- define "hami-vgpu.scheduler" -}}
+{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+The app name for DevicePlugin
+*/}}
+{{- define "hami-vgpu.device-plugin" -}}
+{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+The tls secret name for Scheduler
+*/}}
+{{- define "hami-vgpu.scheduler.tls" -}}
+{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+The webhook name
+*/}}
+{{- define "hami-vgpu.scheduler.webhook" -}}
+{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "hami-vgpu.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "hami-vgpu.labels" -}}
+helm.sh/chart: {{ include "hami-vgpu.chart" . }}
+{{ include "hami-vgpu.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "hami-vgpu.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "hami-vgpu.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+
+{{/*
+ Resolve the tag for kubeScheduler.
+*/}}
+{{- define "resolvedKubeSchedulerTag" -}}
+{{- if .Values.scheduler.kubeScheduler.image.tag }}
+{{- .Values.scheduler.kubeScheduler.image.tag | trim -}}
+{{- else }}
+{{- include "strippedKubeVersion" . | trim -}}
+{{- end }}
+{{- end }}
+
+{{/*
+ Return the stripped Kubernetes version string by removing extra parts after semantic version number.
+ v1.31.1+k3s1 -> v1.31.1
+ v1.30.8-eks-2d5f260 -> v1.30.8
+ v1.31.1 -> v1.31.1
+*/}}
+{{- define "strippedKubeVersion" -}}
+{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }}
+{{- end -}}
+
+{{- define "hami.scheduler.kubeScheduler.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }}
+{{- end -}}
+
+{{- define "hami.scheduler.extender.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }}
+{{- end -}}
+
+{{- define "hami.devicePlugin.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }}
+{{- end -}}
+
+{{- define "hami.devicePlugin.monitor.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }}
+{{- end -}}
+
+{{- define "hami.scheduler.patch.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.scheduler.patch.new.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.scheduler.extender.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.devicePlugin.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.scheduler.patch.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }}
+{{- end -}}
+
+{{- define "hami.scheduler.patch.new.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }}
+{{- end -}}
diff --git a/charts/hami/templates/device-plugin/configmap.yaml b/charts/hami/templates/device-plugin/configmap.yaml
new file mode 100644
index 000000000..75d641f03
--- /dev/null
+++ b/charts/hami/templates/device-plugin/configmap.yaml
@@ -0,0 +1,27 @@
+{{- if .Values.devicePlugin.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "hami-vgpu.device-plugin" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: hami-device-plugin
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+data:
+ config.json: |
+ {
+ "nodeconfig": [
+ {
+ "name": "m5-cloudinfra-online02",
+ "operatingmode": "hami-core",
+ "devicememoryscaling": 1.8,
+ "devicesplitcount": 10,
+ "migstrategy":"none",
+ "filterdevices": {
+ "uuid": [],
+ "index": []
+ }
+ }
+ ]
+ }
+{{- end -}}
\ No newline at end of file
diff --git a/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/charts/hami/templates/device-plugin/daemonsetnvidia.yaml
new file mode 100644
index 000000000..896aa4943
--- /dev/null
+++ b/charts/hami/templates/device-plugin/daemonsetnvidia.yaml
@@ -0,0 +1,182 @@
+{{- if .Values.devicePlugin.enabled }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: {{ include "hami-vgpu.device-plugin" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: hami-device-plugin
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+ {{- with .Values.global.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.global.annotations }}
+ annotations: {{ toYaml .Values.global.annotations | nindent 4}}
+ {{- end }}
+spec:
+ updateStrategy:
+ {{- with .Values.devicePlugin.updateStrategy }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ selector:
+ matchLabels:
+ app.kubernetes.io/component: hami-device-plugin
+ {{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/component: hami-device-plugin
+ hami.io/webhook: ignore
+ {{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
+ annotations:
+ checksum/hami-device-plugin-config: {{ include (print $.Template.BasePath "/device-plugin/configmap.yaml") . | sha256sum }}
+ checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }}
+ {{- if .Values.devicePlugin.podAnnotations }}
+ {{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }}
+ {{- end }}
+ spec:
+ {{- if .Values.devicePlugin.runtimeClassName }}
+ runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }}
+ {{- end }}
+ serviceAccountName: {{ include "hami-vgpu.device-plugin" . }}
+ priorityClassName: system-node-critical
+ hostPID: true
+ hostNetwork: true
+ {{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }}
+ containers:
+ - name: device-plugin
+ image: {{ include "hami.devicePlugin.image" . }}
+ imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }}
+ lifecycle:
+ postStart:
+ exec:
+ command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}]
+ command:
+ - nvidia-device-plugin
+ - --config-file=/device-config.yaml
+ - --mig-strategy={{ .Values.devicePlugin.migStrategy }}
+ - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }}
+ {{- range .Values.devicePlugin.extraArgs }}
+ - {{ . }}
+ {{- end }}
+ env:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ - name: NVIDIA_MIG_MONITOR_DEVICES
+ value: all
+ - name: HOOK_PATH
+ value: {{ .Values.global.gpuHookPath }}
+ {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }}
+ - name: PASS_DEVICE_SPECS
+ value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }}
+ {{- end }}
+ {{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }}
+ - name: ENABLE_TOPOLOGY_SCORE
+ value: "true"
+ {{- end }}
+ securityContext:
+ privileged: true
+ allowPrivilegeEscalation: true
+ capabilities:
+ drop: ["ALL"]
+ add: ["SYS_ADMIN"]
+ resources:
+ {{- toYaml .Values.devicePlugin.resources | nindent 12 }}
+ volumeMounts:
+ - name: device-plugin
+ mountPath: /var/lib/kubelet/device-plugins
+ - name: lib
+ mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }}
+ - name: usrbin
+ mountPath: /usrbin
+ - name: deviceconfig
+ mountPath: /config
+ - name: hosttmp
+ mountPath: /tmp
+ - name: device-config
+ mountPath: /device-config.yaml
+ subPath: device-config.yaml
+ - name: vgpu-monitor
+ image: {{ include "hami.devicePlugin.monitor.image" . }}
+ imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }}
+ command:
+ - "vGPUmonitor"
+ {{- range .Values.devicePlugin.extraArgs }}
+ - {{ . }}
+ {{- end }}
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: ["ALL"]
+ add: ["SYS_ADMIN"]
+ env:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ - name: NVIDIA_VISIBLE_DEVICES
+ value: "all"
+ - name: NVIDIA_MIG_MONITOR_DEVICES
+ value: all
+ - name: HOOK_PATH
+ value: {{ .Values.global.gpuHookPath }}/vgpu
+ - name: HAMI_RESYNC_INTERVAL
+ value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }}
+ resources:
+ {{- toYaml .Values.devicePlugin.vgpuMonitor.resources | nindent 12 }}
+ volumeMounts:
+ - name: ctrs
+ mountPath: {{ .Values.devicePlugin.monitor.ctrPath }}
+ - name: dockers
+ mountPath: /run/docker
+ - name: containerds
+ mountPath: /run/containerd
+ - name: sysinfo
+ mountPath: /sysinfo
+ - name: hostvar
+ mountPath: /hostvar
+ - name: hosttmp
+ mountPath: /tmp
+ volumes:
+ - name: ctrs
+ hostPath:
+ path: {{ .Values.devicePlugin.monitor.ctrPath }}
+ - name: hosttmp
+ hostPath:
+ path: /tmp
+ - name: dockers
+ hostPath:
+ path: /run/docker
+ - name: containerds
+ hostPath:
+ path: /run/containerd
+ - name: device-plugin
+ hostPath:
+ path: {{ .Values.devicePlugin.pluginPath }}
+ - name: lib
+ hostPath:
+ path: {{ .Values.devicePlugin.libPath }}
+ - name: usrbin
+ hostPath:
+ path: /usr/bin
+ - name: sysinfo
+ hostPath:
+ path: /sys
+ - name: hostvar
+ hostPath:
+ path: /var
+ - name: deviceconfig
+ configMap:
+ name: {{ template "hami-vgpu.device-plugin" . }}
+ - name: device-config
+ configMap:
+ name: {{ include "hami-vgpu.scheduler" . }}-device
+ {{- if .Values.devicePlugin.nvidianodeSelector }}
+ nodeSelector: {{ toYaml .Values.devicePlugin.nvidianodeSelector | nindent 8 }}
+ {{- end }}
+ {{- if .Values.devicePlugin.tolerations }}
+ tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }}
+ {{- end }}
+{{- end -}}
diff --git a/charts/hami/templates/device-plugin/monitorrole.yaml b/charts/hami/templates/device-plugin/monitorrole.yaml
new file mode 100644
index 000000000..2a4bfcafc
--- /dev/null
+++ b/charts/hami/templates/device-plugin/monitorrole.yaml
@@ -0,0 +1,29 @@
+{{- if .Values.devicePlugin.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ include "hami-vgpu.device-plugin" . }}-monitor
+rules:
+ - apiGroups:
+ - ""
+ resources:
+ - pods
+ verbs:
+ - get
+ - create
+ - watch
+ - list
+ - update
+ - patch
+ - apiGroups:
+ - ""
+ resources:
+ - nodes
+ verbs:
+ - get
+ - update
+ - list
+ - patch
+{{- end -}}
+
+
diff --git a/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/charts/hami/templates/device-plugin/monitorrolebinding.yaml
new file mode 100644
index 000000000..942f8ec24
--- /dev/null
+++ b/charts/hami/templates/device-plugin/monitorrolebinding.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.devicePlugin.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ include "hami-vgpu.device-plugin" . }}
+ labels:
+ app.kubernetes.io/component: "hami-device-plugin"
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ include "hami-vgpu.device-plugin" . }}-monitor
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "hami-vgpu.device-plugin" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+{{- end -}}
diff --git a/charts/hami/templates/device-plugin/monitorservice.yaml b/charts/hami/templates/device-plugin/monitorservice.yaml
new file mode 100644
index 000000000..bb29e6017
--- /dev/null
+++ b/charts/hami/templates/device-plugin/monitorservice.yaml
@@ -0,0 +1,29 @@
+{{- if .Values.devicePlugin.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "hami-vgpu.device-plugin" . }}-monitor
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: hami-device-plugin
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+ {{- if .Values.devicePlugin.service.labels }} # Use devicePlugin instead of scheduler
+ {{ toYaml .Values.devicePlugin.service.labels | indent 4 }}
+ {{- end }}
+ {{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler
+ annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }}
+ {{- end }}
+spec:
+ type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort
+ ports:
+ - name: monitorport
+ port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992
+ targetPort: 9394
+ {{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
+ nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }}
+ {{- end }}
+ protocol: TCP
+ selector:
+ app.kubernetes.io/component: hami-device-plugin
+ {{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
+{{- end -}}
\ No newline at end of file
diff --git a/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/charts/hami/templates/device-plugin/monitorserviceaccount.yaml
new file mode 100644
index 000000000..2db2739a4
--- /dev/null
+++ b/charts/hami/templates/device-plugin/monitorserviceaccount.yaml
@@ -0,0 +1,10 @@
+{{- if .Values.devicePlugin.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ include "hami-vgpu.device-plugin" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: "hami-device-plugin"
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+{{- end -}}
diff --git a/charts/hami/templates/device-plugin/runtime-class.yaml b/charts/hami/templates/device-plugin/runtime-class.yaml
new file mode 100644
index 000000000..32bb00802
--- /dev/null
+++ b/charts/hami/templates/device-plugin/runtime-class.yaml
@@ -0,0 +1,11 @@
+{{- if .Values.devicePlugin.enabled }}
+{{- if and .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName }}
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+ name: {{ .Values.devicePlugin.runtimeClassName }}
+ annotations:
+ helm.sh/hook: pre-install,pre-upgrade
+handler: nvidia
+{{- end }}
+{{- end -}}
diff --git a/charts/hami/templates/scheduler/certmanager.yaml b/charts/hami/templates/scheduler/certmanager.yaml
new file mode 100644
index 000000000..81a8c9e63
--- /dev/null
+++ b/charts/hami/templates/scheduler/certmanager.yaml
@@ -0,0 +1,31 @@
+{{- if .Values.scheduler.admissionWebhook.enabled }}
+{{- if .Values.scheduler.certManager.enabled }}
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}-serving-cert
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: hami-scheduler
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+spec:
+ dnsNames:
+ - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc
+ - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local
+ issuerRef:
+ kind: Issuer
+ name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
+ secretName: {{ include "hami-vgpu.scheduler.tls" . }}
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: hami-scheduler
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+spec:
+ selfSigned: {}
+{{- end }}
+{{- end }}
diff --git a/charts/hami/templates/scheduler/clusterrole.yaml b/charts/hami/templates/scheduler/clusterrole.yaml
new file mode 100644
index 000000000..6a3d41295
--- /dev/null
+++ b/charts/hami/templates/scheduler/clusterrole.yaml
@@ -0,0 +1,21 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}
+ labels:
+ app.kubernetes.io/component: "hami-scheduler"
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+rules:
+ - apiGroups: [""]
+ resources: ["pods", "configmaps"]
+ verbs: ["get", "list", "watch", "patch"]
+ - apiGroups: [""]
+ resources: ["pods/binding"]
+ verbs: ["create"]
+ - apiGroups: [""]
+ resources: ["nodes"]
+ verbs: ["get", "list", "patch", "watch"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["create", "get", "list"]
+
diff --git a/charts/hami/templates/scheduler/clusterrolebinding.yaml b/charts/hami/templates/scheduler/clusterrolebinding.yaml
new file mode 100644
index 000000000..75831ce1c
--- /dev/null
+++ b/charts/hami/templates/scheduler/clusterrolebinding.yaml
@@ -0,0 +1,47 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}-kube
+ labels:
+ app.kubernetes.io/component: "hami-scheduler"
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: system:kube-scheduler
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "hami-vgpu.scheduler" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}-volume
+ labels:
+ app.kubernetes.io/component: "hami-scheduler"
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: system:volume-scheduler
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "hami-vgpu.scheduler" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}
+ labels:
+ app.kubernetes.io/component: "hami-scheduler"
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: hami-scheduler
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "hami-vgpu.scheduler" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
diff --git a/charts/hami/templates/scheduler/configmap.yaml b/charts/hami/templates/scheduler/configmap.yaml
new file mode 100644
index 000000000..607c0e57c
--- /dev/null
+++ b/charts/hami/templates/scheduler/configmap.yaml
@@ -0,0 +1,127 @@
+{{- if .Values.scheduler.kubeScheduler.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: hami-scheduler
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+data:
+ config.json: |
+ {
+ "kind": "Policy",
+ "apiVersion": "v1",
+ "extenders": [
+ {
+ "urlPrefix": "https://127.0.0.1:443",
+ "filterVerb": "filter",
+ "bindVerb": "bind",
+ "enableHttps": true,
+ "weight": 1,
+ "nodeCacheCapable": true,
+ "httpTimeout": 30000000000,
+ "tlsConfig": {
+ "insecure": true
+ },
+ "managedResources": [
+ {{- if .Values.devices.ascend.enabled }}
+ {{- range .Values.devices.ascend.customresources }}
+ {
+ "name": "{{ . }}",
+ "ignoredByScheduler": true
+ },
+ {{- end }}
+ {{- end }}
+ {{- if .Values.devices.mthreads.enabled }}
+ {{- range .Values.devices.mthreads.customresources }}
+ {
+ "name": "{{ . }}",
+ "ignoredByScheduler": true
+ },
+ {{- end }}
+ {{- end }}
+ {{- if .Values.devices.enflame.enabled }}
+ {{- range .Values.devices.enflame.customresources }}
+ {
+ "name": "{{ . }}",
+ "ignoredByScheduler": true
+ },
+ {{- end }}
+ {{- end }}
+ {{- if .Values.devices.kunlun.enabled }}
+ {{- range .Values.devices.kunlun.customresources }}
+ {
+ "name": "{{ . }}",
+ "ignoredByScheduler": true
+ },
+ {{- end }}
+ {{- end }}
+ {{- range .Values.devices.awsneuron.customresources }}
+ {
+ "name": "{{ . }}",
+ "ignoredByScheduler": true
+ },
+ {{- end }}
+ {
+ "name": "{{ .Values.resourceName }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.resourceMem }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.resourceCores }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.resourceMemPercentage }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.resourcePriority }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.mluResourceName }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.dcuResourceName }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.dcuResourceMem }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.dcuResourceCores }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.iluvatarResourceName }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "metax-tech.com/gpu",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.metaxResourceName }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.metaxResourceCore }}",
+ "ignoredByScheduler": true
+ },
+ {
+ "name": "{{ .Values.metaxResourceMem }}",
+ "ignoredByScheduler": true
+ }
+ ],
+ "ignoreable": false
+ }
+ ]
+ }
+{{- end }}
diff --git a/charts/hami/templates/scheduler/configmapnew.yaml b/charts/hami/templates/scheduler/configmapnew.yaml
new file mode 100644
index 000000000..8c36ebd1f
--- /dev/null
+++ b/charts/hami/templates/scheduler/configmapnew.yaml
@@ -0,0 +1,89 @@
+{{- if .Values.scheduler.kubeScheduler.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}-newversion
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: hami-scheduler
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+data:
+ config.yaml: |
+ {{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}}
+ apiVersion: kubescheduler.config.k8s.io/v1
+ {{- else }}
+ apiVersion: kubescheduler.config.k8s.io/v1beta2
+ {{- end }}
+ kind: KubeSchedulerConfiguration
+ leaderElection:
+ leaderElect: false
+ profiles:
+ - schedulerName: {{ .Values.schedulerName }}
+ extenders:
+ - urlPrefix: "https://127.0.0.1:443"
+ filterVerb: filter
+ bindVerb: bind
+ nodeCacheCapable: true
+ weight: 1
+ httpTimeout: 30s
+ enableHTTPS: true
+ tlsConfig:
+ insecure: true
+ managedResources:
+ - name: {{ .Values.resourceName }}
+ ignoredByScheduler: true
+ - name: {{ .Values.resourceMem }}
+ ignoredByScheduler: true
+ - name: {{ .Values.resourceCores }}
+ ignoredByScheduler: true
+ - name: {{ .Values.resourceMemPercentage }}
+ ignoredByScheduler: true
+ - name: {{ .Values.resourcePriority }}
+ ignoredByScheduler: true
+ - name: {{ .Values.mluResourceName }}
+ ignoredByScheduler: true
+ - name: {{ .Values.dcuResourceName }}
+ ignoredByScheduler: true
+ - name: {{ .Values.dcuResourceMem }}
+ ignoredByScheduler: true
+ - name: {{ .Values.dcuResourceCores }}
+ ignoredByScheduler: true
+ - name: {{ .Values.iluvatarResourceName }}
+ ignoredByScheduler: true
+ - name: "metax-tech.com/gpu"
+ ignoredByScheduler: true
+ - name: {{ .Values.metaxResourceName }}
+ ignoredByScheduler: true
+ - name: {{ .Values.metaxResourceCore }}
+ ignoredByScheduler: true
+ - name: {{ .Values.metaxResourceMem }}
+ ignoredByScheduler: true
+ {{- if .Values.devices.ascend.enabled }}
+ {{- range .Values.devices.ascend.customresources }}
+ - name: {{ . }}
+ ignoredByScheduler: true
+ {{- end }}
+ {{- end }}
+ {{- if .Values.devices.mthreads.enabled }}
+ {{- range .Values.devices.mthreads.customresources }}
+ - name: {{ . }}
+ ignoredByScheduler: true
+ {{- end }}
+ {{- end }}
+ {{- if .Values.devices.enflame.enabled }}
+ {{- range .Values.devices.enflame.customresources }}
+ - name: {{ . }}
+ ignoredByScheduler: true
+ {{- end }}
+ {{- end }}
+ {{- if .Values.devices.kunlun.enabled }}
+ {{- range .Values.devices.kunlun.customresources }}
+ - name: {{ . }}
+ ignoredByScheduler: true
+ {{- end }}
+ {{- end }}
+ {{- range .Values.devices.awsneuron.customresources }}
+ - name: {{ . }}
+ ignoredByScheduler: true
+ {{- end }}
+{{- end }}
diff --git a/charts/hami/templates/scheduler/deployment.yaml b/charts/hami/templates/scheduler/deployment.yaml
new file mode 100644
index 000000000..9f0b4d171
--- /dev/null
+++ b/charts/hami/templates/scheduler/deployment.yaml
@@ -0,0 +1,169 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: hami-scheduler
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+ {{- with .Values.global.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.global.annotations }}
+ annotations: {{ toYaml .Values.global.annotations | nindent 4}}
+ {{- end }}
+spec:
+ {{- if .Values.scheduler.leaderElect }}
+ replicas: {{ .Values.scheduler.replicas }}
+ {{- else }}
+ replicas: 1
+ {{- end }}
+ selector:
+ matchLabels:
+ app.kubernetes.io/component: hami-scheduler
+ {{- include "hami-vgpu.selectorLabels" . | nindent 6 }}
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/component: hami-scheduler
+ {{- include "hami-vgpu.selectorLabels" . | nindent 8 }}
+ hami.io/webhook: ignore
+ annotations:
+ {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+ checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }}
+ {{- else }}
+ checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }}
+ {{- end }}
+ checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }}
+ {{- if .Values.scheduler.podAnnotations }}
+ {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }}
+ {{- end }}
+ spec:
+ serviceAccountName: {{ include "hami-vgpu.scheduler" . }}
+ priorityClassName: system-node-critical
+ {{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }}
+ containers:
+ {{- if .Values.scheduler.kubeScheduler.enabled }}
+ - name: kube-scheduler
+ image: {{ include "hami.scheduler.kubeScheduler.image" . }}
+ imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }}
+ command:
+ - kube-scheduler
+ {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+ {{- range .Values.scheduler.kubeScheduler.extraNewArgs }}
+ - {{ . }}
+ {{- end }}
+ {{- else }}
+ - --scheduler-name={{ .Values.schedulerName }}
+ {{- range .Values.scheduler.kubeScheduler.extraArgs }}
+ - {{ . }}
+ {{- end }}
+ {{- end }}
+ - --leader-elect={{ .Values.scheduler.leaderElect }}
+ - --leader-elect-resource-name={{ .Values.schedulerName }}
+ - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }}
+ resources:
+ {{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }}
+ volumeMounts:
+ - name: scheduler-config
+ mountPath: /config
+ {{- end }}
+ {{- if .Values.scheduler.livenessProbe }}
+ livenessProbe:
+ failureThreshold: 8
+ httpGet:
+ path: /healthz
+ port: 10259
+ scheme: HTTPS
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ successThreshold: 1
+ timeoutSeconds: 15
+ {{- end }}
+ - name: vgpu-scheduler-extender
+ image: {{ include "hami.scheduler.extender.image" . }}
+ imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }}
+ env:
+ {{- if .Values.scheduler.nodeLockExpire }}
+ - name: HAMI_NODELOCK_EXPIRE
+ value: "{{ .Values.scheduler.nodeLockExpire }}"
+ {{- end }}
+ {{- if .Values.global.managedNodeSelectorEnable }}
+ {{- range $key, $value := .Values.global.managedNodeSelector }}
+ - name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }}
+ value: "{{ $value }}"
+ {{- end }}
+ {{- end }}
+ command:
+ - scheduler
+ - --http_bind=0.0.0.0:443
+ - --cert_file=/tls/tls.crt
+ - --key_file=/tls/tls.key
+ - --scheduler-name={{ .Values.schedulerName }}
+ - --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }}
+ - --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }}
+ - --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }}
+ - --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}}
+ - --device-config-file=/device-config.yaml
+ {{- if .Values.devices.ascend.enabled }}
+ - --enable-ascend=true
+ {{- end }}
+ {{- if .Values.scheduler.nodeLabelSelector }}
+ - --node-label-selector={{- $first := true -}}
+ {{- range $key, $value := .Values.scheduler.nodeLabelSelector -}}
+ {{- if not $first }},{{ end -}}
+ {{- $key }}={{ $value -}}
+ {{- $first = false -}}
+ {{- end -}}
+ {{- end }}
+ {{- range .Values.scheduler.extender.extraArgs }}
+ - {{ . }}
+ {{- end }}
+ ports:
+ - name: http
+ containerPort: 443
+ protocol: TCP
+ resources:
+ {{- toYaml .Values.scheduler.extender.resources | nindent 12 }}
+ volumeMounts:
+ - name: tls-config
+ mountPath: /tls
+ - name: device-config
+ mountPath: /device-config.yaml
+ subPath: device-config.yaml
+ {{- if .Values.scheduler.livenessProbe }}
+ livenessProbe:
+ httpGet:
+ path: /healthz
+ port: 443
+ scheme: HTTPS
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ failureThreshold: 3
+ timeoutSeconds: 5
+ {{- end }}
+ volumes:
+ - name: tls-config
+ secret:
+ secretName: {{ template "hami-vgpu.scheduler.tls" . }}
+ {{- if .Values.scheduler.kubeScheduler.enabled }}
+ - name: scheduler-config
+ configMap:
+ {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+ name: {{ template "hami-vgpu.scheduler" . }}-newversion
+ {{- else }}
+ name: {{ template "hami-vgpu.scheduler" . }}
+ {{- end }}
+ {{- end }}
+ - name: device-config
+ configMap:
+ name: {{ include "hami-vgpu.scheduler" . }}-device
+ {{- if .Values.scheduler.nodeSelector }}
+ nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }}
+ {{- end }}
+ {{- if .Values.scheduler.tolerations }}
+ tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }}
+ {{- end }}
+ {{- if .Values.scheduler.nodeName }}
+ nodeName: {{ .Values.scheduler.nodeName }}
+ {{- end }}
diff --git a/charts/hami/templates/scheduler/device-configmap.yaml b/charts/hami/templates/scheduler/device-configmap.yaml
new file mode 100644
index 000000000..2f1606fa9
--- /dev/null
+++ b/charts/hami/templates/scheduler/device-configmap.yaml
@@ -0,0 +1,214 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}-device
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: hami-scheduler
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+data:
+ device-config.yaml: |-
+ {{- if .Files.Glob "files/device-config.yaml" }}
+ {{- .Files.Get "files/device-config.yaml" | nindent 4}}
+ {{- else }}
+ nvidia:
+ resourceCountName: {{ .Values.resourceName }}
+ resourceMemoryName: {{ .Values.resourceMem }}
+ resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }}
+ resourceCoreName: {{ .Values.resourceCores }}
+ resourcePriorityName: {{ .Values.resourcePriority }}
+ overwriteEnv: false
+ defaultMemory: 0
+ defaultCores: 0
+ defaultGPUNum: 1
+ deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }}
+ deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }}
+ deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }}
+ gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }}
+ libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }}
+ runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}"
+ knownMigGeometries:
+ - models: [ "A30" ]
+ allowedGeometries:
+ -
+ - name: 1g.6gb
+ memory: 6144
+ count: 4
+ -
+ - name: 2g.12gb
+ memory: 12288
+ count: 2
+ -
+ - name: 4g.24gb
+ memory: 24576
+ count: 1
+ - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB", "A100-SXM4-40GB" ]
+ allowedGeometries:
+ -
+ - name: 1g.5gb
+ memory: 5120
+ count: 7
+ -
+ - name: 2g.10gb
+ memory: 10240
+ count: 3
+ - name: 1g.5gb
+ memory: 5120
+ count: 1
+ -
+ - name: 3g.20gb
+ memory: 20480
+ count: 2
+ -
+ - name: 7g.40gb
+ memory: 40960
+ count: 1
+ - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
+ allowedGeometries:
+ -
+ - name: 1g.10gb
+ memory: 10240
+ count: 7
+ -
+ - name: 2g.20gb
+ memory: 20480
+ count: 3
+ - name: 1g.10gb
+ memory: 10240
+ count: 1
+ -
+ - name: 3g.40gb
+ memory: 40960
+ count: 2
+ -
+ - name: 7g.79gb
+ memory: 80896
+ count: 1
+ cambricon:
+ resourceCountName: {{ .Values.mluResourceName }}
+ resourceMemoryName: {{ .Values.mluResourceMem }}
+ resourceCoreName: {{ .Values.mluResourceCores }}
+ hygon:
+ resourceCountName: {{ .Values.dcuResourceName }}
+ resourceMemoryName: {{ .Values.dcuResourceMem }}
+ resourceCoreName: {{ .Values.dcuResourceCores }}
+ metax:
+ resourceCountName: "metax-tech.com/gpu"
+ resourceVCountName: {{ .Values.metaxResourceName }}
+ resourceVMemoryName: {{ .Values.metaxResourceMem }}
+ resourceVCoreName: {{ .Values.metaxResourceCore }}
+ sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }}
+ enflame:
+ resourceCountName: "enflame.com/vgcu"
+ resourcePercentageName: "enflame.com/vgcu-percentage"
+ mthreads:
+ resourceCountName: "mthreads.com/vgpu"
+ resourceMemoryName: "mthreads.com/sgpu-memory"
+ resourceCoreName: "mthreads.com/sgpu-core"
+ iluvatar:
+ resourceCountName: {{ .Values.iluvatarResourceName }}
+ resourceMemoryName: {{ .Values.iluvatarResourceMem }}
+ resourceCoreName: {{ .Values.iluvatarResourceCore }}
+ kunlun:
+ resourceCountName: {{ .Values.kunlunResourceName }}
+ awsneuron:
+ resourceCountName: "aws.amazon.com/neuron"
+ resourceCoreName: "aws.amazon.com/neuroncore"
+ vnpus:
+ - chipName: 910B
+ commonWord: Ascend910A
+ resourceName: huawei.com/Ascend910A
+ resourceMemoryName: huawei.com/Ascend910A-memory
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 30
+ templates:
+ - name: vir02
+ memory: 2184
+ aiCore: 2
+ - name: vir04
+ memory: 4369
+ aiCore: 4
+ - name: vir08
+ memory: 8738
+ aiCore: 8
+ - name: vir16
+ memory: 17476
+ aiCore: 16
+ - chipName: 910B2
+ commonWord: Ascend910B2
+ resourceName: huawei.com/Ascend910B2
+ resourceMemoryName: huawei.com/Ascend910B2-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 24
+ aiCPU: 6
+ templates:
+ - name: vir03_1c_8g
+ memory: 8192
+ aiCore: 3
+ aiCPU: 1
+ - name: vir06_1c_16g
+ memory: 16384
+ aiCore: 6
+ aiCPU: 1
+ - name: vir12_3c_32g
+ memory: 32768
+ aiCore: 12
+ aiCPU: 3
+ - chipName: 910B3
+ commonWord: Ascend910B
+ resourceName: huawei.com/Ascend910B
+ resourceMemoryName: huawei.com/Ascend910B-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: vir05_1c_16g
+ memory: 16384
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_32g
+ memory: 32768
+ aiCore: 10
+ aiCPU: 3
+ - chipName: 910B4
+ commonWord: Ascend910B4
+ resourceName: huawei.com/Ascend910B4
+ resourceMemoryName: huawei.com/Ascend910B4-memory
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: vir05_1c_8g
+ memory: 8192
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_16g
+ memory: 16384
+ aiCore: 10
+ aiCPU: 3
+ - chipName: 310P3
+ commonWord: Ascend310P
+ resourceName: huawei.com/Ascend310P
+ resourceMemoryName: huawei.com/Ascend310P-memory
+ memoryAllocatable: 21527
+ memoryCapacity: 24576
+ aiCore: 8
+ aiCPU: 7
+ templates:
+ - name: vir01
+ memory: 3072
+ aiCore: 1
+ aiCPU: 1
+ - name: vir02
+ memory: 6144
+ aiCore: 2
+ aiCPU: 2
+ - name: vir04
+ memory: 12288
+ aiCore: 4
+ aiCPU: 4
+ {{ end }}
diff --git a/deployments/4pd-vgpu/templates/scheduler/job-patch/clusterrole.yaml b/charts/hami/templates/scheduler/job-patch/clusterrole.yaml
similarity index 66%
rename from deployments/4pd-vgpu/templates/scheduler/job-patch/clusterrole.yaml
rename to charts/hami/templates/scheduler/job-patch/clusterrole.yaml
index eb8bb33fc..7bbd7d83e 100644
--- a/deployments/4pd-vgpu/templates/scheduler/job-patch/clusterrole.yaml
+++ b/charts/hami/templates/scheduler/job-patch/clusterrole.yaml
@@ -1,12 +1,14 @@
+{{- if .Values.scheduler.admissionWebhook.enabled }}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
- name: {{ include "4pd-vgpu.fullname" . }}-admission
+ name: {{ include "hami-vgpu.fullname" . }}-admission
annotations:
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
labels:
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
app.kubernetes.io/component: admission-webhook
rules:
- apiGroups:
@@ -22,5 +24,7 @@ rules:
resources: ['podsecuritypolicies']
verbs: ['use']
resourceNames:
- - {{ include "4pd-vgpu.fullname" . }}-admission
+ - {{ include "hami-vgpu.fullname" . }}-admission
+{{- end }}
+{{- end }}
{{- end }}
diff --git a/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml
new file mode 100644
index 000000000..5d64dbb33
--- /dev/null
+++ b/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml
@@ -0,0 +1,22 @@
+{{- if .Values.scheduler.admissionWebhook.enabled }}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ include "hami-vgpu.fullname" . }}-admission
+ annotations:
+ "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+ "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+ labels:
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+ app.kubernetes.io/component: admission-webhook
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ include "hami-vgpu.fullname" . }}-admission
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "hami-vgpu.fullname" . }}-admission
+ namespace: {{ include "hami-vgpu.namespace" . }}
+{{- end }}
+{{- end }}
diff --git a/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml
new file mode 100644
index 000000000..cffe109a9
--- /dev/null
+++ b/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml
@@ -0,0 +1,70 @@
+{{- if .Values.scheduler.admissionWebhook.enabled }}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: {{ include "hami-vgpu.fullname" . }}-admission-create
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ annotations:
+ "helm.sh/hook": pre-install,pre-upgrade
+ "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+ labels:
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+ app.kubernetes.io/component: admission-webhook
+spec:
+ {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
+ # Alpha feature since k8s 1.12
+ ttlSecondsAfterFinished: 0
+ {{- end }}
+ template:
+ metadata:
+ name: {{ include "hami-vgpu.fullname" . }}-admission-create
+ {{- if .Values.scheduler.patch.podAnnotations }}
+ annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
+ {{- end }}
+ labels:
+ {{- include "hami-vgpu.labels" . | nindent 8 }}
+ app.kubernetes.io/component: admission-webhook
+ hami.io/webhook: ignore
+ spec:
+ {{- if .Values.scheduler.patch.priorityClassName }}
+ priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
+ {{- end }}
+ {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+ {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }}
+ {{- else }}
+ {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }}
+ {{- end }}
+ containers:
+ - name: create
+ {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+ image: {{ include "hami.scheduler.patch.new.image" . }}
+ imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }}
+ {{- else }}
+ image: {{ include "hami.scheduler.patch.image" . }}
+ imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }}
+ {{- end }}
+ args:
+ - create
+ - --cert-name=tls.crt
+ - --key-name=tls.key
+ {{- if .Values.scheduler.admissionWebhook.customURL.enabled }}
+ - --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}}
+ {{- else }}
+ - --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }}
+ {{- end }}
+ - --namespace={{ include "hami-vgpu.namespace" . }}
+ - --secret-name={{ include "hami-vgpu.scheduler.tls" . }}
+ restartPolicy: OnFailure
+ serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission
+ {{- if .Values.scheduler.patch.nodeSelector }}
+ nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
+ {{- end }}
+ {{- if .Values.scheduler.patch.tolerations }}
+ tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
+ {{- end }}
+ securityContext:
+ runAsNonRoot: true
+ runAsUser: {{ .Values.scheduler.patch.runAsUser }}
+{{- end }}
+{{- end }}
diff --git a/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml
new file mode 100644
index 000000000..e08f5b644
--- /dev/null
+++ b/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml
@@ -0,0 +1,65 @@
+{{- if .Values.scheduler.admissionWebhook.enabled }}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: {{ include "hami-vgpu.fullname" . }}-admission-patch
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ annotations:
+ "helm.sh/hook": post-install,post-upgrade
+ "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+ labels:
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+ app.kubernetes.io/component: admission-webhook
+spec:
+ {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
+ # Alpha feature since k8s 1.12
+ ttlSecondsAfterFinished: 0
+ {{- end }}
+ template:
+ metadata:
+ name: {{ include "hami-vgpu.fullname" . }}-admission-patch
+ {{- if .Values.scheduler.patch.podAnnotations }}
+ annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
+ {{- end }}
+ labels:
+ {{- include "hami-vgpu.labels" . | nindent 8 }}
+ app.kubernetes.io/component: admission-webhook
+ hami.io/webhook: ignore
+ spec:
+ {{- if .Values.scheduler.patch.priorityClassName }}
+ priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
+ {{- end }}
+ {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+ {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }}
+ {{- else }}
+ {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }}
+ {{- end }}
+ containers:
+ - name: patch
+ {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }}
+ image: {{ include "hami.scheduler.patch.new.image" . }}
+ imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }}
+ {{- else }}
+ image: {{ include "hami.scheduler.patch.image" . }}
+ imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }}
+ {{- end }}
+ args:
+ - patch
+ - --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }}
+ - --namespace={{ include "hami-vgpu.namespace" . }}
+ - --patch-validating=false
+ - --secret-name={{ include "hami-vgpu.scheduler.tls" . }}
+ restartPolicy: OnFailure
+ serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission
+ {{- if .Values.scheduler.patch.nodeSelector }}
+ nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
+ {{- end }}
+ {{- if .Values.scheduler.patch.tolerations }}
+ tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
+ {{- end }}
+ securityContext:
+ runAsNonRoot: true
+ runAsUser: {{ .Values.scheduler.patch.runAsUser }}
+{{- end }}
+{{- end }}
diff --git a/deployments/4pd-vgpu/templates/scheduler/job-patch/psp.yaml b/charts/hami/templates/scheduler/job-patch/psp.yaml
similarity index 66%
rename from deployments/4pd-vgpu/templates/scheduler/job-patch/psp.yaml
rename to charts/hami/templates/scheduler/job-patch/psp.yaml
index 9a616abd7..0bebaec7e 100644
--- a/deployments/4pd-vgpu/templates/scheduler/job-patch/psp.yaml
+++ b/charts/hami/templates/scheduler/job-patch/psp.yaml
@@ -1,12 +1,15 @@
+{{- if .Values.scheduler.admissionWebhook.enabled }}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+{{- if .Values.podSecurityPolicy.enabled }}
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
- name: {{ include "4pd-vgpu.fullname" . }}-admission
+ name: {{ include "hami-vgpu.fullname" . }}-admission
annotations:
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
labels:
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
app.kubernetes.io/component: admission-webhook
spec:
allowPrivilegeEscalation: false
@@ -32,3 +35,6 @@ spec:
- projected
- secret
- downwardAPI
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/deployments/4pd-vgpu/templates/scheduler/job-patch/role.yaml b/charts/hami/templates/scheduler/job-patch/role.yaml
similarity index 53%
rename from deployments/4pd-vgpu/templates/scheduler/job-patch/role.yaml
rename to charts/hami/templates/scheduler/job-patch/role.yaml
index c5bb7e6b0..f5d646e8e 100644
--- a/deployments/4pd-vgpu/templates/scheduler/job-patch/role.yaml
+++ b/charts/hami/templates/scheduler/job-patch/role.yaml
@@ -1,12 +1,15 @@
+{{- if .Values.scheduler.admissionWebhook.enabled }}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
- name: {{ include "4pd-vgpu.fullname" . }}-admission
+ name: {{ include "hami-vgpu.fullname" . }}-admission
+ namespace: {{ include "hami-vgpu.namespace" . }}
annotations:
"helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
labels:
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
app.kubernetes.io/component: admission-webhook
rules:
- apiGroups:
@@ -16,3 +19,5 @@ rules:
verbs:
- get
- create
+{{- end }}
+{{- end }}
diff --git a/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/charts/hami/templates/scheduler/job-patch/rolebinding.yaml
new file mode 100644
index 000000000..e6cf67792
--- /dev/null
+++ b/charts/hami/templates/scheduler/job-patch/rolebinding.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.scheduler.admissionWebhook.enabled }}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ include "hami-vgpu.fullname" . }}-admission
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ annotations:
+ "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+ "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+ labels:
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+ app.kubernetes.io/component: admission-webhook
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ include "hami-vgpu.fullname" . }}-admission
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "hami-vgpu.fullname" . }}-admission
+ namespace: {{ include "hami-vgpu.namespace" . }}
+{{- end }}
+{{- end }}
diff --git a/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml
new file mode 100644
index 000000000..43dbe971c
--- /dev/null
+++ b/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml
@@ -0,0 +1,15 @@
+{{- if .Values.scheduler.admissionWebhook.enabled }}
+{{- if and (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ include "hami-vgpu.fullname" . }}-admission
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ annotations:
+ "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+ "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+ labels:
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+ app.kubernetes.io/component: admission-webhook
+{{- end }}
+{{- end }}
diff --git a/charts/hami/templates/scheduler/role.yaml b/charts/hami/templates/scheduler/role.yaml
new file mode 100644
index 000000000..73ded0009
--- /dev/null
+++ b/charts/hami/templates/scheduler/role.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: "hami-scheduler"
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+rules:
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["create", "get", "update", "patch"]
diff --git a/charts/hami/templates/scheduler/rolebinding.yaml b/charts/hami/templates/scheduler/rolebinding.yaml
new file mode 100644
index 000000000..8a8a7ccf3
--- /dev/null
+++ b/charts/hami/templates/scheduler/rolebinding.yaml
@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: "hami-scheduler"
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ include "hami-vgpu.scheduler" . }}
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "hami-vgpu.scheduler" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
diff --git a/charts/hami/templates/scheduler/service.yaml b/charts/hami/templates/scheduler/service.yaml
new file mode 100644
index 000000000..70378c5cb
--- /dev/null
+++ b/charts/hami/templates/scheduler/service.yaml
@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: hami-scheduler
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
+ {{- if .Values.scheduler.service.labels }}
+ {{ toYaml .Values.scheduler.service.labels | indent 4 }}
+ {{- end }}
+ {{- if .Values.scheduler.service.annotations }}
+ annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }}
+ {{- end }}
+spec:
+ type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort
+ ports:
+ - name: http
+ port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443
+ targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }}
+ {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
+ nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }}
+ {{- end }}
+ protocol: TCP
+ - name: monitor
+ port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993
+ targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }}
+ {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort
+ nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }}
+ {{- end }}
+ protocol: TCP
+ selector:
+ app.kubernetes.io/component: hami-scheduler
+ {{- include "hami-vgpu.selectorLabels" . | nindent 4 }}
\ No newline at end of file
diff --git a/charts/hami/templates/scheduler/serviceaccount.yaml b/charts/hami/templates/scheduler/serviceaccount.yaml
new file mode 100644
index 000000000..f20a6e625
--- /dev/null
+++ b/charts/hami/templates/scheduler/serviceaccount.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ include "hami-vgpu.scheduler" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ labels:
+ app.kubernetes.io/component: "hami-scheduler"
+ {{- include "hami-vgpu.labels" . | nindent 4 }}
diff --git a/charts/hami/templates/scheduler/webhook.yaml b/charts/hami/templates/scheduler/webhook.yaml
new file mode 100644
index 000000000..3e5e92f29
--- /dev/null
+++ b/charts/hami/templates/scheduler/webhook.yaml
@@ -0,0 +1,57 @@
+{{- if .Values.scheduler.admissionWebhook.enabled }}
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+ {{- if .Values.scheduler.certManager.enabled }}
+ annotations:
+ cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert
+ {{- end }}
+ name: {{ include "hami-vgpu.scheduler.webhook" . }}
+webhooks:
+ - admissionReviewVersions:
+ - v1beta1
+ clientConfig:
+ {{- if .Values.scheduler.admissionWebhook.customURL.enabled }}
+ url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}}
+ {{- else }}
+ service:
+ name: {{ include "hami-vgpu.scheduler" . }}
+ namespace: {{ include "hami-vgpu.namespace" . }}
+ path: /webhook
+ port: {{ .Values.scheduler.service.httpPort }}
+ {{- end }}
+ failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }}
+ matchPolicy: Equivalent
+ name: vgpu.hami.io
+ namespaceSelector:
+ matchExpressions:
+ - key: hami.io/webhook
+ operator: NotIn
+ values:
+ - ignore
+ {{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }}
+ - key: kubernetes.io/metadata.name
+ operator: NotIn
+ values:
+ {{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }}
+ {{- end }}
+ objectSelector:
+ matchExpressions:
+ - key: hami.io/webhook
+ operator: NotIn
+ values:
+ - ignore
+ reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }}
+ rules:
+ - apiGroups:
+ - ""
+ apiVersions:
+ - v1
+ operations:
+ - CREATE
+ resources:
+ - pods
+ scope: '*'
+ sideEffects: None
+ timeoutSeconds: 10
+{{- end }}
\ No newline at end of file
diff --git a/charts/hami/values.yaml b/charts/hami/values.yaml
new file mode 100644
index 000000000..8f9ed9e66
--- /dev/null
+++ b/charts/hami/values.yaml
@@ -0,0 +1,380 @@
+## @section Global configuration
+## @param global.imageRegistry Global Docker image registry
+## @param global.imagePullSecrets Global Docker image pull secrets
+global:
+ ## @param global.imageRegistry Global Docker image registry
+ imageRegistry: ""
+ ## E.g.
+ ## imagePullSecrets:
+ ## - myRegistryKeySecretName
+ ## @param global.imagePullSecrets Global Docker image pull secrets
+ imagePullSecrets: []
+ imageTag: "v2.6.0"
+ gpuHookPath: /usr/local
+ labels: {}
+ annotations: {}
+ managedNodeSelectorEnable: false
+ managedNodeSelector:
+ usage: "gpu"
+
+nameOverride: ""
+fullnameOverride: ""
+namespaceOverride: ""
+
+#Nvidia GPU Parameters
+resourceName: "nvidia.com/gpu"
+resourceMem: "nvidia.com/gpumem"
+resourceMemPercentage: "nvidia.com/gpumem-percentage"
+resourceCores: "nvidia.com/gpucores"
+resourcePriority: "nvidia.com/priority"
+
+#MLU Parameters
+mluResourceName: "cambricon.com/vmlu"
+mluResourceMem: "cambricon.com/mlu.smlu.vmemory"
+mluResourceCores: "cambricon.com/mlu.smlu.vcore"
+
+#Hygon DCU Parameters
+dcuResourceName: "hygon.com/dcunum"
+dcuResourceMem: "hygon.com/dcumem"
+dcuResourceCores: "hygon.com/dcucores"
+
+#Iluvatar GPU Parameters
+iluvatarResourceName: "iluvatar.ai/vgpu"
+iluvatarResourceMem: "iluvatar.ai/vcuda-memory"
+iluvatarResourceCore: "iluvatar.ai/vcuda-core"
+
+#Metax sGPU Parameters
+metaxResourceName: "metax-tech.com/sgpu"
+metaxResourceCore: "metax-tech.com/vcore"
+metaxResourceMem: "metax-tech.com/vmemory"
+metaxsGPUTopologyAware: "false"
+
+#Kunlun XPU Parameters
+kunlunResourceName: "kunlunxin.com/xpu"
+
+schedulerName: "hami-scheduler"
+
+podSecurityPolicy:
+ enabled: false
+
+scheduler:
+ # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node.
+ # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default
+ # scheduler pod from the cluster first, we must specify node name to skip the schedule workflow.
+ nodeName: ""
+ #nodeLabelSelector:
+ # "gpu": "on"
+ overwriteEnv: "false"
+ defaultSchedulerPolicy:
+ nodeSchedulerPolicy: binpack
+ gpuSchedulerPolicy: spread
+ metricsBindAddress: ":9395"
+ # If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it
+ forceOverwriteDefaultScheduler: true
+ livenessProbe: false
+ leaderElect: true
+ # when leaderElect is true, replicas is available, otherwise replicas is 1.
+ replicas: 1
+ kubeScheduler:
+ # @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default.
+ enabled: true
+ ## @param image.registry kube scheduler image registry
+ ## @param image.repository kube scheduler image repository
+ ## @param image.tag kube scheduler image tag (immutable tags are recommended)
+ ## @param image.pullPolicy kube scheduler image pull policy
+ ## @param image.pullSecrets Specify docker-registry secret names as an array
+ image:
+ registry: "registry.cn-hangzhou.aliyuncs.com"
+ repository: "google_containers/kube-scheduler"
+ tag: ""
+ ## Specify a imagePullPolicy
+ ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+ ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+ ##
+ pullPolicy: IfNotPresent
+ ## Optionally specify an array of imagePullSecrets.
+ ## Secrets must be manually created in the namespace.
+ ## Example:
+ ## pullSecrets:
+ ## - myRegistryKeySecretName
+ ##
+ pullSecrets: []
+ resources: {}
+ # If you do want to specify resources, uncomment the following lines, adjust them as necessary.
+ # and remove the curly braces after 'resources:'.
+# limits:
+# cpu: 1000m
+# memory: 1000Mi
+# requests:
+# cpu: 100m
+# memory: 100Mi
+ extraNewArgs:
+ - --config=/config/config.yaml
+ - -v=4
+ extraArgs:
+ - --policy-config-file=/config/config.json
+ - -v=4
+ extender:
+ ## @param image.registry scheduler extender image registry
+ ## @param image.repository scheduler extender image repository
+ ## @param image.tag scheduler extender image tag (immutable tags are recommended)
+ ## @param image.pullPolicy scheduler extender image pull policy
+ ## @param image.pullSecrets Specify docker-registry secret names as an array
+ image:
+ registry: "docker.io"
+ repository: "projecthami/hami"
+ tag: ""
+ ## Specify a imagePullPolicy
+ ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+ ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+ ##
+ pullPolicy: IfNotPresent
+ ## Optionally specify an array of imagePullSecrets.
+ ## Secrets must be manually created in the namespace.
+ ## Example:
+ ## pullSecrets:
+ ## - myRegistryKeySecretName
+ ##
+ pullSecrets: []
+ resources: {}
+ # If you do want to specify resources, uncomment the following lines, adjust them as necessary,
+ # and remove the curly braces after 'resources:'.
+# limits:
+# cpu: 1000m
+# memory: 1000Mi
+# requests:
+# cpu: 100m
+# memory: 100Mi
+ extraArgs:
+ - --debug
+ - -v=4
+ nodeLockExpire: "5m"
+ podAnnotations: {}
+ tolerations: []
+ #serviceAccountName: "hami-vgpu-scheduler-sa"
+ admissionWebhook:
+ # If set to false, the admission webhook is not installed and any pods that should use HAMi must be
+ # configured to use the right 'schedulerName' and other device-specific configurations.
+ enabled: true
+ customURL:
+ enabled: false
+ # must be an endpoint using https.
+ # should generate host certs here
+ host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://:/
+ port: 31998
+ path: /webhook
+ whitelistNamespaces:
+ # Specify the namespaces that the webhook will not be applied to.
+ # - default
+ # - kube-system
+ # - istio-system
+ reinvocationPolicy: Never
+ failurePolicy: Ignore
+ ## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate.
+ ## If enabled, always takes precedence over options 2.
+ certManager:
+ enabled: false
+ ## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate.
+ ## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you.
+ patch:
+ enabled: true
+ ## @param image.registry scheduler extender image registry
+ ## @param image.repository scheduler extender image repository
+ ## @param image.tag scheduler extender image tag (immutable tags are recommended)
+ ## @param image.pullPolicy scheduler extender image pull policy
+ ## @param image.pullSecrets Specify docker-registry secret names as an array
+ image:
+ registry: "docker.io"
+ repository: "jettech/kube-webhook-certgen"
+ tag: "v1.5.2"
+ ## Specify a imagePullPolicy
+ ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+ ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+ ##
+ pullPolicy: IfNotPresent
+ ## Optionally specify an array of imagePullSecrets.
+ ## Secrets must be manually created in the namespace.
+ ## Example:
+ ## pullSecrets:
+ ## - myRegistryKeySecretName
+ ##
+ pullSecrets: []
+ ## @param image.registry scheduler extender image registry
+ ## @param image.repository scheduler extender image repository
+ ## @param image.tag scheduler extender image tag (immutable tags are recommended)
+ ## @param image.pullPolicy scheduler extender image pull policy
+ ## @param image.pullSecrets Specify docker-registry secret names as an array
+ imageNew:
+ registry: "docker.io"
+ repository: "liangjw/kube-webhook-certgen"
+ tag: "v1.1.1"
+ ## Specify a imagePullPolicy
+ ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+ ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+ ##
+ pullPolicy: IfNotPresent
+ ## Optionally specify an array of imagePullSecrets.
+ ## Secrets must be manually created in the namespace.
+ ## Example:
+ ## pullSecrets:
+ ## - myRegistryKeySecretName
+ ##
+ pullSecrets: []
+ priorityClassName: ""
+ podAnnotations: {}
+ nodeSelector: {}
+ tolerations: []
+ runAsUser: 2000
+ service:
+ type: NodePort # Default type is NodePort, can be changed to ClusterIP
+ httpPort: 443 # HTTP port
+ schedulerPort: 31998 # NodePort for HTTP
+ monitorPort: 31993 # Monitoring port
+ monitorTargetPort: 9395
+ labels: {}
+ annotations: {}
+
+devicePlugin:
+ enabled: true
+ ## @param image.registry devicePlugin image registry
+ ## @param image.repository devicePlugin image repository
+ ## @param image.tag devicePlugin image tag (immutable tags are recommended)
+ ## @param image.pullPolicy devicePlugin image pull policy
+ ## @param image.pullSecrets Specify docker-registry secret names as an array
+ image:
+ registry: "docker.io"
+ repository: "projecthami/hami"
+ tag: ""
+ ## Specify a imagePullPolicy
+ ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+ ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+ ##
+ pullPolicy: IfNotPresent
+ ## Optionally specify an array of imagePullSecrets.
+ ## Secrets must be manually created in the namespace.
+ ## Example:
+ ## pullSecrets:
+ ## - myRegistryKeySecretName
+ ##
+ pullSecrets: []
+ monitor:
+ ## @param image.registry monitor image registry
+ ## @param image.repository monitor image repository
+ ## @param image.tag monitor image tag (immutable tags are recommended)
+ ## @param image.pullPolicy monitor image pull policy
+ ## @param image.pullSecrets Specify docker-registry secret names as an array
+ image:
+ registry: "docker.io"
+ repository: "projecthami/hami"
+ tag: ""
+ ## Specify a imagePullPolicy
+ ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+ ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+ ##
+ pullPolicy: IfNotPresent
+ ## Optionally specify an array of imagePullSecrets.
+ ## Secrets must be manually created in the namespace.
+ ## Example:
+ ## pullSecrets:
+ ## - myRegistryKeySecretName
+ ##
+ pullSecrets: []
+ ctrPath: /usr/local/vgpu/containers
+ resyncInterval: "5m"
+ deviceSplitCount: 10
+ deviceMemoryScaling: 1
+ deviceCoreScaling: 1
+ # The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs
+ runtimeClassName: ""
+ # Whether to create runtime class, name comes from runtimeClassName when it is set
+ createRuntimeClass: false
+ migStrategy: "none"
+ disablecorelimit: "false"
+ passDeviceSpecsEnabled: false
+ extraArgs:
+ - -v=4
+
+ service:
+ type: NodePort # Default type is NodePort, can be changed to ClusterIP
+ httpPort: 31992
+ labels: {}
+ annotations: {}
+
+ pluginPath: /var/lib/kubelet/device-plugins
+ libPath: /usr/local/vgpu
+
+ podAnnotations: {}
+ nvidianodeSelector:
+ gpu: "on"
+ tolerations: []
+ # The updateStrategy for DevicePlugin DaemonSet.
+ # If you want to update the DaemonSet by manual, set type as "OnDelete".
+ # We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive.
+ # Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod.
+ updateStrategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 1
+
+ resources: {}
+ # If you do want to specify resources, uncomment the following lines, adjust them as necessary.
+ # and remove the curly braces after 'resources:'.
+# limits:
+# cpu: 1000m
+# memory: 1000Mi
+# requests:
+# cpu: 100m
+# memory: 100Mi
+
+ vgpuMonitor:
+ resources: {}
+ # If you do want to specify resources, uncomment the following lines, adjust them as necessary.
+ # and remove the curly braces after 'resources:'.
+# limits:
+# cpu: 1000m
+# memory: 1000Mi
+# requests:
+# cpu: 100m
+# memory: 100Mi
+
+devices:
+ awsneuron:
+ customresources:
+ - aws.amazon.com/neuron
+ - aws.amazon.com/neuroncore
+ kunlun:
+ enabled: true
+ customresources:
+ - kunlunxin.com/xpu
+ enflame:
+ enabled: true
+ customresources:
+ - enflame.com/vgcu
+ - enflame.com/vgcu-percentage
+ mthreads:
+ enabled: true
+ customresources:
+ - mthreads.com/vgpu
+ nvidia:
+ gpuCorePolicy: default
+ libCudaLogLevel: 1
+ ascend:
+ enabled: false
+ image: ""
+ imagePullPolicy: IfNotPresent
+ extraArgs: []
+ nodeSelector:
+ ascend: "on"
+ tolerations: []
+ customresources:
+ - huawei.com/Ascend910A
+ - huawei.com/Ascend910A-memory
+ - huawei.com/Ascend910B2
+ - huawei.com/Ascend910B2-memory
+ - huawei.com/Ascend910B
+ - huawei.com/Ascend910B-memory
+ - huawei.com/Ascend910B4
+ - huawei.com/Ascend910B4-memory
+ - huawei.com/Ascend310P
+ - huawei.com/Ascend310P-memory
+
diff --git a/cmd/device-plugin/main.go b/cmd/device-plugin/main.go
deleted file mode 100644
index 78c7c12ef..000000000
--- a/cmd/device-plugin/main.go
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package main
-
-import (
- "4pd.io/k8s-vgpu/pkg/version"
- "fmt"
- "net"
- "syscall"
-
- "4pd.io/k8s-vgpu/pkg/api"
- "4pd.io/k8s-vgpu/pkg/device-plugin"
- "4pd.io/k8s-vgpu/pkg/device-plugin/config"
- "4pd.io/k8s-vgpu/pkg/util"
- "github.com/NVIDIA/go-gpuallocator/gpuallocator"
- "github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml"
- "github.com/fsnotify/fsnotify"
- "github.com/spf13/cobra"
- "github.com/spf13/viper"
- "google.golang.org/grpc"
- "k8s.io/klog/v2"
- pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
-)
-
-var (
- failOnInitErrorFlag bool
- //nvidiaDriverRootFlag string
- //enableLegacyPreferredFlag bool
-
- rootCmd = &cobra.Command{
- Use: "device-plugin",
- Short: "kubernetes vgpu device-plugin",
- Run: func(cmd *cobra.Command, args []string) {
- if err := start(); err != nil {
- klog.Fatal(err)
- }
- },
- }
-)
-
-func init() {
- // https://github.com/spf13/viper/issues/461
- viper.BindEnv("node-name", "NODE_NAME")
-
- rootCmd.Flags().SortFlags = false
- rootCmd.PersistentFlags().SortFlags = false
-
- rootCmd.Flags().BoolVar(&failOnInitErrorFlag, "fail-on-init-error", true, "fail the plugin if an error is encountered during initialization, otherwise block indefinitely")
- rootCmd.Flags().StringVar(&config.RuntimeSocketFlag, "runtime-socket", "/var/lib/vgpu/vgpu.sock", "runtime socket")
- rootCmd.Flags().UintVar(&config.DeviceSplitCount, "device-split-count", 2, "the number for NVIDIA device split")
- rootCmd.Flags().Float64Var(&config.DeviceMemoryScaling, "device-memory-scaling", 1.0, "the ratio for NVIDIA device memory scaling")
- rootCmd.Flags().Float64Var(&config.DeviceCoresScaling, "device-cores-scaling", 1.0, "the ratio for NVIDIA device cores scaling")
- rootCmd.Flags().StringVar(&config.SchedulerEndpoint, "scheduler-endpoint", "127.0.0.1:9090", "scheduler extender endpoint")
- rootCmd.Flags().IntVar(&config.SchedulerTimeout, "scheduler-timeout", 10, "scheduler connection timeout")
- rootCmd.Flags().StringVar(&config.NodeName, "node-name", viper.GetString("node-name"), "node name")
-
- rootCmd.PersistentFlags().AddGoFlagSet(util.GlobalFlagSet())
- rootCmd.AddCommand(version.VersionCmd)
-}
-
-func start() error {
- klog.Info("Loading NVML")
- if err := nvml.Init(); err != nil {
- klog.Infof("Failed to initialize NVML: %v.", err)
- klog.Infof("If this is a GPU node, did you set the docker default runtime to `nvidia`?")
- klog.Infof("You can check the prerequisites at: https://github.com/NVIDIA/k8s-device-plugin#prerequisites")
- klog.Infof("You can learn how to set the runtime at: https://github.com/NVIDIA/k8s-device-plugin#quick-start")
- klog.Infof("If this is not a GPU node, you should set up a toleration or nodeSelector to only deploy this plugin on GPU nodes")
- if failOnInitErrorFlag {
- return fmt.Errorf("failed to initialize NVML: %v", err)
- }
- select {}
- }
- defer func() { klog.Info("Shutdown of NVML returned:", nvml.Shutdown()) }()
-
- klog.Info("Starting FS watcher.")
- watcher, err := NewFSWatcher(pluginapi.DevicePluginPath)
- if err != nil {
- return fmt.Errorf("failed to create FS watcher: %v", err)
- }
- defer watcher.Close()
-
- klog.Info("Starting OS watcher.")
- sigs := NewOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
-
- cache := device_plugin.NewDeviceCache()
- cache.Start()
- defer cache.Stop()
- register := device_plugin.NewDeviceRegister(cache)
- register.Start()
- defer register.Stop()
- rt := device_plugin.NewVGPURuntimeService(cache)
-
- // start runtime grpc server
- lisGrpc, err := net.Listen("unix", config.RuntimeSocketFlag)
- if err != nil {
- klog.Fatalf("bind unix socket %v failed, %v", err)
- }
- defer lisGrpc.Close()
- runtimeServer := grpc.NewServer()
- api.RegisterVGPURuntimeServiceServer(runtimeServer, rt)
- go func() {
- err := runtimeServer.Serve(lisGrpc)
- if err != nil {
- klog.Fatal(err)
- }
- }()
- defer runtimeServer.Stop()
-
- var plugins []*device_plugin.NvidiaDevicePlugin
-restart:
- // If we are restarting, idempotently stop any running plugins before
- // recreating them below.
- for _, p := range plugins {
- p.Stop()
- }
-
- klog.Info("Retreiving plugins.")
- plugins = []*device_plugin.NvidiaDevicePlugin{
- device_plugin.NewNvidiaDevicePlugin(
- util.ResourceName,
- cache,
- gpuallocator.NewBestEffortPolicy(),
- pluginapi.DevicePluginPath+"nvidia-gpu.sock"),
- }
-
- // Loop through all plugins, starting them if they have any devices
- // to serve. If even one plugin fails to start properly, try
- // starting them all again.
- started := 0
- pluginStartError := make(chan struct{})
- for _, p := range plugins {
- // Just continue if there are no devices to serve for plugin p.
- if len(p.Devices()) == 0 {
- continue
- }
-
- // Start the gRPC server for plugin p and connect it with the kubelet.
- if err := p.Start(); err != nil {
- //klog.SetOutput(os.Stderr)
- klog.Info("Could not contact Kubelet, retrying. Did you enable the device plugin feature gate?")
- klog.Info("You can check the prerequisites at: https://github.com/NVIDIA/k8s-device-plugin#prerequisites")
- klog.Info("You can learn how to set the runtime at: https://github.com/NVIDIA/k8s-device-plugin#quick-start")
- close(pluginStartError)
- goto events
- }
- started++
- }
-
- if started == 0 {
- klog.Info("No devices found. Waiting indefinitely.")
- }
-
-events:
- // Start an infinite loop, waiting for several indicators to either log
- // some messages, trigger a restart of the plugins, or exit the program.
- for {
- select {
- // If there was an error starting any plugins, restart them all.
- case <-pluginStartError:
- goto restart
-
- // Detect a kubelet restart by watching for a newly created
- // 'pluginapi.KubeletSocket' file. When this occurs, restart this loop,
- // restarting all of the plugins in the process.
- case event := <-watcher.Events:
- if event.Name == pluginapi.KubeletSocket && event.Op&fsnotify.Create == fsnotify.Create {
- klog.Infof("inotify: %s created, restarting.", pluginapi.KubeletSocket)
- goto restart
- }
-
- // Watch for any other fs errors and log them.
- case err := <-watcher.Errors:
- klog.Infof("inotify: %s", err)
-
- // Watch for any signals from the OS. On SIGHUP, restart this loop,
- // restarting all of the plugins in the process. On all other
- // signals, exit the loop and exit the program.
- case s := <-sigs:
- switch s {
- case syscall.SIGHUP:
- klog.Info("Received SIGHUP, restarting.")
- goto restart
- default:
- klog.Infof("Received signal %v, shutting down.", s)
- for _, p := range plugins {
- p.Stop()
- }
- break events
- }
- }
- }
- return nil
-}
-
-func main() {
- if err := rootCmd.Execute(); err != nil {
- klog.Fatal(err)
- }
-}
diff --git a/cmd/device-plugin/nvidia/main.go b/cmd/device-plugin/nvidia/main.go
new file mode 100644
index 000000000..7fa133a0c
--- /dev/null
+++ b/cmd/device-plugin/nvidia/main.go
@@ -0,0 +1,383 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "encoding/json"
+ "flag"
+ "fmt"
+ "os"
+ "syscall"
+ "time"
+
+ spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
+ "github.com/fsnotify/fsnotify"
+ cli "github.com/urfave/cli/v2"
+ errorsutil "k8s.io/apimachinery/pkg/util/errors"
+ "k8s.io/klog/v2"
+ kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/info"
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin"
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/rm"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+ flagutil "github.com/Project-HAMi/HAMi/pkg/util/flag"
+)
+
+func main() {
+ var configFile string
+
+ c := cli.NewApp()
+ c.Name = "NVIDIA Device Plugin"
+ c.Usage = "NVIDIA device plugin for Kubernetes"
+ c.Action = func(ctx *cli.Context) error {
+ flagutil.PrintCliFlags(ctx)
+ return start(ctx, c.Flags)
+ }
+ c.Commands = []*cli.Command{
+ {
+ Name: "version",
+ Usage: "Show the version of NVIDIA Device Plugin",
+ Action: func(c *cli.Context) error {
+ fmt.Printf("%s version: %s\n", c.App.Name, info.GetVersionString())
+ return nil
+ },
+ },
+ }
+
+ flagset := flag.NewFlagSet("klog", flag.ExitOnError)
+ klog.InitFlags(flagset)
+
+ c.Before = func(ctx *cli.Context) error {
+ logLevel := ctx.Int("v")
+ if err := flagset.Set("v", fmt.Sprintf("%d", logLevel)); err != nil {
+ return err
+ }
+ return nil
+ }
+
+ c.Flags = []cli.Flag{
+ &cli.StringFlag{
+ Name: "mig-strategy",
+ Value: spec.MigStrategyNone,
+ Usage: "the desired strategy for exposing MIG devices on GPUs that support it:\n\t\t[none | single | mixed]",
+ EnvVars: []string{"MIG_STRATEGY"},
+ },
+ &cli.BoolFlag{
+ Name: "fail-on-init-error",
+ Value: true,
+ Usage: "fail the plugin if an error is encountered during initialization, otherwise block indefinitely",
+ EnvVars: []string{"FAIL_ON_INIT_ERROR"},
+ },
+ &cli.StringFlag{
+ Name: "nvidia-driver-root",
+ Value: "/",
+ Usage: "the root path for the NVIDIA driver installation (typical values are '/' or '/run/nvidia/driver')",
+ EnvVars: []string{"NVIDIA_DRIVER_ROOT"},
+ },
+ &cli.BoolFlag{
+ Name: "pass-device-specs",
+ Value: false,
+ Usage: "pass the list of DeviceSpecs to the kubelet on Allocate()",
+ EnvVars: []string{"PASS_DEVICE_SPECS"},
+ },
+ &cli.StringSliceFlag{
+ Name: "device-list-strategy",
+ Value: cli.NewStringSlice(string(spec.DeviceListStrategyEnvVar)),
+ Usage: "the desired strategy for passing the device list to the underlying runtime:\n\t\t[envvar | volume-mounts | cdi-annotations]",
+ EnvVars: []string{"DEVICE_LIST_STRATEGY"},
+ },
+ &cli.StringFlag{
+ Name: "device-id-strategy",
+ Value: spec.DeviceIDStrategyUUID,
+ Usage: "the desired strategy for passing device IDs to the underlying runtime:\n\t\t[uuid | index]",
+ EnvVars: []string{"DEVICE_ID_STRATEGY"},
+ },
+ &cli.BoolFlag{
+ Name: "gds-enabled",
+ Usage: "ensure that containers are started with NVIDIA_GDS=enabled",
+ EnvVars: []string{"GDS_ENABLED"},
+ },
+ &cli.BoolFlag{
+ Name: "mofed-enabled",
+ Usage: "ensure that containers are started with NVIDIA_MOFED=enabled",
+ EnvVars: []string{"MOFED_ENABLED"},
+ },
+ &cli.StringFlag{
+ Name: "config-file",
+ Usage: "the path to a config file as an alternative to command line options or environment variables",
+ Destination: &configFile,
+ EnvVars: []string{"CONFIG_FILE"},
+ },
+ &cli.StringFlag{
+ Name: "cdi-annotation-prefix",
+ Value: spec.DefaultCDIAnnotationPrefix,
+ Usage: "the prefix to use for CDI container annotation keys",
+ EnvVars: []string{"CDI_ANNOTATION_PREFIX"},
+ },
+ &cli.StringFlag{
+ Name: "nvidia-ctk-path",
+ Value: spec.DefaultNvidiaCTKPath,
+ Usage: "the path to use for the nvidia-ctk in the generated CDI specification",
+ EnvVars: []string{"NVIDIA_CTK_PATH"},
+ },
+ &cli.StringFlag{
+ Name: "container-driver-root",
+ Value: spec.DefaultContainerDriverRoot,
+ Usage: "the path where the NVIDIA driver root is mounted in the container; used for generating CDI specifications",
+ EnvVars: []string{"CONTAINER_DRIVER_ROOT"},
+ },
+ &cli.IntFlag{
+ Name: "v",
+ Usage: "number for the log level verbosity",
+ Value: 0,
+ },
+ }
+ c.Flags = append(c.Flags, addFlags()...)
+ err := c.Run(os.Args)
+ if err != nil {
+ klog.Error(err)
+ os.Exit(1)
+ }
+}
+
+func validateFlags(config *spec.Config) error {
+ _, err := spec.NewDeviceListStrategies(*config.Flags.Plugin.DeviceListStrategy)
+ if err != nil {
+ return fmt.Errorf("invalid --device-list-strategy option: %v", err)
+ }
+
+ if *config.Flags.Plugin.DeviceIDStrategy != spec.DeviceIDStrategyUUID && *config.Flags.Plugin.DeviceIDStrategy != spec.DeviceIDStrategyIndex {
+ return fmt.Errorf("invalid --device-id-strategy option: %v", *config.Flags.Plugin.DeviceIDStrategy)
+ }
+ return nil
+}
+
+func loadConfig(c *cli.Context, flags []cli.Flag) (*spec.Config, error) {
+ config, err := spec.NewConfig(c, flags)
+ if err != nil {
+ return nil, fmt.Errorf("unable to finalize config: %v", err)
+ }
+ err = validateFlags(config)
+ if err != nil {
+ return nil, fmt.Errorf("unable to validate flags: %v", err)
+ }
+ config.Flags.GFD = nil
+ return config, nil
+}
+
+func start(c *cli.Context, flags []cli.Flag) error {
+ klog.Info("Starting FS watcher.")
+ util.NodeName = os.Getenv(util.NodeNameEnvName)
+ client.InitGlobalClient()
+ watcher, err := newFSWatcher(kubeletdevicepluginv1beta1.DevicePluginPath)
+ if err != nil {
+ return fmt.Errorf("failed to create FS watcher: %v", err)
+ }
+ defer watcher.Close()
+ //device.InitDevices()
+
+ /*Loading config files*/
+ klog.Infof("Start working on node %s", util.NodeName)
+ klog.Info("Starting OS watcher.")
+ sigs := newOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+
+ var restarting bool
+ var restartTimeout <-chan time.Time
+ var plugins []plugin.Interface
+restart:
+ // If we are restarting, stop plugins from previous run.
+ if restarting {
+ err := stopPlugins(plugins)
+ if err != nil {
+ return fmt.Errorf("error stopping plugins from previous run: %v", err)
+ }
+ }
+
+ klog.Info("Starting Plugins.")
+ plugins, restartPlugins, err := startPlugins(c, flags, restarting)
+ if err != nil {
+ return fmt.Errorf("error starting plugins: %v", err)
+ }
+
+ if restartPlugins {
+ klog.Info("Failed to start one or more plugins. Retrying in 30s...")
+ restartTimeout = time.After(30 * time.Second)
+ }
+
+ restarting = true
+
+ // Start an infinite loop, waiting for several indicators to either log
+ // some messages, trigger a restart of the plugins, or exit the program.
+ for {
+ select {
+ // If the restart timeout has expired, then restart the plugins
+ case <-restartTimeout:
+ goto restart
+
+ // Detect a kubelet restart by watching for a newly created
+ // 'kubeletdevicepluginv1beta1.KubeletSocket' file. When this occurs, restart this loop,
+ // restarting all of the plugins in the process.
+ case event := <-watcher.Events:
+ if event.Name == kubeletdevicepluginv1beta1.KubeletSocket && event.Op&fsnotify.Create == fsnotify.Create {
+ klog.Infof("inotify: %s created, restarting.", kubeletdevicepluginv1beta1.KubeletSocket)
+ goto restart
+ }
+
+ // Watch for any other fs errors and log them.
+ case err := <-watcher.Errors:
+ klog.Errorf("inotify: %s", err)
+
+ // Watch for any signals from the OS. On SIGHUP, restart this loop,
+ // restarting all of the plugins in the process. On all other
+ // signals, exit the loop and exit the program.
+ case s := <-sigs:
+ switch s {
+ case syscall.SIGHUP:
+ klog.Info("Received SIGHUP, restarting.")
+ goto restart
+ default:
+ klog.Infof("Received signal \"%v\", shutting down.", s)
+ goto exit
+ }
+ }
+ }
+exit:
+ err = stopPlugins(plugins)
+ if err != nil {
+ return fmt.Errorf("error stopping plugins: %v", err)
+ }
+ return nil
+}
+
+func startPlugins(c *cli.Context, flags []cli.Flag, restarting bool) ([]plugin.Interface, bool, error) {
+ // Load the configuration file
+ klog.Info("Loading configuration.")
+ config, err := loadConfig(c, flags)
+ if err != nil {
+ return nil, false, fmt.Errorf("unable to load config: %v", err)
+ }
+ disableResourceRenamingInConfig(config)
+
+ /*Loading config files*/
+ //fmt.Println("NodeName=", config.NodeName)
+ devConfig, err := generateDeviceConfigFromNvidia(config, c, flags)
+ if err != nil {
+ klog.Errorf("failed to load config file %s", err.Error())
+ return nil, false, err
+ }
+
+ // Update the configuration file with default resources.
+ klog.Info("Updating config with default resource matching patterns.")
+ err = rm.AddDefaultResourcesToConfig(&devConfig)
+ if err != nil {
+ return nil, false, fmt.Errorf("unable to add default resources to config: %v", err)
+ }
+
+ // Print the config to the output.
+ configJSON, err := json.MarshalIndent(devConfig, "", " ")
+ if err != nil {
+ return nil, false, fmt.Errorf("failed to marshal config to JSON: %v", err)
+ }
+ klog.Infof("\nRunning with config:\n%v", string(configJSON))
+
+ // Get the set of plugins.
+ klog.Info("Retrieving plugins.")
+ pluginManager, err := NewPluginManager(&devConfig)
+ if err != nil {
+ return nil, false, fmt.Errorf("error creating plugin manager: %v", err)
+ }
+ plugins, err := pluginManager.GetPlugins()
+ if err != nil {
+ return nil, false, fmt.Errorf("error getting plugins: %v", err)
+ }
+
+ // Loop through all plugins, starting them if they have any devices
+ // to serve. If even one plugin fails to start properly, try
+ // starting them all again.
+ started := 0
+ for _, p := range plugins {
+ // Just continue if there are no devices to serve for plugin p.
+ if len(p.Devices()) == 0 {
+ continue
+ }
+
+ // Start the gRPC server for plugin p and connect it with the kubelet.
+ if err := p.Start(); err != nil {
+ klog.Error("Could not contact Kubelet. Did you enable the device plugin feature gate?")
+ klog.Error("You can check the prerequisites at: https://github.com/NVIDIA/k8s-device-plugin#prerequisites")
+ klog.Error("You can learn how to set the runtime at: https://github.com/NVIDIA/k8s-device-plugin#quick-start")
+ return plugins, true, nil
+ }
+ started++
+ }
+
+ if started == 0 {
+ klog.Info("No devices found. Waiting indefinitely.")
+ }
+
+ return plugins, false, nil
+}
+
+func stopPlugins(plugins []plugin.Interface) error {
+ klog.Info("Stopping plugins.")
+ errs := []error{}
+ for _, p := range plugins {
+ err := p.Stop()
+ errs = append(errs, err)
+ }
+ return errorsutil.NewAggregate(errs)
+}
+
+// disableResourceRenamingInConfig temporarily disable the resource renaming feature of the plugin.
+// We plan to reeenable this feature in a future release.
+func disableResourceRenamingInConfig(config *spec.Config) {
+ // Disable resource renaming through config.Resource
+ if len(config.Resources.GPUs) > 0 || len(config.Resources.MIGs) > 0 {
+ klog.Infof("Customizing the 'resources' field is not yet supported in the config. Ignoring...")
+ }
+ config.Resources.GPUs = nil
+ config.Resources.MIGs = nil
+
+ // Disable renaming / device selection in Sharing.TimeSlicing.Resources
+ renameByDefault := config.Sharing.TimeSlicing.RenameByDefault
+ setsNonDefaultRename := false
+ setsDevices := false
+ for i, r := range config.Sharing.TimeSlicing.Resources {
+ if !renameByDefault && r.Rename != "" {
+ setsNonDefaultRename = true
+ config.Sharing.TimeSlicing.Resources[i].Rename = ""
+ }
+ if renameByDefault && r.Rename != r.Name.DefaultSharedRename() {
+ setsNonDefaultRename = true
+ config.Sharing.TimeSlicing.Resources[i].Rename = r.Name.DefaultSharedRename()
+ }
+ if !r.Devices.All {
+ setsDevices = true
+ config.Sharing.TimeSlicing.Resources[i].Devices.All = true
+ config.Sharing.TimeSlicing.Resources[i].Devices.Count = 0
+ config.Sharing.TimeSlicing.Resources[i].Devices.List = nil
+ }
+ }
+ if setsNonDefaultRename {
+ klog.Warning("Setting the 'rename' field in sharing.timeSlicing.resources is not yet supported in the config. Ignoring...")
+ }
+ if setsDevices {
+ klog.Warning("Customizing the 'devices' field in sharing.timeSlicing.resources is not yet supported in the config. Ignoring...")
+ }
+}
diff --git a/cmd/device-plugin/nvidia/plugin-manager.go b/cmd/device-plugin/nvidia/plugin-manager.go
new file mode 100644
index 000000000..1e1c5df88
--- /dev/null
+++ b/cmd/device-plugin/nvidia/plugin-manager.go
@@ -0,0 +1,82 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "fmt"
+
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/cdi"
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
+)
+
+// NewPluginManager creates an NVML-based plugin manager.
+func NewPluginManager(config *nvidia.DeviceConfig) (manager.Interface, error) {
+ var err error
+ switch *config.Flags.MigStrategy {
+ case spec.MigStrategyNone:
+ case spec.MigStrategySingle:
+ case spec.MigStrategyMixed:
+ default:
+ return nil, fmt.Errorf("unknown strategy: %v", *config.Flags.MigStrategy)
+ }
+
+ nvmllib := nvml.New()
+
+ deviceListStrategies, err := spec.NewDeviceListStrategies(*config.Flags.Plugin.DeviceListStrategy)
+ if err != nil {
+ return nil, fmt.Errorf("invalid device list strategy: %v", err)
+ }
+
+ cdiEnabled := deviceListStrategies.AnyCDIEnabled()
+
+ cdiHandler, err := cdi.New(
+ cdi.WithEnabled(cdiEnabled),
+ cdi.WithDriverRoot(*config.Flags.Plugin.ContainerDriverRoot),
+ cdi.WithTargetDriverRoot(*config.Flags.NvidiaDriverRoot),
+ cdi.WithNvidiaCTKPath(*config.Flags.Plugin.NvidiaCTKPath),
+ cdi.WithNvml(nvmllib),
+ cdi.WithDeviceIDStrategy(*config.Flags.Plugin.DeviceIDStrategy),
+ cdi.WithVendor("k8s.device-plugin.nvidia.com"),
+ cdi.WithGdsEnabled(*config.Flags.GDSEnabled),
+ cdi.WithMofedEnabled(*config.Flags.MOFEDEnabled),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("unable to create cdi handler: %v", err)
+ }
+
+ m, err := manager.New(
+ manager.WithNVML(nvmllib),
+ manager.WithCDIEnabled(cdiEnabled),
+ manager.WithCDIHandler(cdiHandler),
+ manager.WithConfig(config),
+ manager.WithFailOnInitError(*config.Flags.FailOnInitError),
+ manager.WithMigStrategy(*config.Flags.MigStrategy),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("unable to create plugin manager: %v", err)
+ }
+
+ if err := m.CreateCDISpecFile(); err != nil {
+ return nil, fmt.Errorf("unable to create cdi spec file: %v", err)
+ }
+
+ return m, nil
+}
diff --git a/cmd/device-plugin/nvidia/vgpucfg.go b/cmd/device-plugin/nvidia/vgpucfg.go
new file mode 100644
index 000000000..a6d2a8307
--- /dev/null
+++ b/cmd/device-plugin/nvidia/vgpucfg.go
@@ -0,0 +1,121 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "fmt"
+ "os"
+ "strings"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
+ cli "github.com/urfave/cli/v2"
+ "k8s.io/klog/v2"
+)
+
+func addFlags() []cli.Flag {
+ addition := []cli.Flag{
+ &cli.StringFlag{
+ Name: "node-name",
+ Value: os.Getenv(util.NodeNameEnvName),
+ Usage: "node name",
+ EnvVars: []string{"NodeName"},
+ },
+ &cli.UintFlag{
+ Name: "device-split-count",
+ Value: 2,
+ Usage: "the number for NVIDIA device split",
+ EnvVars: []string{"DEVICE_SPLIT_COUNT"},
+ },
+ &cli.Float64Flag{
+ Name: "device-memory-scaling",
+ Value: 1.0,
+ Usage: "the ratio for NVIDIA device memory scaling",
+ EnvVars: []string{"DEVICE_MEMORY_SCALING"},
+ },
+ &cli.Float64Flag{
+ Name: "device-cores-scaling",
+ Value: 1.0,
+ Usage: "the ratio for NVIDIA device cores scaling",
+ EnvVars: []string{"DEVICE_CORES_SCALING"},
+ },
+ &cli.BoolFlag{
+ Name: "disable-core-limit",
+ Value: false,
+ Usage: "If set, the core utilization limit will be ignored",
+ EnvVars: []string{"DISABLE_CORE_LIMIT"},
+ },
+ &cli.StringFlag{
+ Name: "resource-name",
+ Value: "nvidia.com/gpu",
+ Usage: "the name of field for number GPU visible in container",
+ },
+ }
+ return addition
+}
+
+// prt returns a reference to whatever type is passed into it.
+func ptr[T any](x T) *T {
+ return &x
+}
+
+// updateFromCLIFlag conditionally updates the config flag at 'pflag' to the value of the CLI flag with name 'flagName'.
+func updateFromCLIFlag[T any](pflag **T, c *cli.Context, flagName string) {
+ if c.IsSet(flagName) || *pflag == (*T)(nil) {
+ switch flag := any(pflag).(type) {
+ case **string:
+ *flag = ptr(c.String(flagName))
+ case **[]string:
+ *flag = ptr(c.StringSlice(flagName))
+ case **bool:
+ *flag = ptr(c.Bool(flagName))
+ case **float64:
+ *flag = ptr(c.Float64(flagName))
+ case **uint:
+ *flag = ptr(c.Uint(flagName))
+ default:
+ panic(fmt.Errorf("unsupported flag type for %v: %T", flagName, flag))
+ }
+ }
+}
+
+func generateDeviceConfigFromNvidia(cfg *spec.Config, c *cli.Context, flags []cli.Flag) (nvidia.DeviceConfig, error) {
+ devcfg := nvidia.DeviceConfig{}
+ devcfg.Config = cfg
+
+ klog.Infoln("flags=", flags)
+ for _, flag := range flags {
+ for _, n := range flag.Names() {
+ // Common flags
+ if strings.Compare(n, "config-file") == 0 {
+ updateFromCLIFlag(&plugin.ConfigFile, c, n)
+ }
+ }
+ }
+
+ config, err := device.LoadConfig(*plugin.ConfigFile)
+ if err != nil {
+ klog.Fatalf("failed to load ascend vnpu config file %s: %v", *plugin.ConfigFile, err)
+ }
+ devcfg.ResourceName = &config.NvidiaConfig.ResourceCountName
+ klog.Infoln("reading config=", config.NvidiaConfig.ResourceCountName, "devcfg", *devcfg.ResourceName, "configfile=", *plugin.ConfigFile)
+ return devcfg, nil
+}
diff --git a/cmd/device-plugin/nvidia/watchers.go b/cmd/device-plugin/nvidia/watchers.go
new file mode 100644
index 000000000..6e90ff4ff
--- /dev/null
+++ b/cmd/device-plugin/nvidia/watchers.go
@@ -0,0 +1,48 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "os"
+ "os/signal"
+
+ "github.com/fsnotify/fsnotify"
+)
+
+func newFSWatcher(files ...string) (*fsnotify.Watcher, error) {
+ watcher, err := fsnotify.NewWatcher()
+ if err != nil {
+ return nil, err
+ }
+
+ for _, f := range files {
+ err = watcher.Add(f)
+ if err != nil {
+ watcher.Close()
+ return nil, err
+ }
+ }
+
+ return watcher, nil
+}
+
+func newOSWatcher(sigs ...os.Signal) chan os.Signal {
+ sigChan := make(chan os.Signal, 1)
+ signal.Notify(sigChan, sigs...)
+
+ return sigChan
+}
diff --git a/cmd/device-plugin/watchers.go b/cmd/device-plugin/watchers.go
deleted file mode 100644
index ec99eb3ff..000000000
--- a/cmd/device-plugin/watchers.go
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "github.com/fsnotify/fsnotify"
- "os"
- "os/signal"
-)
-
-func NewFSWatcher(files ...string) (*fsnotify.Watcher, error) {
- watcher, err := fsnotify.NewWatcher()
- if err != nil {
- return nil, err
- }
-
- for _, f := range files {
- err = watcher.Add(f)
- if err != nil {
- watcher.Close()
- return nil, err
- }
- }
-
- return watcher, nil
-}
-
-func NewOSWatcher(sigs ...os.Signal) chan os.Signal {
- sigChan := make(chan os.Signal, 1)
- signal.Notify(sigChan, sigs...)
-
- return sigChan
-}
diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go
index 01c5df8e4..37c578be1 100644
--- a/cmd/scheduler/main.go
+++ b/cmd/scheduler/main.go
@@ -1,99 +1,153 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
package main
import (
- "4pd.io/k8s-vgpu/pkg/util"
- "4pd.io/k8s-vgpu/pkg/version"
- "net"
- "net/http"
-
- pb "4pd.io/k8s-vgpu/pkg/api"
- "4pd.io/k8s-vgpu/pkg/scheduler"
- "4pd.io/k8s-vgpu/pkg/scheduler/config"
- "4pd.io/k8s-vgpu/pkg/scheduler/routes"
- "github.com/julienschmidt/httprouter"
- "github.com/spf13/cobra"
- "google.golang.org/grpc"
- "k8s.io/klog/v2"
+ "fmt"
+ "net/http"
+ "net/http/pprof"
+ "time"
+
+ "github.com/julienschmidt/httprouter"
+ "github.com/spf13/cobra"
+ klog "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/config"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/routes"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+ "github.com/Project-HAMi/HAMi/pkg/util/flag"
+ "github.com/Project-HAMi/HAMi/pkg/util/nodelock"
+ "github.com/Project-HAMi/HAMi/pkg/version"
)
//var version string
var (
- tlsKeyFile string
- tlsCertFile string
- rootCmd = &cobra.Command{
- Use: "scheduler",
- Short: "kubernetes vgpu scheduler",
- Run: func(cmd *cobra.Command, args []string) {
- start()
- },
- }
+ sher *scheduler.Scheduler
+ tlsKeyFile string
+ tlsCertFile string
+ enableProfiling bool
+ rootCmd = &cobra.Command{
+ Use: "scheduler",
+ Short: "kubernetes vgpu scheduler",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ flag.PrintPFlags(cmd.Flags())
+ return start()
+ },
+ }
)
func init() {
- rootCmd.Flags().SortFlags = false
- rootCmd.PersistentFlags().SortFlags = false
+ rootCmd.Flags().SortFlags = false
+ rootCmd.PersistentFlags().SortFlags = false
+
+ rootCmd.Flags().StringVar(&config.HTTPBind, "http_bind", "127.0.0.1:8080", "http server bind address")
+ rootCmd.Flags().StringVar(&tlsCertFile, "cert_file", "", "tls cert file")
+ rootCmd.Flags().StringVar(&tlsKeyFile, "key_file", "", "tls key file")
+ rootCmd.Flags().StringVar(&config.SchedulerName, "scheduler-name", "", "the name to be added to pod.spec.schedulerName if not empty")
+ rootCmd.Flags().Int32Var(&config.DefaultMem, "default-mem", 0, "default gpu device memory to allocate")
+ rootCmd.Flags().Int32Var(&config.DefaultCores, "default-cores", 0, "default gpu core percentage to allocate")
+ rootCmd.Flags().Int32Var(&config.DefaultResourceNum, "default-gpu", 1, "default gpu to allocate")
+ rootCmd.Flags().StringVar(&config.NodeSchedulerPolicy, "node-scheduler-policy", util.NodeSchedulerPolicyBinpack.String(), "node scheduler policy")
+ rootCmd.Flags().StringVar(&config.GPUSchedulerPolicy, "gpu-scheduler-policy", util.GPUSchedulerPolicySpread.String(), "GPU scheduler policy")
+ rootCmd.Flags().StringVar(&config.MetricsBindAddress, "metrics-bind-address", ":9395", "The TCP address that the scheduler should bind to for serving prometheus metrics(e.g. 127.0.0.1:9395, :9395)")
+ rootCmd.Flags().StringToStringVar(&config.NodeLabelSelector, "node-label-selector", nil, "key=value pairs separated by commas")
+
+ rootCmd.Flags().Float32Var(&config.QPS, "kube-qps", client.DefaultQPS, "QPS to use while talking with kube-apiserver.")
+ rootCmd.Flags().IntVar(&config.Burst, "kube-burst", client.DefaultBurst, "Burst to use while talking with kube-apiserver.")
+ rootCmd.Flags().IntVar(&config.Timeout, "kube-timeout", client.DefaultTimeout, "Timeout to use while talking with kube-apiserver.")
+ rootCmd.Flags().BoolVar(&enableProfiling, "profiling", false, "Enable pprof profiling via HTTP server")
+ rootCmd.Flags().DurationVar(&config.NodeLockTimeout, "node-lock-timeout", time.Minute*5, "timeout for node locks")
+ rootCmd.Flags().BoolVar(&config.ForceOverwriteDefaultScheduler, "force-overwrite-default-scheduler", true, "Overwrite schedulerName in Pod Spec when set to the const DefaultSchedulerName in https://k8s.io/api/core/v1 package")
- rootCmd.Flags().StringVar(&config.GrpcBind, "grpc_bind", "127.0.0.1:9090", "grpc server bind address")
- rootCmd.Flags().StringVar(&config.HttpBind, "http_bind", "127.0.0.1:8080", "http server bind address")
- rootCmd.Flags().StringVar(&tlsCertFile, "cert_file", "", "tls cert file")
- rootCmd.Flags().StringVar(&tlsKeyFile, "key_file", "", "tls key file")
- rootCmd.Flags().StringVar(&config.SchedulerName, "scheduler-name", "", "the name to be added to pod.spec.schedulerName if not empty")
+ rootCmd.PersistentFlags().AddGoFlagSet(device.GlobalFlagSet())
+ rootCmd.AddCommand(version.VersionCmd)
+ rootCmd.Flags().AddGoFlagSet(util.InitKlogFlags())
- rootCmd.PersistentFlags().AddGoFlagSet(util.GlobalFlagSet())
- rootCmd.AddCommand(version.VersionCmd)
}
-func start() {
- sher := scheduler.NewScheduler()
- sher.Start()
- defer sher.Stop()
-
- // start grpc server
- lisGrpc, _ := net.Listen("tcp", config.GrpcBind)
- defer lisGrpc.Close()
- s := grpc.NewServer()
- pb.RegisterDeviceServiceServer(s, sher)
- go func() {
- err := s.Serve(lisGrpc)
- if err != nil {
- klog.Fatal(err)
- }
- }()
-
- // start http server
- router := httprouter.New()
- router.POST("/filter", routes.PredicateRoute(sher))
- router.POST("/webhook", routes.WebHookRoute())
- klog.Info("listen on ", config.HttpBind)
- if len(tlsCertFile) == 0 || len(tlsKeyFile) == 0 {
- if err := http.ListenAndServe(config.HttpBind, router); err != nil {
- klog.Fatal("Listen and Serve error, ", err)
- }
- } else {
- if err := http.ListenAndServeTLS(config.HttpBind, tlsCertFile, tlsKeyFile, router); err != nil {
- klog.Fatal("Listen and Serve error, ", err)
- }
- }
+// injectProfilingRoute injects pprof routes into the router.
+func injectProfilingRoute(router *httprouter.Router) {
+ router.GET("/debug/pprof/*suffix", func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
+ suffix := params.ByName("suffix")
+ switch suffix {
+ case "/cmdline":
+ pprof.Cmdline(w, r)
+ case "/profile":
+ pprof.Profile(w, r)
+ case "/symbol":
+ pprof.Symbol(w, r)
+ case "/trace":
+ pprof.Trace(w, r)
+ default:
+ pprof.Index(w, r)
+ }
+ })
+}
+
+func start() error {
+ // Initialize node lock timeout from config
+ nodelock.NodeLockTimeout = config.NodeLockTimeout
+ klog.InfoS("Set node lock timeout", "timeout", nodelock.NodeLockTimeout)
+ client.InitGlobalClient(
+ client.WithBurst(config.Burst),
+ client.WithQPS(config.QPS),
+ client.WithTimeout(config.Timeout),
+ )
+
+ device.InitDevices()
+ sher = scheduler.NewScheduler()
+ sher.Start()
+ defer sher.Stop()
+
+ // start monitor metrics
+ go sher.RegisterFromNodeAnnotations()
+ go initMetrics(config.MetricsBindAddress)
+
+ // start http server
+ router := httprouter.New()
+ router.POST("/filter", routes.PredicateRoute(sher))
+ router.POST("/bind", routes.Bind(sher))
+ router.POST("/webhook", routes.WebHookRoute())
+ router.GET("/healthz", routes.HealthzRoute())
+ klog.Info("listen on ", config.HTTPBind)
+
+ if enableProfiling {
+ injectProfilingRoute(router)
+ klog.Infof("Profiling enabled, visit %s/debug/pprof/ to view profiles", config.HTTPBind)
+ }
+
+ if len(tlsCertFile) == 0 || len(tlsKeyFile) == 0 {
+ if err := http.ListenAndServe(config.HTTPBind, router); err != nil {
+ return fmt.Errorf("listen and Serve error, %v", err)
+ }
+ } else {
+ if err := http.ListenAndServeTLS(config.HTTPBind, tlsCertFile, tlsKeyFile, router); err != nil {
+ return fmt.Errorf("listen and Serve error, %v", err)
+ }
+ }
+ return nil
}
func main() {
- if err := rootCmd.Execute(); err != nil {
- klog.Fatal(err)
- }
+ if err := rootCmd.Execute(); err != nil {
+ klog.Fatal(err)
+ }
}
diff --git a/cmd/scheduler/metrics.go b/cmd/scheduler/metrics.go
new file mode 100644
index 000000000..83b7f396a
--- /dev/null
+++ b/cmd/scheduler/metrics.go
@@ -0,0 +1,294 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "fmt"
+ "log"
+ "net/http"
+ "strings"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+
+ klog "k8s.io/klog/v2"
+)
+
+// ClusterManager is an example for a system that might have been built without
+// Prometheus in mind. It models a central manager of jobs running in a
+// cluster. Thus, we implement a custom Collector called
+// ClusterManagerCollector, which collects information from a ClusterManager
+// using its provided methods and turns them into Prometheus Metrics for
+// collection.
+//
+// An additional challenge is that multiple instances of the ClusterManager are
+// run within the same binary, each in charge of a different zone. We need to
+// make use of wrapping Registerers to be able to register each
+// ClusterManagerCollector instance with Prometheus.
+type ClusterManager struct {
+ Zone string
+ // Contains many more fields not listed in this example.
+}
+
+// ClusterManagerCollector implements the Collector interface.
+type ClusterManagerCollector struct {
+ ClusterManager *ClusterManager
+}
+
+// Describe is implemented with DescribeByCollect. That's possible because the
+// Collect method will always return the same two metrics with the same two
+// descriptors.
+func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
+ prometheus.DescribeByCollect(cc, ch)
+}
+
+// Collect first triggers the ReallyExpensiveAssessmentOfTheSystemState. Then it
+// creates constant metrics for each host on the fly based on the returned data.
+//
+// Note that Collect could be called concurrently, so we depend on
+// ReallyExpensiveAssessmentOfTheSystemState to be concurrency-safe.
+func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
+ klog.Info("Starting to collect metrics for scheduler")
+ nodevGPUMemoryLimitDesc := prometheus.NewDesc(
+ "GPUDeviceMemoryLimit",
+ "Device memory limit for a certain GPU",
+ []string{"nodeid", "deviceuuid", "deviceidx"}, nil,
+ )
+ nodevGPUCoreLimitDesc := prometheus.NewDesc(
+ "GPUDeviceCoreLimit",
+ "Device memory core limit for a certain GPU",
+ []string{"nodeid", "deviceuuid", "deviceidx"}, nil,
+ )
+ nodevGPUMemoryAllocatedDesc := prometheus.NewDesc(
+ "GPUDeviceMemoryAllocated",
+ "Device memory allocated for a certain GPU",
+ []string{"nodeid", "deviceuuid", "deviceidx", "devicecores"}, nil,
+ )
+ nodevGPUSharedNumDesc := prometheus.NewDesc(
+ "GPUDeviceSharedNum",
+ "Number of containers sharing this GPU",
+ []string{"nodeid", "deviceuuid", "deviceidx"}, nil,
+ )
+
+ nodeGPUCoreAllocatedDesc := prometheus.NewDesc(
+ "GPUDeviceCoreAllocated",
+ "Device core allocated for a certain GPU",
+ []string{"nodeid", "deviceuuid", "deviceidx"}, nil,
+ )
+ nodeGPUOverview := prometheus.NewDesc(
+ "nodeGPUOverview",
+ "GPU overview on a certain node",
+ []string{"nodeid", "deviceuuid", "deviceidx", "devicecores", "sharedcontainers", "devicememorylimit", "devicetype"}, nil,
+ )
+ nodeGPUMemoryPercentage := prometheus.NewDesc(
+ "nodeGPUMemoryPercentage",
+ "GPU Memory Allocated Percentage on a certain GPU",
+ []string{"nodeid", "deviceuuid", "deviceidx"}, nil,
+ )
+ nodeGPUMigInstance := prometheus.NewDesc(
+ "nodeGPUMigInstance",
+ "GPU Sharing mode. 0 for hami-core, 1 for mig, 2 for mps",
+ []string{"nodeid", "deviceuuid", "deviceidx", "migname"}, nil,
+ )
+ nu := sher.InspectAllNodesUsage()
+ for nodeID, val := range *nu {
+ for _, devs := range val.Devices.DeviceLists {
+ if devs.Device.Mode == "mig" {
+ for idx, migs := range devs.Device.MigUsage.UsageList {
+ klog.Infoln("mig instances=", devs.Device.MigUsage)
+ inuse := 0
+ if migs.InUse {
+ inuse = 1
+ }
+ ch <- prometheus.MustNewConstMetric(
+ nodeGPUMigInstance,
+ prometheus.GaugeValue,
+ float64(inuse),
+ nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), migs.Name+"-"+fmt.Sprint(idx),
+ )
+ }
+ }
+
+ ch <- prometheus.MustNewConstMetric(
+ nodevGPUMemoryLimitDesc,
+ prometheus.GaugeValue,
+ float64(devs.Device.Totalmem)*float64(1024)*float64(1024),
+ nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index),
+ )
+ ch <- prometheus.MustNewConstMetric(
+ nodevGPUCoreLimitDesc,
+ prometheus.GaugeValue,
+ float64(devs.Device.Totalcore),
+ nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index),
+ )
+ ch <- prometheus.MustNewConstMetric(
+ nodevGPUMemoryAllocatedDesc,
+ prometheus.GaugeValue,
+ float64(devs.Device.Usedmem)*float64(1024)*float64(1024),
+ nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Usedcores),
+ )
+ ch <- prometheus.MustNewConstMetric(
+ nodevGPUSharedNumDesc,
+ prometheus.GaugeValue,
+ float64(devs.Device.Used),
+ nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index),
+ )
+
+ ch <- prometheus.MustNewConstMetric(
+ nodeGPUCoreAllocatedDesc,
+ prometheus.GaugeValue,
+ float64(devs.Device.Usedcores),
+ nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index),
+ )
+ ch <- prometheus.MustNewConstMetric(
+ nodeGPUOverview,
+ prometheus.GaugeValue,
+ float64(devs.Device.Usedmem)*float64(1024)*float64(1024),
+ nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Usedcores), fmt.Sprint(devs.Device.Used), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type,
+ )
+ ch <- prometheus.MustNewConstMetric(
+ nodeGPUMemoryPercentage,
+ prometheus.GaugeValue,
+ float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem),
+ nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index),
+ )
+ }
+ }
+
+ ctrvGPUDeviceAllocatedDesc := prometheus.NewDesc(
+ "vGPUPodsDeviceAllocated",
+ "vGPU Allocated from pods (This metric will be deprecated in v2.8.0, use vGPUMemoryAllocated and vGPUCoreAllocated instead.)",
+ []string{"deprecated_version", "podnamespace", "nodename", "podname", "containeridx", "deviceuuid", "deviceusedcore"}, nil,
+ )
+ ctrvGPUdeviceAllocatedMemoryPercentageDesc := prometheus.NewDesc(
+ "vGPUMemoryPercentage",
+ "vGPU memory percentage allocated from a container (This metric will be deprecated in v2.8.0, use vGPUMemoryAllocated instead.)",
+ []string{"deprecated_version", "podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil,
+ )
+ ctrvGPUdeviceAllocateCorePercentageDesc := prometheus.NewDesc(
+ "vGPUCorePercentage",
+ "vGPU core allocated from a container (This metric will be deprecated in v2.8.0, use vGPUCoreAllocated instead.)",
+ []string{"deprecated_version", "podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil,
+ )
+ ctrvGPUdeviceAllocatedMemoryDesc := prometheus.NewDesc(
+ "vGPUMemoryAllocated",
+ "vGPU memory allocated from a container",
+ []string{"podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil,
+ )
+ ctrvGPUdeviceAllocatedCoreDesc := prometheus.NewDesc(
+ "vGPUCoreAllocated",
+ "vGPU core allocated from a container",
+ []string{"podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil,
+ )
+ schedpods, _ := sher.GetScheduledPods()
+ for _, val := range schedpods {
+ for _, podSingleDevice := range val.Devices {
+ for ctridx, ctrdevs := range podSingleDevice {
+ for _, ctrdevval := range ctrdevs {
+ klog.V(4).InfoS("Collecting metrics",
+ "namespace", val.Namespace,
+ "podName", val.Name,
+ "deviceUUID", ctrdevval.UUID,
+ "usedCores", ctrdevval.Usedcores,
+ "usedMem", ctrdevval.Usedmem,
+ "nodeID", val.NodeID,
+ )
+ if len(ctrdevval.UUID) == 0 {
+ klog.Warningf("Device UUID is empty, omitting metric collection for namespace=%s, podName=%s, ctridx=%d, nodeID=%s",
+ val.Namespace, val.Name, ctridx, val.NodeID)
+ continue
+ }
+ ch <- prometheus.MustNewConstMetric(
+ ctrvGPUDeviceAllocatedDesc,
+ prometheus.GaugeValue,
+ float64(ctrdevval.Usedmem)*float64(1024)*float64(1024),
+ "v2.8.0", val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID, fmt.Sprint(ctrdevval.Usedcores))
+ ch <- prometheus.MustNewConstMetric(
+ ctrvGPUdeviceAllocatedMemoryDesc,
+ prometheus.GaugeValue,
+ float64(ctrdevval.Usedmem)*float64(1024)*float64(1024),
+ val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID)
+ ch <- prometheus.MustNewConstMetric(
+ ctrvGPUdeviceAllocatedCoreDesc,
+ prometheus.GaugeValue,
+ float64(ctrdevval.Usedcores),
+ val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID)
+ var totaldev int32
+ found := false
+ for _, ni := range *nu {
+ for _, nodedev := range ni.Devices.DeviceLists {
+ //fmt.Println("uuid=", nodedev.ID, ctrdevval.UUID)
+ if strings.Compare(nodedev.Device.ID, ctrdevval.UUID) == 0 {
+ totaldev = nodedev.Device.Totalmem
+ found = true
+ break
+ }
+ }
+ if found {
+ break
+ }
+ }
+ klog.V(4).InfoS("Total memory for device",
+ "deviceUUID", ctrdevval.UUID,
+ "totalMemory", totaldev,
+ "nodeID", val.NodeID,
+ )
+ if totaldev > 0 {
+ ch <- prometheus.MustNewConstMetric(
+ ctrvGPUdeviceAllocatedMemoryPercentageDesc,
+ prometheus.GaugeValue,
+ float64(ctrdevval.Usedmem)/float64(totaldev),
+ "v2.8.0", val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID)
+ }
+ ch <- prometheus.MustNewConstMetric(
+ ctrvGPUdeviceAllocateCorePercentageDesc,
+ prometheus.GaugeValue,
+ float64(ctrdevval.Usedcores),
+ "v2.8.0", val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID)
+ }
+ }
+ }
+ }
+}
+
+// NewClusterManager first creates a Prometheus-ignorant ClusterManager
+// instance. Then, it creates a ClusterManagerCollector for the just created
+// ClusterManager. Finally, it registers the ClusterManagerCollector with a
+// wrapping Registerer that adds the zone as a label. In this way, the metrics
+// collected by different ClusterManagerCollectors do not collide.
+func NewClusterManager(zone string, reg prometheus.Registerer) *ClusterManager {
+ c := &ClusterManager{
+ Zone: zone,
+ }
+ cc := ClusterManagerCollector{ClusterManager: c}
+ prometheus.WrapRegistererWith(prometheus.Labels{"zone": zone}, reg).MustRegister(cc)
+ return c
+}
+
+func initMetrics(bindAddress string) {
+ // Since we are dealing with custom Collector implementations, it might
+ // be a good idea to try it out with a pedantic registry.
+ klog.Info("Initializing metrics for scheduler")
+ reg := prometheus.NewRegistry()
+
+ // Construct cluster managers. In real code, we would assign them to
+ // variables to then do something with them.
+ NewClusterManager("vGPU", reg)
+
+ http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
+ log.Fatal(http.ListenAndServe(bindAddress, nil))
+}
diff --git a/cmd/vGPUmonitor/build.sh b/cmd/vGPUmonitor/build.sh
new file mode 100644
index 000000000..c6bfa7201
--- /dev/null
+++ b/cmd/vGPUmonitor/build.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Copyright 2024 The HAMi Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative noderpc/noderpc.proto
+go build
diff --git a/cmd/vGPUmonitor/feedback.go b/cmd/vGPUmonitor/feedback.go
new file mode 100644
index 000000000..33f9740a7
--- /dev/null
+++ b/cmd/vGPUmonitor/feedback.go
@@ -0,0 +1,215 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/monitor/nvidia"
+)
+
+var cgroupDriver int
+var errTemporaryClosed = errors.New("temporary closed")
+
+//type hostGPUPid struct {
+// hostGPUPid int
+// mtime uint64
+//}
+
+type UtilizationPerDevice []int
+
+func setcGgroupDriver() int {
+ // 1 for cgroupfs 2 for systemd
+ kubeletconfig, err := os.ReadFile("/hostvar/lib/kubelet/config.yaml")
+ if err != nil {
+ return 0
+ }
+ content := string(kubeletconfig)
+ pos := strings.LastIndex(content, "cgroupDriver:")
+ if pos < 0 {
+ return 0
+ }
+ if strings.Contains(content, "systemd") {
+ return 2
+ }
+ if strings.Contains(content, "cgroupfs") {
+ return 1
+ }
+ return 0
+}
+
+func getUsedGPUPid() ([]uint, nvml.Return) {
+ tmp := []nvml.ProcessInfo{}
+ count, err := nvml.DeviceGetCount()
+ if err != nvml.SUCCESS {
+ return []uint{}, err
+ }
+ for i := range count {
+ device, err := nvml.DeviceGetHandleByIndex(i)
+ if err != nvml.SUCCESS {
+ return []uint{}, err
+ }
+ ids, err := device.GetComputeRunningProcesses()
+ if err != nvml.SUCCESS {
+ return []uint{}, err
+ }
+ tmp = append(tmp, ids...)
+ }
+ result := make([]uint, 0)
+ m := make(map[uint]bool)
+ for _, v := range tmp {
+ if _, ok := m[uint(v.Pid)]; !ok {
+ result = append(result, uint(v.Pid))
+ m[uint(v.Pid)] = true
+ }
+ }
+ sort.Slice(tmp, func(i, j int) bool { return tmp[i].Pid > tmp[j].Pid })
+ return result, nvml.SUCCESS
+}
+
+func CheckBlocking(utSwitchOn map[string]UtilizationPerDevice, p int, c *nvidia.ContainerUsage) bool {
+ for i := range c.Info.DeviceMax() {
+ uuid := c.Info.DeviceUUID(i)
+ _, ok := utSwitchOn[uuid]
+ if ok {
+ for i := range p {
+ if utSwitchOn[uuid][i] > 0 {
+ return true
+ }
+ }
+ return false
+ }
+ }
+ return false
+}
+
+// Check whether task with higher priority use GPU or there are other tasks with the same priority.
+func CheckPriority(utSwitchOn map[string]UtilizationPerDevice, p int, c *nvidia.ContainerUsage) bool {
+ for i := range c.Info.DeviceMax() {
+ uuid := c.Info.DeviceUUID(i)
+ _, ok := utSwitchOn[uuid]
+ if ok {
+ for i := range p {
+ if utSwitchOn[uuid][i] > 0 {
+ return true
+ }
+ }
+ if utSwitchOn[uuid][p] > 1 {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func Observe(lister *nvidia.ContainerLister) {
+ utSwitchOn := map[string]UtilizationPerDevice{}
+ containers := lister.ListContainers()
+
+ for _, c := range containers {
+ recentKernel := c.Info.GetRecentKernel()
+ if recentKernel > 0 {
+ recentKernel--
+ if recentKernel > 0 {
+ for i := range c.Info.DeviceMax() {
+ //for _, devuuid := range val.sr.uuids {
+ // Null device condition
+ if !c.Info.IsValidUUID(i) {
+ continue
+ }
+ uuid := c.Info.DeviceUUID(i)
+ if len(utSwitchOn[uuid]) == 0 {
+ utSwitchOn[uuid] = []int{0, 0}
+ }
+ utSwitchOn[uuid][c.Info.GetPriority()]++
+ }
+ }
+ c.Info.SetRecentKernel(recentKernel)
+ }
+ }
+ for idx, c := range containers {
+ priority := c.Info.GetPriority()
+ recentKernel := c.Info.GetRecentKernel()
+ utilizationSwitch := c.Info.GetUtilizationSwitch()
+ if CheckBlocking(utSwitchOn, priority, c) {
+ if recentKernel >= 0 {
+ klog.V(5).Infof("utSwitchon=%v", utSwitchOn)
+ klog.V(5).Infof("Setting Blocking to on %v", idx)
+ c.Info.SetRecentKernel(-1)
+ }
+ } else {
+ if recentKernel < 0 {
+ klog.V(5).Infof("utSwitchon=%v", utSwitchOn)
+ klog.V(5).Infof("Setting Blocking to off %v", idx)
+ c.Info.SetRecentKernel(0)
+ }
+ }
+ if CheckPriority(utSwitchOn, priority, c) {
+ if utilizationSwitch != 1 {
+ klog.V(5).Infof("utSwitchon=%v", utSwitchOn)
+ klog.V(5).Infof("Setting UtilizationSwitch to on %v", idx)
+ c.Info.SetUtilizationSwitch(1)
+ }
+ } else {
+ if utilizationSwitch != 0 {
+ klog.V(5).Infof("utSwitchon=%v", utSwitchOn)
+ klog.V(5).Infof("Setting UtilizationSwitch to off %v", idx)
+ c.Info.SetUtilizationSwitch(0)
+ }
+ }
+ }
+}
+
+func watchAndFeedback(ctx context.Context, lister *nvidia.ContainerLister, migLockSignal <-chan bool) error {
+ klog.Info("Starting watchAndFeedback")
+ if nvret := nvml.Init(); nvret != nvml.SUCCESS {
+ return fmt.Errorf("failed to initialize NVML: %s", nvml.ErrorString(nvret))
+ }
+ defer nvml.Shutdown()
+
+ ticker := time.NewTicker(time.Second * 5)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ klog.Info("Shutting down watchAndFeedback")
+ return nil
+ case signal := <-migLockSignal:
+ if signal {
+ klog.Info("Received MIG apply lock file")
+ return errTemporaryClosed
+ }
+
+ case <-ticker.C:
+ if err := lister.Update(); err != nil {
+ klog.Errorf("Failed to update container list: %v", err)
+ continue
+ }
+ Observe(lister)
+ }
+ }
+}
diff --git a/cmd/vGPUmonitor/main.go b/cmd/vGPUmonitor/main.go
new file mode 100644
index 000000000..2bbbc401e
--- /dev/null
+++ b/cmd/vGPUmonitor/main.go
@@ -0,0 +1,176 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "os"
+ "os/signal"
+ "sync"
+ "syscall"
+
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin"
+ "github.com/Project-HAMi/HAMi/pkg/monitor/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/flag"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+
+ "github.com/spf13/cobra"
+ "k8s.io/klog/v2"
+)
+
+var (
+ rootCmd = &cobra.Command{
+ Use: "vGPUmonitor",
+ Short: "Hami vgpu vGPUmonitor",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ flag.PrintPFlags(cmd.Flags())
+ return start()
+ },
+ }
+)
+
+func init() {
+ rootCmd.Flags().SortFlags = false
+ rootCmd.PersistentFlags().SortFlags = false
+ rootCmd.Flags().AddGoFlagSet(util.InitKlogFlags())
+}
+
+func start() error {
+ if err := ValidateEnvVars(); err != nil {
+ return fmt.Errorf("failed to validate environment variables: %v", err)
+ }
+
+ containerLister, err := nvidia.NewContainerLister()
+ if err != nil {
+ return fmt.Errorf("failed to create container lister: %v", err)
+ }
+
+ cgroupDriver = 0 // Explicitly initialize
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ // Prepare the lock file sub directory.Due to the sequence of startup processes, both the device plugin
+ // and the vGPU monitor should attempt to create this directory by default to ensure its creation.
+ err = plugin.CreateMigApplyLockDir()
+ if err != nil {
+ return fmt.Errorf("failed to create MIG apply lock directory: %v", err)
+ }
+
+ lockChannel, err := plugin.WatchLockFile()
+ if err != nil {
+ return fmt.Errorf("failed to watch lock file: %v", err)
+ }
+
+ var wg sync.WaitGroup
+ errCh := make(chan error, 2)
+
+ // Start the metrics service
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ if err := initMetrics(ctx, containerLister); err != nil {
+ errCh <- err
+ }
+ }()
+
+ // Start the monitoring and feedback service
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for {
+ if err := watchAndFeedback(ctx, containerLister, lockChannel); err != nil {
+ // if err is temporary closed, wait for lock file to be removed
+ if errors.Is(err, errTemporaryClosed) {
+ klog.Info("MIG apply lock file detected, waiting for lock file to be removed")
+ <-lockChannel
+ klog.Info("MIG apply lock file has been removed, restarting watchAndFeedback")
+ continue
+ }
+ errCh <- err
+ return
+ }
+ return
+ }
+ }()
+
+ // Capture system signals
+ signalCh := make(chan os.Signal, 1)
+ signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
+
+ select {
+ case sig := <-signalCh:
+ klog.Infof("Received signal: %s", sig)
+ cancel()
+ case err := <-errCh:
+ klog.Errorf("Received error: %v", err)
+ cancel()
+ }
+
+ // Wait for all goroutines to complete
+ wg.Wait()
+ close(errCh)
+ return nil
+}
+
+func initMetrics(ctx context.Context, containerLister *nvidia.ContainerLister) error {
+ klog.V(4).Info("Initializing metrics for vGPUmonitor")
+ reg := prometheus.NewRegistry()
+ //reg := prometheus.NewPedanticRegistry()
+
+ // Construct cluster managers. In real code, we would assign them to
+ // variables to then do something with them.
+ NewClusterManager("vGPU", reg, containerLister)
+ //NewClusterManager("ca", reg)
+
+ // Uncomment to add the standard process and Go metrics to the custom registry.
+ //reg.MustRegister(
+ // prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
+ // prometheus.NewGoCollector(),
+ //)
+
+ http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
+ server := &http.Server{Addr: ":9394", Handler: nil}
+
+ // Starting the HTTP server in a goroutine
+ go func() {
+ if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ klog.Errorf("Failed to serve metrics: %v", err)
+ }
+ }()
+
+ // Graceful shutdown on context cancellation
+ <-ctx.Done()
+ klog.V(4).Info("Shutting down metrics server")
+ if err := server.Shutdown(context.Background()); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func main() {
+ if err := rootCmd.Execute(); err != nil {
+ klog.Fatal(err)
+ }
+}
diff --git a/cmd/vGPUmonitor/metrics.go b/cmd/vGPUmonitor/metrics.go
new file mode 100644
index 000000000..24574d002
--- /dev/null
+++ b/cmd/vGPUmonitor/metrics.go
@@ -0,0 +1,502 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "fmt"
+ "os"
+ "strings"
+ "time"
+
+ dp "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin"
+ nv "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/monitor/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ "github.com/prometheus/client_golang/prometheus"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/client-go/informers"
+ listerscorev1 "k8s.io/client-go/listers/core/v1"
+ "k8s.io/klog/v2"
+)
+
+// ClusterManager is an example for a system that might have been built without
+// Prometheus in mind. It models a central manager of jobs running in a
+// cluster. Thus, we implement a custom Collector called
+// ClusterManagerCollector, which collects information from a ClusterManager
+// using its provided methods and turns them into Prometheus Metrics for
+// collection.
+//
+// An additional challenge is that multiple instances of the ClusterManager are
+// run within the same binary, each in charge of a different zone. We need to
+// make use of wrapping Registerers to be able to register each
+// ClusterManagerCollector instance with Prometheus.
+type ClusterManager struct {
+ Zone string
+ // Contains many more fields not listed in this example.
+ PodLister listerscorev1.PodLister
+ containerLister *nvidia.ContainerLister
+}
+
+// ReallyExpensiveAssessmentOfTheSystemState is a mock for the data gathering a
+// real cluster manager would have to do. Since it may actually be really
+// expensive, it must only be called once per collection. This implementation,
+// obviously, only returns some made-up data.
+func (c *ClusterManager) ReallyExpensiveAssessmentOfTheSystemState() (
+ oomCountByHost map[string]int, ramUsageByHost map[string]float64,
+) {
+ // Just example fake data.
+ oomCountByHost = map[string]int{
+ "foo.example.org": 42,
+ "bar.example.org": 2001,
+ }
+ ramUsageByHost = map[string]float64{
+ "foo.example.org": 6.023e23,
+ "bar.example.org": 3.14,
+ }
+ return
+}
+
+// ClusterManagerCollector implements the Collector interface.
+type ClusterManagerCollector struct {
+ ClusterManager *ClusterManager
+}
+
+// Descriptors used by the ClusterManagerCollector below.
+var (
+ hostGPUdesc = prometheus.NewDesc(
+ "HostGPUMemoryUsage",
+ "GPU device memory usage",
+ []string{"deviceidx", "deviceuuid"}, nil,
+ )
+
+ hostGPUUtilizationdesc = prometheus.NewDesc(
+ "HostCoreUtilization",
+ "GPU core utilization",
+ []string{"deviceidx", "deviceuuid"}, nil,
+ )
+
+ ctrvGPUdesc = prometheus.NewDesc(
+ "vGPU_device_memory_usage_in_bytes",
+ "vGPU device usage",
+ []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil,
+ )
+
+ ctrvGPUlimitdesc = prometheus.NewDesc(
+ "vGPU_device_memory_limit_in_bytes",
+ "vGPU device limit",
+ []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil,
+ )
+ ctrDeviceMemorydesc = prometheus.NewDesc(
+ "Device_memory_desc_of_container",
+ "Container device meory description",
+ []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid", "context", "module", "data", "offset"}, nil,
+ )
+ ctrDeviceUtilizationdesc = prometheus.NewDesc(
+ "Device_utilization_desc_of_container",
+ "Container device utilization description",
+ []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil,
+ )
+ ctrDeviceLastKernelDesc = prometheus.NewDesc(
+ "Device_last_kernel_of_container",
+ "Container device last kernel description",
+ []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil,
+ )
+ ctrDeviceMigInfo = prometheus.NewDesc(
+ "MigInfo",
+ "Mig device information for container",
+ []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid", "instanceid"}, nil,
+ )
+)
+
+// Describe is implemented with DescribeByCollect. That's possible because the
+// Collect method will always return the same two metrics with the same two
+// descriptors.
+func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
+ ch <- hostGPUdesc
+ ch <- ctrvGPUdesc
+ ch <- ctrvGPUlimitdesc
+ ch <- hostGPUUtilizationdesc
+ //prometheus.DescribeByCollect(cc, ch)
+}
+
+//func parseidstr(podusage string) (string, string, error) {
+// tmp := strings.Split(podusage, "_")
+// if len(tmp) > 1 {
+// return tmp[0], tmp[1], nil
+// } else {
+// return "", "", errors.New("parse error")
+// }
+//}
+//
+//func gettotalusage(usage podusage, vidx int) (deviceMemory, error) {
+// added := deviceMemory{
+// bufferSize: 0,
+// contextSize: 0,
+// moduleSize: 0,
+// offset: 0,
+// total: 0,
+// }
+// for _, val := range usage.sr.procs {
+// added.bufferSize += val.used[vidx].bufferSize
+// added.contextSize += val.used[vidx].contextSize
+// added.moduleSize += val.used[vidx].moduleSize
+// added.offset += val.used[vidx].offset
+// added.total += val.used[vidx].total
+// }
+// return added, nil
+//}
+//
+//func getTotalUtilization(usage podusage, vidx int) deviceUtilization {
+// added := deviceUtilization{
+// decUtil: 0,
+// encUtil: 0,
+// smUtil: 0,
+// }
+// for _, val := range usage.sr.procs {
+// added.decUtil += val.deviceUtil[vidx].decUtil
+// added.encUtil += val.deviceUtil[vidx].encUtil
+// added.smUtil += val.deviceUtil[vidx].smUtil
+// }
+// return added
+//}
+
+// Collect first triggers the ReallyExpensiveAssessmentOfTheSystemState. Then it
+// creates constant metrics for each host on the fly based on the returned data.
+//
+// Note that Collect could be called concurrently, so we depend on
+// ReallyExpensiveAssessmentOfTheSystemState to be concurrency-safe.
+func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
+ klog.Info("Starting to collect metrics for vGPUMonitor")
+
+ // Collect GPU information
+ if err := cc.collectGPUInfo(ch); err != nil {
+ klog.Errorf("Failed to collect GPU info: %v", err)
+ // Decide whether to continue or return based on business requirements
+ }
+
+ // Collect Pod and Container information
+ if err := cc.collectPodAndContainerInfo(ch); err != nil {
+ klog.Errorf("Failed to collect Pod and Container info: %v", err)
+ // Decide whether to continue or return based on business requirements
+ }
+
+ // Collect Pod and Container Mig information
+ if err := cc.collectPodAndContainerMigInfo(ch); err != nil {
+ klog.Errorf("Failed to collect Pod and Container Mig info: %v", err)
+ // Decide whether to continue or return based on business requirements
+ }
+
+ klog.Info("Finished collecting metrics for vGPUMonitor")
+}
+
+func (cc ClusterManagerCollector) collectGPUInfo(ch chan<- prometheus.Metric) error {
+ if err := cc.initNVML(); err != nil {
+ return err
+ }
+ defer nvml.Shutdown()
+
+ devnum, err := cc.getDeviceCount()
+ if err != nil {
+ return err
+ }
+
+ for ii := range devnum {
+ if err := cc.collectGPUDeviceMetrics(ch, ii); err != nil {
+ klog.Error("Failed to collect metrics for GPU device ", ii, ": ", err)
+ }
+ }
+
+ return nil
+}
+
+func (cc ClusterManagerCollector) initNVML() error {
+ nvret := nvml.Init()
+ if nvret != nvml.SUCCESS {
+ return fmt.Errorf("nvml Init err: %s", nvml.ErrorString(nvret))
+ }
+ return nil
+}
+
+func (cc ClusterManagerCollector) getDeviceCount() (int, error) {
+ devnum, nvret := nvml.DeviceGetCount()
+ if nvret != nvml.SUCCESS {
+ return 0, fmt.Errorf("nvml GetDeviceCount err: %s", nvml.ErrorString(nvret))
+ }
+ return devnum, nil
+}
+
+func (cc ClusterManagerCollector) collectGPUDeviceMetrics(ch chan<- prometheus.Metric, index int) error {
+ hdev, nvret := nvml.DeviceGetHandleByIndex(index)
+ if nvret != nvml.SUCCESS {
+ return fmt.Errorf("nvml DeviceGetHandleByIndex err: %s", nvml.ErrorString(nvret))
+ }
+
+ if err := cc.collectGPUMemoryMetrics(ch, hdev, index); err != nil {
+ return err
+ }
+
+ if err := cc.collectGPUUtilizationMetrics(ch, hdev, index); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (cc ClusterManagerCollector) collectGPUMemoryMetrics(ch chan<- prometheus.Metric, hdev nvml.Device, index int) error {
+ memory, ret := hdev.GetMemoryInfo()
+ if ret != nvml.SUCCESS {
+ return fmt.Errorf("nvml get memory error ret=%d", ret)
+ }
+
+ uuid, nvret := hdev.GetUUID()
+ if nvret != nvml.SUCCESS {
+ return fmt.Errorf("nvml GetUUID err: %s", nvml.ErrorString(nvret))
+ }
+
+ ch <- prometheus.MustNewConstMetric(
+ hostGPUdesc,
+ prometheus.GaugeValue,
+ float64(memory.Used),
+ fmt.Sprint(index), uuid,
+ )
+
+ return nil
+}
+
+func (cc ClusterManagerCollector) collectGPUUtilizationMetrics(ch chan<- prometheus.Metric, hdev nvml.Device, index int) error {
+ util, nvret := hdev.GetUtilizationRates()
+ if nvret != nvml.SUCCESS {
+ return fmt.Errorf("nvml GetUtilizationRates err: %s", nvml.ErrorString(nvret))
+ }
+
+ uuid, nvret := hdev.GetUUID()
+ if nvret != nvml.SUCCESS {
+ return fmt.Errorf("nvml GetUUID err: %s", nvml.ErrorString(nvret))
+ }
+
+ ch <- prometheus.MustNewConstMetric(
+ hostGPUUtilizationdesc,
+ prometheus.GaugeValue,
+ float64(util.Gpu),
+ fmt.Sprint(index), uuid,
+ )
+
+ return nil
+}
+
+func (cc ClusterManagerCollector) collectPodAndContainerInfo(ch chan<- prometheus.Metric) error {
+ nodeName := os.Getenv(util.NodeNameEnvName)
+ if nodeName == "" {
+ return fmt.Errorf("node name environment variable %s is not set", util.NodeNameEnvName)
+ }
+
+ pods, err := cc.ClusterManager.PodLister.List(labels.SelectorFromSet(labels.Set{util.AssignedNodeAnnotations: nodeName}))
+ if err != nil {
+ klog.Errorf("Failed to list pods for node %s: %v", nodeName, err)
+ return fmt.Errorf("failed to list pods: %w", err)
+ }
+
+ containers := cc.ClusterManager.containerLister.ListContainers()
+ containerMap := make(map[string][]*nvidia.ContainerUsage) // podUID -> containers
+ for _, c := range containers {
+ if c.Info != nil && c.PodUID != "" {
+ containerMap[c.PodUID] = append(containerMap[c.PodUID], c)
+ }
+ }
+
+ nowSec := time.Now().Unix()
+
+ // Iterate through each Pod
+ for _, pod := range pods {
+ podContainers, found := containerMap[string(pod.UID)]
+ if !found {
+ klog.V(5).Infof("No containers found for pod %s/%s", pod.Namespace, pod.Name)
+ continue
+ }
+
+ klog.V(5).Infof("Processing Pod %s/%s", pod.Namespace, pod.Name)
+
+ // Iterate through each container in the Pod
+ for _, ctr := range pod.Spec.Containers {
+ // Find the matching container
+ for _, c := range podContainers {
+ if c.ContainerName == ctr.Name {
+ klog.V(5).Infof("Processing Container %s in Pod %s/%s", ctr.Name, pod.Namespace, pod.Name)
+ if err := cc.collectContainerMetrics(ch, pod, ctr, c, nowSec); err != nil {
+ klog.Errorf("Failed to collect metrics for container %s in Pod %s/%s: %v", ctr.Name, pod.Namespace, pod.Name, err)
+ }
+ break // Exit the inner loop after finding the matching container
+ }
+ }
+ }
+ }
+
+ klog.V(4).Infof("Finished collecting metrics for %d pods", len(pods))
+ return nil
+}
+
+func (cc ClusterManagerCollector) isPodUIDMatched(pod *corev1.Pod, podUID string) bool {
+ if pod == nil {
+ return false
+ }
+ return string(pod.UID) == podUID
+}
+
+func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.Metric, pod *corev1.Pod, ctr corev1.Container, c *nvidia.ContainerUsage, nowSec int64) error {
+ // Validate inputs
+ if c == nil || c.Info == nil {
+ klog.Errorf("Container or ContainerInfo is nil for Pod %s/%s, Container %s", pod.Namespace, pod.Name, ctr.Name)
+ return fmt.Errorf("container or container info is nil")
+ }
+
+ // Iterate through each device
+ for i := range c.Info.DeviceNum() {
+ uuid := c.Info.DeviceUUID(i)
+ if len(uuid) < 40 {
+ klog.Errorf("Invalid UUID length for device %d in Pod %s/%s, Container %s", i, pod.Namespace, pod.Name, ctr.Name)
+ return fmt.Errorf("invalid UUID length for device %d", i)
+ }
+ uuid = uuid[0:40] // Ensure UUID is truncated to 40 characters
+
+ // Collect device metrics
+ memoryTotal := c.Info.DeviceMemoryTotal(i)
+ memoryLimit := c.Info.DeviceMemoryLimit(i)
+ memoryContextSize := c.Info.DeviceMemoryContextSize(i)
+ memoryModuleSize := c.Info.DeviceMemoryModuleSize(i)
+ memoryBufferSize := c.Info.DeviceMemoryBufferSize(i)
+ memoryOffset := c.Info.DeviceMemoryOffset(i)
+ smUtil := c.Info.DeviceSmUtil(i)
+ lastKernelTime := c.Info.LastKernelTime()
+
+ // Send metrics to Prometheus
+ labels := []string{pod.Namespace, pod.Name, ctr.Name, fmt.Sprint(i), uuid}
+
+ if err := sendMetric(ch, ctrvGPUdesc, prometheus.GaugeValue, float64(memoryTotal), labels...); err != nil {
+ klog.Errorf("Failed to send memoryTotal metric for device %d in Pod %s/%s, Container %s: %v", i, pod.Namespace, pod.Name, ctr.Name, err)
+ return err
+ }
+
+ if err := sendMetric(ch, ctrvGPUlimitdesc, prometheus.GaugeValue, float64(memoryLimit), labels...); err != nil {
+ klog.Errorf("Failed to send memoryLimit metric for device %d in Pod %s/%s, Container %s: %v", i, pod.Namespace, pod.Name, ctr.Name, err)
+ return err
+ }
+
+ // Send memory-related metrics with additional labels
+ memoryLabels := append(labels, fmt.Sprint(memoryContextSize), fmt.Sprint(memoryModuleSize), fmt.Sprint(memoryBufferSize), fmt.Sprint(memoryOffset))
+ if err := sendMetric(ch, ctrDeviceMemorydesc, prometheus.CounterValue, float64(memoryTotal), memoryLabels...); err != nil {
+ klog.Errorf("Failed to send memory-related metrics for device %d in Pod %s/%s, Container %s: %v", i, pod.Namespace, pod.Name, ctr.Name, err)
+ return err
+ }
+
+ if err := sendMetric(ch, ctrDeviceUtilizationdesc, prometheus.GaugeValue, float64(smUtil), labels...); err != nil {
+ klog.Errorf("Failed to send SM utilization metric for device %d in Pod %s/%s, Container %s: %v", i, pod.Namespace, pod.Name, ctr.Name, err)
+ return err
+ }
+
+ // Send last kernel time metric if valid
+ if lastKernelTime > 0 {
+ lastSec := max(nowSec-lastKernelTime, 0)
+ if err := sendMetric(ch, ctrDeviceLastKernelDesc, prometheus.GaugeValue, float64(lastSec), labels...); err != nil {
+ klog.Errorf("Failed to send last kernel time metric for device %d in Pod %s/%s, Container %s: %v", i, pod.Namespace, pod.Name, ctr.Name, err)
+ return err
+ }
+ }
+ }
+
+ klog.V(5).Infof("Successfully collected metrics for Pod %s/%s, Container %s", pod.Namespace, pod.Name, ctr.Name)
+ return nil
+}
+
+func (cc ClusterManagerCollector) collectPodAndContainerMigInfo(ch chan<- prometheus.Metric) error {
+ nodeName := os.Getenv(util.NodeNameEnvName)
+ if nodeName == "" {
+ return fmt.Errorf("node name environment variable %s is not set", util.NodeNameEnvName)
+ }
+
+ pods, err := cc.ClusterManager.PodLister.List(labels.SelectorFromSet(labels.Set{util.AssignedNodeAnnotations: nodeName}))
+ if err != nil {
+ klog.Errorf("Failed to list pods for node %s: %v", nodeName, err)
+ return fmt.Errorf("failed to list pods: %w", err)
+ }
+ for _, pod := range pods {
+ pdevices, err := util.DecodePodDevices(util.SupportDevices, pod.Annotations)
+ if err != nil {
+ return fmt.Errorf("failed to decode pod devices: %w", err)
+ }
+ for ctrIdx, container := range pod.Spec.Containers {
+ for ctrDevIdx, ctrDevices := range pdevices[nv.NvidiaGPUDevice] {
+ if len(ctrDevices) == 0 || ctrIdx != ctrDevIdx {
+ continue
+ }
+ for _, ctrDev := range ctrDevices {
+ if strings.Contains(ctrDev.UUID, "[") {
+ uuid := strings.Split(ctrDev.UUID, "[")[0]
+ _, idx, err := util.ExtractMigTemplatesFromUUID(ctrDev.UUID)
+ if err != nil {
+ klog.Errorf("Failed to get mig template for device %s in Pod %s/%s, container %s: %v", ctrDev.UUID, pod.Namespace, pod.Name, container.Name, err)
+ continue
+ }
+ gpuInstanceId, err := dp.GetMigGpuInstanceIdFromIndex(ctrDev.UUID, idx)
+ if err != nil {
+ klog.Errorf("Failed to get mig InstanceId for device %s in Pod %s/%s, container %s: %v", ctrDev.UUID, pod.Namespace, pod.Name, container.Name, err)
+ continue
+ }
+ labels := []string{pod.Namespace, pod.Name, container.Name, fmt.Sprint(idx), uuid, fmt.Sprint(gpuInstanceId)}
+ if err := sendMetric(ch, ctrDeviceMigInfo, prometheus.GaugeValue, 1, labels...); err != nil {
+ klog.Errorf("Failed to send mig info metric for device %s in Pod %s/%s, container %s: %v", ctrDev.UUID, pod.Namespace, pod.Name, container.Name, err)
+ return err
+ }
+ }
+ }
+ }
+ }
+ }
+ return nil
+}
+
+func sendMetric(ch chan<- prometheus.Metric, desc *prometheus.Desc, valueType prometheus.ValueType, value float64, labels ...string) error {
+ metric, err := prometheus.NewConstMetric(desc, valueType, value, labels...)
+ if err != nil {
+ return fmt.Errorf("failed to create metric: %w", err)
+ }
+ ch <- metric
+ return nil
+}
+
+// NewClusterManager first creates a Prometheus-ignorant ClusterManager
+// instance. Then, it creates a ClusterManagerCollector for the just created
+// ClusterManager. Finally, it registers the ClusterManagerCollector with a
+// wrapping Registerer that adds the zone as a label. In this way, the metrics
+// collected by different ClusterManagerCollectors do not collide.
+func NewClusterManager(zone string, reg prometheus.Registerer, containerLister *nvidia.ContainerLister) *ClusterManager {
+ c := &ClusterManager{
+ Zone: zone,
+ containerLister: containerLister,
+ }
+
+ informerFactory := informers.NewSharedInformerFactoryWithOptions(containerLister.Clientset(), time.Hour*1)
+ c.PodLister = informerFactory.Core().V1().Pods().Lister()
+ stopCh := make(chan struct{})
+ informerFactory.Start(stopCh)
+
+ cc := ClusterManagerCollector{ClusterManager: c}
+ prometheus.WrapRegistererWith(prometheus.Labels{"zone": zone}, reg).MustRegister(cc)
+ return c
+}
diff --git a/cmd/vGPUmonitor/noderpc/noderpc.pb.go b/cmd/vGPUmonitor/noderpc/noderpc.pb.go
new file mode 100644
index 000000000..4c732b88b
--- /dev/null
+++ b/cmd/vGPUmonitor/noderpc/noderpc.pb.go
@@ -0,0 +1,518 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// protoc-gen-go v1.27.1
+// protoc v3.14.0
+// source: noderpc/noderpc.proto
+
+package vGPUmonitor
+
+import (
+ protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+ protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+ reflect "reflect"
+ sync "sync"
+)
+
+const (
+ // Verify that this generated code is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+ // Verify that runtime/protoimpl is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// The sharedProcs contains the sharedRegion
+type ShrregProcSlotT struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Pid int32 `protobuf:"varint,1,opt,name=pid,proto3" json:"pid,omitempty"`
+ Used []uint64 `protobuf:"varint,2,rep,packed,name=used,proto3" json:"used,omitempty"`
+ Status int32 `protobuf:"varint,3,opt,name=status,proto3" json:"status,omitempty"`
+}
+
+func (x *ShrregProcSlotT) Reset() {
+ *x = ShrregProcSlotT{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_noderpc_noderpc_proto_msgTypes[0]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *ShrregProcSlotT) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ShrregProcSlotT) ProtoMessage() {}
+
+func (x *ShrregProcSlotT) ProtoReflect() protoreflect.Message {
+ mi := &file_noderpc_noderpc_proto_msgTypes[0]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use ShrregProcSlotT.ProtoReflect.Descriptor instead.
+func (*ShrregProcSlotT) Descriptor() ([]byte, []int) {
+ return file_noderpc_noderpc_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *ShrregProcSlotT) GetPid() int32 {
+ if x != nil {
+ return x.Pid
+ }
+ return 0
+}
+
+func (x *ShrregProcSlotT) GetUsed() []uint64 {
+ if x != nil {
+ return x.Used
+ }
+ return nil
+}
+
+func (x *ShrregProcSlotT) GetStatus() int32 {
+ if x != nil {
+ return x.Status
+ }
+ return 0
+}
+
+// The sharedRegionT struct is the main struct for monitoring vgpu
+type SharedRegionT struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ InitializedFlag int32 `protobuf:"varint,1,opt,name=initializedFlag,proto3" json:"initializedFlag,omitempty"`
+ OwnerPid uint32 `protobuf:"varint,2,opt,name=ownerPid,proto3" json:"ownerPid,omitempty"`
+ Sem uint32 `protobuf:"varint,3,opt,name=sem,proto3" json:"sem,omitempty"`
+ Limit []uint64 `protobuf:"varint,4,rep,packed,name=limit,proto3" json:"limit,omitempty"`
+ SmLimit []uint64 `protobuf:"varint,5,rep,packed,name=sm_limit,json=smLimit,proto3" json:"sm_limit,omitempty"`
+ Procs []*ShrregProcSlotT `protobuf:"bytes,6,rep,name=procs,proto3" json:"procs,omitempty"`
+}
+
+func (x *SharedRegionT) Reset() {
+ *x = SharedRegionT{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_noderpc_noderpc_proto_msgTypes[1]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *SharedRegionT) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SharedRegionT) ProtoMessage() {}
+
+func (x *SharedRegionT) ProtoReflect() protoreflect.Message {
+ mi := &file_noderpc_noderpc_proto_msgTypes[1]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use SharedRegionT.ProtoReflect.Descriptor instead.
+func (*SharedRegionT) Descriptor() ([]byte, []int) {
+ return file_noderpc_noderpc_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *SharedRegionT) GetInitializedFlag() int32 {
+ if x != nil {
+ return x.InitializedFlag
+ }
+ return 0
+}
+
+func (x *SharedRegionT) GetOwnerPid() uint32 {
+ if x != nil {
+ return x.OwnerPid
+ }
+ return 0
+}
+
+func (x *SharedRegionT) GetSem() uint32 {
+ if x != nil {
+ return x.Sem
+ }
+ return 0
+}
+
+func (x *SharedRegionT) GetLimit() []uint64 {
+ if x != nil {
+ return x.Limit
+ }
+ return nil
+}
+
+func (x *SharedRegionT) GetSmLimit() []uint64 {
+ if x != nil {
+ return x.SmLimit
+ }
+ return nil
+}
+
+func (x *SharedRegionT) GetProcs() []*ShrregProcSlotT {
+ if x != nil {
+ return x.Procs
+ }
+ return nil
+}
+
+type Podusage struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Poduuid string `protobuf:"bytes,1,opt,name=poduuid,proto3" json:"poduuid,omitempty"`
+ Podvgpuinfo *SharedRegionT `protobuf:"bytes,2,opt,name=podvgpuinfo,proto3" json:"podvgpuinfo,omitempty"`
+}
+
+func (x *Podusage) Reset() {
+ *x = Podusage{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_noderpc_noderpc_proto_msgTypes[2]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *Podusage) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Podusage) ProtoMessage() {}
+
+func (x *Podusage) ProtoReflect() protoreflect.Message {
+ mi := &file_noderpc_noderpc_proto_msgTypes[2]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use Podusage.ProtoReflect.Descriptor instead.
+func (*Podusage) Descriptor() ([]byte, []int) {
+ return file_noderpc_noderpc_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *Podusage) GetPoduuid() string {
+ if x != nil {
+ return x.Poduuid
+ }
+ return ""
+}
+
+func (x *Podusage) GetPodvgpuinfo() *SharedRegionT {
+ if x != nil {
+ return x.Podvgpuinfo
+ }
+ return nil
+}
+
+// The request message containing the user's name.
+type GetNodeVGPURequest struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Ctruuid string `protobuf:"bytes,1,opt,name=ctruuid,proto3" json:"ctruuid,omitempty"`
+}
+
+func (x *GetNodeVGPURequest) Reset() {
+ *x = GetNodeVGPURequest{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_noderpc_noderpc_proto_msgTypes[3]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *GetNodeVGPURequest) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GetNodeVGPURequest) ProtoMessage() {}
+
+func (x *GetNodeVGPURequest) ProtoReflect() protoreflect.Message {
+ mi := &file_noderpc_noderpc_proto_msgTypes[3]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use GetNodeVGPURequest.ProtoReflect.Descriptor instead.
+func (*GetNodeVGPURequest) Descriptor() ([]byte, []int) {
+ return file_noderpc_noderpc_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *GetNodeVGPURequest) GetCtruuid() string {
+ if x != nil {
+ return x.Ctruuid
+ }
+ return ""
+}
+
+// The response message containing the greetings
+type GetNodeVGPUReply struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Nodeid string `protobuf:"bytes,1,opt,name=nodeid,proto3" json:"nodeid,omitempty"`
+ Nodevgpuinfo []*Podusage `protobuf:"bytes,2,rep,name=nodevgpuinfo,proto3" json:"nodevgpuinfo,omitempty"`
+}
+
+func (x *GetNodeVGPUReply) Reset() {
+ *x = GetNodeVGPUReply{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_noderpc_noderpc_proto_msgTypes[4]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *GetNodeVGPUReply) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GetNodeVGPUReply) ProtoMessage() {}
+
+func (x *GetNodeVGPUReply) ProtoReflect() protoreflect.Message {
+ mi := &file_noderpc_noderpc_proto_msgTypes[4]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use GetNodeVGPUReply.ProtoReflect.Descriptor instead.
+func (*GetNodeVGPUReply) Descriptor() ([]byte, []int) {
+ return file_noderpc_noderpc_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *GetNodeVGPUReply) GetNodeid() string {
+ if x != nil {
+ return x.Nodeid
+ }
+ return ""
+}
+
+func (x *GetNodeVGPUReply) GetNodevgpuinfo() []*Podusage {
+ if x != nil {
+ return x.Nodevgpuinfo
+ }
+ return nil
+}
+
+var File_noderpc_noderpc_proto protoreflect.FileDescriptor
+
+var file_noderpc_noderpc_proto_rawDesc = []byte{
+ 0x0a, 0x15, 0x6e, 0x6f, 0x64, 0x65, 0x72, 0x70, 0x63, 0x2f, 0x6e, 0x6f, 0x64, 0x65, 0x72, 0x70,
+ 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x72,
+ 0x70, 0x63, 0x22, 0x4f, 0x0a, 0x0f, 0x73, 0x68, 0x72, 0x72, 0x65, 0x67, 0x50, 0x72, 0x6f, 0x63,
+ 0x53, 0x6c, 0x6f, 0x74, 0x54, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01,
+ 0x28, 0x05, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x64, 0x18,
+ 0x02, 0x20, 0x03, 0x28, 0x04, 0x52, 0x04, 0x75, 0x73, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x73,
+ 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61,
+ 0x74, 0x75, 0x73, 0x22, 0xca, 0x01, 0x0a, 0x0d, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x52, 0x65,
+ 0x67, 0x69, 0x6f, 0x6e, 0x54, 0x12, 0x28, 0x0a, 0x0f, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c,
+ 0x69, 0x7a, 0x65, 0x64, 0x46, 0x6c, 0x61, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f,
+ 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x46, 0x6c, 0x61, 0x67, 0x12,
+ 0x1a, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x50, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
+ 0x0d, 0x52, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x50, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x73,
+ 0x65, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x73, 0x65, 0x6d, 0x12, 0x14, 0x0a,
+ 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x04, 0x52, 0x05, 0x6c, 0x69,
+ 0x6d, 0x69, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x6d, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18,
+ 0x05, 0x20, 0x03, 0x28, 0x04, 0x52, 0x07, 0x73, 0x6d, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x30,
+ 0x0a, 0x05, 0x70, 0x72, 0x6f, 0x63, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e,
+ 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x68, 0x72, 0x72, 0x65, 0x67,
+ 0x50, 0x72, 0x6f, 0x63, 0x53, 0x6c, 0x6f, 0x74, 0x54, 0x52, 0x05, 0x70, 0x72, 0x6f, 0x63, 0x73,
+ 0x22, 0x60, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x75, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07,
+ 0x70, 0x6f, 0x64, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70,
+ 0x6f, 0x64, 0x75, 0x75, 0x69, 0x64, 0x12, 0x3a, 0x0a, 0x0b, 0x70, 0x6f, 0x64, 0x76, 0x67, 0x70,
+ 0x75, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x70, 0x6c,
+ 0x75, 0x67, 0x69, 0x6e, 0x72, 0x70, 0x63, 0x2e, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x52, 0x65,
+ 0x67, 0x69, 0x6f, 0x6e, 0x54, 0x52, 0x0b, 0x70, 0x6f, 0x64, 0x76, 0x67, 0x70, 0x75, 0x69, 0x6e,
+ 0x66, 0x6f, 0x22, 0x2e, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x56, 0x47, 0x50,
+ 0x55, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x74, 0x72, 0x75,
+ 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x74, 0x72, 0x75, 0x75,
+ 0x69, 0x64, 0x22, 0x63, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x56, 0x47, 0x50,
+ 0x55, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x69, 0x64,
+ 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x69, 0x64, 0x12, 0x37,
+ 0x0a, 0x0c, 0x6e, 0x6f, 0x64, 0x65, 0x76, 0x67, 0x70, 0x75, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02,
+ 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x72, 0x70, 0x63,
+ 0x2e, 0x70, 0x6f, 0x64, 0x75, 0x73, 0x61, 0x67, 0x65, 0x52, 0x0c, 0x6e, 0x6f, 0x64, 0x65, 0x76,
+ 0x67, 0x70, 0x75, 0x69, 0x6e, 0x66, 0x6f, 0x32, 0x5b, 0x0a, 0x0c, 0x4e, 0x6f, 0x64, 0x65, 0x56,
+ 0x47, 0x50, 0x55, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4b, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4e, 0x6f,
+ 0x64, 0x65, 0x56, 0x47, 0x50, 0x55, 0x12, 0x1d, 0x2e, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x72,
+ 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x56, 0x47, 0x50, 0x55, 0x52, 0x65,
+ 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x72, 0x70,
+ 0x63, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x56, 0x47, 0x50, 0x55, 0x52, 0x65, 0x70,
+ 0x6c, 0x79, 0x22, 0x00, 0x42, 0x4b, 0x0a, 0x1b, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
+ 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f,
+ 0x72, 0x6c, 0x64, 0x42, 0x0f, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x50,
+ 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x19, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x34,
+ 0x70, 0x64, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x47, 0x50, 0x55, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f,
+ 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+ file_noderpc_noderpc_proto_rawDescOnce sync.Once
+ file_noderpc_noderpc_proto_rawDescData = file_noderpc_noderpc_proto_rawDesc
+)
+
+func file_noderpc_noderpc_proto_rawDescGZIP() []byte {
+ file_noderpc_noderpc_proto_rawDescOnce.Do(func() {
+ file_noderpc_noderpc_proto_rawDescData = protoimpl.X.CompressGZIP(file_noderpc_noderpc_proto_rawDescData)
+ })
+ return file_noderpc_noderpc_proto_rawDescData
+}
+
+var file_noderpc_noderpc_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
+var file_noderpc_noderpc_proto_goTypes = []interface{}{
+ (*ShrregProcSlotT)(nil), // 0: pluginrpc.shrregProcSlotT
+ (*SharedRegionT)(nil), // 1: pluginrpc.sharedRegionT
+ (*Podusage)(nil), // 2: pluginrpc.podusage
+ (*GetNodeVGPURequest)(nil), // 3: pluginrpc.GetNodeVGPURequest
+ (*GetNodeVGPUReply)(nil), // 4: pluginrpc.GetNodeVGPUReply
+}
+var file_noderpc_noderpc_proto_depIdxs = []int32{
+ 0, // 0: pluginrpc.sharedRegionT.procs:type_name -> pluginrpc.shrregProcSlotT
+ 1, // 1: pluginrpc.podusage.podvgpuinfo:type_name -> pluginrpc.sharedRegionT
+ 2, // 2: pluginrpc.GetNodeVGPUReply.nodevgpuinfo:type_name -> pluginrpc.podusage
+ 3, // 3: pluginrpc.NodeVGPUInfo.GetNodeVGPU:input_type -> pluginrpc.GetNodeVGPURequest
+ 4, // 4: pluginrpc.NodeVGPUInfo.GetNodeVGPU:output_type -> pluginrpc.GetNodeVGPUReply
+ 4, // [4:5] is the sub-list for method output_type
+ 3, // [3:4] is the sub-list for method input_type
+ 3, // [3:3] is the sub-list for extension type_name
+ 3, // [3:3] is the sub-list for extension extendee
+ 0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_noderpc_noderpc_proto_init() }
+func file_noderpc_noderpc_proto_init() {
+ if File_noderpc_noderpc_proto != nil {
+ return
+ }
+ if !protoimpl.UnsafeEnabled {
+ file_noderpc_noderpc_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*ShrregProcSlotT); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_noderpc_noderpc_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*SharedRegionT); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_noderpc_noderpc_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*Podusage); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_noderpc_noderpc_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*GetNodeVGPURequest); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_noderpc_noderpc_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*GetNodeVGPUReply); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ }
+ type x struct{}
+ out := protoimpl.TypeBuilder{
+ File: protoimpl.DescBuilder{
+ GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+ RawDescriptor: file_noderpc_noderpc_proto_rawDesc,
+ NumEnums: 0,
+ NumMessages: 5,
+ NumExtensions: 0,
+ NumServices: 1,
+ },
+ GoTypes: file_noderpc_noderpc_proto_goTypes,
+ DependencyIndexes: file_noderpc_noderpc_proto_depIdxs,
+ MessageInfos: file_noderpc_noderpc_proto_msgTypes,
+ }.Build()
+ File_noderpc_noderpc_proto = out.File
+ file_noderpc_noderpc_proto_rawDesc = nil
+ file_noderpc_noderpc_proto_goTypes = nil
+ file_noderpc_noderpc_proto_depIdxs = nil
+}
diff --git a/cmd/vGPUmonitor/noderpc/noderpc.proto b/cmd/vGPUmonitor/noderpc/noderpc.proto
new file mode 100644
index 000000000..611c3ec8c
--- /dev/null
+++ b/cmd/vGPUmonitor/noderpc/noderpc.proto
@@ -0,0 +1,61 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+option go_package = "gitlab.4pd.io/vGPUmonitor";
+option java_multiple_files = true;
+option java_package = "io.grpc.examples.helloworld";
+option java_outer_classname = "HelloWorldProto";
+
+package pluginrpc;
+
+// The greeting service definition.
+service NodeVGPUInfo {
+ // Sends a greeting
+ rpc GetNodeVGPU (GetNodeVGPURequest) returns (GetNodeVGPUReply) {}
+}
+
+// The sharedProcs contains the sharedRegion
+message shrregProcSlotT {
+ int32 pid = 1;
+ repeated uint64 used = 2;
+ int32 status = 3;
+}
+
+// The sharedRegionT struct is the main struct for monitoring vgpu
+message sharedRegionT {
+ int32 initializedFlag = 1;
+ uint32 ownerPid = 2;
+ uint32 sem = 3;
+ repeated uint64 limit = 4;
+ repeated uint64 sm_limit = 5;
+ repeated shrregProcSlotT procs = 6;
+}
+
+message podusage {
+ string poduuid = 1;
+ sharedRegionT podvgpuinfo = 2;
+}
+
+// The request message containing the user's name.
+message GetNodeVGPURequest {
+ string ctruuid = 1;
+}
+
+// The response message containing the greetings
+message GetNodeVGPUReply {
+ string nodeid = 1;
+ repeated podusage nodevgpuinfo = 2;
+}
diff --git a/cmd/vGPUmonitor/noderpc/noderpc_grpc.pb.go b/cmd/vGPUmonitor/noderpc/noderpc_grpc.pb.go
new file mode 100644
index 000000000..346ed4ce8
--- /dev/null
+++ b/cmd/vGPUmonitor/noderpc/noderpc_grpc.pb.go
@@ -0,0 +1,103 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package vGPUmonitor
+
+import (
+ context "context"
+ grpc "google.golang.org/grpc"
+ codes "google.golang.org/grpc/codes"
+ status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
+const _ = grpc.SupportPackageIsVersion7
+
+// NodeVGPUInfoClient is the client API for NodeVGPUInfo service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type NodeVGPUInfoClient interface {
+ // Sends a greeting
+ GetNodeVGPU(ctx context.Context, in *GetNodeVGPURequest, opts ...grpc.CallOption) (*GetNodeVGPUReply, error)
+}
+
+type nodeVGPUInfoClient struct {
+ cc grpc.ClientConnInterface
+}
+
+func NewNodeVGPUInfoClient(cc grpc.ClientConnInterface) NodeVGPUInfoClient {
+ return &nodeVGPUInfoClient{cc}
+}
+
+func (c *nodeVGPUInfoClient) GetNodeVGPU(ctx context.Context, in *GetNodeVGPURequest, opts ...grpc.CallOption) (*GetNodeVGPUReply, error) {
+ out := new(GetNodeVGPUReply)
+ err := c.cc.Invoke(ctx, "/pluginrpc.NodeVGPUInfo/GetNodeVGPU", in, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+// NodeVGPUInfoServer is the server API for NodeVGPUInfo service.
+// All implementations must embed UnimplementedNodeVGPUInfoServer
+// for forward compatibility
+type NodeVGPUInfoServer interface {
+ // Sends a greeting
+ GetNodeVGPU(context.Context, *GetNodeVGPURequest) (*GetNodeVGPUReply, error)
+ mustEmbedUnimplementedNodeVGPUInfoServer()
+}
+
+// UnimplementedNodeVGPUInfoServer must be embedded to have forward compatible implementations.
+type UnimplementedNodeVGPUInfoServer struct {
+}
+
+func (UnimplementedNodeVGPUInfoServer) GetNodeVGPU(context.Context, *GetNodeVGPURequest) (*GetNodeVGPUReply, error) {
+ return nil, status.Errorf(codes.Unimplemented, "method GetNodeVGPU not implemented")
+}
+func (UnimplementedNodeVGPUInfoServer) mustEmbedUnimplementedNodeVGPUInfoServer() {}
+
+// UnsafeNodeVGPUInfoServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to NodeVGPUInfoServer will
+// result in compilation errors.
+type UnsafeNodeVGPUInfoServer interface {
+ mustEmbedUnimplementedNodeVGPUInfoServer()
+}
+
+func RegisterNodeVGPUInfoServer(s grpc.ServiceRegistrar, srv NodeVGPUInfoServer) {
+ s.RegisterService(&NodeVGPUInfo_ServiceDesc, srv)
+}
+
+func _NodeVGPUInfo_GetNodeVGPU_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(GetNodeVGPURequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(NodeVGPUInfoServer).GetNodeVGPU(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: "/pluginrpc.NodeVGPUInfo/GetNodeVGPU",
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(NodeVGPUInfoServer).GetNodeVGPU(ctx, req.(*GetNodeVGPURequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
+// NodeVGPUInfo_ServiceDesc is the grpc.ServiceDesc for NodeVGPUInfo service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var NodeVGPUInfo_ServiceDesc = grpc.ServiceDesc{
+ ServiceName: "pluginrpc.NodeVGPUInfo",
+ HandlerType: (*NodeVGPUInfoServer)(nil),
+ Methods: []grpc.MethodDesc{
+ {
+ MethodName: "GetNodeVGPU",
+ Handler: _NodeVGPUInfo_GetNodeVGPU_Handler,
+ },
+ },
+ Streams: []grpc.StreamDesc{},
+ Metadata: "noderpc/noderpc.proto",
+}
diff --git a/cmd/vGPUmonitor/testcollector/main.go b/cmd/vGPUmonitor/testcollector/main.go
new file mode 100644
index 000000000..c95d36a8f
--- /dev/null
+++ b/cmd/vGPUmonitor/testcollector/main.go
@@ -0,0 +1,146 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "log"
+ "net/http"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/collectors"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// ClusterManager is an example for a system that might have been built without
+// Prometheus in mind. It models a central manager of jobs running in a
+// cluster. Thus, we implement a custom Collector called
+// ClusterManagerCollector, which collects information from a ClusterManager
+// using its provided methods and turns them into Prometheus Metrics for
+// collection.
+//
+// An additional challenge is that multiple instances of the ClusterManager are
+// run within the same binary, each in charge of a different zone. We need to
+// make use of wrapping Registerers to be able to register each
+// ClusterManagerCollector instance with Prometheus.
+type ClusterManager struct {
+ Zone string
+ // Contains many more fields not listed in this example.
+}
+
+// ReallyExpensiveAssessmentOfTheSystemState is a mock for the data gathering a
+// real cluster manager would have to do. Since it may actually be really
+// expensive, it must only be called once per collection. This implementation,
+// obviously, only returns some made-up data.
+func (c *ClusterManager) ReallyExpensiveAssessmentOfTheSystemState() (
+ oomCountByHost map[string]int, ramUsageByHost map[string]float64,
+) {
+ // Just example fake data.
+ oomCountByHost = map[string]int{
+ "foo.example.org": 42,
+ "bar.example.org": 2001,
+ }
+ ramUsageByHost = map[string]float64{
+ "foo.example.org": 6.023e23,
+ "bar.example.org": 3.14,
+ }
+ return
+}
+
+// ClusterManagerCollector implements the Collector interface.
+type ClusterManagerCollector struct {
+ ClusterManager *ClusterManager
+}
+
+// Descriptors used by the ClusterManagerCollector below.
+var (
+ oomCountDesc = prometheus.NewDesc(
+ "clustermanager_oom_crashes_total",
+ "Number of OOM crashes.",
+ []string{"host"}, nil,
+ )
+ ramUsageDesc = prometheus.NewDesc(
+ "clustermanager_ram_usage_bytes",
+ "RAM usage as reported to the cluster manager.",
+ []string{"host"}, nil,
+ )
+)
+
+// Describe is implemented with DescribeByCollect. That's possible because the
+// Collect method will always return the same two metrics with the same two
+// descriptors.
+func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
+ prometheus.DescribeByCollect(cc, ch)
+}
+
+// Collect first triggers the ReallyExpensiveAssessmentOfTheSystemState. Then it
+// creates constant metrics for each host on the fly based on the returned data.
+//
+// Note that Collect could be called concurrently, so we depend on
+// ReallyExpensiveAssessmentOfTheSystemState to be concurrency-safe.
+func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
+ oomCountByHost, ramUsageByHost := cc.ClusterManager.ReallyExpensiveAssessmentOfTheSystemState()
+ for host, oomCount := range oomCountByHost {
+ ch <- prometheus.MustNewConstMetric(
+ oomCountDesc,
+ prometheus.CounterValue,
+ float64(oomCount),
+ host,
+ )
+ }
+ for host, ramUsage := range ramUsageByHost {
+ ch <- prometheus.MustNewConstMetric(
+ ramUsageDesc,
+ prometheus.GaugeValue,
+ ramUsage,
+ host,
+ )
+ }
+}
+
+// NewClusterManager first creates a Prometheus-ignorant ClusterManager
+// instance. Then, it creates a ClusterManagerCollector for the just created
+// ClusterManager. Finally, it registers the ClusterManagerCollector with a
+// wrapping Registerer that adds the zone as a label. In this way, the metrics
+// collected by different ClusterManagerCollectors do not collide.
+func NewClusterManager(zone string, reg prometheus.Registerer) *ClusterManager {
+ c := &ClusterManager{
+ Zone: zone,
+ }
+ cc := ClusterManagerCollector{ClusterManager: c}
+ prometheus.WrapRegistererWith(prometheus.Labels{"zone": zone}, reg).MustRegister(cc)
+ return c
+}
+
+func main() {
+ // Since we are dealing with custom Collector implementations, it might
+ // be a good idea to try it out with a pedantic registry.
+ reg := prometheus.NewPedanticRegistry()
+
+ // Construct cluster managers. In real code, we would assign them to
+ // variables to then do something with them.
+ NewClusterManager("db", reg)
+ NewClusterManager("ca", reg)
+
+ // Add the standard process and Go metrics to the custom registry.
+ reg.MustRegister(
+ collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+ collectors.NewGoCollector(),
+ )
+
+ http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
+ log.Fatal(http.ListenAndServe(":8080", nil))
+}
diff --git a/cmd/vGPUmonitor/validation.go b/cmd/vGPUmonitor/validation.go
new file mode 100644
index 000000000..fbba208b6
--- /dev/null
+++ b/cmd/vGPUmonitor/validation.go
@@ -0,0 +1,37 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "fmt"
+ "os"
+)
+
+var requiredEnvVars = map[string]bool{
+ "HOOK_PATH": true,
+ "OTHER_ENV_VAR": false,
+}
+
+func ValidateEnvVars() error {
+ for envVar, required := range requiredEnvVars {
+ _, exists := os.LookupEnv(envVar)
+ if required && !exists {
+ return fmt.Errorf("required environment variable %s not set", envVar)
+ }
+ }
+ return nil
+}
diff --git a/deployments/4pd-vgpu/Chart.yaml b/deployments/4pd-vgpu/Chart.yaml
deleted file mode 100644
index 7a0a9ed2b..000000000
--- a/deployments/4pd-vgpu/Chart.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-apiVersion: v2
-name: 4pd-vgpu
-version: 0.0.1
-kubeVersion: ">= 1.16.0"
-description: 4paradigm vgpu for kubernetes
-keywords:
- - vgpu
- - gpu
-type: application
-maintainers:
- - name: 4pd
- email: opensource@4paradigm.com
-appVersion: 0.0.1
diff --git a/deployments/4pd-vgpu/templates/_helpers.tpl b/deployments/4pd-vgpu/templates/_helpers.tpl
deleted file mode 100644
index 22626a782..000000000
--- a/deployments/4pd-vgpu/templates/_helpers.tpl
+++ /dev/null
@@ -1,87 +0,0 @@
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "4pd-vgpu.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "4pd-vgpu.fullname" -}}
-{{- if .Values.fullnameOverride -}}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-The app name for Scheduler
-*/}}
-{{- define "4pd-vgpu.scheduler" -}}
-{{- printf "%s-scheduler" ( include "4pd-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-The app name for DevicePlugin
-*/}}
-{{- define "4pd-vgpu.device-plugin" -}}
-{{- printf "%s-device-plugin" ( include "4pd-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-The tls secret name for Scheduler
-*/}}
-{{- define "4pd-vgpu.scheduler.tls" -}}
-{{- printf "%s-scheduler-tls" ( include "4pd-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-The webhook name
-*/}}
-{{- define "4pd-vgpu.scheduler.webhook" -}}
-{{- printf "%s-webhook" ( include "4pd-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "4pd-vgpu.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "4pd-vgpu.labels" -}}
-helm.sh/chart: {{ include "4pd-vgpu.chart" . }}
-{{ include "4pd-vgpu.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "4pd-vgpu.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "4pd-vgpu.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}
-
-{{/*
-Image registry secret name
-*/}}
-{{- define "4pd-vgpu.imagePullSecrets" -}}
-imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 2 }}
-{{- end }}
-
diff --git a/deployments/4pd-vgpu/templates/device-plugin/daemonset.yaml b/deployments/4pd-vgpu/templates/device-plugin/daemonset.yaml
deleted file mode 100644
index fd3e93b5f..000000000
--- a/deployments/4pd-vgpu/templates/device-plugin/daemonset.yaml
+++ /dev/null
@@ -1,74 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
- name: {{ include "4pd-vgpu.device-plugin" . }}
- labels:
- app.kubernetes.io/component: 4pd-device-plugin
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
- {{- with .Values.global.labels }}
- {{- toYaml . | nindent 4 }}
- {{- end }}
- {{- if .Values.global.annotations }}
- annotations: {{ toYaml .Values.global.annotations | nindent 4}}
- {{- end }}
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/component: 4pd-device-plugin
- {{- include "4pd-vgpu.selectorLabels" . | nindent 6 }}
- template:
- metadata:
- labels:
- app.kubernetes.io/component: 4pd-device-plugin
- 4pd.io/webhook: ignore
- {{- include "4pd-vgpu.selectorLabels" . | nindent 8 }}
- {{- if .Values.devicePlugin.podAnnotations }}
- annotations: {{ toYaml .Values.devicePlugin.podAnnotations | nindent 8 }}
- {{- end }}
- spec:
- {{- include "4pd-vgpu.imagePullSecrets" . | nindent 6}}
- # serviceAccountName:
- priorityClassName: system-node-critical
- containers:
- - name: device-plugin
- image: {{ .Values.devicePlugin.image | quote }}
- imagePullPolicy: {{ .Values.devicePlugin.imagePullPolicy | quote }}
- command:
- - device-plugin
- - --resource-name={{ .Values.resourceName }}
- - --scheduler-endpoint={{ printf "%s:%d" ( include "4pd-vgpu.scheduler" . ) ( int .Values.scheduler.service.grpcPort ) }}
- {{- range .Values.devicePlugin.extraArgs }}
- - {{ . }}
- {{- end }}
- env:
- - name: NODE_NAME
- valueFrom:
- fieldRef:
- fieldPath: spec.nodeName
- volumeMounts:
- - name: device-plugin
- mountPath: /var/lib/kubelet/device-plugins
- - name: lib
- mountPath: /usr/local/vgpu
- - name: sock
- mountPath: /var/lib/vgpu
- volumes:
- - name: device-plugin
- hostPath:
- path: {{ .Values.devicePlugin.pluginPath }}
- - name: lib
- hostPath:
- path: /usr/local/vgpu
- #path: {{ .Values.devicePlugin.libPath }}
- - name: sock
- hostPath:
- path: /var/lib/vgpu
- #path: {{ .Values.devicePlugin.sockPath }}
- {{- if .Values.devicePlugin.nodeSelector }}
- nodeSelector: {{ toYaml .Values.devicePlugin.nodeSelector | nindent 8 }}
- {{- end }}
- {{- if .Values.devicePlugin.tolerations }}
- tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }}
- {{- end }}
-
-
diff --git a/deployments/4pd-vgpu/templates/scheduler/configmap.yaml b/deployments/4pd-vgpu/templates/scheduler/configmap.yaml
deleted file mode 100644
index ad432f153..000000000
--- a/deployments/4pd-vgpu/templates/scheduler/configmap.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: {{ include "4pd-vgpu.scheduler" . }}
- labels:
- app.kubernetes.io/component: 4pd-scheduler
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
-data:
- config.yaml: |
- apiVersion: kubescheduler.config.k8s.io/v1beta1
- kind: KubeSchedulerConfiguration
- healthzBindAddress: 0.0.0.0:10251
- leaderElection:
- leaderElect: false
- metricsBindAddress: 0.0.0.0:10251
- profiles:
- - schedulerName: {{ .Values.schedulerName }}
- extenders:
- - urlPrefix: "https://127.0.0.1:443"
- filterVerb: filter
- nodeCacheCapable: true
- weight: 1
- httpTimeout: 30s
- enableHTTPS: true
- tlsConfig:
- insecure: true
- managedResources:
- - name: {{ .Values.resourceName }}
- ignoredByScheduler: true
diff --git a/deployments/4pd-vgpu/templates/scheduler/deployment.yaml b/deployments/4pd-vgpu/templates/scheduler/deployment.yaml
deleted file mode 100644
index 489c9b348..000000000
--- a/deployments/4pd-vgpu/templates/scheduler/deployment.yaml
+++ /dev/null
@@ -1,82 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: {{ include "4pd-vgpu.scheduler" . }}
- labels:
- app.kubernetes.io/component: 4pd-scheduler
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
- {{- with .Values.global.labels }}
- {{- toYaml . | nindent 4 }}
- {{- end }}
- {{- if .Values.global.annotations }}
- annotations: {{ toYaml .Values.global.annotations | nindent 4}}
- {{- end }}
-spec:
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/component: 4pd-scheduler
- {{- include "4pd-vgpu.selectorLabels" . | nindent 6 }}
- template:
- metadata:
- labels:
- app.kubernetes.io/component: 4pd-scheduler
- {{- include "4pd-vgpu.selectorLabels" . | nindent 8 }}
- 4pd.io/webhook: ignore
- {{- if .Values.scheduler.podAnnotations }}
- annotations: {{ toYaml .Values.scheduler.podAnnotations | nindent 8 }}
- {{- end }}
- spec:
- {{- include "4pd-vgpu.imagePullSecrets" . | nindent 6}}
- serviceAccountName: {{ include "4pd-vgpu.scheduler" . }}
- priorityClassName: system-node-critical
- containers:
- - name: kube-scheduler
- image: {{ .Values.scheduler.kubeScheduler.image | quote }}
- imagePullPolicy: {{ .Values.scheduler.kubeScheduler.imagePullPolicy | quote }}
- command:
- - kube-scheduler
- - --config=/config/config.yaml
- {{- range .Values.scheduler.kubeScheduler.extraArgs }}
- - {{ . }}
- {{- end }}
- volumeMounts:
- - name: scheduler-config
- mountPath: /config
- - name: vgpu-scheduler-extender
- image: {{ .Values.scheduler.extender.image | quote }}
- imagePullPolicy: {{ .Values.scheduler.extender.imagePullPolicy | quote }}
- command:
- - scheduler
- - --resource-name={{ .Values.resourceName }}
- - --http_bind=0.0.0.0:443
- - --grpc_bind=0.0.0.0:1080
- - --cert_file=/tls/tls.crt
- - --key_file=/tls/tls.key
- - --scheduler-name={{ .Values.schedulerName }}
- {{- range .Values.scheduler.extender.extraArgs }}
- - {{ . }}
- {{- end }}
- ports:
- - name: http
- containerPort: 443
- protocol: TCP
- - name: grpc
- containerPort: 1080
- protocol: TCP
- volumeMounts:
- - name: tls-config
- mountPath: /tls
- volumes:
- - name: tls-config
- secret:
- secretName: {{ template "4pd-vgpu.scheduler.tls" . }}
- - name: scheduler-config
- configMap:
- name: {{ template "4pd-vgpu.scheduler" . }}
- {{- if .Values.scheduler.nodeSelector }}
- nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }}
- {{- end }}
- {{- if .Values.scheduler.tolerations }}
- tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }}
- {{- end }}
diff --git a/deployments/4pd-vgpu/templates/scheduler/job-patch/clusterrolebinding.yaml b/deployments/4pd-vgpu/templates/scheduler/job-patch/clusterrolebinding.yaml
deleted file mode 100644
index cbff8d4be..000000000
--- a/deployments/4pd-vgpu/templates/scheduler/job-patch/clusterrolebinding.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- name: {{ include "4pd-vgpu.fullname" . }}-admission
- annotations:
- "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
- "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
- labels:
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
- app.kubernetes.io/component: admission-webhook
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: {{ include "4pd-vgpu.fullname" . }}-admission
-subjects:
- - kind: ServiceAccount
- name: {{ include "4pd-vgpu.fullname" . }}-admission
- namespace: {{ .Release.Namespace | quote }}
diff --git a/deployments/4pd-vgpu/templates/scheduler/job-patch/job-createSecret.yaml b/deployments/4pd-vgpu/templates/scheduler/job-patch/job-createSecret.yaml
deleted file mode 100644
index 0cc35df1a..000000000
--- a/deployments/4pd-vgpu/templates/scheduler/job-patch/job-createSecret.yaml
+++ /dev/null
@@ -1,52 +0,0 @@
-apiVersion: batch/v1
-kind: Job
-metadata:
- name: {{ include "4pd-vgpu.fullname" . }}-admission-create
- annotations:
- "helm.sh/hook": pre-install,pre-upgrade
- "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
- labels:
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
- app.kubernetes.io/component: admission-webhook
-spec:
- {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
- # Alpha feature since k8s 1.12
- ttlSecondsAfterFinished: 0
- {{- end }}
- template:
- metadata:
- name: {{ include "4pd-vgpu.fullname" . }}-admission-create
- {{- if .Values.scheduler.patch.podAnnotations }}
- annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
- {{- end }}
- labels:
- {{- include "4pd-vgpu.labels" . | nindent 8 }}
- app.kubernetes.io/component: admission-webhook
- 4pd.io/webhook: ignore
- spec:
- {{- include "4pd-vgpu.imagePullSecrets" . | nindent 6}}
- {{- if .Values.scheduler.patch.priorityClassName }}
- priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
- {{- end }}
- containers:
- - name: create
- image: {{ .Values.scheduler.patch.image }}
- imagePullPolicy: {{ .Values.scheduler.patch.imagePullPolicy }}
- args:
- - create
- - --cert-name=tls.crt
- - --key-name=tls.key
- - --host={{ printf "%s.%s.svc,127.0.0.1" (include "4pd-vgpu.scheduler" .) .Release.Namespace }}
- - --namespace={{ .Release.Namespace }}
- - --secret-name={{ include "4pd-vgpu.scheduler.tls" . }}
- restartPolicy: OnFailure
- serviceAccountName: {{ include "4pd-vgpu.fullname" . }}-admission
- {{- if .Values.scheduler.patch.nodeSelector }}
- nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
- {{- end }}
- {{- if .Values.scheduler.patch.tolerations }}
- tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
- {{- end }}
- securityContext:
- runAsNonRoot: true
- runAsUser: {{ .Values.scheduler.patch.runAsUser }}
diff --git a/deployments/4pd-vgpu/templates/scheduler/job-patch/job-patchWebhook.yaml b/deployments/4pd-vgpu/templates/scheduler/job-patch/job-patchWebhook.yaml
deleted file mode 100644
index 2700546bf..000000000
--- a/deployments/4pd-vgpu/templates/scheduler/job-patch/job-patchWebhook.yaml
+++ /dev/null
@@ -1,52 +0,0 @@
-apiVersion: batch/v1
-kind: Job
-metadata:
- name: {{ include "4pd-vgpu.fullname" . }}-admission-patch
- annotations:
- "helm.sh/hook": post-install,post-upgrade
- "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
- labels:
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
- app.kubernetes.io/component: admission-webhook
-spec:
- {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }}
- # Alpha feature since k8s 1.12
- ttlSecondsAfterFinished: 0
- {{- end }}
- template:
- metadata:
- name: {{ include "4pd-vgpu.fullname" . }}-admission-patch
- {{- if .Values.scheduler.patch.podAnnotations }}
- annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }}
- {{- end }}
- labels:
- {{- include "4pd-vgpu.labels" . | nindent 8 }}
- app.kubernetes.io/component: admission-webhook
- 4pd.io/webhook: ignore
- spec:
- {{- include "4pd-vgpu.imagePullSecrets" . | nindent 6}}
- {{- if .Values.scheduler.patch.priorityClassName }}
- priorityClassName: {{ .Values.scheduler.patch.priorityClassName }}
- {{- end }}
- containers:
- - name: patch
- image: {{ .Values.scheduler.patch.image }}
- imagePullPolicy: {{ .Values.scheduler.patch.imagePullPolicy }}
- args:
- - patch
- - --webhook-name={{ include "4pd-vgpu.scheduler.webhook" . }}
- - --namespace={{ .Release.Namespace }}
- - --patch-validating=false
- - --secret-name={{ include "4pd-vgpu.scheduler.tls" . }}
- - --patch-failure-policy=Fail
- restartPolicy: OnFailure
- serviceAccountName: {{ include "4pd-vgpu.fullname" . }}-admission
- {{- if .Values.scheduler.patch.nodeSelector }}
- nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }}
- {{- end }}
- {{- if .Values.scheduler.patch.tolerations }}
- tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }}
- {{- end }}
- securityContext:
- runAsNonRoot: true
- runAsUser: {{ .Values.scheduler.patch.runAsUser }}
diff --git a/deployments/4pd-vgpu/templates/scheduler/job-patch/rolebinding.yaml b/deployments/4pd-vgpu/templates/scheduler/job-patch/rolebinding.yaml
deleted file mode 100644
index d60c77dd9..000000000
--- a/deployments/4pd-vgpu/templates/scheduler/job-patch/rolebinding.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: {{ include "4pd-vgpu.fullname" . }}-admission
- annotations:
- "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
- "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
- labels:
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
- app.kubernetes.io/component: admission-webhook
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: Role
- name: {{ include "4pd-vgpu.fullname" . }}-admission
-subjects:
- - kind: ServiceAccount
- name: {{ include "4pd-vgpu.fullname" . }}-admission
- namespace: {{ .Release.Namespace | quote }}
diff --git a/deployments/4pd-vgpu/templates/scheduler/job-patch/serviceaccount.yaml b/deployments/4pd-vgpu/templates/scheduler/job-patch/serviceaccount.yaml
deleted file mode 100644
index 5e4a897ad..000000000
--- a/deployments/4pd-vgpu/templates/scheduler/job-patch/serviceaccount.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: {{ include "4pd-vgpu.fullname" . }}-admission
- annotations:
- "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
- "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
- labels:
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
- app.kubernetes.io/component: admission-webhook
diff --git a/deployments/4pd-vgpu/templates/scheduler/rolebinding.yaml b/deployments/4pd-vgpu/templates/scheduler/rolebinding.yaml
deleted file mode 100644
index 0cc0c7619..000000000
--- a/deployments/4pd-vgpu/templates/scheduler/rolebinding.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- name: {{ include "4pd-vgpu.scheduler" . }}
- labels:
- app.kubernetes.io/component: "4pd-scheduler"
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: cluster-admin
-subjects:
- - kind: ServiceAccount
- name: {{ include "4pd-vgpu.scheduler" . }}
- namespace: {{ .Release.Namespace | quote }}
diff --git a/deployments/4pd-vgpu/templates/scheduler/service.yaml b/deployments/4pd-vgpu/templates/scheduler/service.yaml
deleted file mode 100644
index 989969810..000000000
--- a/deployments/4pd-vgpu/templates/scheduler/service.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
- name: {{ include "4pd-vgpu.scheduler" . }}
- labels:
- app.kubernetes.io/component: 4pd-scheduler
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
- {{- if .Values.scheduler.service.labels }}
- {{ toYaml .Values.scheduler.service.labels | indent 4 }}
- {{- end }}
- {{- if .Values.scheduler.service.annotations }}
- annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }}
- {{- end }}
-spec:
- type: ClusterIP
- ports:
- - name: http
- port: {{ .Values.scheduler.service.httpPort }}
- targetPort: 443
- protocol: TCP
- - name: grpc
- port: {{ .Values.scheduler.service.grpcPort }}
- targetPort: 1080
- protocol: TCP
- selector:
- app.kubernetes.io/component: 4pd-scheduler
- {{- include "4pd-vgpu.selectorLabels" . | nindent 4 }}
-
diff --git a/deployments/4pd-vgpu/templates/scheduler/serviceaccount.yaml b/deployments/4pd-vgpu/templates/scheduler/serviceaccount.yaml
deleted file mode 100644
index 2846e6fab..000000000
--- a/deployments/4pd-vgpu/templates/scheduler/serviceaccount.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: {{ include "4pd-vgpu.scheduler" . }}
- namespace: {{ .Release.Namespace | quote }}
- labels:
- app.kubernetes.io/component: "4pd-scheduler"
- {{- include "4pd-vgpu.labels" . | nindent 4 }}
diff --git a/deployments/4pd-vgpu/templates/scheduler/webhook.yaml b/deployments/4pd-vgpu/templates/scheduler/webhook.yaml
deleted file mode 100644
index d044b737c..000000000
--- a/deployments/4pd-vgpu/templates/scheduler/webhook.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-apiVersion: admissionregistration.k8s.io/v1
-kind: MutatingWebhookConfiguration
-metadata:
- name: {{ include "4pd-vgpu.scheduler.webhook" . }}
-webhooks:
- - admissionReviewVersions:
- - v1
- clientConfig:
- service:
- name: {{ include "4pd-vgpu.scheduler" . }}
- namespace: {{ .Release.Namespace }}
- path: /webhook
- port: {{ .Values.scheduler.service.httpPort }}
- failurePolicy: Fail
- matchPolicy: Equivalent
- name: vgpu.4pd.io
- namespaceSelector:
- matchExpressions:
- - key: 4pd.io/webhook
- operator: NotIn
- values:
- - ignore
- objectSelector:
- matchExpressions:
- - key: 4pd.io/webhook
- operator: NotIn
- values:
- - ignore
- reinvocationPolicy: Never
- rules:
- - apiGroups:
- - ""
- apiVersions:
- - v1
- operations:
- - CREATE
- resources:
- - pods
- scope: '*'
- sideEffects: None
- timeoutSeconds: 10
diff --git a/deployments/4pd-vgpu/values.yaml b/deployments/4pd-vgpu/values.yaml
deleted file mode 100644
index 4b61feffe..000000000
--- a/deployments/4pd-vgpu/values.yaml
+++ /dev/null
@@ -1,64 +0,0 @@
-# Default values for 4pd-vgpu.
-
-nameOverride: ""
-fullnameOverride: ""
-
-imagePullSecrets: []
-#
-resourceName: "nvidia.com/gpu"
-schedulerName: "4pd-scheduler"
-
-podSecurityPolicy:
- enabled: false
-
-global:
- labels: {}
- annotations: {}
-
-scheduler:
- kubeScheduler:
- image: "4pdosc/kube-scheduler:v1.20.9"
- imagePullPolicy: IfNotPresent
- extraArgs:
- - -v=4
- extender:
- image: "m7-ieg-pico-test01:5000/k8s-vgpu:master"
- imagePullPolicy: Always
- extraArgs:
- - --debug
- - -v=4
- podAnnotations: {}
- nodeSelector: {}
- tolerations: []
- #serviceAccountName: "4pd-vgpu-scheduler-sa"
- patch:
- image: docker.io/jettech/kube-webhook-certgen:v1.5.2
- imagePullPolicy: IfNotPresent
- priorityClassName: ""
- podAnnotations: {}
- nodeSelector: {}
- tolerations: []
- runAsUser: 2000
-
- service:
- httpPort: 443
- grpcPort: 1080
- labels: {}
- annotations: {}
-
-devicePlugin:
- image: "m7-ieg-pico-test01:5000/k8s-vgpu:master"
- imagePullPolicy: Always
- extraArgs:
- - --device-split-count=2
- - -v=4
-
- pluginPath: /var/lib/kubelet/device-plugins
- #libPath: /usr/local/vgpu
- #sockPath: /var/lib/vgpu
-
- podAnnotations: {}
- nodeSelector:
- gpu: "on"
- tolerations: []
-
diff --git a/docker/Dockerfile b/docker/Dockerfile
index aaf921882..4cae8a0d1 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,29 +1,41 @@
-ARG GOLANG_IMAGE=golang:1.16.6-buster
-ARG NVIDIA_IMAGE=nvidia/cuda:11.2.1-base-ubuntu20.04
-FROM $GOLANG_IMAGE AS build
+ARG GOLANG_IMAGE=golang:1.24.6-bullseye
+ARG NVIDIA_IMAGE=nvidia/cuda:12.2.0-devel-ubuntu20.04
+FROM $GOLANG_IMAGE AS build
+FROM $GOLANG_IMAGE AS gobuild
+ARG GOPROXY
+ARG VERSION
ADD . /k8s-vgpu
-ENV GOPRIVATE="gitlab.4pd.io/*"
-ARG GOPROXY=https://goproxy.cn,direct
-ARG VERSION="unknown"
-RUN cd /k8s-vgpu && make all
-
-
-FROM $NVIDIA_IMAGE
-
+#RUN --mount=type=cache,target=/go/pkg/mod \
+# cd /k8s-vgpu && make all
+RUN cd /k8s-vgpu && make all VERSION=$VERSION
+RUN go install github.com/NVIDIA/mig-parted/cmd/nvidia-mig-parted@v0.10.0
+
+FROM $NVIDIA_IMAGE AS nvbuild
+COPY ./libvgpu /libvgpu
+WORKDIR /libvgpu
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get -y update; apt-get -y install cmake
+RUN bash ./build.sh
+
+FROM nvidia/cuda:12.6.3-base-ubi8
+RUN dnf update -y -x cuda-toolkit-config-common -x cuda-toolkit-12-config-common
+RUN rm -rf /usr/local/cuda-12.6/compat/libcuda.so*
ENV NVIDIA_DISABLE_REQUIRE="true"
ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=utility
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
-ARG VERSION="unknown"
+ARG VERSION
LABEL version="$VERSION"
-LABEL maintainer="opensource@4paradigm.com"
-
+LABEL maintainer="info@dynamia.ai"
COPY ./LICENSE /k8s-vgpu/LICENSE
-COPY --from=build /k8s-vgpu/bin /k8s-vgpu/bin
+COPY --from=gobuild /k8s-vgpu/bin /k8s-vgpu/bin
+COPY --from=gobuild /go/bin/nvidia-mig-parted /k8s-vgpu/bin/
COPY ./docker/entrypoint.sh /k8s-vgpu/bin/entrypoint.sh
COPY ./lib /k8s-vgpu/lib
+COPY --from=nvbuild /libvgpu/build/libvgpu.so /k8s-vgpu/lib/nvidia/libvgpu.so."$VERSION"
+COPY ./docker/vgpu-init.sh /k8s-vgpu/bin/vgpu-init.sh
ENV PATH="/k8s-vgpu/bin:${PATH}"
-
-ENTRYPOINT ["entrypoint.sh"]
+ARG DEST_DIR
+ENTRYPOINT ["/bin/bash", "-c", "entrypoint.sh $DEST_DIR"]
diff --git a/docker/Dockerfile.hamicore b/docker/Dockerfile.hamicore
new file mode 100644
index 000000000..7dde65ae2
--- /dev/null
+++ b/docker/Dockerfile.hamicore
@@ -0,0 +1,19 @@
+ARG NVIDIA_IMAGE=nvidia/cuda:12.2.0-devel-ubuntu20.04
+
+FROM $NVIDIA_IMAGE AS nvbuild
+COPY ./libvgpu /libvgpu
+WORKDIR /libvgpu
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get -y update; apt-get -y install cmake
+RUN bash ./build.sh
+
+FROM nvidia/cuda:12.6.3-base-ubi8
+RUN rm -rf /usr/local/cuda-12.6/compat/libcuda.so*
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
+
+ARG VERSION
+LABEL version="$VERSION"
+LABEL maintainer="projecthami@dynamia.ai"
+COPY --from=nvbuild /libvgpu/build/libvgpu.so /k8s-vgpu/lib/nvidia/libvgpu.so."$VERSION"
diff --git a/docker/Dockerfile.hamimaster b/docker/Dockerfile.hamimaster
new file mode 100644
index 000000000..4b3e50430
--- /dev/null
+++ b/docker/Dockerfile.hamimaster
@@ -0,0 +1,32 @@
+ARG GOLANG_IMAGE
+ARG HAMICORE_IMAGE
+FROM $GOLANG_IMAGE AS build
+FROM $HAMICORE_IMAGE AS corebuild
+
+FROM $GOLANG_IMAGE AS GOBUILD
+ADD . /k8s-vgpu
+ARG VERSION
+RUN go env -w GO111MODULE=on
+RUN cd /k8s-vgpu && make all VERSION=$VERSION
+RUN go install github.com/NVIDIA/mig-parted/cmd/nvidia-mig-parted@v0.10.0
+
+FROM nvidia/cuda:12.6.3-base-ubuntu22.04
+RUN rm -rf /usr/local/cuda-12.6/compat/libcuda.so*
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
+
+ARG VERSION
+LABEL version="$VERSION"
+LABEL maintainer="opensource@4paradigm.com"
+COPY ./LICENSE /k8s-vgpu/LICENSE
+COPY --from=GOBUILD /k8s-vgpu/bin /k8s-vgpu/bin
+COPY --from=GOBUILD /go/bin/nvidia-mig-parted /k8s-vgpu/bin/
+COPY ./docker/entrypoint.sh /k8s-vgpu/bin/entrypoint.sh
+COPY ./docker/vgpu-init.sh /k8s-vgpu/bin/vgpu-init.sh
+COPY ./lib /k8s-vgpu/lib
+COPY --from=corebuild /k8s-vgpu/lib/nvidia/libvgpu.so."$VERSION" /k8s-vgpu/lib/nvidia/libvgpu.so."$VERSION"
+
+ENV PATH="/k8s-vgpu/bin:${PATH}"
+ARG DEST_DIR
+ENTRYPOINT ["/bin/bash", "-c", "entrypoint.sh $DEST_DIR"]
diff --git a/docker/Dockerfile.withlib b/docker/Dockerfile.withlib
new file mode 100644
index 000000000..19182cf2a
--- /dev/null
+++ b/docker/Dockerfile.withlib
@@ -0,0 +1,34 @@
+ARG GOLANG_IMAGE
+ARG NVIDIA_IMAGE
+FROM $GOLANG_IMAGE AS build
+
+FROM $GOLANG_IMAGE AS GOBUILD
+ADD . /k8s-vgpu
+ARG GOPROXY=https://goproxy.cn,direct
+ARG VERSION
+RUN go env -w GO111MODULE=on
+RUN cd /k8s-vgpu && make all VERSION=$VERSION
+RUN go install github.com/NVIDIA/mig-parted/cmd/nvidia-mig-parted@v0.10.0
+
+FROM nvidia/cuda:12.6.3-base-ubuntu22.04
+RUN rm -rf /usr/local/cuda-12.6/compat/libcuda.so*
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
+
+ARG VERSION
+LABEL version="$VERSION"
+LABEL maintainer="info@dynamia.ai"
+COPY ./LICENSE /k8s-vgpu/LICENSE
+COPY --from=GOBUILD /k8s-vgpu/bin /k8s-vgpu/bin
+COPY --from=GOBUILD /go/bin/nvidia-mig-parted /k8s-vgpu/bin/
+COPY ./docker/entrypoint.sh /k8s-vgpu/bin/entrypoint.sh
+COPY ./docker/vgpu-init.sh /k8s-vgpu/bin/vgpu-init.sh
+COPY ./lib /k8s-vgpu/lib
+COPY ./libvgpu.so /k8s-vgpu/lib/nvidia/libvgpu.so."$VERSION"
+COPY ./license /k8s-vgpu/lib/nvidia/
+COPY ./vgpuvalidator /k8s-vgpu/lib/nvidia
+
+ENV PATH="/k8s-vgpu/bin:${PATH}"
+ARG DEST_DIR
+ENTRYPOINT ["/bin/bash", "-c", "entrypoint.sh $DEST_DIR"]
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 3a7fa08cd..4b9322b11 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -1,6 +1,6 @@
#!/bin/bash
#
-# Copyright © 2021 peizhaoyou
+# Copyright 2024 The HAMi Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
-if [ $1 == "device-plugin" ]; then
- cp -f /k8s-vgpu/lib/* /usr/local/vgpu/
-fi
-exec "$@"
\ No newline at end of file
+# if [ $1 == "device-plugin" ]; then
+# cp -f /k8s-vgpu/lib/* $DEST_DIR/vgpu
+# fi
+exec "$@"
diff --git a/docker/vgpu-init.sh b/docker/vgpu-init.sh
new file mode 100755
index 000000000..972a05df6
--- /dev/null
+++ b/docker/vgpu-init.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+
+# Check if the destination directory is provided as an argument
+if [ -z "$1" ]; then
+ echo "Usage: $0 "
+ exit 1
+fi
+
+# Source directory
+SOURCE_DIR="/k8s-vgpu/lib/nvidia/"
+
+# Destination directory from the argument
+DEST_DIR="$1"
+
+
+# Check if the destination directory exists, create it if it doesn't
+if [ ! -d "$DEST_DIR" ]; then
+ mkdir -p "$DEST_DIR"
+fi
+
+# Traverse all files in the source directory
+find "$SOURCE_DIR" -type f | while read -r source_file; do
+ # Get the relative path of the source file
+ relative_path="${source_file#$SOURCE_DIR}"
+
+ # Construct the destination file path
+ dest_file="$DEST_DIR$relative_path"
+
+ # If the destination file doesn't exist, copy the source file
+ if [ ! -f "$dest_file" ]; then
+ # Create the parent directory of the destination file if it doesn't exist
+ mkdir -p "$(dirname "$dest_file")"
+
+ # Copy the file from source to destination
+ cp "$source_file" "$dest_file"
+ echo "Copied: $source_file -> $dest_file"
+ else
+ # Compare MD5 values of source and destination files
+ source_md5=$(md5sum "$source_file" | cut -d ' ' -f 1)
+ dest_md5=$(md5sum "$dest_file" | cut -d ' ' -f 1)
+
+ # If MD5 values are different, copy the file
+ if [ "$source_md5" != "$dest_md5" ]; then
+ cp "$source_file" "$dest_file"
+ echo "Copied: $source_file -> $dest_file"
+ else
+ echo "Skipped (same MD5): $source_file"
+ fi
+ fi
+done
diff --git a/docs/CHANGELOG/CHANGELOG-0.0.0.md b/docs/CHANGELOG/CHANGELOG-0.0.0.md
new file mode 100644
index 000000000..d48b69147
--- /dev/null
+++ b/docs/CHANGELOG/CHANGELOG-0.0.0.md
@@ -0,0 +1,29 @@
+
+
+**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [v0.0.0](#v000)
+ - [Downloads for v0.0.0](#downloads-for-v000)
+ - [Changelog since v0.0.0](#changelog-since-v000)
+ - [Changes by Kind](#changes-by-kind)
+ - [Bug Fixes](#bug-fixes)
+ - [Others](#others)
+
+
+
+# v0.0.0
+## Downloads for v0.0.0
+
+Download v0.0.0 in the [v0.0.0 release page](https://github.com/Project-HAMi/HAMi/releases/tag/v0.0.0).
+
+## Changelog since v0.0.0
+### Changes by Kind
+#### Bug Fixes
+None.
+
+### Deprecation
+None.
+
+#### Others
+None.
+
diff --git a/docs/ascend910b-support.md b/docs/ascend910b-support.md
new file mode 100644
index 000000000..7f45c319f
--- /dev/null
+++ b/docs/ascend910b-support.md
@@ -0,0 +1,359 @@
+# Introduction to huawei.com/Ascend910A, Ascend910B Series, and Ascend310P Support
+
+HAMi supports virtualization of Huawei Ascend 910A, 910B series devices (910B, 910B2, 910B3, 910B4), and 310P devices, providing several features similar to vGPU, including:
+
+* **_NPU sharing_**: Each task can allocate a portion of Ascend NPU instead of a whole NLU card, thus NPU can be shared among multiple tasks.
+
+* **_Device Memory Control_**: Ascend NPUs can be allocated with certain device memory size and guarantee it that it does not exceed the boundary.
+
+* **_Device Core Control_**: Ascend NPUs can be allocated with certain compute cores and guarantee it that it does not exceed the boundary.
+
+## Prerequisites
+
+* Ascend docker runtime
+* Driver version > 24.1.rc1
+* Ascend device type: 910B, 910B2, 910B3, 910B4, 310P
+
+## Enabling NPU Sharing
+
+* Install the chart using helm, See 'enabling vGPU support in kubernetes' section [here](https://github.com/Project-HAMi/HAMi#enabling-vgpu-support-in-kubernetes)
+
+* Label the Ascend 910B node with the following command:
+
+```bash
+kubectl label node {ascend-node} accelerator=huawei-Ascend910
+```
+
+* Deploy [Ascend docker runtime](https://gitee.com/ascend/ascend-docker-runtime)
+
+* Deploy [ascend-device-plugin](https://github.com/Project-HAMi/ascend-device-plugin)
+
+## Customizing NPU Virtualization Parameters
+
+HAMi includes a built-in [virtualization configuration file](https://github.com/Project-HAMi/HAMi/blob/master/charts/hami/templates/scheduler/device-configmap.yaml) for NPUs.
+
+HAMi also supports customizing virtualization parameters through the following method:
+
+
+ customize ascend config
+
+ ### Create a new directory in hami charts
+
+ The directory structure is as follows:
+
+ ```bash
+ tree -L 1
+ .
+ ├── Chart.yaml
+ ├── files
+ ├── templates
+ └── values.yaml
+ ```
+
+ ### Create device-config.yaml
+
+ The content is as follows:
+
+ ```yaml
+ vnpus:
+ - chipName: 910B
+ commonWord: Ascend910A
+ resourceName: huawei.com/Ascend910A
+ resourceMemoryName: huawei.com/Ascend910A-memory
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 30
+ templates:
+ - name: vir02
+ memory: 2184
+ aiCore: 2
+ - name: vir04
+ memory: 4369
+ aiCore: 4
+ - name: vir08
+ memory: 8738
+ aiCore: 8
+ - name: vir16
+ memory: 17476
+ aiCore: 16
+ - chipName: 910B2
+ commonWord: Ascend910B2
+ resourceName: huawei.com/Ascend910B2
+ resourceMemoryName: huawei.com/Ascend910B2-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 24
+ aiCPU: 6
+ templates:
+ - name: vir03_1c_8g
+ memory: 8192
+ aiCore: 3
+ aiCPU: 1
+ - name: vir06_1c_16g
+ memory: 16384
+ aiCore: 6
+ aiCPU: 1
+ - name: vir12_3c_32g
+ memory: 32768
+ aiCore: 12
+ aiCPU: 3
+ - chipName: 910B3
+ commonWord: Ascend910B
+ resourceName: huawei.com/Ascend910B
+ resourceMemoryName: huawei.com/Ascend910B-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: vir05_1c_16g
+ memory: 16384
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_32g
+ memory: 32768
+ aiCore: 10
+ aiCPU: 3
+ - chipName: 910B4
+ commonWord: Ascend910B4
+ resourceName: huawei.com/Ascend910B4
+ resourceMemoryName: huawei.com/Ascend910B4-memory
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: vir05_1c_8g
+ memory: 8192
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_16g
+ memory: 16384
+ aiCore: 10
+ aiCPU: 3
+ - chipName: 310P3
+ commonWord: Ascend310P
+ resourceName: huawei.com/Ascend310P
+ resourceMemoryName: huawei.com/Ascend310P-memory
+ memoryAllocatable: 21527
+ memoryCapacity: 24576
+ aiCore: 8
+ aiCPU: 7
+ templates:
+ - name: vir01
+ memory: 3072
+ aiCore: 1
+ aiCPU: 1
+ - name: vir02
+ memory: 6144
+ aiCore: 2
+ aiCPU: 2
+ - name: vir04
+ memory: 12288
+ aiCore: 4
+ aiCPU: 4
+ ```
+
+ ### Install and update with Helm
+
+ Helm installation and updates will be based on the configuration in this file, overwriting the built-in configuration of Helm.
+
+
+
+## Virtualization Template Overview
+
+HAMi supports configuring NPU resource allocation through predefined device templates. Each template includes the following:
+
+- Template name (name): Unique identifier for the template.
+- Memory size (memory): Device memory allocated to the template (in MB).
+- AI core count (aiCore): Number of AI cores allocated to the template.
+- AI CPU core count (aiCPU): Number of AI CPU cores allocated to the template (supported by some models).
+
+When a user requests a specific memory size, the system automatically aligns the requested memory to the nearest template size. For example, if a user requests 2000MB of memory, the system will select the smallest template with memory size greater than or equal to 2000MB.
+
+For specific configurations, refer to the [official Ascend virtualization templates](https://www.hiascend.com/document/detail/zh/computepoweralloca/300/cpaug/cpaug/cpaug_00005.html).
+
+## Device Granularity Partitioning
+
+Refer to the aiCore ratio in each type configuration (chipName) and the aiCore under the template.
+
+### Ascend910 Series Device Granularity Partitioning
+
+- Ascend910A devices support 4 granularity partitions: 1/15, 2/15, 4/15, and 8/15 of a card. Allocated memory automatically aligns to the nearest granularity above the requested amount.
+- Ascend910B2 devices support 3 granularity partitions: 1/8, 1/4, and 1/2 of a card. Allocated memory automatically aligns to the nearest granularity above the requested amount.
+- Ascend910B3 and Ascend910B4 devices support 2 granularity partitions: 1/4 and 1/2 of a card. Allocated memory automatically aligns to the nearest granularity above the requested amount.
+
+### Ascend310P Device Granularity Partitioning
+
+Ascend310P devices (Atlas inference series products) support multiple granularity partitions, including 1/8, 1/4, and 1/2 of a card. Allocated memory automatically aligns to the nearest granularity above the requested amount.
+
+## Running NPU Workloads
+
+You can request Ascend 910B resources using the `huawei.com/ascend910B` and `huawei.com/ascend910B-memory` resource types:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910A-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910A: 1 # requesting 1 vGPUs
+ huawei.com/Ascend910A-memory: 2000 # requesting 2000m device memory
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910B2-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910B2: 1 # requesting 1 vGPUs
+ huawei.com/Ascend910B2-memory: 2000 # requesting 2000m device memory
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910B-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910B: 1 # requesting 1 vGPUs
+ huawei.com/Ascend910B-memory: 2000 # requesting 2000m device memory
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910B4-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910B4: 1 # requesting 1 vGPUs
+ huawei.com/Ascend910B4-memory: 2000 # requesting 2000m device memory
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend310P-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend310P: 1 # requesting 1 vGPUs
+ huawei.com/Ascend310P-memory: 2000 # requesting 2000m device memory
+```
+
+## Device Health Monitoring
+
+HAMi supports health monitoring for Ascend NPU devices, ensuring only healthy devices are allocated to Pods. Health monitoring includes:
+
+- Device status verification
+- Device resource availability verification
+- Device driver status verification
+
+## Resource Usage Statistics
+
+HAMi supports statistics collection for Ascend NPU device resource usage, including:
+
+- Device memory usage
+- AI core usage
+- AI CPU core usage
+- Device utilization
+
+These statistics can be used for resource scheduling decisions and performance optimization.
+
+## Node Locking Mechanism
+
+HAMi implements a node locking mechanism to prevent resource allocation conflicts. When a Pod requests Ascend NPU resources, the system locks the corresponding node to prevent other Pods from using the same device resources simultaneously.
+
+
+## Device UUID Selection
+
+You can specify which Ascend devices to use or exclude using Pod annotations:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend-pod
+ annotations:
+ # Use specific Ascend devices (comma-separated list)
+ hami.io/use-Ascend910B-uuid: "device-uuid-1,device-uuid-2"
+ # Or exclude specific Ascend devices (comma-separated list)
+ hami.io/no-use-Ascend910B-uuid: "device-uuid-3,device-uuid-4"
+spec:
+ # ... rest of pod spec
+```
+
+> **NOTE:** The device UUID format depends on the device type, such as `Ascend910B`, `Ascend910B2`, `Ascend910B3`, `Ascend910B4`, etc. You can find the available device UUIDs in the node status.
+
+### Usage Example
+
+Here is a complete example demonstrating how to use the UUID selection feature:
+
+
+ Custom Configuration
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend-pod
+ annotations:
+ hami.io/use-Ascend910B-uuid: "device-uuid-1,device-uuid-2"
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910B: 1
+ huawei.com/Ascend910B-memory: 2000
+```
+
+In this example, the Pod will only run on Ascend910B devices with UUIDs `device-uuid-1` or `device-uuid-2`.
+
+#### Finding Device UUIDs
+
+You can find the Ascend device UUIDs on a node using the following command:
+
+```bash
+kubectl describe node | grep -A 10 "Allocated resources"
+```
+
+Or by viewing the node annotations:
+
+```bash
+kubectl get node -o yaml | grep -A 10 "annotations:"
+```
+
+In the node annotations, look for `hami.io/node-register-Ascend910B` or similar annotations, which contain the device UUID information.
+
+
+
+## Notes
+
+- NPU sharing is not supported in init containers.
+- `huawei.com/Ascend910B-memory` is only effective when `huawei.com/Ascend910B=1`.
+- Multi-device requests (`huawei.com/Ascend910B > 1`) do not support vNPU mode.
\ No newline at end of file
diff --git a/docs/ascend910b-support_cn.md b/docs/ascend910b-support_cn.md
new file mode 100644
index 000000000..aa8ab0ed6
--- /dev/null
+++ b/docs/ascend910b-support_cn.md
@@ -0,0 +1,361 @@
+# huawei.com/Ascend910A、Ascend910B系列 和 Ascend310P 支持简介
+
+HAMi 支持复用华为昇腾 910A、910B 系列设备（910B、910B2、910B3、910B4）和 310P 设备，并为此提供以下几种与 vGPU 类似的复用功能，包括：
+
+* **_NPU 共享_**: 每个任务可以只占用一部分显卡，多个任务可以共享一张显卡；
+
+* **_可限制分配的显存大小_**: 你可以用显存值（例如 3000M）来分配 NPU，本组件会确保任务使用的显存不会超过分配数值；
+
+* **_可限制分配的算力大小_**: 你可以用固定数量来分配 NPU 的 AI 核心和 AI CPU 核心，本组件会确保任务使用的算力不会超过分配数值。
+
+## 节点需求
+
+* Ascend docker runtime
+* driver version > 24.1.rc1
+* Ascend device type: 910B,910B2,910B3,910B4,310P
+
+## 开启 NPU 复用
+
+* 通过 helm 部署本组件, 参照[主文档中的开启 vGPU 支持章节](https://github.com/Project-HAMi/HAMi/blob/master/README_cn.md#kubernetes开启vgpu支持)。
+
+* 使用以下指令，为 Ascend 910B 所在节点打上 label：
+
+```bash
+kubectl label node {ascend-node} accelerator=huawei-Ascend910
+```
+
+* 部署[Ascend docker runtime](https://gitee.com/ascend/ascend-docker-runtime)
+
+* 部署 [ascend-device-plugin](https://github.com/Project-HAMi/ascend-device-plugin)
+
+## 自定义 NPU 虚拟化参数
+
+HAMi 目前有一个 NPU 内置[虚拟化配置文件](https://github.com/Project-HAMi/HAMi/blob/master/charts/hami/templates/scheduler/device-configmap.yaml).
+
+当然 HAMi 也支持通过以下方式自定义虚拟化参数:
+
+
+ 自定义配置
+
+ ### 在 HAMi charts 创建 files 的目录
+
+ 创建后的目录架构应为如下所示：
+
+ ```bash
+ tree -L 1
+ .
+ ├── Chart.yaml
+ ├── files
+ ├── templates
+ └── values.yaml
+ ```
+
+ ### 在 files 目录下创建 device-config.yaml
+
+ 配置文件如下所示，可以按需调整：
+
+ ```yaml
+ vnpus:
+ - chipName: 910B
+ commonWord: Ascend910A
+ resourceName: huawei.com/Ascend910A
+ resourceMemoryName: huawei.com/Ascend910A-memory
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 30
+ templates:
+ - name: vir02
+ memory: 2184
+ aiCore: 2
+ - name: vir04
+ memory: 4369
+ aiCore: 4
+ - name: vir08
+ memory: 8738
+ aiCore: 8
+ - name: vir16
+ memory: 17476
+ aiCore: 16
+ - chipName: 910B2
+ commonWord: Ascend910B2
+ resourceName: huawei.com/Ascend910B2
+ resourceMemoryName: huawei.com/Ascend910B2-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 24
+ aiCPU: 6
+ templates:
+ - name: vir03_1c_8g
+ memory: 8192
+ aiCore: 3
+ aiCPU: 1
+ - name: vir06_1c_16g
+ memory: 16384
+ aiCore: 6
+ aiCPU: 1
+ - name: vir12_3c_32g
+ memory: 32768
+ aiCore: 12
+ aiCPU: 3
+ - chipName: 910B3
+ commonWord: Ascend910B
+ resourceName: huawei.com/Ascend910B
+ resourceMemoryName: huawei.com/Ascend910B-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: vir05_1c_16g
+ memory: 16384
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_32g
+ memory: 32768
+ aiCore: 10
+ aiCPU: 3
+ - chipName: 910B4
+ commonWord: Ascend910B4
+ resourceName: huawei.com/Ascend910B4
+ resourceMemoryName: huawei.com/Ascend910B4-memory
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: vir05_1c_8g
+ memory: 8192
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_16g
+ memory: 16384
+ aiCore: 10
+ aiCPU: 3
+ - chipName: 310P3
+ commonWord: Ascend310P
+ resourceName: huawei.com/Ascend310P
+ resourceMemoryName: huawei.com/Ascend310P-memory
+ memoryAllocatable: 21527
+ memoryCapacity: 24576
+ aiCore: 8
+ aiCPU: 7
+ templates:
+ - name: vir01
+ memory: 3072
+ aiCore: 1
+ aiCPU: 1
+ - name: vir02
+ memory: 6144
+ aiCore: 2
+ aiCPU: 2
+ - name: vir04
+ memory: 12288
+ aiCore: 4
+ aiCPU: 4
+ ```
+
+ ### Helm 安装和更新
+
+ Helm 安装、更新将基于该配置文件，覆盖默认的配置文件
+
+
+
+## 虚拟化模板说明
+
+HAMi 支持通过预定义的设备模板来配置 NPU 资源分配。每个模板包含以下内容：
+
+- 模板名称（name）：模板的唯一标识符；
+- 内存大小（memory）：分配给该模板的设备内存大小（单位：MB）；
+- AI 核心数量（aiCore）：分配给该模板的 AI 核心数量；
+- AI CPU 核心数量（aiCPU）：分配给该模板的 AI CPU 核心数量（部分型号支持）。
+
+当用户请求特定大小的内存时，系统会自动将请求的内存大小对齐到最接近的模板大小。例如，如果用户请求 2000MB 内存，系统会选择内存大小大于或等于 2000MB 的最小模板。
+
+具体配置，请参考：[昇腾官方的虚拟化模板](https://www.hiascend.com/document/detail/zh/computepoweralloca/300/cpaug/cpaug/cpaug_00005.html)
+
+## 设备粒度切分
+
+具体参考每个类型配置（chipName）中的aiCore 和 template 下的 aiCore 配比。
+
+### Ascend910 系列设备粒度切分
+
+- Ascend910A 设备支持 4 种粒度的切分，分别是 1/15、2/15 卡、 4/15 卡和 8/15 卡，分配的显存会自动对齐到在分配额之上最近的粒度上。
+- Ascend910B2 设备支持 3 种粒度的切分，分别是 1/8、1/4 卡和 1/2 卡，分配的显存会自动对齐到在分配额之上最近的粒度上。
+- Ascend910B3 和 Ascend910B4 设备支持 2 种粒度的切分，分别是 1/4 卡和 1/2 卡，分配的显存会自动对齐到在分配额之上最近的粒度上。
+
+### Ascend310P 设备粒度切分
+
+Ascend310P 设备（Atlas 推理系列产品）支持多种粒度的切分，包括 1/8 卡、1/4 卡和 1/2 卡，分配的显存会自动对齐到在分配额之上最近的粒度上。
+
+## 运行 NPU 任务
+
+可通过使用 `huawei.com/ascend910B` 和 `huawei.com/ascend910B-memory` 资源类型，来请求 Ascend 910B：
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910A-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910A: 1 # requesting 1 vGPUs
+ huawei.com/Ascend910A-memory: 2000 # requesting 2000m device memory
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910B2-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910B2: 1 # requesting 1 vGPUs
+ huawei.com/Ascend910B2-memory: 2000 # requesting 2000m device memory
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910B-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910B: 1 # requesting 1 vGPUs
+ huawei.com/Ascend910B-memory: 2000 # requesting 2000m device memory
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910B4-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910B4: 1 # requesting 1 vGPUs
+ huawei.com/Ascend910B4-memory: 2000 # requesting 2000m device memory
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend310P-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend310P: 1 # requesting 1 vGPUs
+ huawei.com/Ascend310P-memory: 2000 # requesting 2000m device memory
+```
+
+## 设备健康检查
+
+HAMi 支持对 Ascend NPU 设备进行健康检查，确保只有健康的设备被分配给 Pod。健康检查包括以下内容：
+
+- 设备状态检查；
+- 设备资源可用性检查；
+- 设备驱动状态检查。
+
+## 资源使用统计
+
+HAMi 支持对 Ascend NPU 设备的资源使用情况进行统计，包括：
+
+- 设备内存使用情况；
+- AI 核心使用情况；
+- AI CPU 核心使用情况；
+- 设备利用率。
+
+这些统计信息可以用于资源调度决策和性能优化。
+
+## 节点锁定机制
+
+HAMi 实现了节点锁定机制，确保在分配设备资源时不会发生冲突。当 Pod 请求 Ascend NPU 资源时，系统会锁定相应的节点，防止其他 Pod 同时使用相同的设备资源。
+
+
+## 设备 UUID 选择
+
+你可以通过 Pod 注解来指定要使用或排除特定的 Ascend 设备：
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend-pod
+ annotations:
+ # 使用特定的 Ascend 设备（逗号分隔的列表）
+ hami.io/use-Ascend910B-uuid: "device-uuid-1,device-uuid-2"
+ # 或者排除特定的 Ascend 设备（逗号分隔的列表）
+ hami.io/no-use-Ascend910B-uuid: "device-uuid-3,device-uuid-4"
+spec:
+ # ... 其余 Pod 配置
+```
+
+> **注意**：设备 UUID 格式取决于设备类型，例如 `Ascend910B`、`Ascend910B2`、`Ascend910B3`、`Ascend910B4` 等。你可以在节点状态中找到可用的设备 UUID。
+
+
+
+### 使用示例
+
+以下是一个完整的示例，展示如何使用 UUID 选择功能：
+
+
+ 自定义配置
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend-pod
+ annotations:
+ hami.io/use-Ascend910B-uuid: "device-uuid-1,device-uuid-2"
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910B: 1
+ huawei.com/Ascend910B-memory: 2000
+```
+
+在这个示例中，Pod 将只在 UUID 为 `device-uuid-1` 或 `device-uuid-2` 的 Ascend910B 设备上运行。
+
+#### 查找设备 UUID
+
+你可以通过以下命令查找节点上的 Ascend 设备 UUID：
+
+```bash
+kubectl describe node | grep -A 10 "Allocated resources"
+```
+
+或者使用以下命令查看节点的注解：
+
+```bash
+kubectl get node -o yaml | grep -A 10 "annotations:"
+```
+
+在节点注解中，查找 `hami.io/node-register-Ascend910B` 或类似的注解，其中包含设备 UUID 信息。
+
+
+
+## 注意事项
+
+- 在 init container 中无法使用 NPU 复用功能；
+- `huawei.com/Ascend910B-memory` 仅在 `huawei.com/Ascend910B=1` 时有效；
+- 多设备请求（`huawei.com/Ascend910B > 1`）不支持 vNPU 模式。
\ No newline at end of file
diff --git a/docs/benchmark.md b/docs/benchmark.md
new file mode 100644
index 000000000..a288e0eb5
--- /dev/null
+++ b/docs/benchmark.md
@@ -0,0 +1,49 @@
+## Benchmarks
+
+Three instances from ai-benchmark have been used to evaluate vGPU-device-plugin performance as follows:
+
+| Test Environment | description |
+| ---------------- | :------------------------------------------------------: |
+| Kubernetes version | v1.12.9 |
+| Docker version | 18.09.1 |
+| GPU Type | Tesla V100 |
+| GPU Num | 2 |
+
+| Test instance | description |
+| ------------- | :---------------------------------------------------------: |
+| nvidia-device-plugin | k8s + nvidia k8s-device-plugin |
+| vGPU-device-plugin | k8s + VGPU k8s-device-plugin，without virtual device memory |
+| vGPU-device-plugin(virtual device memory) | k8s + VGPU k8s-device-plugin，with virtual device memory |
+
+Test Cases:
+
+| test id | case | type | params |
+| ------- | :-----------: | :-------: | :---------------------: |
+| 1.1 | Resnet-V2-50 | inference | batch=50,size=346*346 |
+| 1.2 | Resnet-V2-50 | training | batch=20,size=346*346 |
+| 2.1 | Resnet-V2-152 | inference | batch=10,size=256*256 |
+| 2.2 | Resnet-V2-152 | training | batch=10,size=256*256 |
+| 3.1 | VGG-16 | inference | batch=20,size=224*224 |
+| 3.2 | VGG-16 | training | batch=2,size=224*224 |
+| 4.1 | DeepLab | inference | batch=2,size=512*512 |
+| 4.2 | DeepLab | training | batch=1,size=384*384 |
+| 5.1 | LSTM | inference | batch=100,size=1024*300 |
+| 5.2 | LSTM | training | batch=10,size=1024*300 |
+
+Test Result: 
+
+
+
+To reproduce:
+
+1. install k8s-vGPU-scheduler, and configure properly
+2. run benchmark job
+
+```
+$ kubectl apply -f benchmarks/ai-benchmark/ai-benchmark.yml
+```
+
+3. View the result by using kubctl logs
+
+```
+$ kubectl logs [pod id]
\ No newline at end of file
diff --git a/docs/benchmark_cn.md b/docs/benchmark_cn.md
new file mode 100644
index 000000000..c1f5f1fa8
--- /dev/null
+++ b/docs/benchmark_cn.md
@@ -0,0 +1,50 @@
+## 性能测试
+
+在测试报告中，我们一共在下面五种场景都执行了ai-benchmark 测试脚本，并汇总最终结果：
+
+| 测试环境 | 环境描述 |
+| ---------------- | :------------------------------------------------------: |
+| Kubernetes version | v1.12.9 |
+| Docker version | 18.09.1 |
+| GPU Type | Tesla V100 |
+| GPU Num | 2 |
+
+| 测试名称 | 测试用例 |
+| -------- | :------------------------------------------------: |
+| Nvidia-device-plugin | k8s + nvidia官方k8s-device-plugin |
+| vGPU-device-plugin | k8s + VGPU k8s-device-plugin，无虚拟显存 |
+| vGPU-device-plugin(virtual device memory) | k8s + VGPU k8s-device-plugin，高负载，开启虚拟显存 |
+
+测试内容
+
+| test id | 名称 | 类型 | 参数 |
+| ------- | :-----------: | :-------: | :---------------------: |
+| 1.1 | Resnet-V2-50 | inference | batch=50,size=346*346 |
+| 1.2 | Resnet-V2-50 | training | batch=20,size=346*346 |
+| 2.1 | Resnet-V2-152 | inference | batch=10,size=256*256 |
+| 2.2 | Resnet-V2-152 | training | batch=10,size=256*256 |
+| 3.1 | VGG-16 | inference | batch=20,size=224*224 |
+| 3.2 | VGG-16 | training | batch=2,size=224*224 |
+| 4.1 | DeepLab | inference | batch=2,size=512*512 |
+| 4.2 | DeepLab | training | batch=1,size=384*384 |
+| 5.1 | LSTM | inference | batch=100,size=1024*300 |
+| 5.2 | LSTM | training | batch=10,size=1024*300 |
+
+测试结果： 
+
+
+
+测试步骤：
+
+1. 安装nvidia-device-plugin，并配置相应的参数
+2. 运行benchmark任务
+
+```
+$ kubectl apply -f benchmarks/ai-benchmark/ai-benchmark.yml
+```
+
+3. 通过kubctl logs 查看结果
+
+```
+$ kubectl logs [pod id]
+```
\ No newline at end of file
diff --git a/docs/cambricon-mlu-support.md b/docs/cambricon-mlu-support.md
new file mode 100644
index 000000000..2f0ea2185
--- /dev/null
+++ b/docs/cambricon-mlu-support.md
@@ -0,0 +1,81 @@
+## Introduction
+
+**We now support cambricon.com/mlu by implementing most device-sharing features as nvidia-GPU**, including:
+
+***MLU sharing***: Each task can allocate a portion of MLU instead of a whole MLU card, thus MLU can be shared among multiple tasks.
+
+***Device Memory Control***: MLUs can be allocated with certain device memory size and guarantee it that it does not exceed the boundary.
+
+***Device Core Control***: MLUs can be allocated with certain compute cores and guarantee it that it does not exceed the boundary.
+
+***MLU Type Specification***: You can specify which type of MLU to use or to avoid for a certain task, by setting "cambricon.com/use-mlutype" or "cambricon.com/nouse-mlutype" annotations.
+
+
+## Prerequisites
+
+* neuware-mlu370-driver > 5.10
+* cntoolkit > 2.5.3
+
+## Enabling MLU-sharing Support
+
+* Install the chart using helm, See 'enabling vGPU support in kubernetes' section [here](https://github.com/Project-HAMi/HAMi#enabling-vgpu-support-in-kubernetes)
+
+* Activate the smlu mode for each MLUs on that node
+
+```sh
+cnmon set -c 0 -smlu on
+cnmon set -c 1 -smlu on
+# ...
+```
+
+* Get cambricon-device-plugin from your device provider and specify the following parameters during deployment:
+
+`mode=dynamic-smlu`, `min-dsmlu-unit=256`
+
+These two parameters represent enabling the dynamic smlu function and setting the minimum allocable memory unit to 256 MB, respectively. You can refer to the document from device provider for more details
+
+* Deploy the cambricon-device-plugin you just specified
+
+```
+kubectl apply -f cambricon-device-plugin-daemonset.yaml
+```
+
+## Running MLU jobs
+
+Cambricon MLUs can now be requested by a container
+using the `cambricon.com/vmlu` ,`cambricon.com/mlu.smlu.vmemory` and `cambricon.com/mlu.smlu.vcore` resource type:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: binpack-1
+ labels:
+ app: binpack-1
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: binpack-1
+ template:
+ metadata:
+ labels:
+ app: binpack-1
+ spec:
+ containers:
+ - name: c-1
+ image: ubuntu:18.04
+ command: ["sleep"]
+ args: ["100000"]
+ resources:
+ limits:
+ cambricon.com/vmlu: "1"
+ cambricon.com/mlu.smlu.vmemory: "20"
+ cambricon.com/mlu.smlu.vcore: "10"
+```
+
+## Notes
+
+1. Mlu-sharing in init container is not supported, pods with "combricon.com/mlumem" in init container will never be scheduled.
+
+2. `cambricon.com/mlu.smlu.vmemory`, `cambricon.com/mlu.smlu.vcore` only work when `cambricon.com/vmlu=1`, otherwise, whole MLUs are allocated when `cambricon.com/vmlu>1` regardless of `cambricon.com/mlu.smlu.vmemory` and `cambricon.com/mlu.smlu.vcore`.
\ No newline at end of file
diff --git a/docs/cambricon-mlu-support_cn.md b/docs/cambricon-mlu-support_cn.md
new file mode 100644
index 000000000..d7ce59485
--- /dev/null
+++ b/docs/cambricon-mlu-support_cn.md
@@ -0,0 +1,75 @@
+## 简介
+
+本组件支持复用寒武纪MLU设备，并为此提供以下几种与vGPU类似的复用功能，包括：
+
+***MLU 共享***: 每个任务可以只占用一部分显卡，多个任务可以共享一张显卡
+
+***可限制分配的显存大小***: 你现在可以用显存值（例如3000M）来分配MLU，本组件会确保任务使用的显存不会超过分配数值
+
+***可限制分配的算力大小***: 你现在可以用百分比来分配MLU的算力，本组件会确保任务使用的算力不会超过分配数值
+
+***指定MLU型号***：当前任务可以通过设置annotation("cambricon.com/use-mlutype","cambricon.com/nouse-mlutype")的方式，来选择使用或者不使用某些具体型号的MLU
+
+## 节点需求
+
+* neuware-mlu370-driver > 5.10
+* cntoolkit > 2.5.3
+
+## 开启MLU复用
+
+* 通过helm部署本组件, 参照[主文档中的开启vgpu支持章节](https://github.com/Project-HAMi/HAMi/blob/master/README_cn.md#kubernetes开启vgpu支持)
+
+* 使用以下指令，为MLU节点打上label
+```
+kubectl label node {mlu-node} mlu=on
+```
+
+* 从您的设备提供商处获取cambricon-device-plugin，并配置以下两个参数：
+
+`mode=dynamic-smlu`, `min-dsmlu-unit=256`
+
+它们分别代表开启MLU复用功能，与设置最小可分配的内存单元为256M，您可以参考设备提供方的文档来获取更多的配置信息。
+
+* 部署配置后的`cambricon-device-plugin`
+
+```
+kubectl apply -f cambricon-device-plugin-daemonset.yaml
+```
+
+
+## 运行MLU任务
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: binpack-1
+ labels:
+ app: binpack-1
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: binpack-1
+ template:
+ metadata:
+ labels:
+ app: binpack-1
+ spec:
+ containers:
+ - name: c-1
+ image: ubuntu:18.04
+ command: ["sleep"]
+ args: ["100000"]
+ resources:
+ limits:
+ cambricon.com/vmlu: "1"
+ cambricon.com/mlu.smlu.vmemory: "20"
+ cambricon.com/mlu.smlu.vcore: "10"
+```
+
+## 注意事项
+
+1. 在init container中无法使用MLU复用功能，否则该任务不会被调度
+
+2. 只有申请单MLU的任务可以指定显存`mlu.smlu.vmemory`和算力`mlu.smlu.vcore`的数值，若申请的MLU数量大于1，则所有申请的MLU都会被整卡分配
diff --git a/docs/config.md b/docs/config.md
new file mode 100644
index 000000000..e9033f23a
--- /dev/null
+++ b/docs/config.md
@@ -0,0 +1,163 @@
+# Global Config
+
+## Device Configs: ConfigMap
+
+**Note:**
+All the configurations listed below are managed within the hami-scheduler-device ConfigMap.
+You can update these configurations using one of the following methods:
+
+1. Directly edit the ConfigMap: If HAMi has already been successfully installed, you can manually update the hami-scheduler-device ConfigMap using the `kubectl edit` command to manually update the hami-scheduler-device ConfigMap.
+
+ ```bash
+ kubectl edit configmap hami-scheduler-device -n
+ ```
+
+ After making changes, restart the related HAMi components to apply the updated configurations.
+
+2. Modify Helm Chart: Update the corresponding values in the [ConfigMap](../charts/hami/templates/scheduler/device-configmap.yaml), then reapply the Helm Chart to regenerate the ConfigMap.
+
+* `nvidia.deviceMemoryScaling`:
+ Float type, by default: 1. The ratio for NVIDIA device memory scaling, can be greater than 1 (enable virtual device memory, experimental feature). For NVIDIA GPU with *M* memory, if we set `nvidia.deviceMemoryScaling` argument to *S*, vGPUs splitted by this GPU will totally get `S * M` memory in Kubernetes with our device plugin.
+* `nvidia.deviceSplitCount`:
+ Integer type, by default: equals 10. Maximum tasks assigned to a simple GPU device.
+* `nvidia.migstrategy`:
+ String type, "none" for ignoring MIG features or "mixed" for allocating MIG device by seperate resources. Default "none"
+* `nvidia.disablecorelimit`:
+ String type, "true" for disable core limit, "false" for enable core limit, default: false
+* `nvidia.defaultMem`:
+ Integer type, by default: 0. The default device memory of the current task, in MB.'0' means use 100% device memory
+* `nvidia.defaultCores`:
+ Integer type, by default: equals 0. Percentage of GPU cores reserved for the current task. If assigned to 0, it may fit in any GPU with enough device memory. If assigned to 100, it will use an entire GPU card exclusively.
+* `nvidia.defaultGPUNum`:
+ Integer type, by default: equals 1, if configuration value is 0, then the configuration value will not take effect and will be filtered. when a user does not set nvidia.com/gpu this key in pod resource, webhook should check nvidia.com/gpumem、resource-mem-percentage、nvidia.com/gpucores this three key, anyone a key having value, webhook should add nvidia.com/gpu key and this default value to resources limits map.
+* `nvidia.resourceCountName`:
+ String type, vgpu number resource name, default: "nvidia.com/gpu"
+* `nvidia.resourceMemoryName`:
+ String type, vgpu memory size resource name, default: "nvidia.com/gpumem"
+* `nvidia.resourceMemoryPercentageName`:
+ String type, vgpu memory fraction resource name, default: "nvidia.com/gpumem-percentage"
+* `nvidia.resourceCoreName`:
+ String type, vgpu cores resource name, default: "nvidia.com/gpucores"
+* `nvidia.resourcePriorityName`:
+ String type, vgpu task priority name, default: "nvidia.com/priority"
+
+## Node Configs: ConfigMap
+HAMi allows configuring per-node behavior for device plugin. Edit
+```sh
+kubectl -n kube-system edit cm hami-device-plugin
+```
+* `name`: Name of the node.
+* `operatingmode`: Operating mode of the node, can be "hami-core" or "mig", default: "hami-core".
+* `devicememoryscaling`: Overcommit ratio of device memory.
+* `devicecorescaling`: Overcommit ratio of device core.
+* `devicesplitcount`: Allowed number of tasks sharing a device.
+* `filterdevices`: Devices that are not registered to HAMi.
+ * `uuid`: UUIDs of devices to ignore
+ * `index`: Indexes of devices to ignore.
+ * A device is ignored by HAMi if it's in `uuid` or `index` list.
+
+## Chart Configs: parameters
+
+you can customize your vGPU support by setting the following parameters using `-set`, for example
+
+```bash
+helm install hami hami-charts/hami --set devicePlugin.deviceMemoryScaling=5 ...
+```
+
+* `devicePlugin.service.schedulerPort`:
+ Integer type, by default: 31998, scheduler webhook service nodePort.
+* `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy`: String type, default value is "binpack", representing the GPU node scheduling policy. "binpack" means trying to allocate tasks to the same GPU node as much as possible, while "spread" means trying to allocate tasks to different GPU nodes as much as possible.
+* `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy`: String type, default value is "spread", representing the GPU scheduling policy. "binpack" means trying to allocate tasks to the same GPU as much as possible, while "spread" means trying to allocate tasks to different GPUs as much as possible.
+
+**Webhook TLS Certificate Configs**
+
+In Kubernetes, in order for the API server to communicate with the webhook component, the webhook requires a TLS certificate that the API server is configured to trust. HAMi scheduler provides two methods to generate/configure the required TLS certificate.
+
+* `scheduler.patch.enabled`:
+ Boolean type, default value is true, if true, helm will use kube-webhook-certgen ([job-patch](../charts/hami/templates/scheduler/job-patch/job-createSecret.yaml)) to generate a self-signed certificate and create a secret.
+* `scheduler.certManager.enabled`:
+ Boolean type, default value is false, if true, cert-manager will generate a self-signed certificate. **Note: This option requires cert-manager to be installed in your cluster first.** _See [cert-manager installation](https://cert-manager.io/docs/installation/kubernetes/) for more details._
+
+## Pod configs: annotations
+
+* `nvidia.com/use-gpuuuid`:
+
+ String type, ie: "GPU-AAA,GPU-BBB"
+
+ If set, devices allocated by this pod must be one of UUIDs defined in this string.
+
+* `nvidia.com/nouse-gpuuuid`:
+
+ String type, ie: "GPU-AAA,GPU-BBB"
+
+ If set, devices allocated by this pod will NOT in UUIDs defined in this string.
+
+* `nvidia.com/use-gpuindexes`:
+
+ String type, ie: "0,1"
+
+ If set, devices allocated by this pod must be one of indexes defined in this string.
+
+* `nvidia.com/nouse-gpuindexes`:
+
+ String type, ie: "0,1"
+
+ If set, devices allocated by this pod will NOT in indexes defined in this string.
+
+* `nvidia.com/nouse-gputype`:
+
+ String type, ie: "Tesla V100-PCIE-32GB, NVIDIA A10"
+
+ If set, devices allocated by this pod will NOT in types defined in this string.
+
+* `nvidia.com/use-gputype`:
+
+ String type, ie: "Tesla V100-PCIE-32GB, NVIDIA A10"
+
+ If set, devices allocated by this pod MUST be one of types defined in this string.
+
+* `hami.io/node-scheduler-policy`:
+
+ String type, "binpack" or "spread"
+
+ - binpack: the scheduler will try to allocate the pod to used GPU nodes for execution.
+ - spread: the scheduler will try to allocate the pod to different GPU nodes for execution.
+
+* `hami.io/gpu-scheduler-policy`:
+
+ String type, "binpack" or "spread"
+
+ - binpack: the scheduler will try to allocate the pod to the same GPU card for execution.
+ - spread:the scheduler will try to allocate the pod to different GPU card for execution.
+
+* `nvidia.com/vgpu-mode`:
+
+ String type, "hami-core" or "mig"
+
+ Which type of vgpu instance this pod wish to use
+
+## Container configs: env
+
+* `GPU_CORE_UTILIZATION_POLICY`:
+> Currently this parameter can be specified during `helm install` and then automatically injected into the container environment variables, through `--set devices.nvidia.gpuCorePolicy=force`
+
+ String type, "default", "force", "disable"
+
+ - default: "default"
+ - "default" means the dafault utilization policy
+ - "force" means the container will always limit the core utilization below "nvidia.com/gpucores"
+ - "disable" means the container will ignore the utilization limitation set by "nvidia.com/gpucores" during task execution
+
+* `CUDA_DISABLE_CONTROL`:
+
+ Bool type, "true", "false"
+
+ - default: false
+ - "true" means the HAMi-core will not be used inside container, as a result, there will be no resource isolation and limitation in that container, only for debug.
+
+* `CUDA_DISABLE_CONTROL`:
+
+ Bool type, "true", "false"
+
+ - default: false
+ - "true" means the HAMi-core will not be used inside container, as a result, there will be no resource isolation and limitation in that container, only for debug.
diff --git a/docs/config_cn.md b/docs/config_cn.md
new file mode 100644
index 000000000..97d76455d
--- /dev/null
+++ b/docs/config_cn.md
@@ -0,0 +1,159 @@
+# 全局配置
+
+## 设备配置
+
+**注意:**
+以下所有配置均在名称为 `hami-scheduler-device` 的 ConfigMap 中进行定义，你可以通过以下的方式进行更新：
+
+1. 直接热更新 ConfigMap：成功部署 HAMi 后，你可以通过 `kubectl edit` 指令手动热更新 hami-scheduler 中的数据，如以下指令所示：
+
+ ```bash
+ kubectl edit configmap hami-scheduler-device -n
+ ```
+
+ 更新完毕后，需要重启 hami-Scheduler 和对应节点上的 hami-device-plugin
+
+2. 修改 Helm Chart: 在[这里](../charts/hami/templates/scheduler/device-configmap.yaml)更新对应的字段，并进行部署
+
+* `nvidia.deviceSplitCount`：
+ 整数类型，预设值是 10。GPU 的分割数，每一张 GPU 都不能分配超过其配置数目的任务。若其配置为 N 的话，每个 GPU 上最多可以同时存在 N 个任务。
+* `nvidia.deviceMemoryScaling`：
+ 浮点数类型，预设值是 1。NVIDIA 装置显存使用比例，可以大于 1（启用虚拟显存，实验功能）。对于有 *M* 显存大小的 NVIDIA GPU，
+ 如果我们配置`nvidia.deviceMemoryScaling`参数为 *S*，在部署了我们装置插件的 Kubenetes 集群中，这张 GPU 分出的 vGPU 将总共包含 `S * M` 显存。
+* `nvidia.migStrategy`：
+ 字符串类型，目前支持 "none“ 与 “mixed“ 两种工作方式，前者忽略 MIG 设备，后者使用专门的资源名称指定 MIG 设备，使用详情请参考 mix_example.yaml，默认为 "none"
+* `nvidia.disablecorelimit`：
+ 字符串类型，"true" 为关闭算力限制，"false" 为启动算力限制，默认为 "false"
+* `nvidia.defaultMem`：
+ 整数类型，预设值为 0，表示不配置显存时使用的默认显存大小，单位为 MB。当值为 0 时，代表使用全部的显存。
+* `nvidia.defaultCores`：
+ 整数类型 (0-100)，默认为 0，表示默认为每个任务预留的百分比算力。若设置为 0，则代表任务可能会被分配到任一满足显存需求的 GPU 中，若设置为 100，代表该任务独享整张显卡
+* `nvidia.defaultGPUNum`：
+ 整数类型，默认为 1，如果配置为 0，则配置不会生效。当用户在 Pod 资源中没有设置 nvidia.com/gpu 这个 key 时，webhook 会检查 nvidia.com/gpumem、
+ resource-mem-percentage、nvidia.com/gpucores 这三个 key 中的任何一个 key 有值，webhook 都会添加 nvidia.com/gpu 键和此默认值到 resources limit 中。
+* `nvidia.resourceCountName`：
+ 字符串类型，申请 vgpu 个数的资源名，默认："nvidia.com/gpu"
+* `nvidia.resourceMemoryName`：
+ 字符串类型，申请 vgpu 显存大小资源名，默认："nvidia.com/gpumem"
+* `nvidia.resourceMemoryPercentageName`：
+ 字符串类型，申请 vgpu 显存比例资源名，默认："nvidia.com/gpumem-percentage"
+* `nvidia.resourceCoreName`：
+ 字符串类型，申请 vgpu 算力资源名，默认："nvidia.com/gpucores"
+* `nvidia.resourcePriorityName`：
+ 字符串类型，表示申请任务的任务优先级，默认："nvidia.com/priority"
+
+## 节点配置
+
+HAMi 允许为每个节点的 device plugin 配置不同的行为，只需要修改 configmap，如下:
+```sh
+kubectl -n edit cm hami-device-plugin
+```
+* `name`: 要配置的节点名称。
+* `operatingmode`: 节点的运行模式，可以是 "hami-core" 或者 "mig", 默认: "hami-core"。
+* `devicememoryscaling`: 节点内存的超配率。
+* `devicecorescaling`: 节点算力的超配率。
+* `devicesplitcount`: 每个设备允许被分配的任务数。
+* `filterdevices`: 节点上不被 HAMi 管理的设备。
+ * `uuid`: 所要排除设备的 UUID。
+ * `index`: 所要排除设备的索引。
+ * 一个设备只要在 `uuid` 或者 `index` 列表中，就不会被 HAMi 管理。
+
+## Chart 参数
+
+你可以在安装过程中，通过 `-set` 来修改以下的客制化参数，例如：
+
+1. 直接编辑 ConfigMap：如果 HAMi 已成功安装，你可以使用 `kubectl edit` 命令手动更新 hami-scheduler-device ConfigMap。
+
+ ```bash
+ kubectl edit configmap hami-scheduler-device -n
+ ```
+
+ 更改后，重启相关的 HAMi 组件以应用更新的配置。
+
+2. 修改 Helm Chart：更新 [ConfigMap](../charts/hami/templates/scheduler/device-configmap.yaml)
+ 中相应的值，然后重新应用 Helm Chart 以重新生成 ConfigMap。
+
+你可以在安装过程中，通过 `-set` 来修改以下的定制参数，例如：
+
+```bash
+helm install vgpu vgpu-charts/vgpu --set devicePlugin.deviceMemoryScaling=5 ...
+```
+
+* `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy`：字符串类型，预设值为 "binpack" 表示 GPU 节点调度策略，
+ "binpack"表示尽量将任务分配到同一个 GPU 节点上，"spread"表示尽量将任务分配到不同 GPU 节点上。
+* `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy`：字符串类型，预设值为 "spread" 表示 GPU 调度策略，
+ "binpack"表示尽量将任务分配到同一个 GPU 上，"spread"表示尽量将任务分配到不同 GPU 上。
+
+**Webhook TLS 证书配置**
+
+在 Kubernetes 中，为了让 API server 能够与 webhook 组件通信，webhook 需要一个 API server 信任的 TLS 证书。HAMi scheduler 提供了两种生成/配置所需 TLS 证书的方法。
+
+* `scheduler.patch.enabled`：
+ 布尔类型，默认值为 true。如果设置为 true，helm 将使用 kube-webhook-certgen ([job-patch](../charts/hami/templates/scheduler/job-patch/job-createSecret.yaml)) 生成自签名证书并创建 secret。
+* `scheduler.certManager.enabled`：
+ 布尔类型，默认值为 false。如果设置为 true，cert-manager 将生成自签名证书。**注意：此选项需要先在集群中安装 cert-manager。** _更多详情请参见 [cert-manager 安装说明](https://cert-manager.io/docs/installation/kubernetes/)。_
+
+
+# Pod 配置（在注解中指定）
+
+* `nvidia.com/use-gpuuuid`：
+
+ 字符串类型，如: "GPU-AAA,GPU-BBB"
+
+ 如果设置，该任务申请的设备只能是字符串中定义的设备之一。
+
+* `nvidia.com/nouse-gpuuuid`：
+
+ 字符串类型，如: "GPU-AAA,GPU-BBB"
+
+ 如果设置，该任务不能使用字符串中定义的任何设备
+
+* `nvidia.com/nouse-gputype`：
+
+ 字符串类型，如: "Tesla V100-PCIE-32GB，NVIDIA A10"
+
+ 如果设置，该任务不能使用字符串中定义的任何设备型号
+
+* `nvidia.com/use-gputype`：
+
+ 字符串类型，如: "Tesla V100-PCIE-32GB，NVIDIA A10"
+
+ 如果设置，该任务申请的设备只能使用字符串中定义的设备型号。
+
+* `hami.io/gpu-scheduler-policy`：
+
+ 字符串类型，"binpack" 或 "spread"
+
+ - spread:，调度器会尽量将任务均匀地分配在不同 GPU 中
+ - binpack: 调度器会尽量将任务分配在已分配的 GPU 中，从而减少碎片
+
+* `hami.io/node-scheduler-policy`：
+
+ 字符串类型，"binpack" 或 "spread"
+
+ - spread: 调度器会尽量将任务均匀地分配到不同节点上
+ - binpack: 调度器会尽量将任务分配在已分配任务的节点上，从而减少碎片
+
+* `nvidia.com/vgpu-mode`：
+
+ 字符串类型，"hami-core" 或 "mig"
+
+ 该任务希望使用的 vgpu 类型
+
+## 容器配置（在容器的环境变量中指定）
+
+* `GPU_CORE_UTILIZATION_POLICY`
+ > 当前这个参数可以在 `helm install` 的时候指定，然后自动注入到容器环境变量中, 通过 `--set devices.nvidia.gpuCorePolicy=force`
+
+ 字符串类型，"default"，"force"，"disable"
+
+ - 默认为"default" 代表容器算力限制策略，"default" 为默认
+ - "force" 为强制限制算力，一般用于测试算力限制的功能
+ - "disable" 为忽略算力限制
+
+* `CUDA_DISABLE_CONTROL`：
+
+ 布尔类型，"true"，"false"
+
+ - 默认为 false
+ - 若设置为 true，则代表屏蔽掉容器层的资源隔离机制，需要注意的是，这个参数只有在容器创建时指定才会生效，一般用于调试
diff --git a/docs/dashboard.md b/docs/dashboard.md
new file mode 100644
index 000000000..3cda2a1e4
--- /dev/null
+++ b/docs/dashboard.md
@@ -0,0 +1,237 @@
+# Abstract
+
+For the sake of simplicity, this article provides only one possible way to ultimately use prometheus to capture monitoring metrics as a data source and grafana to present monitoring information.
+
+Many users feedback from creating issues that they do not know how to install and configure related components, resulting in failure to use related dashboard normally. The installation and configuration steps are described as follows, Hope you use it smoothly! Any feedback is welcome.
+
+This article assumes that Kubernetes cluster and HAMi has been deployed successfully. The following components are installed in a kubernetes cluster. The components or software versions are as follows:
+
+| components or software name | version | remark |
+| --------------------------- | ------------------- | ---------------- |
+| kubernetes cluster | v1.23.10 | in AMD64 servers |
+| kube-prometheus stack | branch release-0.11 | |
+| dcgm-exporter | tag 3.2.5-3.1.7 | |
+
+# Deploy and configure kube-prometheus stack
+
+## Deploy kube-prometheus stack
+
+**Note:** See the version compatibility matrix for kubernetes and kube-prometheus stack in：https://github.com/prometheus-operator/kube-prometheus?tab=readme-ov-file#compatibility
+
+```shell
+#Clone kube-prometheus code repository(using release-0.11 here)
+git clone -b release-0.11 https://github.com/prometheus-operator/kube-prometheus.git
+cd kube-prometheus
+
+#Change type of grafana service into NodePort by Adding "type: NodePort" under spec section
+vi manifests/grafana-service.yaml
+...
+spec:
+ type: NodePort
+...
+#Similarly, change the prometheus and alertmanager service types to NodePort. Their configuration files are prometheus-service.yaml and alertmanager-service.yaml, respectively, in the manifests directory
+
+#Do deployment
+kubectl create -f manifests/setup/
+kubectl create -f manifests/.
+
+#All resouce objects are been created under the monitoring namespace, you can check them and their status by run the following command
+kubectl -n monitoring get all
+```
+
+```shell
+#Once all resouce objects under the monitoring namespace are in right status, you can obtain svc information for grafana, prometheus, and alertmanager in the following way
+root@controller01:~/kube-prometheus# kubectl -n monitoring get svc | egrep "NAME|grafana|prometheus-k8s|alertmanager-main"
+NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
+alertmanager-main NodePort 10.233.5.65 9093:30093/TCP,8080:30401/TCP 19h
+grafana NodePort 10.233.56.112 3000:30300/TCP 19h
+prometheus-k8s NodePort 10.233.38.113 9090:30090/TCP,8080:31273/TCP 19h
+```
+
+If ip address of controller node is 10.0.0.21, then grafana, prometheus, and alertmanager can be accessed using the following urls: http://10.0.0.21:30300 , http://10.0.0.21:30090 , and http://10.0.0.21:30093 , and the default user name and password for accessing grafana are admin
+
+## Configure grafana
+
+### Create Datasource ALL
+
+Go to the "Configuration" -> "Data soutces" page in grafana and create a datasource named "ALL", and keep the value of HTTP.URL be same with the counterpart in default "prometheus" datasource.
+
+### Import dashboard
+
+Go to the "Configuration" -> "Data soutces" page in grafana and import the dashboard from https://grafana.com/grafana/dashboards/22043-hami-vgpu-metrics-dashboard/ , and a dashboard page named "hami-vgpu-metrics-dashboard" will be created. 22043-hami-vgpu-metrics-dashboard is valid in grafana8.5.5 and grafana9.1.0, and it's grealty possible that this dashboard is vaild in grafana version later than 9.1.0. Now data of some panels in this dashboard page are missing, which requires you read the rest of the document.
+
+For versions earlier than grafana8.5.5, such as grafana7.5.17, please refer to：https://grafana.com/grafana/dashboards/21833-hami-vgpu-dashboard/
+
+# Deploy dcgm-exporter
+
+```shell
+#Clone dcgm-exporter code repository(the compatibility matrix for dcgm-export and kubernetes is not been founded in its official website, using "tag 3.2.5-3.1.7" here)
+git clone -b 3.2.5-3.1.7 https://github.com/NVIDIA/dcgm-exporter.git
+cd dcgm-exporter
+
+#Modified file "deployment/values.yaml", add configurations under relabelings in serviceMonitor section, these configurations are necessary for dashboard page display
+#There is only one serviceMonitor in file "deployment/values.yaml", and the contents of the modified file are as follows
+vi deployment/values.yaml
+...
+serviceMonitor:
+ apiVersion: "monitoring.coreos.com/v1"
+ enabled: true
+ interval: 15s
+ honorLabels: false
+ additionalLabels: {}
+ #monitoring: prometheus
+ relabelings:
+ - sourceLabels: [__meta_kubernetes_endpoints_name]
+ regex: dcgm-exporter
+ replacement: $1
+ action: keep
+ - sourceLabels: [__meta_kubernetes_pod_node_name]
+ regex: (.*)
+ targetLabel: node_name
+ replacement: ${1}
+ action: replace
+ - sourceLabels: [__meta_kubernetes_pod_host_ip]
+ regex: (.*)
+ targetLabel: ip
+ replacement: $1
+ action: replace
+...
+
+#Install dcgm-exporter under monitoring namespace with helm
+helm install dcgm-exporter deployment/ -n monitoring
+
+#Check the result of running "helm install ..."
+helm list -n monitoring
+NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
+dcgm-exporter monitoring 1 2024-10-02 16:32:35.691073696 +0800 CST deployed dcgm-exporter-3.1.7 3.1.7
+#Make sure pod(s)/dcgm-exporter-xxx is(are) in running status
+kubectl -n monitoring get pods | egrep "NAME|dcgm-exporter"
+NAME READY STATUS RESTARTS AGE
+dcgm-exporter-p4fdp 1/1 Running 0 4h14m
+```
+
+# Create ServiceMonitor
+
+```shell
+#Create the file hami-device-plugin-svc-monitor.yaml
+root@controller01:~# touch hami-device-plugin-svc-monitor.yaml
+#The content of the file hami-device-plugin-svc-monitor.yaml
+root@controller01:~# cat hami-device-plugin-svc-monitor.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ name: hami-device-plugin-svc-monitor
+ namespace: kube-system
+spec:
+ selector:
+ matchLabels:
+ app.kubernetes.io/component: hami-device-plugin
+ namespaceSelector:
+ matchNames:
+ - "kube-system"
+ endpoints:
+ - path: /metrics
+ port: monitorport
+ interval: "15s"
+ honorLabels: false
+ relabelings:
+ - sourceLabels: [__meta_kubernetes_endpoints_name]
+ regex: hami-.*
+ replacement: $1
+ action: keep
+ - sourceLabels: [__meta_kubernetes_pod_node_name]
+ regex: (.*)
+ targetLabel: node_name
+ replacement: ${1}
+ action: replace
+ - sourceLabels: [__meta_kubernetes_pod_host_ip]
+ regex: (.*)
+ targetLabel: ip
+ replacement: $1
+ action: replace
+
+#apply the file hami-device-plugin-svc-monitor.yaml
+root@controller01:~# kubectl apply -f hami-device-plugin-svc-monitor.yaml
+```
+
+```shell
+#Create the file hami-scheduler-svc-monitor.yaml
+root@controller01:~# touch hami-device-plugin-svc-monitor.yaml
+#The content of the file hami-scheduler-svc-monitor.yaml
+root@controller01:~# cat hami-scheduler-svc-monitor.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ name: hami-scheduler-svc-monitor
+ namespace: kube-system
+spec:
+ selector:
+ matchLabels:
+ app.kubernetes.io/component: hami-scheduler
+ namespaceSelector:
+ matchNames:
+ - "kube-system"
+ endpoints:
+ - path: /metrics
+ port: monitor
+ interval: "15s"
+ honorLabels: false
+ relabelings:
+ - sourceLabels: [__meta_kubernetes_endpoints_name]
+ regex: hami-.*
+ replacement: $1
+ action: keep
+ - sourceLabels: [__meta_kubernetes_pod_node_name]
+ regex: (.*)
+ targetLabel: node_name
+ replacement: ${1}
+ action: replace
+ - sourceLabels: [__meta_kubernetes_pod_host_ip]
+ regex: (.*)
+ targetLabel: ip
+ replacement: $1
+ action: replace
+
+#apply the file hami-scheduler-svc-monitor.yaml
+root@controller01:~# kubectl apply -f hami-scheduler-svc-monitor.yaml
+```
+
+```shell
+#Check the servicemonitors
+root@controller01:~# kubectl -n kube-system get servicemonitor
+NAME AGE
+hami-device-plugin-svc-monitor 28h
+hami-scheduler-svc-monitor 28h
+```
+
+# Confirm the final monitoring effect
+
+```shell
+#Create the file gpu-pod.yaml
+root@controller01:~# touch gpu-pod.yaml
+root@controller01:~# cat gpu-pod.yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod-01
+spec:
+ restartPolicy: Never
+ containers:
+ - name: cuda-container
+ image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda10.2
+ resources:
+ limits:
+ nvidia.com/vgpu: 2 # requesting 2 vGPUs
+ nvidia.com/gpumem: 3000 # Each vGPU contains 3000m device memory （Optional,Integer）
+ nvidia.com/gpucores: 10 # Each vGPU uses 30% of the entire GPU （Optional,Integer)
+
+#apply the file gpu-pod.yaml
+root@controller01:~# kubectl apply -f gpu-pod.yaml
+root@controller01:~# kubectl get pods -o wide
+NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
+gpu-pod-01 0/1 Completed 0 52s 10.233.81.70 controller01
+```
+
+You can see the monitoring details in the dashboard. The contents are as follows:
+
+
\ No newline at end of file
diff --git a/docs/dashboard_cn.md b/docs/dashboard_cn.md
new file mode 100644
index 000000000..b4afa90a1
--- /dev/null
+++ b/docs/dashboard_cn.md
@@ -0,0 +1,237 @@
+# 说明
+
+为了阐述的简明性，本文只提供一种可行的办法，最终实现使用prometheus抓取监控指标并作为数据源、使用grafana来展示监控信息的目的。
+
+很多使用者通过issue反馈不知如何安装与配置相关组件导致无法正常使用相关dashboard，现将安装与配置步骤叙述如下，希望大家使用顺利！如有问题欢迎创建issue反馈，如有不足欢迎补充！
+
+本文假设读者已经部署好Kubernetes集群、HAMi。以下涉及到的相关组件都是在kubernetes集群内安装的，相关组件或软件版本信息如下：
+
+| 组件或软件名称 | 版本 | 备注 |
+| --------------------- | ----------------- | --------------------- |
+| kubernetes集群 | v1.23.10 | AMD64构架服务器环境下 |
+| kube-prometheus stack | 分支 release-0.11 | |
+| dcgm-exporter | tag 3.2.5-3.1.7 | |
+
+# 部署与配置kube-prometheus stack
+
+## 部署kube-prometheus stack
+
+**注：** kubernetes与kube-prometheus stack的版本兼容矩阵请查看 https://github.com/prometheus-operator/kube-prometheus?tab=readme-ov-file#compatibility ，请根据自己的kubernetes版本选择合适版本的kube-prometheus stack
+
+```shell
+#下载kube-prometheus代码仓库（此处使用分支 release-0.11）
+git clone -b release-0.11 https://github.com/prometheus-operator/kube-prometheus.git
+cd kube-prometheus
+
+#修改下grafana的service类型为NodePort。即在spec下添加type配置项
+vi manifests/grafana-service.yaml
+...
+spec:
+ type: NodePort
+...
+#类似的方法修改prometheus与alertmanager service类型为NodePort，它们的配置文件分别是manifests目录下的prometheus-service.yaml与alertmanager-service.yaml
+
+#执行部署
+kubectl create -f manifests/setup/
+kubectl create -f manifests/.
+
+#创建的所有资源对象都在monitorin命名空间下，使用如下命令查看资源对象的运行状态
+kubectl -n monitoring get all
+```
+
+```shell
+#等monitorin命名空间下所有资源对象处于正常运行状态后，使用如下方式获取grafana、prometheus与alertmanager的svc信息
+root@controller01:~/kube-prometheus# kubectl -n monitoring get svc | egrep "NAME|grafana|prometheus-k8s|alertmanager-main"
+NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
+alertmanager-main NodePort 10.233.5.65 9093:30093/TCP,8080:30401/TCP 19h
+grafana NodePort 10.233.56.112 3000:30300/TCP 19h
+prometheus-k8s NodePort 10.233.38.113 9090:30090/TCP,8080:31273/TCP 19h
+```
+
+此时，假如控制节点的ip是10.0.0.21，则可以分别使用如下url访问grafana、prometheus与alertmanager：http://10.0.0.21:30300 、http://10.0.0.21:30090 、http://10.0.0.21:30093 ，其中访问grafana的默认用户名与密码都是admin
+
+## 配置grafana
+
+### 创建数据源ALL
+
+访问”Configuration“->“Data soutces”页面，创建一个名为"ALL"的数据源，其中HTTP.URL的值保持跟默认创建的数据源“prometheus”中的一样即可为 “http://prometheus-k8s.monitoring.svc:9090” ，然后保存上述数据源“ALL”
+
+### 导入HAMi默认的dashboard
+
+访问“Dashboards”->“Browse”页面，导入此dashboard：https://grafana.com/grafana/dashboards/22043-hami-vgpu-metrics-dashboard/ ，grafana中将创建一个名为“hami-vgpu-metrics-dashboard”的dashboard，22043这个编号对应的dashboard在grafana8.5.5与grafana9.1.0验证过，在grafana9.1.0之后应该也能用。此时此页面中有一些Panel如vGPUCorePercentage还没有数据，请继续看完此文档，执行完"部署dcgm-exporter"与“创建ServiceMonitor”中的步骤之后Panel数据将正常显示。
+
+对于grafana8.5.5之前的版本如grafana7.5.17，请使用此dashboard：https://grafana.com/grafana/dashboards/21833-hami-vgpu-dashboard/
+
+# 部署dcgm-exporter
+
+```shell
+#下载dcgm-exporter代码仓库（未看到与kubernetes的兼容矩阵说明，此处使用tag 3.2.5-3.1.7）
+git clone -b 3.2.5-3.1.7 https://github.com/NVIDIA/dcgm-exporter.git
+cd dcgm-exporter
+
+#修改 deployment/values.yaml 文件，为其中的serviceMonitor添加relabelings配置，这样dcgm-exporter的监控指标才有node_name与ip属性
+#只有这一处serviceMonitor，relabelings内的内容都是添加的，添加后内容如下
+root@controller01:~/dcgm-exporter# vi deployment/values.yaml
+...
+serviceMonitor:
+ apiVersion: "monitoring.coreos.com/v1"
+ enabled: true
+ interval: 15s
+ honorLabels: false
+ additionalLabels: {}
+ #monitoring: prometheus
+ relabelings:
+ - sourceLabels: [__meta_kubernetes_endpoints_name]
+ regex: dcgm-exporter
+ replacement: $1
+ action: keep
+ - sourceLabels: [__meta_kubernetes_pod_node_name]
+ regex: (.*)
+ targetLabel: node_name
+ replacement: ${1}
+ action: replace
+ - sourceLabels: [__meta_kubernetes_pod_host_ip]
+ regex: (.*)
+ targetLabel: ip
+ replacement: $1
+ action: replace
+...
+
+#使用helm工具在monitoring空间下安装dcgm-exporter
+root@controller01:~/dcgm-exporter# helm install dcgm-exporter deployment/ -n monitoring
+
+#查看helm install结果
+root@controller01:~/dcgm-exporter# helm list -n monitoring
+NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
+dcgm-exporter monitoring 1 2024-10-02 16:32:35.691073696 +0800 CST deployed dcgm-exporter-3.1.7 3.1.7
+#还需确认dcgm-exporter 相关的pod已经处于running状态
+root@controller01:~/dcgm-exporter# kubectl -n monitoring get pods | egrep "NAME|dcgm-exporter"
+NAME READY STATUS RESTARTS AGE
+dcgm-exporter-p4fdp 1/1 Running 0 4h14m
+```
+
+# 创建ServiceMonitor
+
+```shell
+#创建文件hami-device-plugin-svc-monitor.yaml
+root@controller01:~# touch hami-device-plugin-svc-monitor.yaml
+#文件hami-device-plugin-svc-monitor.yaml内容如下
+root@controller01:~# cat hami-device-plugin-svc-monitor.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ name: hami-device-plugin-svc-monitor
+ namespace: kube-system
+spec:
+ selector:
+ matchLabels:
+ app.kubernetes.io/component: hami-device-plugin
+ namespaceSelector:
+ matchNames:
+ - "kube-system"
+ endpoints:
+ - path: /metrics
+ port: monitorport
+ interval: "15s"
+ honorLabels: false
+ relabelings:
+ - sourceLabels: [__meta_kubernetes_endpoints_name]
+ regex: hami-.*
+ replacement: $1
+ action: keep
+ - sourceLabels: [__meta_kubernetes_pod_node_name]
+ regex: (.*)
+ targetLabel: node_name
+ replacement: ${1}
+ action: replace
+ - sourceLabels: [__meta_kubernetes_pod_host_ip]
+ regex: (.*)
+ targetLabel: ip
+ replacement: $1
+ action: replace
+
+#应用此文件
+root@controller01:~# kubectl apply -f hami-device-plugin-svc-monitor.yaml
+```
+
+```shell
+#创建文件hami-scheduler-svc-monitor.yaml
+root@controller01:~# touch hami-device-plugin-svc-monitor.yaml
+#文件hami-scheduler-svc-monitor.yaml内容如下
+root@controller01:~# cat hami-scheduler-svc-monitor.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ name: hami-scheduler-svc-monitor
+ namespace: kube-system
+spec:
+ selector:
+ matchLabels:
+ app.kubernetes.io/component: hami-scheduler
+ namespaceSelector:
+ matchNames:
+ - "kube-system"
+ endpoints:
+ - path: /metrics
+ port: monitor
+ interval: "15s"
+ honorLabels: false
+ relabelings:
+ - sourceLabels: [__meta_kubernetes_endpoints_name]
+ regex: hami-.*
+ replacement: $1
+ action: keep
+ - sourceLabels: [__meta_kubernetes_pod_node_name]
+ regex: (.*)
+ targetLabel: node_name
+ replacement: ${1}
+ action: replace
+ - sourceLabels: [__meta_kubernetes_pod_host_ip]
+ regex: (.*)
+ targetLabel: ip
+ replacement: $1
+ action: replace
+
+#应用此文件
+root@controller01:~# kubectl apply -f hami-scheduler-svc-monitor.yaml
+```
+
+```shell
+#确认创建的ServiceMonitor
+root@controller01:~# kubectl -n kube-system get servicemonitor
+NAME AGE
+hami-device-plugin-svc-monitor 28h
+hami-scheduler-svc-monitor 28h
+```
+
+# 确认最终监控效果
+
+```shell
+#创建文件gpu-pod.yaml，尝试使用HAMi虚拟出来的NVIDIA vGPU
+root@controller01:~# touch gpu-pod.yaml
+root@controller01:~# cat gpu-pod.yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod-01
+spec:
+ restartPolicy: Never
+ containers:
+ - name: cuda-container
+ image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda10.2
+ resources:
+ limits:
+ nvidia.com/vgpu: 2 # 请求2个vGPUs
+ nvidia.com/gpumem: 3000 # 每个vGPU申请3000m显存 （可选，整数类型）
+ nvidia.com/gpucores: 10 # 每个vGPU的算力为10%实际显卡的算力 （可选，整数类型）
+
+#应用此文件
+root@controller01:~# kubectl apply -f gpu-pod.yaml
+root@controller01:~# kubectl get pods -o wide
+NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
+gpu-pod-01 0/1 Completed 0 52s 10.233.81.70 controller01
+```
+
+此时，应该可以dashboard中看到监控详情。内容大概如下
+
+
\ No newline at end of file
diff --git a/docs/develop/design.md b/docs/develop/design.md
new file mode 100644
index 000000000..1dfb7114d
--- /dev/null
+++ b/docs/develop/design.md
@@ -0,0 +1,28 @@
+# Design
+
+
+
+The architect of HAMi is shown in the figure above, It is organized in the form of "chart".
+
+- MutatingWebhook
+
+The MutatingWebhook checks the validity of each task, and set the "schedulerName" to "HAMi scheduler" if the resource requests have been recognized by HAMi
+If Not, the MutatingWebhook does nothing and pass this task to default-scheduler.
+
+- Scheduler
+
+HAMi support default kube-scheduler and volcano-scheduler, it implements an extender and register 'Filter' and 'Score' methods to deal with sharable devices.
+When a pod with sharable device request arrives, 'Filter' searches the cluster and returns a list of 'available' nodes. 'Score' scores each node 'Filter' returned, and pick the highest one to host the pod. It patches the schedule decision on corresponding pod annotations, for the detailed protocol, see [protocol.md](protocol.md)
+
+- DevicePlugin
+
+When the schedule decision is made, scheduler calls devicePlugin on that node to generate environment variables and mounts according to pod annotations.
+Please note that, the DP used here is a customized version, you need to install according to [README](../../README.md) document with that device. Most officaial DP will not fit in HAMi, and will result in unexpected behaviour
+
+- InContainer Control
+
+The implementation of in-container hard limit is different for diffent devices. For example, HAMi-Core is responsible for NVIDIA devices. libvgpu-control.so is responsible for iluvatar devices, etc. HAMi needs to pass the correct environment variables in order for it to operate.
+
+
+
+In summary, The flowchart of pod is descirbed as the figure above.
diff --git a/docs/develop/dynamic-mig.md b/docs/develop/dynamic-mig.md
new file mode 100644
index 000000000..2cbcdffdf
--- /dev/null
+++ b/docs/develop/dynamic-mig.md
@@ -0,0 +1,158 @@
+# NVIDIA GPU MPS and MIG dynamic slice plugin
+
+## Special Thanks
+
+This feature will not be implemented without the help of @sailorvii.
+
+## Introduction
+
+The NVIDIA GPU build-in sharing method includes: time-slice, MPS and MIG. The context switch for time slice sharing would waste some time, so we chose the MPS and MIG. The GPU MIG profile is variable, the user could acquire the MIG device in the profile definition, but current implementation only defines the dedicated profile before the user requirement. That limits the usage of MIG. We want to develop an automatic slice plugin and create the slice when the user require it.
+For the scheduling method, node-level binpack and spread will be supported. Referring to the binpack plugin, we consider the CPU, Mem, GPU memory and other user-defined resource.
+HAMi is done by using [hami-core](https://github.com/Project-HAMi/HAMi-core), which is a cuda-hacking library. But mig is also widely used across the world. A unified API for dynamic-mig and hami-core is needed.
+
+## Targets
+
+- CPU, Mem, and GPU combined schedule
+- GPU dynamic slice: Hami-core and MIG
+- Support node-level binpack and spread by GPU memory, CPU and Mem
+- A unified vGPU Pool different virtualization technics
+- Tasks can choose to use MIG, use HAMi-core, or use both.
+
+### Config maps
+- hami-scheduler-device-configMap
+This configmap defines the plugin configurations including resourceName, and MIG geometries, and node-level configurations.
+
+```yaml
+apiVersion: v1
+data:
+ device-config.yaml: |
+ nvidia:
+ resourceCountName: nvidia.com/gpu
+ resourceMemoryName: nvidia.com/gpumem
+ resourceCoreName: nvidia.com/gpucores
+ knownMigGeometries:
+ - models: [ "A30" ]
+ allowedGeometries:
+ -
+ - name: 1g.6gb
+ memory: 6144
+ count: 4
+ -
+ - name: 2g.12gb
+ memory: 12288
+ count: 2
+ -
+ - name: 4g.24gb
+ memory: 24576
+ count: 1
+ - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB", "A100-SXM4-40GB" ]
+ allowedGeometries:
+ -
+ - name: 1g.5gb
+ memory: 5120
+ count: 7
+ -
+ - name: 2g.10gb
+ memory: 10240
+ count: 3
+ - name: 1g.5gb
+ memory: 5120
+ count: 1
+ -
+ - name: 3g.20gb
+ memory: 20480
+ count: 2
+ -
+ - name: 7g.40gb
+ memory: 40960
+ count: 1
+ - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
+ allowedGeometries:
+ -
+ - name: 1g.10gb
+ memory: 10240
+ count: 7
+ -
+ - name: 2g.20gb
+ memory: 20480
+ count: 3
+ - name: 1g.10gb
+ memory: 10240
+ count: 1
+ -
+ - name: 3g.40gb
+ memory: 40960
+ count: 2
+ -
+ - name: 7g.79gb
+ memory: 80896
+ count: 1
+ nodeconfig:
+ - name: nodeA
+ operatingmode: hami-core
+ - name: nodeB
+ operatingmode: mig
+```
+
+## Structure
+
+
+
+## Examples
+
+Dynamic mig is compatable with hami tasks, as the example below:
+Just Setting `nvidia.com/gpu` and `nvidia.com/gpumem`.
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+spec:
+ containers:
+ - name: ubuntu-container1
+ image: ubuntu:20.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # requesting 2 vGPUs
+ nvidia.com/gpumem: 8000 # Each vGPU contains 8000m device memory （Optional,Integer)
+```
+
+A task can decide only to use `mig` or `hami-core` by setting `annotations.nvidia.com/vgpu-mode` to corresponding value, as the example below shows:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+ annotations:
+ nvidia.com/vgpu-mode: "mig"
+spec:
+ containers:
+ - name: ubuntu-container1
+ image: ubuntu:20.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # requesting 2 vGPUs
+ nvidia.com/gpumem: 8000 # Each vGPU contains 8000m device memory （Optional,Integer
+```
+
+## Procedures
+
+The Procedure of a vGPU task which uses dynamic-mig is shown below:
+
+
+
+Note that after submited a task, deviceshare plugin will iterate over templates defined in configMap `hami-scheduler-device`, and find the first available template to fit. You can always change the content of that configMap, and restart vc-scheduler to customize.
+
+If you submit the example on an empty A100-PCIE-40GB node, then it will select a GPU and chosse MIG template below:
+
+```yaml
+ 2g.10gb : 3
+ 1g.5gb : 1
+```
+
+Then start the container with 2g.10gb instances * 2
+
diff --git a/docs/develop/imgs/flowchart.jpeg b/docs/develop/imgs/flowchart.jpeg
new file mode 100644
index 000000000..1cbe0a590
Binary files /dev/null and b/docs/develop/imgs/flowchart.jpeg differ
diff --git a/docs/develop/imgs/gpu-scheduler-policy-demo.png b/docs/develop/imgs/gpu-scheduler-policy-demo.png
new file mode 100644
index 000000000..fe122ec42
Binary files /dev/null and b/docs/develop/imgs/gpu-scheduler-policy-demo.png differ
diff --git a/docs/develop/imgs/hami-dynamic-mig-procedure.png b/docs/develop/imgs/hami-dynamic-mig-procedure.png
new file mode 100644
index 000000000..55995bbea
Binary files /dev/null and b/docs/develop/imgs/hami-dynamic-mig-procedure.png differ
diff --git a/docs/develop/imgs/hami-dynamic-mig-structure.png b/docs/develop/imgs/hami-dynamic-mig-structure.png
new file mode 100644
index 000000000..7e38d1be2
Binary files /dev/null and b/docs/develop/imgs/hami-dynamic-mig-structure.png differ
diff --git a/docs/develop/imgs/node-shceduler-policy-demo.png b/docs/develop/imgs/node-shceduler-policy-demo.png
new file mode 100644
index 000000000..5e33fc402
Binary files /dev/null and b/docs/develop/imgs/node-shceduler-policy-demo.png differ
diff --git a/docs/develop/imgs/offline_validation.png b/docs/develop/imgs/offline_validation.png
new file mode 100644
index 000000000..8dec96255
Binary files /dev/null and b/docs/develop/imgs/offline_validation.png differ
diff --git a/docs/develop/imgs/protocol_pod.png b/docs/develop/imgs/protocol_pod.png
new file mode 100644
index 000000000..0fff3c6b6
Binary files /dev/null and b/docs/develop/imgs/protocol_pod.png differ
diff --git a/docs/develop/imgs/protocol_register.png b/docs/develop/imgs/protocol_register.png
new file mode 100644
index 000000000..94c2529e3
Binary files /dev/null and b/docs/develop/imgs/protocol_register.png differ
diff --git a/docs/develop/imgs/scheduler-policy-story.png b/docs/develop/imgs/scheduler-policy-story.png
new file mode 100644
index 000000000..91039543b
Binary files /dev/null and b/docs/develop/imgs/scheduler-policy-story.png differ
diff --git a/docs/develop/protocol.md b/docs/develop/protocol.md
new file mode 100644
index 000000000..967640d7b
--- /dev/null
+++ b/docs/develop/protocol.md
@@ -0,0 +1,67 @@
+# Protocol
+
+## Device Register
+
+
+
+HAMi needs to know the spec of each AI devices in the cluster in order to schedule properly. During device registration, device-plugin needs to keep patching the spec of each device into node annotations every 30 seconds, in the format of the following:
+
+```
+hami.io/node-handshake-{device-type}: Reported_{device_node_current_timestamp}
+hami.io/node-{deivce-type}-register: {Device 1}:{Device2}:...:{Device N}
+```
+
+The definiation of each device is in the following format:
+```
+{Device UUID},{device split count},{device memory limit},{device core limit},{device type},{device numa},{healthy}
+```
+
+An example is shown below:
+```
+hami.io/node-handshake-nvidia: Reported 2024-01-23 04:30:04.434037031 +0000 UTC m=+1104711.777756895
+hami.io/node-handshake-mlu: Requesting_2024.01.10 04:06:57
+hami.io/node-mlu-register: MLU-45013011-2257-0000-0000-000000000000,10,23308,0,MLU-MLU370-X4,0,false:MLU-54043011-2257-0000-0000-000000000000,10,23308,0,
+hami.io/node-nvidia-register: GPU-00552014-5c87-89ac-b1a6-7b53aa24b0ec,10,32768,100,NVIDIA-Tesla V100-PCIE-32GB,0,true:GPU-0fc3eda5-e98b-a25b-5b0d-cf5c855d1448,10,32768,100,NVIDIA-Tesla V100-PCIE-32GB,0,true:
+
+```
+In this example, this node has two different AI devices, 2 Nvidia-V100 GPUs, and 2 Cambircon 370-X4 MLUs
+
+Note that a device node may become unavailable due to hardware or network failure, if a node hasn't registered in last 5 minutes, scheduler will mark that node as 'unavailable'.
+
+Since system clock on scheduler node and 'device' node may not align properly, scheduler node will patch the following device node annotations every 30s
+
+```
+hami.io/node-handshake-{device-type}: Requesting_{scheduler_node_current_timestamp}
+```
+
+If hami.io/node-handshake annotations remains in "Requesting_xxxx" and {scheduler current timestamp} > 5 mins + {scheduler timestamp in annotations}, then this device on that node will be marked "unavailable" in scheduler.
+
+
+## Schedule Decision
+
+
+
+HAMi scheduler needs to patch schedule decisions into pod annotations, in the format of the following:
+
+```
+hami.io/devices-to-allocate:{ctr1 request}:{ctr2 request}:...{Last ctr request}:
+hami.io/device-node: {schedule decision node}
+hami.io/device-schedule-time: {timestamp}
+```
+
+each container request is in the following format:
+
+```
+{device UUID},{device type keywork},{device memory request}:{device core request}
+```
+
+for example:
+
+A pod with 2 containers, first container requests 1 GPU with 3G device Memory, second container requests 1 GPU with 5G device Memory, then the patched annotations will be like the
+
+```
+hami.io/devices-to-allocate: GPU-0fc3eda5-e98b-a25b-5b0d-cf5c855d1448,NVIDIA,3000,0:GPU-0fc3eda5-e98b-a25b-5b0d-cf5c855d1448,NVIDIA,5000,0:
+hami.io/vgpu-node: node67-4v100
+hami.io/vgpu-time: 1705054796
+```
+
diff --git a/docs/develop/roadmap.md b/docs/develop/roadmap.md
new file mode 100644
index 000000000..1f3c951c4
--- /dev/null
+++ b/docs/develop/roadmap.md
@@ -0,0 +1,27 @@
+# roadmap
+
+Heterogeneous AI Computing device to support
+
+| Production | manufactor | Type |MemoryIsolation | CoreIsolation | MultiCard support |
+|-------------|------------|-------------|-----------|---------------|-------------------|
+| GPU | NVIDIA | All | ✅ | ✅ | ✅ |
+| MLU | Cambricon | 370, 590 | ✅ | ✅ | ❌ |
+| GCU | Enflame | S60 | ✅ | ✅ | ❌ |
+| DCU | Hygon | Z100, Z100L | ✅ | ✅ | ❌ |
+| Ascend | Huawei | 910B | ✅ | ✅ | ❌ |
+| GPU | iluvatar | All | ✅ | ✅ | ❌ |
+| DPU | Teco | Checking | In progress | In progress | ❌ |
+
+
+- [ ] Support video codec processing
+- [ ] Support Multi-Instance GPUs (MIG)
+- [ ] Support Flexible scheduling policies
+ - [x] binpack
+ - [x] spread
+ - [ ] numa affinity
+- [ ] integrated gpu-operator
+- [ ] Rich observability support
+- [ ] DRA Support
+- [ ] Support Intel GPU device
+- [ ] Support AMD GPU device
+- [x] Support Enflame GCU device
diff --git a/docs/develop/scheduler-policy.md b/docs/develop/scheduler-policy.md
new file mode 100644
index 000000000..ed209ba0b
--- /dev/null
+++ b/docs/develop/scheduler-policy.md
@@ -0,0 +1,167 @@
+# Scheduler Policy Design
+
+## Summary
+
+Current in a cluster with many GPU nodes, nodes are not `binpack` or `spread` when making scheduling decisions, nor are GPU cards `binpack` or `spread` when using vGPU.
+
+## Proposal
+
+We add a `node-scheduler-policy` and `gpu-scheduler-policy` to config, then scheduler to use this policy can impl node `binpack` or `spread` or GPU `binpack` or `spread`. and
+use can set Pod annotation to change this default policy, use `hami.io/node-scheduler-policy` and `hami.io/gpu-scheduler-policy` to overlay scheduler config.
+
+### User Stories
+
+This is a GPU cluster, having two node, the following story takes this cluster as a prerequisite.
+
+
+
+#### Story 1
+
+node binpack, use one node’s GPU card whenever possible, egs:
+- cluster resources:
+ - node1: GPU having 4 GPU device
+ - node2: GPU having 4 GPU device
+
+- request:
+ - pod1: User 1 GPU
+ - pod2: User 1 GPU
+
+- scheduler result:
+ - pod1: scheduler to node1
+ - pod2: scheduler to node1
+
+#### Story 2
+
+node spread, use GPU cards from different nodes as much as possible, egs:
+
+- cluster resources:
+ - node1: GPU having 4 GPU device
+ - node2: GPU having 4 GPU device
+
+- request:
+ - pod1: User 1 GPU
+ - pod2: User 1 GPU
+
+- scheduler result:
+ - pod1: scheduler to node1
+ - pod2: scheduler to node2
+
+#### Story 3
+
+GPU binpack, use the same GPU card as much as possible, egs:
+
+- cluster resources:
+ - node1: GPU having 4 GPU device, they are GPU1,GPU2,GPU3,GPU4
+
+- request:
+ - pod1: User 1 GPU, gpucore is 20%, gpumem-percentage is 20%
+ - pod2: User 1 GPU, gpucore is 20%, gpumem-percentage is 20%
+
+- scheduler result:
+ - pod1: scheduler to node1, select GPU1 this device
+ - pod2: scheduler to node1, select GPU1 this device
+
+#### Story 4
+
+GPU spread, use different GPU cards when possible, egs:
+
+- cluster resources:
+ - node1: GPU having 4 GPU device, they are GPU1,GPU2,GPU3,GPU4
+
+- request:
+ - pod1: User 1 GPU, gpucore is 20%, gpumem-percentage is 20%
+ - pod2: User 1 GPU, gpucore is 20%, gpumem-percentage is 20%
+
+- scheduler result:
+ - pod1: scheduler to node1, select GPU1 this device
+ - pod2: scheduler to node1, select GPU2 this device
+
+## Design Details
+
+### Node-scheduler-policy
+
+
+
+#### Binpack
+
+Binpack mainly considers node resource usage. The more full the usage, the higher the score.
+
+```
+score: ((request + used) / allocatable) * 10
+```
+
+1. Binpack scoring information for Node 1 is as follows
+
+```
+Node1 score: ((1+3)/4) * 10= 10
+```
+
+2. Binpack scoring information for Node 2 is as follows
+
+```
+Node2 score: ((1+2)/4) * 10= 7.5
+```
+
+So, in `Binpack` policy we can select `Node1`.
+
+#### Spread
+
+Spread mainly considers node resource usage. The less it is used, the higher the score.
+
+```
+score: ((request + used) / allocatable) * 10
+```
+
+1. Spread scoring information for Node 1 is as follows
+```
+Node1 score: ((1+3)/4) * 10= 10
+```
+
+2. Spread scoring information for Node 2 is as follows
+```
+Node2 score: ((1+2)/4) * 10= 7.5
+```
+
+So, in `Spread` policy we can select `Node2`.
+
+### GPU-scheduler-policy
+
+
+
+#### Binpack
+
+Binpack mainly focuses on the computing power and video memory usage of each card. The more it is used, the higher the score.
+```
+score: ((request.core + used.core) / allocatable.core + (request.mem + used.mem) / allocatable.mem)) * 10
+```
+
+1. Binpack scoring information for GPU 1 is as follows
+```
+GPU1 Score: ((20+10)/100 + (1000+2000)/8000)) * 10 = 6.75
+```
+
+2. Binpack scoring information for GPU 2 is as follows
+```
+GPU2 Score: ((20+70)/100 + (1000+6000)/8000)) * 10 = 17.75
+```
+
+So, in `Binpack` policy we can select `GPU2`.
+
+#### Spread
+
+Spread mainly focuses on the computing power and video memory usage of each card. The less it is used, the higher the score.
+```
+score: ((request.core + used.core) / allocatable.core + (request.mem + used.mem) / allocatable.mem)) * 10
+```
+
+1. Spread scoring information for GPU 1 is as follows
+```
+GPU1 Score: ((20+10)/100 + (1000+2000)/8000)) * 10 = 6.75
+```
+
+2. Spread scoring information for GPU 2 is as follows
+```
+GPU2 Score: ((20+70)/100 + (1000+6000)/8000)) * 10 = 17.75
+```
+
+So, in `Spread` policy we can select `GPU1`.
\ No newline at end of file
diff --git a/docs/develop/tasklist.md b/docs/develop/tasklist.md
new file mode 100644
index 000000000..576cc8c91
--- /dev/null
+++ b/docs/develop/tasklist.md
@@ -0,0 +1,118 @@
+# Tasks
+
+## Support Moore threads MTT S4000
+
+```yaml
+resources:
+requests:
+ mthreads.com/gpu: ${num}
+ mthreads.com/vcuda-core: ${core}
+ mthreads.com/vcuda-memory: ${mem}
+limits:
+ mthreads.com/gpu: ${num}
+ mthreads.com/vcuda-core: ${core}
+ mthreads.com/vcuda-memory: ${mem}
+```
+
+## Support Birentech Model 110
+
+```yaml
+resources:
+requests:
+ birentech.com/gpu: ${num}
+ birentech.com/vcuda-core: ${core}
+ birentech.com/vcuda-memory: ${mem}
+limits:
+ birentech.com/gpu: ${num}
+ birentech.com/vcuda-core: ${core}
+ birentech.com/vcuda-memory: ${mem}
+```
+
+## Support iluvatar MR-V100
+
+```yaml
+resources:
+requests:
+ iluvatar.ai/gpu: ${num}
+ iluvatar.ai/vcuda-core: ${core}
+ iluvatar.ai/vcuda-memory: ${mem}
+limits:
+ iluvatar.ai/gpu: ${num}
+ iluvatar.ai/vcuda-core: ${core}
+ iluvatar.ai/vcuda-memory: ${mem}
+```
+
+## Support HuaWei Ascend 910B device
+
+```yaml
+resources:
+ requests:
+ ascend.com/npu: ${num}
+ ascend.com/npu-core: ${core}
+ ascend.com/npu-mem: ${mem}
+ limits:
+ ascend.com/npu: ${num}
+ ascend.com/npu-core: ${core}
+ ascend.com/npu-mem: ${mem}
+```
+
+## Support resourceQuota for Kubernetes
+
+Description: ResourceQuota is frequently used in kubernetes namespace. Since the number of virtual devices doesn't mean anything, we need to support the limitation in deviceMemory.
+
+For example, the following resourceQuota
+
+```yaml
+cat < compute-resources.yaml
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+ name: compute-resources
+spec:
+ hard:
+ requests.cpu: "1"
+ requests.memory: 1Gi
+ limits.cpu: "2"
+ limits.memory: 2Gi
+ requests.nvidia.com/gpu-memory: 30000
+EOF
+```
+
+with the following command
+
+```bash
+kubectl create -f ./compute-resources.yaml --namespace=myspace
+```
+
+will limit the maxinum device memory allocated to namespace 'myspace' to 30G
+
+## Support multiple schedule policies
+
+Description: HAMi needs to support multiple schedule policies, to provide meets the need in complex senarios, a pod can select a schedule policy in annotations field.
+
+The effect of each schedule policy is shown in the table below
+
+| Schedule Policy | Effect |
+| -------- | ------- |
+| best-fit | the fewer device memory remains, the higher score |
+| idle-first | idle GPU has higher score |
+| numa-first | for multiple GPU allocations, GPUs on the same numa have higher score |
+
+For example, if a pod want to select a 'best-fit' schedule policy, it can specify .metadata.annotations as the code below:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ nvidia.com/schedule-policy: "best-fit"
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command:["bash"，"-c"，"sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # declare how many physical GPUs the pod needs
+```
diff --git a/docs/dynamic-mig-support.md b/docs/dynamic-mig-support.md
new file mode 100644
index 000000000..4e9aaa44a
--- /dev/null
+++ b/docs/dynamic-mig-support.md
@@ -0,0 +1,194 @@
+## Introduction
+
+**We now support dynamic-mig by using mig-parted to adjust mig-devices dynamically**, including:
+
+***Dynamic MIG instance management***: User don't need to operate on GPU node, using 'nvidia-smi -i 0 -mig 1' or other command to manage MIG instance, all will be done by HAMi-device-plugin.
+
+***Dynamic MIG Adjustment***: Each MIG device managed by HAMi will dyamically adjust their MIG template according to tasks submitted when necessary.
+
+***Device MIG Observation***: Each MIG instance generated by HAMi will be shown in scheduler-monitor, including task information. user can get a clear overview of MIG nodes.
+
+***Compatable with HAMi-core nodes***: HAMi can manage a unified GPU pool of `HAMi-core node` and `mig node`. A task can be scheduled to either node if not appointed manually by using `nvidia.com/vgpu-mode` annotation.
+
+***Unified API with HAMi-core***: Zero work needs to be done to make the job compatible with dynamic-mig feature.
+
+## Prerequisites
+
+* NVIDIA Blackwell and Hopper™ and Ampere Devices
+* HAMi > v2.5.0
+* Nvidia-container-toolkit
+
+## Enabling Dynamic-mig Support
+
+* Install the chart using helm, See 'enabling vGPU support in kubernetes' section [here](https://github.com/Project-HAMi/HAMi#enabling-vgpu-support-in-kubernetes)
+
+* Configure `mode` in device-plugin configMap to `mig` for MIG nodes. For more detailed configurations, refer to [node configs](./config.md)
+```
+kubectl describe cm hami-device-plugin -n kube-system
+```
+
+```json
+{
+ "nodeconfig": [
+ {
+ "name": "MIG-NODE-A",
+ "operatingmode": "mig",
+ "filterdevices": {
+ "uuid": [],
+ "index": []
+ }
+ }
+ ]
+}
+```
+
+* Restart the following pods for the change to take effect:
+ * hami-scheduler
+ * hami-device-plugin on 'MIG-NODE-A'
+
+## Custom mig configuration (Optional)
+HAMi currently has a [built-in mig configuration](https://github.com/Project-HAMi/HAMi/blob/master/charts/hami/templates/scheduler/device-configmap.yaml) for MIG.
+
+You can customize the mig configuration by following the steps below:
+
+ ### Change the content of 'device-configmap.yaml' in charts/hami/templates/scheduler, the as follows
+
+ ```yaml
+ nvidia:
+ resourceCountName: {{ .Values.resourceName }}
+ resourceMemoryName: {{ .Values.resourceMem }}
+ resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }}
+ resourceCoreName: {{ .Values.resourceCores }}
+ resourcePriorityName: {{ .Values.resourcePriority }}
+ overwriteEnv: false
+ defaultMemory: 0
+ defaultCores: 0
+ defaultGPUNum: 1
+ deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }}
+ deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }}
+ deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }}
+ knownMigGeometries:
+ - models: [ "A30" ]
+ allowedGeometries:
+ -
+ - name: 1g.6gb
+ memory: 6144
+ count: 4
+ -
+ - name: 2g.12gb
+ memory: 12288
+ count: 2
+ -
+ - name: 4g.24gb
+ memory: 24576
+ count: 1
+ - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB", "A100-SXM4-40GB" ]
+ allowedGeometries:
+ -
+ - name: 1g.5gb
+ memory: 5120
+ count: 7
+ -
+ - name: 2g.10gb
+ memory: 10240
+ count: 3
+ - name: 1g.5gb
+ memory: 5120
+ count: 1
+ -
+ - name: 3g.20gb
+ memory: 20480
+ count: 2
+ -
+ - name: 7g.40gb
+ memory: 40960
+ count: 1
+ - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
+ allowedGeometries:
+ -
+ - name: 1g.10gb
+ memory: 10240
+ count: 7
+ -
+ - name: 2g.20gb
+ memory: 20480
+ count: 3
+ - name: 1g.10gb
+ memory: 10240
+ count: 1
+ -
+ - name: 3g.40gb
+ memory: 40960
+ count: 2
+ -
+ - name: 7g.79gb
+ memory: 80896
+ count: 1
+ ```
+
+ > **Note** Helm installation and updates will be based on the configuration in this file, overwriting the built-in configuration of Helm
+
+ > **Note** Be aware HAMi will find and use the first MIG template suitable to the task in the order of this configMap
+
+## Running MIG jobs
+
+MIG instance can now be requested by a container the same way as using `hami-core`
+simply by specifying the `nvidia.com/gpu` and `nvidia.com/gpumem` resource type.
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ nvidia.com/vgpu-mode: "mig" #(Optional), if not set, this pod can be assigned to a MIG instance or a hami-core instance
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2
+ nvidia.com/gpumem: 8000
+```
+
+In this example above, the task allocates two mig instances, each with at least 8G device memory.
+
+## Monitor MIG Instance
+
+MIG Instance managed by HAMi will be displayed in scheduler monitor(scheduler node ip:31993/metrics), as follows:
+
+```bash
+# HELP nodeGPUMigInstance GPU Sharing mode. 0 for hami-core, 1 for mig, 2 for mps
+# TYPE nodeGPUMigInstance gauge
+nodeGPUMigInstance{deviceidx="0",deviceuuid="GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313",migname="3g.20gb-0",nodeid="aio-node15",zone="vGPU"} 1
+nodeGPUMigInstance{deviceidx="0",deviceuuid="GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313",migname="3g.20gb-1",nodeid="aio-node15",zone="vGPU"} 0
+nodeGPUMigInstance{deviceidx="1",deviceuuid="GPU-30f90f49-43ab-0a78-bf5c-93ed41ef2da2",migname="3g.20gb-0",nodeid="aio-node15",zone="vGPU"} 1
+nodeGPUMigInstance{deviceidx="1",deviceuuid="GPU-30f90f49-43ab-0a78-bf5c-93ed41ef2da2",migname="3g.20gb-1",nodeid="aio-node15",zone="vGPU"} 1
+```
+
+## Notes
+
+1. You don't need to do anything on MIG node, all are managed by mig-parted in hami-device-plugin.
+
+2. Nvidia devices before Ampere architect can't use 'mig' mode
+
+3. You won't see any mig resources(ie, `nvidia.com/mig-1g.10gb`) on node, hami uses a unified resource name for both 'mig' and 'hami-core' node
+
+## Known Issues
+
+### MIG Partitioning Failures in Mixed Deployments
+
+* Scenario: When running workloads with HAMi and other NVIDIA-native components (e.g., dcgm-exporter) mixed on the same GPU node, MIG partitioning is highly likely to fail.
+
+* Error Indication: If the hami-device-plugin logs show the error: nvidia-mig-parted failed with exit...
+
+* Resolution:
+ 1. Use the `lsof` command to view device usage (e.g., for GPU 0):
+ ```
+ lsof /dev/nvidia0
+ ```
+ 2. Stop all non-HAMi processes occupying the target GPU devices.
+
+Based on the output of lsof, stop all processes unrelated to HAMi that are occupying /dev/nvidiaX devices; then recreate the workload.
\ No newline at end of file
diff --git a/docs/dynamic-mig-support_cn.md b/docs/dynamic-mig-support_cn.md
new file mode 100644
index 000000000..dfc584757
--- /dev/null
+++ b/docs/dynamic-mig-support_cn.md
@@ -0,0 +1,193 @@
+## 动态MIG功能简介
+
+**HAMi将在2.5版本后支持动态MIG切分模式**, 其功能包括:
+
+***动态MIG实例管理***: 用户不需要在节点上事先生成MIG实例，HAMi会根据任务需要自动创建
+
+***动态切换MIG切分方案***: HAMi会根据设备上的任务情况和新任务的需求，动态的切换MIG模版
+
+***MIG实例监控***: 每个由HAMi管理的MIG实例都可以在调度器监控中找到，用户可以通过该监控清晰地获取整个集群的MIG视图
+
+***可与使用hami-core的节点进行统一的资源池化***: HAMi将MIG与hami-core这两种切分方案进行了统一的池化处理，若任务未指定切分模式的话，分配给hami-core或者mig都是有可能的
+
+***统一的API***: 使用动态MIG功能完全不需要进行任务层的适配工作
+
+## 需求
+
+* NVIDIA Blackwell and Hopper™ and Ampere Devices
+* HAMi > v2.5.0
+* Nvidia-container-toolkit
+
+## 开启动态MIG功能
+
+* 通过[这里](https://github.com/Project-HAMi/HAMi#enabling-vgpu-support-in-kubernetes)的文档部署HAMi
+
+* 通过以下指令修改configMap，并将节点的工作模式修改为`mig`。详细配置说明见 [node config](./config_cn.md)
+```
+kubectl describe cm hami-device-plugin -n kube-system
+```
+
+```json
+{
+ "nodeconfig": [
+ {
+ "name": "MIG-NODE-A",
+ "operatingmode": "mig",
+ "filterdevices": {
+ "uuid": [],
+ "index": []
+ }
+ }
+ ]
+}
+```
+
+* 重启以下2个pod使修改后的配置生效:
+ * hami-scheduler
+ * 在'MIG-NODE-A'上的hami-device-plugin
+
+## 修改MIG模版列表 (可选)
+
+HAMi目前包含[MIG配置模版](https://github.com/Project-HAMi/HAMi/blob/master/charts/hami/templates/scheduler/device-configmap.yaml)
+
+你可以根据自己的集群环境，通过以下的方式去进行修改:
+
+ ### 修改`charts/hami/templates/scheduler`路径下的`device-configmap.yaml`
+
+ ```yaml
+ nvidia:
+ resourceCountName: {{ .Values.resourceName }}
+ resourceMemoryName: {{ .Values.resourceMem }}
+ resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }}
+ resourceCoreName: {{ .Values.resourceCores }}
+ resourcePriorityName: {{ .Values.resourcePriority }}
+ overwriteEnv: false
+ defaultMemory: 0
+ defaultCores: 0
+ defaultGPUNum: 1
+ deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }}
+ deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }}
+ deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }}
+ knownMigGeometries:
+ - models: [ "A30" ]
+ allowedGeometries:
+ -
+ - name: 1g.6gb
+ memory: 6144
+ count: 4
+ -
+ - name: 2g.12gb
+ memory: 12288
+ count: 2
+ -
+ - name: 4g.24gb
+ memory: 24576
+ count: 1
+ - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB", "A100-SXM4-40GB" ]
+ allowedGeometries:
+ -
+ - name: 1g.5gb
+ memory: 5120
+ count: 7
+ -
+ - name: 2g.10gb
+ memory: 10240
+ count: 3
+ - name: 1g.5gb
+ memory: 5120
+ count: 1
+ -
+ - name: 3g.20gb
+ memory: 20480
+ count: 2
+ -
+ - name: 7g.40gb
+ memory: 40960
+ count: 1
+ - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"]
+ allowedGeometries:
+ -
+ - name: 1g.10gb
+ memory: 10240
+ count: 7
+ -
+ - name: 2g.20gb
+ memory: 20480
+ count: 3
+ - name: 1g.10gb
+ memory: 10240
+ count: 1
+ -
+ - name: 3g.40gb
+ memory: 40960
+ count: 2
+ -
+ - name: 7g.79gb
+ memory: 80896
+ count: 1
+ ```
+ > **Note** 修改后可以通过更新或重新部署chart来生效
+
+ > **Note** 在收到任务请求后，HAMi会在上述定义的MIG模版中的依次查找，直到找到一个可以运行任务的模版
+
+## 使用MIG模式运行任务
+
+MIG实例子可以通过和使用hami-core相同的方式进行申请，只需要指定`nvidia.com/gpu`和`nvidia.com/gpumem`即可
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ nvidia.com/vgpu-mode: "mig" #(Optional), if not set, this pod can be assigned to a MIG instance or a hami-core instance
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2
+ nvidia.com/gpumem: 8000
+```
+
+在上面的例子中，该任务申请了2个MIG实例，每个实例至少需要8G显存
+
+## 监控MIG实例
+
+由HAMi管理和生成的MIG实例可以从调度器监控中看到（scheduler node ip:31993/metrics），如下所示：
+
+```bash
+# HELP nodeGPUMigInstance GPU Sharing mode. 0 for hami-core, 1 for mig, 2 for mps
+# TYPE nodeGPUMigInstance gauge
+nodeGPUMigInstance{deviceidx="0",deviceuuid="GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313",migname="3g.20gb-0",nodeid="aio-node15",zone="vGPU"} 1
+nodeGPUMigInstance{deviceidx="0",deviceuuid="GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313",migname="3g.20gb-1",nodeid="aio-node15",zone="vGPU"} 0
+nodeGPUMigInstance{deviceidx="1",deviceuuid="GPU-30f90f49-43ab-0a78-bf5c-93ed41ef2da2",migname="3g.20gb-0",nodeid="aio-node15",zone="vGPU"} 1
+nodeGPUMigInstance{deviceidx="1",deviceuuid="GPU-30f90f49-43ab-0a78-bf5c-93ed41ef2da2",migname="3g.20gb-1",nodeid="aio-node15",zone="vGPU"} 1
+```
+
+## 备注
+
+1. 你不需要在MIG节点上进行任何操作，所有MIG实例的创建和维护都是由hami-vgpu-device-plugin进行的
+
+2. 安培架构之前的NVIDIA设备无法使用`MIG`模式
+
+3. 你不会在节点上看到MIG资源名(例如, `nvidia.com/mig-1g.10gb`)，HAMi对于hami-core和mig使用统一的资源名进行管理
+
+## 已知问题
+
+### 混合部署中的 MIG 切分失败问题
+
+* 场景： 当在同一个 GPU 节点上混合使用 HAMi 和其他 NVIDIA 原生组件（如 dcgm-exporter等）运行任务时，极有可能导致 MIG 切分失败。
+
+* 错误现象： 如果 hami-device-plugin 日志中出现错误 nvidia-mig-parted failed with exit...。
+
+* 解决方法：
+ 1. 使用 lsof 命令查看设备占用情况 (例如，对于 GPU 0)：
+ ```
+ lsof /dev/nvidia0
+ ```
+ 2. 停止占用目标 GPU 设备的非 HAMi 进程。
+
+根据 lsof 的输出结果，停止所有与 HAMi 无关且正在占用 /dev/nvidiaX 设备的进程 然后重新创建部署即可。
\ No newline at end of file
diff --git a/docs/enflame-gcu-suport.md b/docs/enflame-gcu-suport.md
new file mode 100644
index 000000000..b00a27309
--- /dev/null
+++ b/docs/enflame-gcu-suport.md
@@ -0,0 +1,120 @@
+## Introduction
+
+**We now support sharing on enflame.com/gcu(i.e S60) by implementing most device-sharing features as nvidia-GPU**, including:
+
+***GCU sharing***: Each task can allocate a portion of GCU instead of a whole GCU card, thus GCU can be shared among multiple tasks.
+
+***Device Memory and Core Control***: GCUs can be allocated with certain percentage of device memory and core, we make sure that it does not exceed the boundary.
+
+***Device UUID Selection***: You can specify which GCU devices to use or exclude using annotations.
+
+***Very Easy to use***: You don't need to modify your task yaml to use our scheduler. All your GPU jobs will be automatically supported after installation.
+
+## Prerequisites
+
+* Enflame gcushare-device-plugin >= 2.1.6 (please consult your device provider, gcushare has two components: gcushare-scheduler-plugin and gcushare-device-plugin, we only need gcushare-device-plugin here )
+* driver version >= 1.2.3.14
+* kubernetes >= 1.24
+* enflame-container-toolkit >=2.0.50
+
+## Enabling GCU-sharing Support
+
+* Deploy gcushare-device-plugin on enflame nodes (Please consult your device provider to aquire its package and document)
+
+> **NOTICE:** *Install only gpushare-device-plugin, don't install gpu-scheduler-plugin package.*
+
+> **NOTE:** The default resource names are:
+> - `enflame.com/vgcu` for GCU count, only support 1 now.
+> - `enflame.com/vgcu-percentage` for the percentage of memory and cores in a gcu slice.
+>
+> You can customize these names by modifying `hami-scheduler-device` configMap above.
+
+* Set 'devices.enflame.enabled=true' when deploy HAMi
+
+```
+helm install hami hami-charts/hami --set devices.enflame.enabled=true -n kube-system
+```
+
+## Device Granularity
+
+HAMi divides each Enflame GCU into 100 units for resource allocation. When you request a portion of a GPU, you're actually requesting a certain number of these units.
+
+### GCU Slice Allocation
+
+- Each unit of `enflame.com/vgcu-percentage` represents 1% device memory and 1% core
+- If you don't specify a memory request, the system will default to using 100% of the available memory
+- Memory allocation is enforced with hard limits to ensure tasks don't exceed their allocated memory
+- Core allocation is enforced with hard limits to ensure tasks don't exceed their allocated cores
+
+## Running Enflame jobs
+
+Enflame GCUs can now be requested by a container
+using the `enflame.com/vgcu` and `enflame.com/vgcu-percentage` resource type:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gcushare-pod-2
+ namespace: kube-system
+spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: pod-gcu-example1
+ image: ubuntu:18.04
+ imagePullPolicy: IfNotPresent
+ command:
+ - sleep
+ args:
+ - '100000'
+ resources:
+ limits:
+ enflame.com/vgcu: 1
+ enflame.com/vgcu-percentage: 22
+```
+
+> **NOTICE:** *You can find more examples in [examples/enflame folder](../examples/enflame/)*
+
+## Device UUID Selection
+
+You can specify which GPU devices to use or exclude using annotations:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: poddemo
+ annotations:
+ # Use specific GPU devices (comma-separated list)
+ enflame.com/use-gpuuuid: "node1-enflame-0,node1-enflame-1"
+ # Or exclude specific GPU devices (comma-separated list)
+ enflame.com/nouse-gpuuuid: "node1-enflame-2,node1-enflame-3"
+spec:
+ # ... rest of pod spec
+```
+
+> **NOTE:** The device ID format is `{node-name}-enflame-{index}`. You can find the available device IDs in the node status.
+
+### Finding Device UUIDs
+
+You can find the UUIDs of Enflame GCUs on a node using the following command:
+
+```bash
+kubectl get pod -o yaml | grep -A 10 "hami.io/-devices-allocated"
+```
+
+Or by examining the node annotations:
+
+```bash
+kubectl get node -o yaml | grep -A 10 "hami.io/node-register-"
+```
+
+Look for annotations containing device information in the node status.
+
+## Notes
+
+1. GCUshare takes effect only for containers that apply for one GCU(i.e enflame.com/vgcu=1 ).
+
+2. Multiple GCU allocation in one container is not supported yet
+
+3. `efsmi` inside container shows the total device memory, which is NOT a bug, device memory will be properly limited when running tasks.
\ No newline at end of file
diff --git a/docs/enflame-gcu-support_cn.md b/docs/enflame-gcu-support_cn.md
new file mode 100644
index 000000000..b398844eb
--- /dev/null
+++ b/docs/enflame-gcu-support_cn.md
@@ -0,0 +1,117 @@
+## 简介
+
+本组件支持复用燧原GCU设备(S60)，并为此提供以下几种与vGPU类似的复用功能，包括：
+
+***GPU 共享***: 每个任务可以只占用一部分显卡，多个任务可以共享一张显卡
+
+***百分比切片能力***: 你现在可以用百分比来申请一个GCU切片（例如20%），本组件会确保任务使用的显存和算力不会超过这个百分比对应的数值
+
+***设备 UUID 选择***: 你可以通过注解指定使用或排除特定的GCU设备
+
+***方便易用***: 部署本组件后，只需要部署厂家提供的gcushare-device-plugin即可使用
+
+
+## 节点需求
+
+* Enflame gcushare-device-plugin >= 2.1.6
+* driver version >= 1.2.3.14
+* kubernetes >= 1.24
+* enflame-container-toolkit >=2.0.50
+
+## 开启GCU复用
+
+* 部署'gcushare-device-plugin'，燧原的GCU共享需要配合厂家提供的'gcushare-device-plugin'一起使用，请联系设备提供方获取
+
+> **注意:** *只需要安装gcushare-device-plugin，不要安装gcushare-scheduler-plugin.*
+
+* 在安装HAMi时配置参数'devices.enflame.enabled=true'
+
+```
+helm install hami hami-charts/hami --set devices.enflame.enabled=true -n kube-system
+```
+
+> **说明:** 默认资源名称如下：
+> - `enflame.com/vgcu` 用于GCU数量，这里只能为1
+> - `enflame.com/vgcu-percentage` 用于生成共享GCU切片
+>
+> 你可以通过修改`hami-scheduler-device`配置，来修改这些资源名称
+
+## 设备粒度切分
+
+HAMi 将每个燧原 GCU 划分为 100 个单元进行资源分配。当你请求一部分 GPU 时，实际上是在请求这些单元中的一定数量。
+
+### 内存和核心分配
+
+- 每个 `enflame.com/vgcu-percentage` 单位代表1%的算力和1%的显存
+- 如果不指定内存请求，系统将默认使用 100% 的可用内存
+- 内存与核心的分配通过硬限制强制执行，确保任务不会超过其分配的内存与核心
+
+## 运行GCU任务
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gcushare-pod-2
+ namespace: kube-system
+spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: pod-gcu-example1
+ image: ubuntu:18.04
+ imagePullPolicy: IfNotPresent
+ command:
+ - sleep
+ args:
+ - '100000'
+ resources:
+ limits:
+ enflame.com/vgcu: 1
+ enflame.com/vgcu-percentage: 22
+```
+> **注意:** *查看更多的[用例](../examples/enflame/).*
+
+## 设备 UUID 选择
+
+你可以通过 Pod 注解来指定要使用或排除特定的 GPU 设备：
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: poddemo
+ annotations:
+ # Use specific GPU devices (comma-separated list)
+ enflame.com/use-gpuuuid: "node1-enflame-0,node1-enflame-1"
+ # Or exclude specific GPU devices (comma-separated list)
+ enflame.com/nouse-gpuuuid: "node1-enflame-2,node1-enflame-3"
+spec:
+ # ... rest of pod spec
+```
+
+> **说明:** 设备 ID 格式为 `{节点名称}-enflame-{索引}`。你可以在节点状态中找到可用的设备 ID。
+
+### 查找设备 UUID
+
+你可以使用以下命令查找节点上的燧原 GCU 设备 UUID：
+
+```bash
+kubectl get pod -o yaml | grep -A 10 "hami.io/-devices-allocated"
+```
+
+或者通过检查节点注解：
+
+```bash
+kubectl get node -o yaml | grep -A 10 "hami.io/node-register-"
+```
+
+在节点注解中查找包含设备信息的注解。
+
+
+## 注意事项
+
+1. 共享模式只对申请一张GPU的容器生效（enflame.com/vgcu=1）。
+
+2. 目前暂时不支持一个容器中申请多个GCU设备。
+
+3. 任务中使用`efsmi`可以看到全部的显存，但这并非异常，显存会在任务使用过程中被正确限制。
\ No newline at end of file
diff --git a/docs/gpu-dashboard.json b/docs/gpu-dashboard.json
new file mode 100644
index 000000000..e5fb491d4
--- /dev/null
+++ b/docs/gpu-dashboard.json
@@ -0,0 +1,1834 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_ALL",
+ "label": "ALL",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__elements": [],
+ "__requires": [
+ {
+ "type": "panel",
+ "id": "bargauge",
+ "name": "Bar gauge",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "gauge",
+ "name": "Gauge",
+ "version": ""
+ },
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "8.5.5"
+ },
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph (old)",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "stat",
+ "name": "Stat",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "table",
+ "name": "Table",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "$$hashKey": "object:192",
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "This dashboard is gpu metrics dashboard base on NVIDIA DCGM Exporter and HAMi/k8s-vgpu-scheduler This dashboard is gpu metrics dashboard base on NVIDIA DCGM Exporter and HAMi/k8s-vgpu-scheduler, and was modified from https://grafana.com/grafana/dashboards/21833-hami-vgpu-dashboard/ ",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "gnetId": 21833,
+ "graphTooltip": 0,
+ "id": null,
+ "iteration": 1728370823317,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 0,
+ "y": 0
+ },
+ "id": 47,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "8.5.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "count(DCGM_FI_DEV_DEC_UTIL{node_name=~\"${node_name}\",UUID=~\"${uuid}\"})",
+ "instant": true,
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "GPU Total",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 2,
+ "x": 3,
+ "y": 0
+ },
+ "id": 59,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "8.5.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(count_values(\"DCGM_FI_DEV_XID_ERRORS\", DCGM_FI_DEV_XID_ERRORS{node_name=~\"${node_name}\",UUID=~\"${uuid}\"} > 0)) or on() vector(0)",
+ "instant": true,
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "XID Error",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 83
+ },
+ {
+ "color": "red",
+ "value": 87
+ }
+ ]
+ },
+ "unit": "celsius"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 5,
+ "y": 0
+ },
+ "id": 55,
+ "options": {
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true,
+ "text": {}
+ },
+ "pluginVersion": "8.5.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "avg(DCGM_FI_DEV_GPU_TEMP{node_name=~\"${node_name}\",UUID=~\"${uuid}\"})",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Average GPU Temp",
+ "type": "gauge"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "max": 2400,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 1800
+ },
+ {
+ "color": "red",
+ "value": 2200
+ }
+ ]
+ },
+ "unit": "watt"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 3,
+ "x": 8,
+ "y": 0
+ },
+ "id": 57,
+ "links": [],
+ "options": {
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "sum"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true,
+ "text": {}
+ },
+ "pluginVersion": "8.5.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(DCGM_FI_DEV_POWER_USAGE{node_name=~\"${node_name}\",UUID=~\"${uuid}\"})",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "GPU power usage",
+ "type": "gauge"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 5,
+ "w": 13,
+ "x": 11,
+ "y": 0
+ },
+ "hiddenSeries": false,
+ "id": 12,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": 150,
+ "sort": "current",
+ "sortDesc": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "DCGM_FI_DEV_GPU_TEMP{node_name=~\"${node_name}\",UUID=~\"${uuid}\"}",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "{{node_name}} uid:{{UUID}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "GPU temp(DCGM)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:97",
+ "format": "celsius",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:98",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "filterable": false,
+ "inspect": false
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 11,
+ "x": 0,
+ "y": 5
+ },
+ "id": 53,
+ "options": {
+ "footer": {
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "instance"
+ }
+ ]
+ },
+ "pluginVersion": "8.5.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "nodeGPUOverview{nodeid=~\"${node_name}\",deviceuuid=~\"${uuid}\"}",
+ "instant": true,
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "nodeGPUOverview",
+ "transformations": [
+ {
+ "id": "labelsToFields",
+ "options": {}
+ },
+ {
+ "id": "merge",
+ "options": {}
+ },
+ {
+ "id": "merge",
+ "options": {}
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Time": true,
+ "Value": true,
+ "branch": true,
+ "container": true,
+ "dc": true,
+ "devicecores": true,
+ "deviceidx": true,
+ "devicememorylimit": false,
+ "goversion": true,
+ "instance": true,
+ "ip": true,
+ "job": true,
+ "namespace": true,
+ "node_name": true,
+ "nodeid": false,
+ "pod": true,
+ "project": true,
+ "revision": true,
+ "sharedcontainers": false,
+ "zone": true
+ },
+ "indexByName": {
+ "Time": 2,
+ "Value": 3,
+ "dc": 4,
+ "devicecores": 5,
+ "deviceidx": 6,
+ "devicememorylimit": 11,
+ "devicetype": 10,
+ "deviceuuid": 12,
+ "instance": 1,
+ "ip": 9,
+ "job": 0,
+ "node_name": 8,
+ "nodeid": 7,
+ "project": 13,
+ "sharedcontainers": 14,
+ "zone": 15
+ },
+ "renameByName": {
+ "devicememorylimit": "vram",
+ "devicetype": "",
+ "node_name": "",
+ "sharedcontainers": ""
+ }
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "align": "auto",
+ "displayMode": "auto",
+ "filterable": false,
+ "inspect": false
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 13,
+ "x": 11,
+ "y": 5
+ },
+ "id": 61,
+ "options": {
+ "footer": {
+ "fields": "",
+ "reducer": [
+ "sum"
+ ],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "instance"
+ }
+ ]
+ },
+ "pluginVersion": "8.5.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "DCGM_FI_DEV_POWER_USAGE{node_name=~\"${node_name}\", UUID=~\"${uuid}\"}",
+ "instant": true,
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "nodeGPUList(DCGM)",
+ "transformations": [
+ {
+ "id": "labelsToFields",
+ "options": {}
+ },
+ {
+ "id": "merge",
+ "options": {}
+ },
+ {
+ "id": "merge",
+ "options": {}
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "DCGM_FI_CUDA_DRIVER_VERSION": false,
+ "DCGM_FI_DEV_BRAND": true,
+ "DCGM_FI_DEV_MINOR_NUMBER": true,
+ "DCGM_FI_DEV_NAME": true,
+ "DCGM_FI_DEV_SERIAL": true,
+ "DCGM_FI_DRIVER_VERSION": false,
+ "DCGM_FI_PROCESS_NAME": true,
+ "Hostname": true,
+ "Time": true,
+ "UUID": false,
+ "Value": true,
+ "branch": true,
+ "container": true,
+ "dc": true,
+ "device": true,
+ "devicecores": true,
+ "deviceidx": true,
+ "goversion": true,
+ "gpu": true,
+ "instance": true,
+ "ip": true,
+ "job": true,
+ "modelName": false,
+ "namespace": true,
+ "node_name": false,
+ "nodeid": false,
+ "pod": true,
+ "project": true,
+ "revision": true,
+ "sharedcontainers": false,
+ "zone": true
+ },
+ "indexByName": {
+ "DCGM_FI_CUDA_DRIVER_VERSION": 11,
+ "DCGM_FI_DEV_BRAND": 13,
+ "DCGM_FI_DEV_MINOR_NUMBER": 14,
+ "DCGM_FI_DEV_NAME": 15,
+ "DCGM_FI_DEV_SERIAL": 16,
+ "DCGM_FI_DRIVER_VERSION": 12,
+ "DCGM_FI_PROCESS_NAME": 17,
+ "Hostname": 6,
+ "Time": 1,
+ "UUID": 19,
+ "Value": 2,
+ "dc": 3,
+ "device": 10,
+ "gpu": 7,
+ "instance": 5,
+ "ip": 18,
+ "job": 0,
+ "modelName": 9,
+ "node_name": 8,
+ "project": 4
+ },
+ "renameByName": {
+ "devicememorylimit": "vram",
+ "devicetype": "",
+ "node_name": "",
+ "sharedcontainers": ""
+ }
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 11,
+ "x": 0,
+ "y": 10
+ },
+ "id": 49,
+ "interval": "",
+ "links": [],
+ "options": {
+ "displayMode": "basic",
+ "minVizHeight": 10,
+ "minVizWidth": 0,
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "text": {}
+ },
+ "pluginVersion": "8.5.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "sum by (deviceuuid, nodename) (vGPUCorePercentage{nodename=~\"${node_name}\"})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": " {{nodename}}:{{deviceuuid}}",
+ "refId": "A"
+ }
+ ],
+ "title": "vGPUCorePercentage",
+ "type": "bargauge"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 7,
+ "x": 11,
+ "y": 10
+ },
+ "hiddenSeries": false,
+ "id": 18,
+ "legend": {
+ "avg": false,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "DCGM_FI_DEV_FB_USED{node_name=~\"${node_name}\", UUID=~\"${uuid}\"}",
+ "interval": "",
+ "legendFormat": "{{node_name}} uid:{{UUID}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "GPU FB used(DCGM)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:618",
+ "format": "decmbytes",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:619",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 6,
+ "x": 18,
+ "y": 10
+ },
+ "hiddenSeries": false,
+ "id": 6,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "DCGM_FI_DEV_GPU_UTIL{node_name=~\"${node_name}\", UUID=~\"${uuid}\"}",
+ "interval": "",
+ "legendFormat": "{{node_name}} uid:{{UUID}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "GPU util(DCGM)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:699",
+ "format": "percent",
+ "logBase": 1,
+ "max": "100",
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:700",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 11,
+ "x": 0,
+ "y": 15
+ },
+ "id": 51,
+ "interval": "",
+ "links": [],
+ "options": {
+ "displayMode": "basic",
+ "minVizHeight": 10,
+ "minVizWidth": 0,
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "text": {}
+ },
+ "pluginVersion": "8.5.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "sum by (deviceuuid, nodename) (vGPUMemoryPercentage{nodename=~\"${node_name}\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{nodename}}:{{deviceuuid}}",
+ "refId": "A"
+ }
+ ],
+ "title": "vGPUMemoryPercentage",
+ "type": "bargauge"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 7,
+ "x": 11,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 10,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "DCGM_FI_DEV_POWER_USAGE{node_name=~\"${node_name}\",UUID=~\"${uuid}\"}",
+ "interval": "",
+ "legendFormat": "{{node_name}} uid:{{UUID}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "GPU power usage(DCGM)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:214",
+ "format": "watt",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:215",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 6,
+ "x": 18,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 65,
+ "interval": "",
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "sideWidth": 80,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "DCGM_FI_DEV_SM_CLOCK{node_name=~\"${node_name}\", UUID=~\"${uuid}\"} * 1000000",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{node_name}} uid:{{UUID}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "GPU SM Clock(DCGM)",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:536",
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:537",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "max": 10,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "index": 0
+ },
+ {
+ "color": "red",
+ "index": 1,
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 11,
+ "x": 0,
+ "y": 20
+ },
+ "id": 36,
+ "interval": "",
+ "links": [],
+ "options": {
+ "displayMode": "basic",
+ "minVizHeight": 10,
+ "minVizWidth": 0,
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "text": {}
+ },
+ "pluginVersion": "8.5.5",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "count by (node_name) (vGPU_device_memory_usage_in_bytes{node_name=~\"${node_name}\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{node_name}}",
+ "refId": "A"
+ }
+ ],
+ "title": "vgpu used by nodes",
+ "type": "bargauge"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 11,
+ "x": 0,
+ "y": 25
+ },
+ "hiddenSeries": false,
+ "id": 24,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": 150,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "sum by (podname) (Device_memory_desc_of_container{node_name=~\"${node_name}\",deviceuuid=~\"${uuid}\"})",
+ "interval": "",
+ "legendFormat": "{{podname}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "HAMi-Memory desc of container ",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:779",
+ "format": "bytes",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:780",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "decimals": 150,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 13,
+ "x": 11,
+ "y": 25
+ },
+ "hiddenSeries": false,
+ "id": 38,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": 150,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "sum by (podname) (Device_utilization_desc_of_container{node_name=~\"${node_name}\",deviceuuid=~\"${uuid}\"})",
+ "interval": "",
+ "legendFormat": "{{podname}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "HAMi-GPU util desc of container",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1243",
+ "format": "percent",
+ "logBase": 1,
+ "max": "100",
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1244",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 11,
+ "x": 0,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 22,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": 150,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "HostGPUMemoryUsage{node_name=~\"${node_name}\",deviceuuid=~\"${uuid}\"}",
+ "interval": "",
+ "legendFormat": "{{node_name}} uid:{{deviceuuid}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "HAMi-Host memory usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1087",
+ "format": "bytes",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1088",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 13,
+ "x": 11,
+ "y": 35
+ },
+ "hiddenSeries": false,
+ "id": 20,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sideWidth": 150,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "8.5.5",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "exemplar": true,
+ "expr": "HostCoreUtilization{node_name=~\"${node_name}\",deviceuuid=~\"${uuid}\"}",
+ "interval": "",
+ "legendFormat": "{{node_name}} uid:{{deviceuuid}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "HAMi-Host core util",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1243",
+ "format": "percent",
+ "logBase": 1,
+ "max": "100",
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1244",
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "refresh": "5s",
+ "schemaVersion": 36,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "definition": "label_values({__name__=~\"DCGM_FI_DEV_FB_FREE|vGPU_device_memory_limit_in_bytes\"}, node_name)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "node_name",
+ "options": [],
+ "query": {
+ "query": "label_values({__name__=~\"DCGM_FI_DEV_FB_FREE|vGPU_device_memory_limit_in_bytes\"}, node_name)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_ALL}"
+ },
+ "definition": "label_values(DCGM_FI_DEV_FB_FREE{node_name=~\"$node_name\"},UUID)",
+ "hide": 0,
+ "includeAll": true,
+ "multi": true,
+ "name": "uuid",
+ "options": [],
+ "query": {
+ "query": "label_values(DCGM_FI_DEV_FB_FREE{node_name=~\"$node_name\"},UUID)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-12h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "hami-vgpu-metrics-dashboard",
+ "uid": "Oxed_c6Wz22",
+ "version": 2,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/docs/how-to-profiling-scheduler.md b/docs/how-to-profiling-scheduler.md
new file mode 100644
index 000000000..1308edf04
--- /dev/null
+++ b/docs/how-to-profiling-scheduler.md
@@ -0,0 +1,213 @@
+# How to profiling HAMi scheduler
+
+## Prerequisite
+### Enable profiling
+- Add the `--profiling` flag to the `extraArgs` field of `scheduler.extender` in the Helm Chart to make pprof available via HTTP(s) server on ``.
+``` yaml
+scheduler:
+ ...
+ extender:
+ extraArgs:
+ - --debug
+ - -v=4
+ - --profiling
+```
+
+### Prepare profiling environment
+- [Install Go](https://go.dev/doc/install) on your system.
+- Get HAMi [source code](https://github.com/Project-HAMi/HAMi) and place it at `/k8s-vgpu`
+- Install dependencies by running:
+``` shell
+cd /k8s-vgpu
+make tidy
+go install github.com/NVIDIA/mig-parted/cmd/nvidia-mig-parted@v0.10.0
+```
+
+### (Optional) Prepare profiling image
+- Get HAMi source code
+- Checkout the target version
+- Building image
+``` Dockerfile
+FROM golang:1.24.4-bullseye
+ADD . /k8s-vgpu
+RUN cd /k8s-vgpu && make tidy
+RUN go install github.com/NVIDIA/mig-parted/cmd/nvidia-mig-parted@v0.10.0
+```
+
+## Profiling scheduler
+**Note**: If HAMi source code and dependencies are correctly placed, you can use the `list` command in pprof to view source code. Otherwise, you may encounter a `no such file or directory` error.
+For detailed information about pprof, refer to the [official documentation](https://pkg.go.dev/net/http/pprof)
+
+### CPU Profiling
+Run the following command to capture a 120-second CPU profile:
+```bash
+go tool pprof --seconds 120 https+insecure:///debug/pprof/profile`
+```
+Example output:
+```bash
+root@hami-pprof-76cfcb66f6-jpjnm:/# go tool pprof --seconds 120 https+insecure:://10.42.0.24/debug/pprof/profile
+Fetching profile over HTTP from https+insecure://10.42.0.24/debug/pprof/profile?seconds=120
+Please wait... (2m0s)
+Saved profile in /root/pprof/pprof.scheduler.samples.cpu.002.pb.gz
+File: scheduler
+Type: cpu
+Time: 2025-04-01 07:08:42 UTC
+Duration: 120s, Total samples = 10ms (0.0083%)
+Entering interactive mode (type "help" for commands, "o" for options)
+(pprof) top
+Showing nodes accounting for 10ms, 100% of 10ms total
+Showing top 10 nodes out of 12
+ flat flat% sum% cum cum%
+ 10ms 100% 100% 10ms 100% sigs.k8s.io/json/internal/golang/encoding/json.unquoteBytes
+ 0 0% 100% 10ms 100% k8s.io/apimachinery/pkg/runtime.Decode (inline)
+ 0 0% 100% 10ms 100% k8s.io/apimachinery/pkg/runtime.WithoutVersionDecoder.Decode
+ 0 0% 100% 10ms 100% k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).Decode
+ 0 0% 100% 10ms 100% k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).unmarshal
+ 0 0% 100% 10ms 100% k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive
+ 0 0% 100% 10ms 100% k8s.io/client-go/rest/watch.(*Decoder).Decode
+ 0 0% 100% 10ms 100% sigs.k8s.io/json.UnmarshalCaseSensitivePreserveInts (inline)
+ 0 0% 100% 10ms 100% sigs.k8s.io/json/internal/golang/encoding/json.(*decodeState).object
+ 0 0% 100% 10ms 100% sigs.k8s.io/json/internal/golang/encoding/json.(*decodeState).unmarshal
+```
+
+### Memory Profiling
+To analyze memory usage, you have two options:
+- Current live objects
+```
+go tool pprof https+insecure:///debug/pprof/heap
+```
+- Cumulative allocation history
+```
+go tool pprof https+insecure:///debug/pprof/allocs
+```
+Example output from the allocation profile:
+
+```bash
+root@hami-scheduler-ffd687cb7-7gqm2:/# /usr/local/go/bin/go tool pprof --seconds 120 https+insecure://10.42.0.24/debug/pprof/allocs
+Fetching profile over HTTP from https+insecure://10.42.0.24/debug/pprof/allocs?seconds=120
+Saved profile in /root/pprof/pprof.scheduler.alloc_objects.alloc_space.inuse_objects.inuse_space.041.pb.gz
+File: scheduler
+Type: alloc_space
+Time: 2025-04-01 07:03:05 UTC
+Entering interactive mode (type "help" for commands, "o" for options)
+(pprof) top
+Showing nodes accounting for 4383.93MB, 69.18% of 6336.84MB total
+Dropped 376 nodes (cum <= 31.68MB)
+Showing top 10 nodes out of 164
+ flat flat% sum% cum cum%
+ 1114.44MB 17.59% 17.59% 1114.94MB 17.59% io.ReadAll
+ 980.52MB 15.47% 33.06% 980.52MB 15.47% sync.(*Pool).pinSlow
+ 606.88MB 9.58% 42.64% 606.88MB 9.58% golang.org/x/net/http2.init.func5
+ 357.15MB 5.64% 48.27% 357.15MB 5.64% k8s.io/apimachinery/pkg/runtime.(*RawExtension).UnmarshalJSON
+ 293.20MB 4.63% 52.90% 293.20MB 4.63% reflect.mapassign_faststr0
+ 265.58MB 4.19% 57.09% 265.58MB 4.19% reflect.unsafe_NewArray
+ 234.07MB 3.69% 60.78% 461.59MB 7.28% sigs.k8s.io/json/internal/golang/encoding/json.(*decodeState).literalStore
+ 210.54MB 3.32% 64.11% 3409.63MB 53.81% github.com/Project-HAMi/HAMi/pkg/scheduler.(*Scheduler).RegisterFromNodeAnnotations
+ 162.02MB 2.56% 66.66% 331.76MB 5.24% github.com/Project-HAMi/HAMi/pkg/scheduler.(*Scheduler).getNodesUsage
+ 159.52MB 2.52% 69.18% 225.53MB 3.56% encoding/json.Unmarshal
+(pprof) list RegisterFromNodeAnnotations
+Total: 6.21GB
+ROUTINE ======================== github.com/Project-HAMi/HAMi/pkg/scheduler.(*Scheduler).RegisterFromNodeAnnotations in /k8s-vgpu/pkg/scheduler/scheduler.go
+ 210.54MB 3.33GB (flat, cum) 53.73% of Total
+ . . 158:func (s *Scheduler) RegisterFromNodeAnnotations() {
+ . . 159: klog.InfoS("Entering RegisterFromNodeAnnotations")
+ . . 160: defer klog.InfoS("Exiting RegisterFromNodeAnnotations")
+ . . 161: ticker := time.NewTicker(time.Second * 15)
+ . . 162: defer ticker.Stop()
+ . . 163: printedLog := map[string]bool{}
+ . . 164: for {
+ 512.05kB 512.05kB 165: select {
+ . . 166: case <-s.nodeNotify:
+ . . 167: klog.V(5).InfoS("Received node notification")
+ . . 168: case <-ticker.C:
+ . 46.84MB 169: klog.InfoS("Ticker triggered")
+ . . 170: case <-s.stopCh:
+ . . 171: klog.InfoS("Received stop signal, exiting RegisterFromNodeAnnotations")
+ . . 172: return
+ . . 173: }
+ . . 174: labelSelector := labels.Everything()
+ . . 175: if len(config.NodeLabelSelector) > 0 {
+ . . 176: labelSelector = (labels.Set)(config.NodeLabelSelector).AsSelector()
+ . . 177: klog.InfoS("Using label selector", "selector", labelSelector.String())
+ . . 178: }
+ . . 179: rawNodes, err := s.nodeLister.List(labelSelector)
+ . . 180: if err != nil {
+ . . 181: klog.ErrorS(err, "Failed to list nodes with selector", "selector", labelSelector.String())
+ . . 182: continue
+ . . 183: }
+ 1MB 1MB 184: klog.V(5).InfoS("Listed nodes", "nodeCount", len(rawNodes))
+ . . 185: var nodeNames []string
+ . . 186: for _, val := range rawNodes {
+ 1.50MB 1.50MB 187: nodeNames = append(nodeNames, val.Name)
+ 5.50MB 5.50MB 188: klog.V(5).InfoS("Processing node", "nodeName", val.Name)
+ . . 189:
+ . . 190: for devhandsk, devInstance := range device.GetDevices() {
+ 36.50MB 36.50MB 191: klog.V(5).InfoS("Checking device health", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . . 192:
+ . 4MB 193: health, needUpdate := devInstance.CheckHealth(devhandsk, val)
+ 72.51MB 72.51MB 194: klog.V(5).InfoS("Device health check result", "nodeName", val.Name, "deviceVendor", devhandsk, "health", health, "needUpdate", needUpdate)
+ . . 195:
+ . . 196: if !health {
+ . . 197: klog.Warning("Device is unhealthy, cleaning up node", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . . 198: err := devInstance.NodeCleanUp(val.Name)
+ . . 199: if err != nil {
+ . . 200: klog.ErrorS(err, "Node cleanup failed", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . . 201: }
+ . . 202:
+ . . 203: info, ok := s.nodes[val.Name]
+ . . 204: if ok {
+ . . 205: klog.InfoS("Removing device from node", "nodeName", val.Name, "deviceVendor", devhandsk, "remainingDevices", s.nodes[val.Name].Devices)
+ . . 206: s.rmNodeDevice(val.Name, info, devhandsk)
+ . . 207: }
+ . . 208: continue
+ . . 209: }
+ . . 210: if !needUpdate {
+ 8MB 8MB 211: klog.V(5).InfoS("No update needed for device", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . . 212: continue
+ . . 213: }
+ . . 214: _, ok := util.HandshakeAnnos[devhandsk]
+ . . 215: if ok {
+ 1.50MB 1.50MB 216: tmppat := make(map[string]string)
+ 5.50MB 5.50MB 217: tmppat[util.HandshakeAnnos[devhandsk]] = "Requesting_" + time.Now().Format(time.DateTime)
+ 2MB 203.15MB 218: klog.InfoS("New timestamp for annotation", "nodeName", val.Name, "annotationKey", util.HandshakeAnnos[devhandsk], "annotationValue", tmppat[util.HandshakeAnnos[devhandsk]])
+ . 1.25GB 219: n, err := util.GetNode(val.Name)
+ . . 220: if err != nil {
+ . . 221: klog.ErrorS(err, "Failed to get node", "nodeName", val.Name)
+ . . 222: continue
+ . . 223: }
+ 512.03kB 512.03kB 224: klog.V(5).InfoS("Patching node annotations", "nodeName", val.Name, "annotations", tmppat)
+ . 1.21GB 225: if err := util.PatchNodeAnnotations(n, tmppat); err != nil {
+ . . 226: klog.ErrorS(err, "Failed to patch node annotations", "nodeName", val.Name)
+ . . 227: }
+ . . 228: }
+ 11.50MB 11.50MB 229: nodeInfo := &util.NodeInfo{}
+ . . 230: nodeInfo.ID = val.Name
+ . . 231: nodeInfo.Node = val
+ 24.50MB 24.50MB 232: klog.V(5).InfoS("Fetching node devices", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . 78.03MB 233: nodedevices, err := devInstance.GetNodeDevices(*val)
+ . . 234: if err != nil {
+ . . 235: klog.ErrorS(err, "Failed to get node devices", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . . 236: continue
+ . . 237: }
+ . . 238: nodeInfo.Devices = make([]util.DeviceInfo, 0)
+ . . 239: for _, deviceinfo := range nodedevices {
+ 36.03MB 36.03MB 240: nodeInfo.Devices = append(nodeInfo.Devices, *deviceinfo)
+ . . 241: }
+ . 21.52MB 242: s.addNode(val.Name, nodeInfo)
+ . . 243: if s.nodes[val.Name] != nil && len(nodeInfo.Devices) > 0 {
+ . . 244: if printedLog[val.Name] {
+ 3.50MB 3.50MB 245: klog.V(5).InfoS("Node device updated", "nodeName", val.Name, "deviceVendor", devhandsk, "nodeInfo", nodeInfo, "totalDevices", s.nodes[val.Name].Devices)
+ . . 246: } else {
+ . . 247: klog.InfoS("Node device added", "nodeName", val.Name, "deviceVendor", devhandsk, "nodeInfo", nodeInfo, "totalDevices", s.nodes[val.Name].Devices)
+ . . 248: printedLog[val.Name] = true
+ . . 249: }
+ . . 250: }
+ . . 251: }
+ . . 252: }
+ . 332.76MB 253: _, _, err = s.getNodesUsage(&nodeNames, nil)
+ . . 254: if err != nil {
+ . . 255: klog.ErrorS(err, "Failed to get node usage", "nodeNames", nodeNames)
+ . . 256: }
+ . . 257: }
+ . . 258:}
+```
diff --git a/docs/how-to-profiling-scheduler_cn.md b/docs/how-to-profiling-scheduler_cn.md
new file mode 100644
index 000000000..dc1d64dcc
--- /dev/null
+++ b/docs/how-to-profiling-scheduler_cn.md
@@ -0,0 +1,192 @@
+# HAMi 调度器性能分析指南
+
+## 前提条件
+### 启用性能分析
+- 在 Helm Chart 的 `scheduler.extender` 的 `extraArgs` 字段中添加 `--profiling` 标志，使 pprof 在 上的 HTTP(s) 服务器可用可以被访问到
+```yaml
+scheduler:
+ ...
+ extender:
+ extraArgs:
+ - --debug
+ - -v=4
+ - --profiling
+```
+
+### 准备性能分析环境
+- 在您的系统上[安装 Go](https://go.dev/doc/install)。
+- 获取 HAMi [源代码](https://github.com/Project-HAMi/HAMi)并将其放置在 `/k8s-vgpu` 目录下
+- 通过运行以下命令安装依赖项：
+```shell
+cd /k8s-vgpu
+make tidy
+go install github.com/NVIDIA/mig-parted/cmd/nvidia-mig-parted@v0.10.0
+```
+
+### （可选）准备性能分析镜像
+- 获取 HAMi 源代码
+- 检出目标版本
+- 构建镜像
+```Dockerfile
+FROM golang:1.24.4-bullseye
+ADD . /k8s-vgpu
+RUN cd /k8s-vgpu && make tidy
+RUN go install github.com/NVIDIA/mig-parted/cmd/nvidia-mig-parted@v0.10.0
+```
+
+## 对调度器进行性能分析
+**注意**：如果 HAMi 源代码和依赖项正确放置，您可以在 pprof 中使用 `list` 命令查看源代码。否则，您可能会遇到 `no such file or directory` 错误。
+有关 pprof 的详细信息，请参阅[官方文档](https://pkg.go.dev/net/http/pprof)
+
+### CPU 性能分析
+运行以下命令捕获 120 秒的 CPU 性能分析：
+```bash
+go tool pprof --seconds 120 https+insecure:///debug/pprof/profile`
+```
+示例输出：
+```bash
+root@hami-pprof-76cfcb66f6-jpjnm:/# go tool pprof --seconds 120 https+insecure://10.42.0.24/debug/pprof/profile
+Fetching profile over HTTP from https+insecure://10.42.0.24/debug/pprof/profile?seconds=120
+Please wait... (2m0s)
+Saved profile in /root/pprof/pprof.scheduler.samples.cpu.002.pb.gz
+File: scheduler
+Type: cpu
+Time: 2025-04-01 07:08:42 UTC
+Duration: 120s, Total samples = 10ms (0.0083%)
+Entering interactive mode (type "help" for commands, "o" for options)
+(pprof) top
+Showing nodes accounting for 10ms, 100% of 10ms total
+Showing top 10 nodes out of 12
+ flat flat% sum% cum cum%
+ 10ms 100% 100% 10ms 100% sigs.k8s.io/json/internal/golang/encoding/json.unquoteBytes
+ 0 0% 100% 10ms 100% k8s.io/apimachinery/pkg/runtime.Decode (inline)
+ 0 0% 100% 10ms 100% k8s.io/apimachinery/pkg/runtime.WithoutVersionDecoder.Decode
+ 0 0% 100% 10ms 100% k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).Decode
+ 0 0% 100% 10ms 100% k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).unmarshal
+ 0 0% 100% 10ms 100% k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive
+ 0 0% 100% 10ms 100% k8s.io/client-go/rest/watch.(*Decoder).Decode
+ 0 0% 100% 10ms 100% sigs.k8s.io/json.UnmarshalCaseSensitivePreserveInts (inline)
+ 0 0% 100% 10ms 100% sigs.k8s.io/json/internal/golang/encoding/json.(*decodeState).object
+ 0 0% 100% 10ms 100% sigs.k8s.io/json/internal/golang/encoding/json.(*decodeState).unmarshal
+```
+
+### 内存性能分析
+要分析内存使用情况，您有两个选项：
+- 当前活动对象
+```
+go tool pprof https+insecure:///debug/pprof/heap
+```
+- 累计分配历史
+```
+go tool pprof https+insecure:///debug/pprof/allocs
+```
+分配配置文件的示例输出：
+
+```bash
+root@hami-pprof-76cfcb66f6-jpjnm:/# go tool pprof https+insecure://10.42.0.24/debug/pprof/allocs
+Fetching profile over HTTP from https+insecure://10.42.0.24/debug/pprof/allocs
+Saved profile in /root/pprof/pprof.scheduler.alloc_objects.alloc_space.inuse_objects.inuse_space.041.pb.gz
+File: scheduler
+Type: alloc_space
+Time: 2025-04-01 07:03:05 UTC
+Entering interactive mode (type "help" for commands, "o" for options)
+(pprof) top
+Showing nodes accounting for 4383.93MB, 69.18% of 6336.84MB total
+Dropped 376 nodes (cum <= 31.68MB)
+Showing top 10 nodes out of 164
+ flat flat% sum% cum cum%
+ 1114.44MB 17.59% 17.59% 1114.94MB 17.59% io.ReadAll
+ 980.52MB 15.47% 33.06% 980.52MB 15.47% sync.(*Pool).pinSlow
+ 606.88MB 9.58% 42.64% 606.88MB 9.58% golang.org/x/net/http2.init.func5
+ 357.15MB 5.64% 48.27% 357.15MB 5.64% k8s.io/apimachinery/pkg/runtime.(*RawExtension).UnmarshalJSON
+ 293.20MB 4.63% 52.90% 293.20MB 4.63% reflect.mapassign_faststr0
+ 265.58MB 4.19% 57.09% 265.58MB 4.19% reflect.unsafe_NewArray
+ 234.07MB 3.69% 60.78% 461.59MB 7.28% sigs.k8s.io/json/internal/golang/encoding/json.(*decodeState).literalStore
+ 210.54MB 3.32% 64.11% 3409.63MB 53.81% github.com/Project-HAMi/HAMi/pkg/scheduler.(*Scheduler).RegisterFromNodeAnnotations
+ 162.02MB 2.56% 66.66% 331.76MB 5.24% github.com/Project-HAMi/HAMi/pkg/scheduler.(*Scheduler).getNodesUsage
+ 159.52MB 2.52% 69.18% 225.53MB 3.56% encoding/json.Unmarshal
+(pprof) list RegisterFromNodeAnnotations
+Total: 6.21GB
+ROUTINE ======================== github.com/Project-HAMi/HAMi/pkg/scheduler.(*Scheduler).RegisterFromNodeAnnotations in /k8s-vgpu/pkg/scheduler/scheduler.go
+ 210.54MB 3.33GB (flat, cum) 53.73% of Total
+ . . 158:func (s *Scheduler) RegisterFromNodeAnnotations() {
+ . . 159: klog.InfoS("Entering RegisterFromNodeAnnotations")
+ . . 160: defer klog.InfoS("Exiting RegisterFromNodeAnnotations")
+ . . 161: ticker := time.NewTicker(time.Second * 15)
+ . . 162: defer ticker.Stop()
+ . . 163: printedLog := map[string]bool{}
+ . . 164: for {
+ 512.05kB 512.05kB 165: select {
+ . . 166: case <-s.nodeNotify:
+ . . 167: klog.V(5).InfoS("Received node notification")
+ . . 168: case <-ticker.C:
+ . 46.84MB 169: klog.InfoS("Ticker triggered")
+ . . 170: case <-s.stopCh:
+ . . 171: klog.InfoS("Received stop signal, exiting RegisterFromNodeAnnotations")
+ . . 172: return
+ . . 173: }
+ . . 174: labelSelector := labels.Everything()
+ . . 175: if len(config.NodeLabelSelector) > 0 {
+ . . 176: labelSelector = (labels.Set)(config.NodeLabelSelector).AsSelector()
+ . . 177: klog.InfoS("Using label selector", "selector", labelSelector.String())
+ . . 178: }
+ . . 179: rawNodes, err := s.nodeLister.List(labelSelector)
+ . . 180: if err != nil {
+ . . 181: klog.ErrorS(err, "Failed to list nodes with selector", "selector", labelSelector.String())
+ . . 182: continue
+ . . 183: }
+ 1MB 1MB 184: klog.V(5).InfoS("Listed nodes", "nodeCount", len(rawNodes))
+ . . 185: var nodeNames []string
+ . . 186: for _, val := range rawNodes {
+ 1.50MB 1.50MB 187: nodeNames = append(nodeNames, val.Name)
+ 5.50MB 5.50MB 188: klog.V(5).InfoS("Processing node", "nodeName", val.Name)
+ . . 189:
+ . . 190: for devhandsk, devInstance := range device.GetDevices() {
+ 36.50MB 36.50MB 191: klog.V(5).InfoS("Checking device health", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . . 192:
+ . 4MB 193: health, needUpdate := devInstance.CheckHealth(devhandsk, val)
+ 72.51MB 72.51MB 194: klog.V(5).InfoS("Device health check result", "nodeName", val.Name, "deviceVendor", devhandsk, "health", health, "needUpdate", needUpdate)
+ . . 195:
+ . . 196: if !health {
+ . . 197: klog.Warning("Device is unhealthy, cleaning up node", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . . 198: err := devInstance.NodeCleanUp(val.Name)
+ . . 199: if err != nil {
+ . . 200: klog.ErrorS(err, "Node cleanup failed", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . . 201: }
+ . . 202:
+ . . 203: info, ok := s.nodes[val.Name]
+ . . 204: if ok {
+ . . 205: klog.InfoS("Removing device from node", "nodeName", val.Name, "deviceVendor", devhandsk, "remainingDevices", s.nodes[val.Name].Devices)
+ . . 206: s.rmNodeDevice(val.Name, info, devhandsk)
+ . . 207: }
+ . . 208: continue
+ . . 209: }
+ . . 210: if !needUpdate {
+ 8MB 8MB 211: klog.V(5).InfoS("No update needed for device", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . . 212: continue
+ . . 213: }
+ . . 214: _, ok := util.HandshakeAnnos[devhandsk]
+ . . 215: if ok {
+ 1.50MB 1.50MB 216: tmppat := make(map[string]string)
+ 5.50MB 5.50MB 217: tmppat[util.HandshakeAnnos[devhandsk]] = "Requesting_" + time.Now().Format(time.DateTime)
+ 2MB 203.15MB 218: klog.InfoS("New timestamp for annotation", "nodeName", val.Name, "annotationKey", util.HandshakeAnnos[devhandsk], "annotationValue", tmppat[util.HandshakeAnnos[devhandsk]])
+ . 1.25GB 219: n, err := util.GetNode(val.Name)
+ . . 220: if err != nil {
+ . . 221: klog.ErrorS(err, "Failed to get node", "nodeName", val.Name)
+ . . 222: continue
+ . . 223: }
+ 512.03kB 512.03kB 224: klog.V(5).InfoS("Patching node annotations", "nodeName", val.Name, "annotations", tmppat)
+ . 1.21GB 225: if err := util.PatchNodeAnnotations(n, tmppat); err != nil {
+ . . 226: klog.ErrorS(err, "Failed to patch node annotations", "nodeName", val.Name)
+ . . 227: }
+ . . 228: }
+ 11.50MB 11.50MB 229: nodeInfo := &util.NodeInfo{}
+ . . 230: nodeInfo.ID = val.Name
+ . . 231: nodeInfo.Node = val
+ 24.50MB 24.50MB 232: klog.V(5).InfoS("Fetching node devices", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . 78.03MB 233: nodedevices, err := devInstance.GetNodeDevices(*val)
+ . . 234: if err != nil {
+ . . 235: klog.ErrorS(err, "Failed to get node devices", "nodeName", val.Name, "deviceVendor", devhandsk)
+ . . 236: continue
+ . . 237: }
+ . . 238: nodeInfo.Devices = make([]util.DeviceInfo, 0)
diff --git a/docs/how-to-use-volcano-vgpu.md b/docs/how-to-use-volcano-vgpu.md
new file mode 100644
index 000000000..d6edcf616
--- /dev/null
+++ b/docs/how-to-use-volcano-vgpu.md
@@ -0,0 +1,130 @@
+# Volcano vgpu device plugin for Kubernetes
+
+**Note**:
+
+You *DON'T* need to install HAMi when using volcano-vgpu, only use
+[Volcano vgpu device-plugin](https://github.com/Project-HAMi/volcano-vgpu-device-plugin) is good enough. It can provide device-sharing mechanism for NVIDIA devices managed by volcano.
+
+This is based on [Nvidia Device Plugin](https://github.com/NVIDIA/k8s-device-plugin), it uses [HAMi-core](https://github.com/Project-HAMi/HAMi-core) to support hard isolation of GPU card.
+
+Volcano vgpu is only available in volcano > 1.9
+
+## Quick Start
+
+### Install Volcano
+
+helm repo add volcano-sh https://volcano-sh.github.io/helm-charts
+helm install volcano volcano-sh/volcano -n volcano-system --create-namespace
+
+### Configure scheduler
+
+update the scheduler configuration:
+
+```shell script
+kubectl edit cm -n volcano-system volcano-scheduler-configmap
+```
+
+```yaml
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: volcano-scheduler-configmap
+ namespace: volcano-system
+data:
+ volcano-scheduler.conf: |
+ actions: "enqueue, allocate, backfill"
+ tiers:
+ - plugins:
+ - name: priority
+ - name: gang
+ - name: conformance
+ - plugins:
+ - name: drf
+ - name: deviceshare
+ arguments:
+ deviceshare.VGPUEnable: true # enable vgpu
+ - name: predicates
+ - name: proportion
+ - name: nodeorder
+ - name: binpack
+```
+
+### Enabling GPU Support in Kubernetes
+
+Once you have enabled this option on *all* the GPU nodes you wish to use,
+you can then enable GPU support in your cluster by deploying the following Daemonset:
+
+```
+$ kubectl create -f https://raw.githubusercontent.com/Project-HAMi/volcano-vgpu-device-plugin/main/volcano-vgpu-device-plugin.yml
+```
+
+### Verify environment is ready
+
+Check the node status, it is ok if `volcano.sh/vgpu-number` is included in the allocatable resources.
+
+```shell script
+$ kubectl get node {node name} -oyaml
+...
+status:
+ addresses:
+ - address: 172.17.0.3
+ type: InternalIP
+ - address: volcano-control-plane
+ type: Hostname
+ allocatable:
+ cpu: "4"
+ ephemeral-storage: 123722704Ki
+ hugepages-1Gi: "0"
+ hugepages-2Mi: "0"
+ memory: 8174332Ki
+ pods: "110"
+ volcano.sh/gpu-number: "10" # vGPU resource
+ capacity:
+ cpu: "4"
+ ephemeral-storage: 123722704Ki
+ hugepages-1Gi: "0"
+ hugepages-2Mi: "0"
+ memory: 8174332Ki
+ pods: "110"
+ volcano.sh/gpu-memory: "89424"
+ volcano.sh/gpu-number: "10" # vGPU resource
+```
+
+### Running VGPU Jobs
+
+VGPU can be requested by both set "volcano.sh/vgpu-number" , "volcano.sh/vgpu-cores" and "volcano.sh/vgpu-memory" in resource.limit
+
+```shell script
+$ cat < **WARNING:** *if you don't request GPUs when using the device plugin with NVIDIA images all
+> the GPUs on the machine will be exposed inside your container.
+> The number of vgpu used by a container can not exceed the number of gpus on that node.*
+
+### Monitor
+
+volcano-scheduler-metrics records every GPU usage and limitation, visit the following address to get these metrics.
+
+```
+curl {volcano scheduler cluster ip}:8080/metrics
+```
+
diff --git a/docs/hygon-dcu-support.md b/docs/hygon-dcu-support.md
new file mode 100644
index 000000000..c33f0a1ae
--- /dev/null
+++ b/docs/hygon-dcu-support.md
@@ -0,0 +1,77 @@
+## Introduction
+
+**We now support hygon.com/dcu by implementing most device-sharing features as nvidia-GPU**, including:
+
+***DCU sharing***: Each task can allocate a portion of DCU instead of a whole DCU card, thus DCU can be shared among multiple tasks.
+
+***Device Memory Control***: DCUs can be allocated with certain device memory size on certain type(i.e Z100) and have made it that it does not exceed the boundary.
+
+***Device compute core limitation***: DCUs can be allocated with certain percentage of device core(i.e hygon.com/dcucores:60 indicate this container uses 60% compute cores of this device)
+
+***DCU Type Specification***: You can specify which type of DCU to use or to avoid for a certain task, by setting "hygon.com/use-dcutype" or "hygon.com/nouse-dcutype" annotations.
+
+## Prerequisites
+
+* dtk driver >= 24.04
+* hy-smi v1.6.0
+
+## Enabling DCU-sharing Support
+
+* Deploy the dcu-vgpu-device-plugin [here](https://github.com/Project-HAMi/dcu-vgpu-device-plugin)
+
+
+## Running DCU jobs
+
+Hygon DCUs can now be requested by a container
+using the `hygon.com/dcunum` , `hygon.com/dcumem` and `hygon.com/dcucores` resource type:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: alexnet-tf-gpu-pod-mem
+ labels:
+ purpose: demo-tf-amdgpu
+spec:
+ containers:
+ - name: alexnet-tf-gpu-container
+ image: pytorch:resnet50
+ workingDir: /root
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ hygon.com/dcunum: 1 # requesting a GPU
+ hygon.com/dcumem: 2000 # each dcu require 2000 MiB device memory
+ hygon.com/dcucores: 60 # each dcu use 60% of total compute cores
+
+```
+
+## Enable vDCU inside container
+
+You need to enable vDCU inside container in order to use it.
+```
+source /opt/hygondriver/env.sh
+```
+
+check if you have successfully enabled vDCU by using following command
+
+```
+hy-virtual -show-device-info
+```
+
+If you have an output like this, then you have successfully enabled vDCU inside container.
+
+```
+Device 0:
+ Actual Device: 0
+ Compute units: 60
+ Global memory: 2097152000 bytes
+```
+
+Launch your DCU tasks like you usually do
+
+## Notes
+
+1. DCU-sharing in init container is not supported, pods with "hygon.com/dcumem" in init container will never be scheduled.
+
+2. Only one vdcu can be aquired per container. If you want to mount multiple dcu devices, then you shouldn't set `hygon.com/dcumem` or `hygon.com/dcucores`
diff --git a/docs/hygon-dcu-support_cn.md b/docs/hygon-dcu-support_cn.md
new file mode 100644
index 000000000..f83bb17d6
--- /dev/null
+++ b/docs/hygon-dcu-support_cn.md
@@ -0,0 +1,71 @@
+## 简介
+
+本组件支持复用海光DCU设备，并为此提供以下几种与vGPU类似的复用功能，包括：
+
+***DCU 共享***: 每个任务可以只占用一部分显卡，多个任务可以共享一张显卡
+
+***可限制分配的显存大小***: 你现在可以用显存值（例如3000M）来分配DCU，本组件会确保任务使用的显存不会超过分配数值
+
+***可限制计算单元数量***: 你现在可以指定任务使用的算力比例（例如60即代表使用60%算力）来分配DCU，本组件会确保任务使用的算力不会超过分配数值
+
+***指定DCU型号***：当前任务可以通过设置annotation("hygon.com/use-dcutype","hygon.com/nouse-dcutype")的方式，来选择使用或者不使用某些具体型号的DCU
+
+## 节点需求
+
+* dtk driver >= 24.04
+* hy-smi v1.6.0
+
+## 开启DCU复用
+
+* 部署[dcu-vgpu-device-plugin](https://github.com/Project-HAMi/dcu-vgpu-device-plugin)
+
+## 运行DCU任务
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: alexnet-tf-gpu-pod-mem
+ labels:
+ purpose: demo-tf-amdgpu
+spec:
+ containers:
+ - name: alexnet-tf-gpu-container
+ image: pytorch:resnet50
+ workingDir: /root
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ hygon.com/dcunum: 1 # requesting a GPU
+ hygon.com/dcumem: 2000 # each dcu require 2000 MiB device memory
+ hygon.com/dcucores: 60 # each dcu use 60% of total compute cores
+
+```
+
+## 容器内开启虚拟DCU功能
+
+使用vDCU首先需要激活虚拟环境
+```
+source /opt/hygondriver/env.sh
+```
+
+随后，使用hdmcli指令查看虚拟设备是否已经激活
+```
+hy-virtual -show-device-info
+```
+
+若输出如下，则代表虚拟设备已经成功激活
+```
+Device 0:
+ Actual Device: 0
+ Compute units: 60
+ Global memory: 2097152000 bytes
+```
+
+接下来正常启动DCU任务即可
+
+## 注意事项
+
+1. 在init container中无法使用DCU复用功能，否则该任务不会被调度
+
+2. 每个容器最多只能使用一个虚拟DCU设备, 如果您希望在容器中挂载多个DCU设备，则不能使用`hygon.com/dcumem`和`hygon.com/dcucores`字段
diff --git a/docs/iluvatar-gpu-support.md b/docs/iluvatar-gpu-support.md
new file mode 100644
index 000000000..d094ba9bb
--- /dev/null
+++ b/docs/iluvatar-gpu-support.md
@@ -0,0 +1,149 @@
+## Introduction
+
+**We now support iluvatar.ai/gpu(i.e MR-V100、BI-V150、BI-V100) by implementing most device-sharing features as nvidia-GPU**, including:
+
+***GPU sharing***: Each task can allocate a portion of GPU instead of a whole GPU card, thus GPU can be shared among multiple tasks.
+
+***Device Memory Control***: GPUs can be allocated with certain device memory size and have made it that it does not exceed the boundary.
+
+***Device Core Control***: GPUs can be allocated with limited compute cores and have made it that it does not exceed the boundary.
+
+***Device UUID Selection***: You can specify which GPU devices to use or exclude using annotations.
+
+***Very Easy to use***: You don't need to modify your task yaml to use our scheduler. All your GPU jobs will be automatically supported after installation.
+
+## Prerequisites
+
+* Iluvatar gpu-manager (please consult your device provider)
+* driver version > 3.1.0
+
+## Enabling GPU-sharing Support
+
+* Deploy gpu-manager on iluvatar nodes (Please consult your device provider to aquire its package and document)
+
+> **NOTICE:** *Install only gpu-manager, don't install gpu-admission package.*
+
+* Identify the resource name about core and memory usage(i.e 'iluvatar.ai/vcuda-core', 'iluvatar.ai/vcuda-memory')
+
+* set the 'iluvatarResourceMem' and 'iluvatarResourceCore' parameters when install hami
+
+```
+helm install hami hami-charts/hami --set scheduler.kubeScheduler.imageTag={your kubernetes version} --set iluvatarResourceMem=iluvatar.ai/vcuda-memory --set iluvatarResourceCore=iluvatar.ai/vcuda-core -n kube-system
+```
+
+> **NOTE:** The default resource names are:
+> - `iluvatar.ai/vgpu` for GPU count
+> - `iluvatar.ai/vcuda-memory` for memory allocation
+> - `iluvatar.ai/vcuda-core` for core allocation
+>
+> You can customize these names using the parameters above.
+
+## Device Granularity
+
+HAMi divides each Iluvatar GPU into 100 units for resource allocation. When you request a portion of a GPU, you're actually requesting a certain number of these units.
+
+### Memory Allocation
+
+- Each unit of `iluvatar.ai/vcuda-memory` represents 256MB of device memory
+- If you don't specify a memory request, the system will default to using 100% of the available memory
+- Memory allocation is enforced with hard limits to ensure tasks don't exceed their allocated memory
+
+### Core Allocation
+
+- Each unit of `iluvatar.ai/vcuda-core` represents 1% of the available compute cores
+- Core allocation is enforced with hard limits to ensure tasks don't exceed their allocated cores
+- When requesting multiple GPUs, the system will automatically set the core resources based on the number of GPUs requested
+
+## Running Iluvatar jobs
+
+Iluvatar GPUs can now be requested by a container
+using the `iluvatar.ai/vgpu`, `iluvatar.ai/vcuda-memory` and `iluvatar.ai/vcuda-core` resource type:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: poddemo
+spec:
+ restartPolicy: Never
+ containers:
+ - name: poddemo
+ image: harbor.4pd.io/vgpu/corex_transformers@sha256:36a01ec452e6ee63c7aa08bfa1fa16d469ad19cc1e6000cf120ada83e4ceec1e
+ command:
+ - bash
+ args:
+ - -c
+ - |
+ set -ex
+ echo "export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH">> /root/.bashrc
+ cp -f /usr/local/iluvatar/lib64/libcuda.* /usr/local/corex/lib64/
+ cp -f /usr/local/iluvatar/lib64/libixml.* /usr/local/corex/lib64/
+ source /root/.bashrc
+ sleep 360000
+ resources:
+ requests:
+ iluvatar.ai/vgpu: 1
+ iluvatar.ai/vcuda-core: 50
+ iluvatar.ai/vcuda-memory: 64
+ limits:
+ iluvatar.ai/vgpu: 1
+ iluvatar.ai/vcuda-core: 50
+ iluvatar.ai/vcuda-memory: 64
+```
+
+> **NOTICE1:** *Each unit of vcuda-memory indicates 256M device memory*
+
+> **NOTICE2:** *You can find more examples in [examples/iluvatar folder](../examples/iluvatar/)*
+
+## Device UUID Selection
+
+You can specify which GPU devices to use or exclude using annotations:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: poddemo
+ annotations:
+ # Use specific GPU devices (comma-separated list)
+ iluvatar.ai/use-gpuuuid: "node1-iluvatar-0,node1-iluvatar-1"
+ # Or exclude specific GPU devices (comma-separated list)
+ iluvatar.ai/nouse-gpuuuid: "node1-iluvatar-2,node1-iluvatar-3"
+spec:
+ # ... rest of pod spec
+```
+
+> **NOTE:** The device ID format is `{node-name}-iluvatar-{index}`. You can find the available device IDs in the node status.
+
+### Finding Device UUIDs
+
+You can find the UUIDs of Iluvatar GPUs on a node using the following command:
+
+```bash
+kubectl get pod -o yaml | grep -A 10 "hami.io/-devices-allocated"
+```
+
+Or by examining the node annotations:
+
+```bash
+kubectl get node -o yaml | grep -A 10 "hami.io/node-register-"
+```
+
+Look for annotations containing device information in the node status.
+
+## Notes
+
+1. You need to set the following prestart command in order for the device-share to work properly
+```sh
+ set -ex
+ echo "export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH">> /root/.bashrc
+ cp -f /usr/local/iluvatar/lib64/libcuda.* /usr/local/corex/lib64/
+ cp -f /usr/local/iluvatar/lib64/libixml.* /usr/local/corex/lib64/
+ source /root/.bashrc
+```
+
+2. Virtualization takes effect only for containers that apply for one GPU(i.e iluvatar.ai/vgpu=1 ). When requesting multiple GPUs, the system will automatically set the core resources based on the number of GPUs requested.
+
+3. The `iluvatar.ai/vcuda-memory` resource is only effective when `iluvatar.ai/vgpu=1`.
+
+4. Multi-device requests (`iluvatar.ai/vgpu > 1`) do not support vGPU mode.
\ No newline at end of file
diff --git a/docs/iluvatar-gpu-support_cn.md b/docs/iluvatar-gpu-support_cn.md
new file mode 100644
index 000000000..164ebf542
--- /dev/null
+++ b/docs/iluvatar-gpu-support_cn.md
@@ -0,0 +1,148 @@
+## 简介
+
+本组件支持复用天数智芯GPU设备(MR-V100、BI-V150、BI-V100)，并为此提供以下几种与vGPU类似的复用功能，包括：
+
+***GPU 共享***: 每个任务可以只占用一部分显卡，多个任务可以共享一张显卡
+
+***可限制分配的显存大小***: 你现在可以用显存值（例如3000M）来分配GPU，本组件会确保任务使用的显存不会超过分配数值
+
+***可限制分配的算力核组比例***: 你现在可以用算力比例（例如60%）来分配GPU，本组件会确保任务使用的显存不会超过分配数值
+
+***设备 UUID 选择***: 你可以通过注解指定使用或排除特定的 GPU 设备
+
+***方便易用***: 部署本组件后，只需要部署厂家提供的gpu-manager即可使用
+
+
+## 节点需求
+
+* Iluvatar gpu-manager (please consult your device provider)
+* driver version > 3.1.0
+
+## 开启GPU复用
+
+* 部署'gpu-manager'，天数智芯的GPU共享需要配合厂家提供的'gpu-manager'一起使用，请联系设备提供方获取
+
+> **注意:** *只需要安装gpu-manager，不要安装gpu-admission.*
+
+* 部署'gpu-manager'之后，你需要确认显存和核组对应的资源名称(例如 'iluvatar.ai/vcuda-core', 'iluvatar.ai/vcuda-memory')
+
+* 在安装HAMi时配置'iluvatarResourceMem'和'iluvatarResourceCore'参数
+
+```
+helm install hami hami-charts/hami --set scheduler.kubeScheduler.imageTag={your kubernetes version} --set iluvatarResourceMem=iluvatar.ai/vcuda-memory --set iluvatarResourceCore=iluvatar.ai/vcuda-core -n kube-system
+```
+
+> **说明:** 默认资源名称如下：
+> - `iluvatar.ai/vgpu` 用于 GPU 数量
+> - `iluvatar.ai/vcuda-memory` 用于内存分配
+> - `iluvatar.ai/vcuda-core` 用于核心分配
+>
+> 你可以通过上述参数自定义这些名称。
+
+## 设备粒度切分
+
+HAMi 将每个天数智芯 GPU 划分为 100 个单元进行资源分配。当你请求一部分 GPU 时，实际上是在请求这些单元中的一定数量。
+
+### 内存分配
+
+- 每个 `iluvatar.ai/vcuda-memory` 单位代表 256MB 的设备内存
+- 如果不指定内存请求，系统将默认使用 100% 的可用内存
+- 内存分配通过硬限制强制执行，确保任务不会超过其分配的内存
+
+### 核心分配
+
+- 每个 `iluvatar.ai/vcuda-core` 单位代表 1% 的可用计算核心
+- 核心分配通过硬限制强制执行，确保任务不会超过其分配的核心
+- 当请求多个 GPU 时，系统会根据请求的 GPU 数量自动设置核心资源
+
+## 运行GPU任务
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: poddemo
+spec:
+ restartPolicy: Never
+ containers:
+ - name: poddemo
+ image: harbor.4pd.io/vgpu/corex_transformers@sha256:36a01ec452e6ee63c7aa08bfa1fa16d469ad19cc1e6000cf120ada83e4ceec1e
+ command:
+ - bash
+ args:
+ - -c
+ - |
+ set -ex
+ echo "export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH">> /root/.bashrc
+ cp -f /usr/local/iluvatar/lib64/libcuda.* /usr/local/corex/lib64/
+ cp -f /usr/local/iluvatar/lib64/libixml.* /usr/local/corex/lib64/
+ source /root/.bashrc
+ sleep 360000
+ resources:
+ requests:
+ iluvatar.ai/vgpu: 1
+ iluvatar.ai/vcuda-core: 50
+ iluvatar.ai/vcuda-memory: 64
+ limits:
+ iluvatar.ai/vgpu: 1
+ iluvatar.ai/vcuda-core: 50
+ iluvatar.ai/vcuda-memory: 64
+```
+
+> **注意1:** *每一单位的vcuda-memory代表256M的显存.*
+
+> **注意2:** *查看更多的[用例](../examples/iluvatar/).*
+
+## 设备 UUID 选择
+
+你可以通过 Pod 注解来指定要使用或排除特定的 GPU 设备：
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: poddemo
+ annotations:
+ # 使用特定的 GPU 设备（逗号分隔的列表）
+ iluvatar.ai/use-gpuuuid: "node1-iluvatar-0,node1-iluvatar-1"
+ # 或者排除特定的 GPU 设备（逗号分隔的列表）
+ iluvatar.ai/nouse-gpuuuid: "node1-iluvatar-2,node1-iluvatar-3"
+spec:
+ # ... 其余 Pod 配置
+```
+
+> **说明:** 设备 ID 格式为 `{节点名称}-iluvatar-{索引}`。你可以在节点状态中找到可用的设备 ID。
+
+### 查找设备 UUID
+
+你可以使用以下命令查找节点上的天数智芯 GPU 设备 UUID：
+
+```bash
+kubectl get pod -o yaml | grep -A 10 "hami.io/-devices-allocated"
+```
+
+或者通过检查节点注解：
+
+```bash
+kubectl get node -o yaml | grep -A 10 "hami.io/node-register-"
+```
+
+在节点注解中查找包含设备信息的注解。
+
+
+## 注意事项
+
+1. 你需要在容器中进行如下的设置才能正常的使用共享功能
+```sh
+ set -ex
+ echo "export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH">> /root/.bashrc
+ cp -f /usr/local/iluvatar/lib64/libcuda.* /usr/local/corex/lib64/
+ cp -f /usr/local/iluvatar/lib64/libixml.* /usr/local/corex/lib64/
+ source /root/.bashrc
+```
+
+2. 共享模式只对申请一张GPU的容器生效（iluvatar.ai/vgpu=1）。当请求多个 GPU 时，系统会根据请求的 GPU 数量自动设置核心资源。
+
+3. `iluvatar.ai/vcuda-memory` 资源仅在 `iluvatar.ai/vgpu=1` 时有效。
+
+4. 多设备请求（`iluvatar.ai/vgpu > 1`）不支持 vGPU 模式。
\ No newline at end of file
diff --git a/docs/metax-support.md b/docs/metax-support.md
new file mode 100644
index 000000000..be57567c2
--- /dev/null
+++ b/docs/metax-support.md
@@ -0,0 +1,115 @@
+## Introduction
+
+We support metax.com/gpu as follows:
+
+- support metax.com/gpu by implementing most device-sharing features as nvidia-GPU
+- support metax.com/gpu by implementing topo-awareness among metax GPUs
+
+## support metax.com/gpu by implementing most device-sharing features as nvidia-GPU
+
+device-sharing features include the following:
+
+***GPU sharing***: Each task can allocate a portion of GPU instead of a whole GPU card, thus GPU can be shared among multiple tasks.
+
+***Device Memory Control***: GPUs can be allocated with certain device memory size and have made it that it does not exceed the boundary.
+
+***Device compute core limitation***: GPUs can be allocated with certain percentage of device core(60 indicate this container uses 60% compute cores of this device)
+
+### Prerequisites
+
+* Metax Driver >= 2.32.0
+* Metax GPU Operator >= 0.10.2
+* Kubernetes >= 1.23
+
+### Enabling GPU-sharing Support
+
+* Deploy Metax GPU Operator on metax nodes (Please consult your device provider to aquire its package and document)
+
+* Deploy HAMi according to README.md
+
+### Running Metax jobs
+
+Metax GPUs can now be requested by a container
+using the `metax-tech.com/sgpu` resource type:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/sgpu: 1 # requesting 1 GPU
+ metax-tech.com/vcore: 60 # each GPU use 60% of total compute cores
+ metax-tech.com/vmemory: 4 # each GPU require 4 GiB device memory
+```
+
+> **NOTICE1:** *You can find more examples in [examples/metax folder](../examples/metax/sgpu)*
+
+## support metax.com/gpu by implementing topo-awareness among metax GPUs
+
+When multiple GPUs are configured on a single server, the GPU cards are connected to the same PCIe Switch or MetaXLink depending on whether they are connected
+, there is a near-far relationship. This forms a topology among all the cards on the server, as shown in the following figure:
+
+
+
+A user job requests a certain number of metax-tech.com/gpu resources, Kubernetes schedule pods to the appropriate node. gpu-device further processes the logic of allocating the remaining resources on the resource node following criterias below:
+1. MetaXLink takes precedence over PCIe Switch in two way:
+– A connection is considered a MetaXLink connection when there is a MetaXLink connection and a PCIe Switch connection between the two cards.
+– When both the MetaXLink and the PCIe Switch can meet the job request
+Equipped with MetaXLink interconnected resources.
+
+2. When using `node-scheduler-policy=spread` , Allocate Metax resources to be under the same Metaxlink or Paiswich as much as possible, as the following figure shows:
+
+
+
+3. When using `node-scheduler-policy=binpack`, Assign GPU resources, so minimize the damage to MetaxXLink topology, as the following figure shows:
+
+
+
+### Important Notes
+
+1. Device sharing is not supported yet.
+
+2. These features are tested on MXC500
+
+### Prerequisites
+
+* Metax GPU extensions >= 0.8.0
+* Kubernetes >= 1.23
+
+### Enabling topo-awareness scheduling
+
+* Deploy Metax GPU Extensions on metax nodes (Please consult your device provider to aquire its package and document)
+
+* Deploy HAMi according to README.md
+
+### Running Metax jobs
+
+Metax GPUs can now be requested by a container
+using the `metax-tech.com/gpu` resource type:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+ annotations: hami.io/node-scheduler-policy: "spread" # when this parameter is set to spread, the scheduler will try to find the best topology for this task.
+spec:
+ containers:
+ - name: ubuntu-container
+ image: cr.metax-tech.com/public-ai-release/c500/colossalai:2.24.0.5-py38-ubuntu20.04-amd64
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/gpu: 1 # requesting 1 vGPUs
+```
+
+> **NOTICE2:** *You can find more examples in [examples/metax folder](../examples/metax/gpu)*
diff --git a/docs/metax-support_cn.md b/docs/metax-support_cn.md
new file mode 100644
index 000000000..08f6c6175
--- /dev/null
+++ b/docs/metax-support_cn.md
@@ -0,0 +1,114 @@
+## 简介
+
+我们对沐曦设备做如下支持：
+
+- 复用沐曦GPU设备，提供与vGPU类似的复用功能
+- 基于拓扑结构，对沐曦设备进行优化调度
+
+## 复用沐曦GPU设备，提供与vGPU类似的复用功能
+
+复用功能包括以下：
+
+***GPU 共享***: 每个任务可以只占用一部分显卡，多个任务可以共享一张显卡
+
+***可限制分配的显存大小***: 你现在可以用显存值（例如4G）来分配GPU，本组件会确保任务使用的显存不会超过分配数值
+
+***可限制计算单元数量***: 你现在可以指定任务使用的算力比例（例如60即代表使用60%算力）来分配GPU，本组件会确保任务使用的算力不会超过分配数值
+
+### 需求
+
+* Metax Driver >= 2.32.0
+* Metax GPU Operator >= 0.10.2
+* Kubernetes >= 1.23
+
+### 开启复用沐曦设备
+
+* 部署Metax GPU Operator (请联系您的设备提供方获取)
+* 根据readme.md部署HAMi
+
+### 运行沐曦任务
+
+一个典型的沐曦任务如下所示：
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/sgpu: 1 # requesting 1 GPU
+ metax-tech.com/vcore: 60 # each GPU use 60% of total compute cores
+ metax-tech.com/vmemory: 4 # each GPU require 4 GiB device memory
+```
+
+> **NOTICE1:** *你可以在这里找到更多样例 [examples/metax folder](../examples/metax/sgpu)*
+
+## 基于拓扑结构，对沐曦设备进行优化调度
+
+在单台服务器上配置多张 GPU 时，GPU 卡间根据双方是否连接在相同的 PCIe Switch 或 MetaXLink
+下，存在近远（带宽高低）关系。服务器上所有卡间据此形成一张拓扑，如下图所示。
+
+
+
+用户作业请求一定数量的 metax-tech.com/gpu 资源，Kubernetes 选择剩余资源数量满足要求的
+节点，并将 Pod 调度到相应节点。gpu‑device 进一步处理资源节点上剩余资源的分配逻辑，并按照以
+下优先级逻辑为作业容器分配 GPU 设备：
+1. MetaXLink 优先级高于 PCIe Switch，包含两层含义：
+– 两卡之间同时存在 MetaXLink 连接以及 PCIe Switch 连接时，认定为 MetaXLink 连接。
+– 服务器剩余 GPU 资源中 MetaXLink 互联资源与 PCIe Switch 互联资源均能满足作业请求时，分
+配 MetaXLink 互联资源。
+
+2. 当任务使用 `node-scheduler-policy=spread` ,分配GPU资源尽可能位于相同 MetaXLink或PCIe Switch下，如下图所示:
+
+
+
+3. 当使用 `node-scheduler-policy=binpack`,分配GPU资源后，剩余资源尽可能完整，如下图所示：
+
+
+
+### 注意：
+
+1. 暂时不支持沐曦设备的切片，只能申请整卡
+
+2. 本功能基于MXC500进行测试
+
+### 需求
+
+* Metax GPU extensions >= 0.8.0
+* Kubernetes >= 1.23
+
+### 开启针对沐曦设备的拓扑调度优化
+
+* 部署Metax GPU extensions (请联系您的设备提供方获取)
+
+* 根据readme.md部署HAMi
+
+### 运行沐曦任务
+
+一个典型的沐曦任务如下所示：
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+ annotations: hami.io/node-scheduler-policy: "spread" # when this parameter is set to spread, the scheduler will try to find the best topology for this task.
+spec:
+ containers:
+ - name: ubuntu-container
+ image: cr.metax-tech.com/public-ai-release/c500/colossalai:2.24.0.5-py38-ubuntu20.04-amd64
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/gpu: 1 # requesting 1 vGPUs
+```
+
+> **NOTICE2:** *你可以在这里找到更多样例 [examples/metax folder](../examples/metax/gpu)*
diff --git a/docs/mind-map/HAMI-VGPU-mind-map-Chinese.png b/docs/mind-map/HAMI-VGPU-mind-map-Chinese.png
new file mode 100644
index 000000000..bec3cc30d
Binary files /dev/null and b/docs/mind-map/HAMI-VGPU-mind-map-Chinese.png differ
diff --git a/docs/mind-map/HAMI-VGPU-mind-map-Chinese.xmind b/docs/mind-map/HAMI-VGPU-mind-map-Chinese.xmind
new file mode 100644
index 000000000..f662288f3
Binary files /dev/null and b/docs/mind-map/HAMI-VGPU-mind-map-Chinese.xmind differ
diff --git a/docs/mind-map/HAMI-VGPU-mind-map-English.png b/docs/mind-map/HAMI-VGPU-mind-map-English.png
new file mode 100644
index 000000000..1eccc30de
Binary files /dev/null and b/docs/mind-map/HAMI-VGPU-mind-map-English.png differ
diff --git a/docs/mind-map/HAMI-VGPU-mind-map-English.xmind b/docs/mind-map/HAMI-VGPU-mind-map-English.xmind
new file mode 100644
index 000000000..94206a34b
Binary files /dev/null and b/docs/mind-map/HAMI-VGPU-mind-map-English.xmind differ
diff --git a/docs/mind-map/readme b/docs/mind-map/readme
new file mode 100644
index 000000000..ec9319ae2
--- /dev/null
+++ b/docs/mind-map/readme
@@ -0,0 +1,9 @@
+- 根据交流群里各位大佬的交流梳理此份思维导图（尤其感谢 @意琦行 大佬梳理的博客）
+- 英文版由 @隽戈 大佬提供
+- 若有问题处，各位大佬可随时提出
+
+-Based on the communication among the experts in the communication group, this mind map has been compiled (especially thanks to @意琦行 expert for organizing the blog)
+-The English version is provided by @隽戈
+-If there are any issues, anyone can raise them at any time
+
+
diff --git a/docs/mthreads-support.md b/docs/mthreads-support.md
new file mode 100644
index 000000000..711aacc79
--- /dev/null
+++ b/docs/mthreads-support.md
@@ -0,0 +1,65 @@
+## Introduction
+
+**We now support mthreads.com/vgpu by implementing most device-sharing features as nvidia-GPU**, including:
+
+***GPU sharing***: Each task can allocate a portion of GPU instead of a whole GPU card, thus GPU can be shared among multiple tasks.
+
+***Device Memory Control***: GPUs can be allocated with certain device memory size on certain type(i.e MTT S4000) and have made it that it does not exceed the boundary.
+
+***Device Core Control***: GPUs can be allocated with limited compute cores on certain type(i.e MTT S4000) and have made it that it does not exceed the boundary.
+
+## Important Notes
+
+1. Device sharing for multi-cards is not supported.
+
+2. Only one mthreads device can be shared in a pod(even there are multiple containers).
+
+3. Support allocating exclusive mthreads GPU by specifying mthreads.com/vgpu only.
+
+4. These features are tested on MTT S4000
+
+## Prerequisites
+
+* [MT CloudNative Toolkits > 1.9.0](https://docs.mthreads.com/cloud-native/cloud-native-doc-online/)
+* driver version >= 1.2.0
+
+## Enabling GPU-sharing Support
+
+* Deploy MT-CloudNative Toolkit on mthreads nodes (Please consult your device provider to aquire its package and document)
+
+> **NOTICE:** *You can remove mt-mutating-webhook and mt-gpu-scheduler after installation(optional).*
+
+* set the 'devices.mthreads.enabled = true' when installing hami
+
+```
+helm install hami hami-charts/hami --set scheduler.kubeScheduler.imageTag={your kubernetes version} --set device.mthreads.enabled=true -n kube-system
+```
+
+## Running Mthreads jobs
+
+Mthreads GPUs can now be requested by a container
+using the `mthreads.com/vgpu`, `mthreads.com/sgpu-memory` and `mthreads.com/sgpu-core` resource type:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpushare-pod-default
+spec:
+ restartPolicy: OnFailure
+ containers:
+ - image: core.harbor.zlidc.mthreads.com:30003/mt-ai/lm-qy2:v17-mpc
+ imagePullPolicy: IfNotPresent
+ name: gpushare-pod-1
+ command: ["sleep"]
+ args: ["100000"]
+ resources:
+ limits:
+ mthreads.com/vgpu: 1
+ mthreads.com/sgpu-memory: 32
+ mthreads.com/sgpu-core: 8
+```
+
+> **NOTICE1:** *Each unit of sgpu-memory indicates 512M device memory*
+
+> **NOTICE2:** *You can find more examples in [examples/mthreads folder](../examples/mthreads/)*
diff --git a/docs/mthreads-support_cn.md b/docs/mthreads-support_cn.md
new file mode 100644
index 000000000..e8acdad1d
--- /dev/null
+++ b/docs/mthreads-support_cn.md
@@ -0,0 +1,64 @@
+## 简介
+
+本组件支持复用摩尔线程GPU设备，并为此提供以下几种与vGPU类似的复用功能，包括：
+
+***GPU 共享***: 每个任务可以只占用一部分显卡，多个任务可以共享一张显卡
+
+***可限制分配的显存大小***: 你现在可以用显存值（例如3000M）来分配MLU，本组件会确保任务使用的显存不会超过分配数值、
+
+***可限制分配的算力核组比例***: 你现在可以用算力核组数量（例如8个）来分配GPU，本组件会确保任务使用的显存不会超过分配数值
+
+## 注意事项
+
+1. 暂时不支持多卡切片，多卡任务只能分配整卡
+
+2. 一个pod只能使用一个GPU生成的切片，即使该pod中有多个容器
+
+3. 支持独占模式，只指定`mthreads.com/vgpu`即为独占申请
+
+4. 本特性目前只支持MTT S4000设备
+
+## 节点需求
+
+* [MT CloudNative Toolkits > 1.9.0](https://docs.mthreads.com/cloud-native/cloud-native-doc-online/)
+* 驱动版本 >= 1.2.0
+
+## 开启GPU复用
+
+* 部署'MT-CloudNative Toolkit'，摩尔线程的GPU共享需要配合厂家提供的'MT-CloudNative Toolkit'一起使用，请联系设备提供方获取
+
+> **注意:** *（可选），部署完之后，卸载掉mt-mutating-webhook与mt-scheduler组件，因为这部分功能将由HAMi调度器提供*
+
+* 在安装HAMi时配置'devices.mthreads.enabled = true'参数
+
+```
+helm install hami hami-charts/hami --set scheduler.kubeScheduler.imageTag={your kubernetes version} --set device.mthreads.enabled=true -n kube-system
+```
+
+## 运行GPU任务
+
+通过指定`mthreads.com/vgpu`, `mthreads.com/sgpu-memory` and `mthreads.com/sgpu-core`这3个参数，可以确定容器申请的切片个数，对应的显存和算力核组
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpushare-pod-default
+spec:
+ restartPolicy: OnFailure
+ containers:
+ - image: core.harbor.zlidc.mthreads.com:30003/mt-ai/lm-qy2:v17-mpc
+ imagePullPolicy: IfNotPresent
+ name: gpushare-pod-1
+ command: ["sleep"]
+ args: ["100000"]
+ resources:
+ limits:
+ mthreads.com/vgpu: 1
+ mthreads.com/sgpu-memory: 32
+ mthreads.com/sgpu-core: 8
+```
+
+> **注意1:** *每一单位的sgpu-memory代表512M的显存.*
+
+> **注意2:** *查看更多的[用例](../examples/mthreads/).*
diff --git a/docs/offline-install.md b/docs/offline-install.md
new file mode 100644
index 000000000..d0bca1cc5
--- /dev/null
+++ b/docs/offline-install.md
@@ -0,0 +1,60 @@
+# Offline-install Maunal
+
+For some cluster that don't have external web access, you can install HAMi by the following step:
+
+1. Refer to [README.md](../README.md) until step 'Install and Uninstall'
+
+2. pull the following images and save them into a '.tar' file, then move it into your cluster
+
+Image list:
+```
+projecthami/hami:{HAMi version}
+docker.io/jettech/kube-webhook-certgen:v1.5.2
+liangjw/kube-webhook-certgen:v1.1.1
+registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:{your kubernetes version}
+```
+
+```
+docker pull {iamge} && docker save {image_name} -o {image_name}.tar
+```
+
+3. Load these images using docker load, tag these images with your registry, and push them into your registry
+
+```
+docker load -i {HAMi_image}.tar
+docker tag projecthami/hami:{HAMi version} {your_inner_registry}/hami:{HAMi version}
+docker push {your_inner_registry}/hami:{HAMi version}
+docker tag docker.io/jettech/kube-webhook-certgen:v1.5.2 {your inner_regisry}/kube-webhook-certgen:v1.5.2
+docker push {your inner_regisry}/kube-webhook-certgen:v1.5.2
+docker tag liangjw/kube-webhook-certgen:v1.1.1 {your_inner_registry}/kube-webhook-certgen:v1.1.1
+docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:{your kubernetes version} {your_inner_registry}/kube-scheduler:{your kubernetes version}
+docker push {your_inner_registry}/kube-scheduler:{your kubernetes version}
+```
+
+4. Download the charts folder from [github](https://github.com/Project-HAMi/HAMi/tree/master/charts), place it into ${CHART_PATH} inside cluser, then edit the following fields in ${CHART_PATH}/hami/values.yaml.
+
+```
+scheduler.kubeScheduler.image
+scheduler.extender.image
+scheduler.patch.image
+scheduler.patch.imageNew
+scheduler.devicePlugin.image
+scheduler.devicePlugin.monitorimage
+```
+
+5. Execute the following command in your /root/HAMi/chart folder
+
+```
+helm install hami hami --set scheduler.kubeScheduler.imageTag={your k8s server version} -n kube-system
+```
+
+6. Verify your installation
+
+execute the following command
+```
+kubectl get pods -n kube-system
+```
+
+If you can see both the 'device-plugin' and 'schduler' running, then HAMi is installed successfully, as the figure shown below:
+
+
diff --git a/docs/proposals/e2e_test.md b/docs/proposals/e2e_test.md
new file mode 100644
index 000000000..b3c4b7228
--- /dev/null
+++ b/docs/proposals/e2e_test.md
@@ -0,0 +1,175 @@
+# Support E2E Testing
+
+
+
+- [Summary](#summary)
+- [Motivation](#motivation)
+ - [Goals](#goals)
+ - [Non-Goals](#non-goals)
+- [Proposal](#proposal)
+ - [Test Scope](#test-scope)
+ - [Implementation Details](#implementation-details)
+ - [User Stories (Optional)](#user-stories-optional)
+ - [Story 1](#story-1)
+ - [Story 2](#story-2)
+ - [Story 3](#story-3)
+ - [Risks and Mitigations](#risks-and-mitigations)
+- [Design Details](#design-details)
+
+
+
+## Summary
+
+
+
+This KEP proposes to support End-to-End (E2E) testing for HAMi, ensuring its functionality and compatibility
+within the Kubernetes ecosystem.
+
+It introduces mechanisms to validate the entire workflow of the feature and
+guarantee that it meets production-level requirements.
+
+## Motivation
+
+
+
+End-to-end (E2E) tests validate the complete functionality of a system, ensuring that the end-user experience
+aligns with developer specifications.
+
+While unit and integration tests provide valuable feedback, they are often insufficient in distributed systems.
+Minor changes may pass unit and integration tests but still introduce unforeseen issues at the system level.
+
+Comprehensive E2E test coverage is essential to mitigate the risks of regressions, improve reliability,
+and maintain confidence in the system's seamless integration with Kubernetes.
+Without it, HAMi's robustness and user trust may be compromised.
+
+### Goals
+
+
+
+- Setup E2E testing basic environment.
+- Define the scope and scenarios for E2E testing of HAMi.
+- Implement E2E tests that cover key workflows and edge cases.
+- Ensure compatibility with the Kubernetes.
+- Establish a reliable and repeatable test framework for future enhancements.
+
+### Non-Goals
+
+
+
+- Unit or integration testing of the feature (covered elsewhere).
+- Performance benchmarking beyond basic scenarios.
+
+## Proposal
+
+
+
+This proposal aims to integrate E2E testing for HAMi. Tests will be implemented using the
+Kubernetes E2E testing framework (e.g. Ginkgo) and adhere to community best practices.
+
+### Test Scope
+
+- Core functionality: Validate basic operations and workflows of the feature.
+- Edge cases: Test unusual scenarios or invalid inputs to ensure robustness.
+- Compatibility:
+ - Verify that the feature integrates with different heterogeneous devices.
+ - Verify that the feature integrates with different Kubernetes versions.
+ - Verify that the feature integrates with different CUDA versions. (Optional)
+- Error handling: Ensure appropriate error messages and recovery mechanisms are in place.
+
+### Implementation Details
+
+- Test environment will hold in local environment.
+- Tests will be written using the [Ginkgo](https://onsi.github.io/ginkgo/) testing framework.
+- All tests will use isolated namespaces to avoid conflicts.
+- Resource cleanup will be automated after each test run.
+- CI integration will ensure tests run against PRs, daily builds, and releases.
+
+### User Stories (Optional)
+
+
+
+#### Story 1
+Automating E2E testing with helm deployment
+
+#### Story 2
+Automating E2E testing with resource validation
+
+### Story 3
+Automating E2E testing with Kubernetes resource deploy
+
+### Risks and Mitigations
+
+
+
+- Resource Limitations
+ - During E2E testing, testing clusters may encounter resource constraints,
+ such as insufficient CPU, memory, or storage. This could lead to test failures,
+ degraded performance, or timeouts during deployments.
+- Environment Instability
+ - Instabilities in the testing environment, such as network latency, intermittent network failures,
+ or cluster node failures, can cause tests to fail or behave inconsistently.
+
+## Design Details
+
+
+
+
+
+
diff --git a/docs/proposals/e2e_test.png b/docs/proposals/e2e_test.png
new file mode 100644
index 000000000..bd129f94a
Binary files /dev/null and b/docs/proposals/e2e_test.png differ
diff --git a/docs/proposals/gpu-topo-policy.md b/docs/proposals/gpu-topo-policy.md
new file mode 100644
index 000000000..247d57562
--- /dev/null
+++ b/docs/proposals/gpu-topo-policy.md
@@ -0,0 +1,197 @@
+# GPU topology scheduling strategy
+
+## Use Case
+
+### Node Select
+
+If multiple nodes meet the requirements, the minimum meeting node should be selected first::
+- `Node1`
+```json
+[
+ {
+ "uuid": "gpu0",
+ "score": {
+ "gpu1": "100",
+ "gpu2": "100",
+ "gpu3": "200"
+ }
+ },
+ {
+ "uuid": "gpu1",
+ "score": {
+ "gpu0": "100",
+ "gpu2": "200",
+ "gpu3": "100"
+ }
+ },
+ {
+ "uuid": "gpu2",
+ "score": {
+ "gpu0": "100",
+ "gpu1": "200",
+ "gpu3": "200"
+ }
+ },
+ {
+ "uuid": "gpu3",
+ "score": {
+ "gpu0": "200",
+ "gpu1": "100",
+ "gpu2": "200"
+ }
+ }
+]
+```
+
+
+- `Node2`
+```json
+[
+ {
+ "uuid": "gpu0",
+ "score": {
+ "gpu1": "100",
+ "gpu2": "100",
+ "gpu3": "200",
+ "gpu4": "200",
+ "gpu5": "200"
+ }
+ },
+ {
+ "uuid": "gpu1",
+ "score": {
+ "gpu0": "100",
+ "gpu2": "200",
+ "gpu3": "100",
+ "gpu4": "200",
+ "gpu5": "200"
+ }
+ },
+ {
+ "uuid": "gpu2",
+ "score": {
+ "gpu0": "100",
+ "gpu1": "200",
+ "gpu3": "200",
+ "gpu4": "200",
+ "gpu5": "200"
+ }
+ },
+ {
+ "uuid": "gpu3",
+ "score": {
+ "gpu0": "200",
+ "gpu1": "100",
+ "gpu2": "200",
+ "gpu4": "200",
+ "gpu5": "200"
+ }
+ },
+ {
+ "uuid": "gpu4",
+ "score": {
+ "gpu0": "200",
+ "gpu1": "100",
+ "gpu2": "200",
+ "gpu3": "200",
+ "gpu5": "200"
+ }
+ },
+ {
+ "uuid": "gpu5",
+ "score": {
+ "gpu0": "200",
+ "gpu1": "100",
+ "gpu2": "200",
+ "gpu3": "200",
+ "gpu4": "200"
+ }
+ }
+]
+```
+
+For example, the two nodes above should be given priority `Node1`.
+
+
+### Device Select, One Pod One Device
+
+1. When a Pod only needs one card, the card with the worst communication with other cards should be given priority if the video memory and computing power are sufficient, such as：
+- `Node1`
+```json
+[
+ {
+ "uuid": "gpu0",
+ "score": {
+ "gpu1": "100",
+ "gpu2": "100",
+ "gpu3": "200"
+ }
+ },
+ {
+ "uuid": "gpu1",
+ "score": {
+ "gpu0": "100",
+ "gpu2": "200",
+ "gpu3": "100"
+ }
+ },
+ {
+ "uuid": "gpu2",
+ "score": {
+ "gpu0": "100",
+ "gpu1": "200",
+ "gpu3": "200"
+ }
+ },
+ {
+ "uuid": "gpu3",
+ "score": {
+ "gpu0": "200",
+ "gpu1": "100",
+ "gpu2": "200"
+ }
+ }
+]
+```
+There are only four cards on this node, and `gpu0` and `gpu1` have the worst connectivity with other cards, so these two cards are preferred in single-card mode.
+
+### Device Select, One Pod More Than One Device
+1. When a Pod only needs multiple cards, if the video memory and computing power are sufficient, the card with the best communication with other cards should be given priority, such as：
+- `Node1`
+```json
+[
+ {
+ "uuid": "gpu0",
+ "score": {
+ "gpu1": "100",
+ "gpu2": "100",
+ "gpu3": "200"
+ }
+ },
+ {
+ "uuid": "gpu1",
+ "score": {
+ "gpu0": "100",
+ "gpu2": "200",
+ "gpu3": "100"
+ }
+ },
+ {
+ "uuid": "gpu2",
+ "score": {
+ "gpu0": "100",
+ "gpu1": "200",
+ "gpu3": "200"
+ }
+ },
+ {
+ "uuid": "gpu3",
+ "score": {
+ "gpu0": "200",
+ "gpu1": "100",
+ "gpu2": "200"
+ }
+ }
+]
+```
+There are only four cards on this node, and `gpu2` and `gpu3` have the best connectivity with other cards, so these two cards are given priority in multi-card mode.
diff --git a/docs/proposals/gpu_utilization.png b/docs/proposals/gpu_utilization.png
new file mode 100644
index 000000000..ff0d997f0
Binary files /dev/null and b/docs/proposals/gpu_utilization.png differ
diff --git a/docs/proposals/gpu_utilization_cn.md b/docs/proposals/gpu_utilization_cn.md
new file mode 100644
index 000000000..8071dbe31
--- /dev/null
+++ b/docs/proposals/gpu_utilization_cn.md
@@ -0,0 +1,189 @@
+---
+title: Support GPU Utilization Metrics
+authors:
+- @chaunceyjiang
+ reviewers:
+- TBD
+ approvers:
+- TBD
+
+creation-date: 2024-04-10
+
+---
+
+# Support GPU Utilization Metrics
+
+## Summary
+Currently, HMAi supports dividing a Nvidia GPU card into several vGPU cards to efficiently utilize the
+efficiency of the GPU. When I assign a vGPU to a Pod, HAMi cannot provide information on the Pod's
+utilization of the vGPU. This results in users being unable to observe the usage situation of the Pod's vGPU.
+
+This KEP proposes support for monitoring vGPU utilization.
+
+
+## Motivation
+
+### Goals
+- Support for monitoring vGPU utilization
+
+### Non-Goals
+
+Does not support monitoring of GPU utilization for non-Nvidia GPUs.
+
+## Proposal
+
+### User Stories (Optional)
+
+
+#### Story 1
+
+I have partitioned a Nvidia GPU card into 4 parts and deployed 2 Pods on this card.
+Currently, I want to observe the GPU usage of these two Pods separately, in order to assess
+whether my business logic is reasonable.
+
+Currently, HAMi provides a `HostCoreUtilization` usage rate for the entire GPU card,
+but it still cannot observe the use of GPUs from each Pod's perspective.
+
+### Notes/Constraints/Caveats (Optional)
+
+
+### Risks and Mitigations
+
+Because the design scheme will expand the fields of the `struct shared_region` structure, there may be potential incompatibilities.
+
+## Design Details
+
+Modify the shared_region, add a `gpu_util` field to record the current pid's GPU usage.
+```c++
+typedef struct {
+ uint64_t dec_util;
+ uint64_t enc_util;
+ uint64_t sm_util;
+} device_gpu_t;
+
+typedef struct {
+ int32_t pid;
+ int32_t hostpid;
+ device_memory_t used[CUDA_DEVICE_MAX_COUNT];
+ uint64_t monitorused[CUDA_DEVICE_MAX_COUNT];
+ int32_t status;
+ device_gpu_t gpu_util[CUDA_DEVICE_MAX_COUNT]; // new field
+} shrreg_proc_slot_t;
+
+
+int set_gpu_device_gpu_monitor(int32_t pid,int dev, unsigned int smUtil){ // new function
+ //LOG_WARN("set_gpu_device_memory_monitor:%d %d %lu",pid,dev,monitor);
+ int i;
+ ensure_initialized();
+ lock_shrreg();
+ for (i=0;iproc_num;i++){
+ if (region_info.shared_region->procs[i].hostpid == pid){
+ LOG_INFO("set_gpu_device_gpu_monitor:%d %d %lu->%lu",pid,dev,region_info.shared_region->procs[i].gpuUsed[dev].smUtil,smUtil);
+ region_info.shared_region->procs[i].gpu_util[dev].smUtil = smUtil;
+ break;
+ }
+ }
+ unlock_shrreg();
+ return 1;
+}
+```
+
+Modify the `get_used_gpu_utilization` method, to record the GPU usage rate of the current pid.
+
+```c++
+
+int get_used_gpu_utilization(int *userutil,int *sysprocnum) {
+ ...
+ unsigned int nvmlCounts;
+ CHECK_NVML_API(nvmlDeviceGetCount(&nvmlCounts));
+
+ int devi,cudadev;
+ for (devi=0;devi
+
+
\ No newline at end of file
diff --git a/docs/proposals/nvidia-gpu-topology-scheduler.md b/docs/proposals/nvidia-gpu-topology-scheduler.md
new file mode 100644
index 000000000..187829049
--- /dev/null
+++ b/docs/proposals/nvidia-gpu-topology-scheduler.md
@@ -0,0 +1,34 @@
+# GPU Topology-Aware Scheduling Introduction
+
+HAMi supports GPU topology-aware scheduling in vGPU environments. HAMi can optimize GPU card scheduling based on the topological relationships between GPUs, thereby improving GPU resource utilization and performance.
+
+You can use the `nvidia-smi topo -m` command to view the topological relationships between GPUs.
+
+## How to Enable Nvidia GPU Topology-Aware Scheduling
+When installing HAMi, set `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` to `topology-aware`.
+```bash
+helm install hami hami-charts/hami --set scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy=topology-aware -n kube-system
+```
+If HAMi is already installed, you can enable it through the following methods:
+
+1. Device-plugin configuration
+
+ Set the environment variable `ENABLE_TOPOLOGY_SCORE: 'true'` in the DaemonSet `hami-device-plugin`.
+
+2. Global scheduler settings
+
+ Add `gpu-scheduler-policy=topology-aware` when starting `hami-scheduler`.
+
+3. Pod-level individual settings
+
+ Set `hami.io/gpu-scheduler-policy: topology-aware` in the Pod annotations.
+
+4. Submit the Pod and check the logs of `hami-scheduler`. HAMi will allocate the optimal GPU combination for you.
+
+ The log level must be greater than 5.
+```bash
+I0703 08:34:27.032644 1 device.go:708] "device allocate success" pod="default/testpod" best device combination={"NVIDIA":[{"Idx":7,"UUID":"GPU-dsaf","Type":"NVIDIA","Usedmem":1024,"Usedcores":0,"CustomInfo":null},{"Idx":5,"UUID":"GPU-gads","Type":"NVIDIA","Usedmem":1024,"Usedcores":0,"CustomInfo":null}]}
+```
+
+## How It Works
+For details, refer to the translated document `gpu-topo-policy.md`.
\ No newline at end of file
diff --git a/docs/proposals/nvidia-gpu-topology-scheduler_cn.md b/docs/proposals/nvidia-gpu-topology-scheduler_cn.md
new file mode 100644
index 000000000..7765babba
--- /dev/null
+++ b/docs/proposals/nvidia-gpu-topology-scheduler_cn.md
@@ -0,0 +1,33 @@
+# GPU 拓扑感知介绍
+
+HAMi 支持 vGPU 环境下的 GPU 拓扑感知调度。HAMi 可以根据 GPU 之间的拓扑关系优化 GPU 卡的调度，从而提高 GPU 资源的利用率和性能。
+
+您可以使用 `nvidia-smi topo -m` 命令查看 GPU 之间的拓扑关系。
+
+## 如何启用 Nvidia GPU 拓扑感知调度
+安装 hami 时，设置 `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` 为 `topology`。
+```bash
+helm install hami hami-charts/hami --set scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy=topology -n kube-system
+```
+如果您已经安装了 hami，可以通过以下方式：
+1. device-plugin 配置
+
+ 在 daemonset hami-device-plugin 中设置环境变量 `ENABLE_TOPOLOGY_SCORE: 'true'`。
+
+2. Scheduler 全局设置
+
+ hami-scheduler 启动时新增 `gpu-scheduler-policy=togology`
+
+3. Pod 级别单独设置
+
+ 在 Annotations 中设置 `hami.io/gpu-scheduler-policy: topology`
+
+4. 提交Pod，检查 hami-scheduler 的日志，HAMi 已为您分配最优的 GPU 组合。
+
+ 需要日志级别大于 5
+```bash
+I0703 08:34:27.032644 1 device.go:708] "device allocate success" pod="default/testpod" best device combination={"NVIDIA":[{"Idx":7,"UUID":"GPU-dsaf","Type":"NVIDIA","Usedmem":1024,"Usedcores":0,"CustomInfo":null},{"Idx":5,"UUID":"GPU-gads","Type":"NVIDIA","Usedmem":1024,"Usedcores":0,"CustomInfo":null}]}
+```
+
+## 工作原理
+详见 gpu-topo-policy.md
\ No newline at end of file
diff --git a/docs/release-process.md b/docs/release-process.md
new file mode 100644
index 000000000..c4657fc02
--- /dev/null
+++ b/docs/release-process.md
@@ -0,0 +1,119 @@
+# workflow for release
+
+## TLDR: The content is very long-winded, you can go directly to the [core content](#release-process)
+
+## Release Notes and Assets
+
+Release notes are available on GitHub at .
+
+## Release Management
+
+This section provides guidelines on release timelines and release branch maintenance.
+
+### Release Timelines
+
+HAMi uses the Semantic Versioning 2.0.0 [schema](https://semver.org/). This project follows a given version number MAJOR.MINOR.PATCH.
+
+### MAJOR release
+
+Major releases contain large features, design and architectural changes, and may include incompatible API changes. Major releases are low frequency and stable over a long period of time.
+
+### MINOR release
+
+Minor releases contain features, enhancements, and fixes that are introduced in a backwards-compatible manner. Since HAMi is a fast growing project and features continue to iterate rapidly, having a minor release approximately every few months helps balance speed and stability.
+
+* Roughly every 1-3 months
+
+### PATCH release
+
+Patch releases are for backwards-compatible bug fixes and very minor enhancements which do not impact stability or compatibility. Typically only critical fixes are selected for patch releases. Usually there will be at least one patch release in a minor release cycle.
+
+* When critical fixes are required, or roughly each month
+
+### Versioning
+
+HAMi uses GitHub tags to manage versions. New releases and release candidates are published using the wildcard tag`v..`.
+
+Whenever a PR is merged into the master branch, CI will pull the latest code, generate an image and upload it to the image repository. The latest image of HAMi components can usually be downloaded online using the latest tag.
+Whenever a release is released, the image will also be released, and the tag is the same as the tag of the release above.
+
+### Branches and PRs
+
+Release branches and PRs are managed as follows:
+
+* All changes are always first committed to `master`.
+* Branches are created for each major or minor release.
+* The branch name will contain the version, for example release-v0.23
+* Patch releases are created from a release branch.
+* For critical fixes that need to be included in a patch release, PRs should always be first merged to master and then cherry-picked to the release branch. PRs need to be guaranteed to have a release note written and these descriptions will be reflected in the next patch release.
+* For complex changes, specially critical bugfixes, separate PRs may be required for master and release branches.
+
+## release process
+
+This is the architecture diagram of the HAMi release process
+
+
+
+## [Manual] pre-steps
+
+Update version information through PR and create a new tag
+
+* update 'version' and 'appVersion' filed in 'charts/hami/Chart.yaml' by pr
+* update 'version' in 'charts/hami/values.yaml' by pr
+* update version in '/VERSION' by pr
+* a version tag should be set on right branch. The version should go with
+ * v0.1.0
+ * v0.2.0
+
+## [automatic] push a version tag
+
+> planing indicates that the plan is in progress and has not yet started.
+
+If a tag vx.x.x is pushed , the following steps will automatically run:
+
+1. check the tag name is same with
+
+2. generate pre-release-notes
+
+3. create a branch named 'release-vx.x.x'
+
+4. build the images with the pushed tag, and push to dockerhub registry
+
+5. build the helm chart with the pushed tag
+ 1. Update HAMi charts version
+ 2. build the chart package with the pushed tag, and submit a PR to branch 'gh-pages'
+
+ you cloud get the chart with command `helm repo add hami https://Project-HAMi.github.io/HAMi`
+
+6. generate release-notes and upload helm charts to github release
+ 1. upload HAMi chart tgz to release
+ 2. **[planing]** save changelogs to
+
+7. **[planing]** build HAMi website
+
+8. **[planing]** Running e2e tests
+
+9. **[planing]** Run version compatibility e2e test
+
+For the detail, refer to
+
+## [Manual] post
+
+***Although AI is smart enough, the HAMi maintainer team needs to ensure that the version is released reliably. This is why we need to do it manually at the end.***
+
+🔥🔥🔥 Finally, if the github action release runs successfully, the version rlease is complete and has passed the e2e test (including images, charts, release-notes, etc.)
+
+Finally, we need to make sure that the GITHUB release information is what we expected (release-note content, Assets, etc.). If it is confirmed, modify the version to the official version instead of the pre-release version.
+
+
+### task list
+
+* [ ] Push the image to ghcr
+
+* [ ] save changelogs to
+
+* [ ] build HAMi website
+
+* [ ] Running e2e tests
+
+* [ ] Run version compatibility e2e test
diff --git a/docs/scheduler-event-log.md b/docs/scheduler-event-log.md
new file mode 100644
index 000000000..3514836d9
--- /dev/null
+++ b/docs/scheduler-event-log.md
@@ -0,0 +1,129 @@
+# Scheduler event
+## Current Status
+### Ambiguous event descriptions make problem diagnosis difficult
+
+When a user submits a job scheduled by hami-scheduler, the Pod remains in Pending state. The Pod events only show
+generic messages like "no available node, all node scores do not meet", without providing sufficient details to
+help users identify root causes.
+
+If the Pod schedules successfully but ends up on unexpected nodes, users need visibility into:
+- Number of failed/successful node candidates
+- Detailed scoring metrics for candidate nodes
+
+Below shows a user example with pending Pod events and hami-scheduler logs:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+spec:
+ containers:
+ - name: worker01
+ image: ubuntu:22.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: "1"
+ nvidia.com/gpumem: "3000"
+ nvidia.com/gpucores: "30"
+```
+
+```event
+$ kubectl describe pod gpu-pod
+...
+Events:
+ Type Reason Age From Message
+ Warning FailedScheduling 10s hami-scheduler 0/1 nodes available: 1 node unregistered
+ Warning FilteringFailed 11s hami-scheduler no available node, all node scores do not meet
+```
+
+```log
+$ kubectl logs -f hami-scheduler-d69cb679b-9vtdg -c vgpu-scheduler-extender
+I0422 13:42:30.272812 1 pod.go:44] "collect requestreqs" counts=[{"NVIDIA":{"Nums":2,"Type":"NVIDIA","Memreq":3000,"MemPercentagereq":101,"Coresreq":30}}]
+I0422 13:42:30.272827 1 scheduler.go:499] All node scores do not meet for pod gpu-pod
+I0422 13:42:30.273047 1 event.go:307] "Event occurred" object="default/gpu-pod" type="Warning" reason="FilteringFailed" message="no available node, all node scores do not meet"
+```
+
+### Interleaved logs from multiple tasks make node scoring analysis impossible
+
+Concurrent node scoring creates interleaved logs across multiple Pods/nodes. For example:
+- 10-node cluster with 8 GPUs/node generates 80 log entries per failed Pod
+- v5-level logs show device details but lack Pod/node context
+- Default v4-level logs lack sufficient diagnostic details
+
+## Proposal
+### Event Enhancements
+Display in Pod events:
+- Number of failed nodes (for both success/failure cases)
+- Scores of successful nodes (when scheduled)
+
+Failure example:
+```
+Events:
+ Type Reason Age From Message
+ Warning FilteringFailed 2m45s hami-scheduler no available node, %d nodes do not meet
+```
+
+Success example:
+```
+Events:
+ Type Reason Age From Message
+ Normal FilteringSucceed 2m45s hami-scheduler find fit node(node3), 7 nodes not fit, 2 nodes fit(node3:0.98,node4:0.65)
+```
+
+### Log Improvements
+Two-level logging system:
+
+**v4-level (Node summary):**
+- Failed nodes: Aggregate rejection reasons
+- Successful nodes: Display scores
+
+**v5-level (Device details):**
+- Per-device failure reasons with standardized error codes
+
+Log format specification:
+` `
+
+Standardized error codes:
+```
+request devices nums cannot exceed... → NodeInsufficientDevice
+card type mismatch... → CardTypeMismatch
+the container wants exclusive access... → ExclusiveDeviceAllocateConflict
+card Insufficient remaining memory → CardInsufficientMemory
+card insufficient remaining core → CardInsufficientCore
+Numa not fit → NumaNotFit
+can't allocate core=0 job... → CardComputeUnitsExhausted
+```
+
+Example logs:
+```
+(v=5) I0422 02:15:42.349712 1 score.go:210] NodeInsufficientDevice pod="llm/llm/deepseek-5996b8569d-kgwgx" node="node2" request devices nums=2 node device nums=1
+(v=5) I0422 02:15:42.349712 1 score.go:99] CardTypeMismatch pod="llm/deepseek-5996b8569d-kgwgx" node="node1" device="GPU-0fc3eda5-e98b-a25b-5b0d-cf5c855d1448" DCU="DCU-f4502784-0000-0000-0000-000000000000"
+(v=5) I0422 02:15:42.349712 1 score.go:499] CardInsufficientMemory pod="llm/deepseek-5996b8569d-kgwgx" node="node3" device="GPU-62b7408e-edb2-41d1-bc91-f46165c61130" device index=0 device total memory=50 device used memory=0 request memory=1000
+(v=5) I0422 02:15:42.349712 1 score.go:99] CardTypeMismatch pod="llm/deepseek-5996b8569d-kgwgx" node="node2" device="GPU-006aa3c3-b59a-4fc1-8480-c5676c7bedbe" DCU="DCU-7b8fb3af-fa73-4717-8ce9-2798684431b0"
+(v=5) I0422 02:15:42.349712 1 score.go:205] NumaNotFit pod="llm/deepseek-5996b8569d-kgwgx" node="node1" device="GPU-cc244283-5652-4c35-81b0-0f54d75c0a56", k.nums=1 numa=true prevnuma=1 device numa=0
+(v=5) I0422 02:15:42.349712 1 score.go:205] NumaNotFit pod="llm/deepseek-5996b8569d-kgwgx" node="node2" device="GPU-acaecde2-cfed-4240-8d86-3155a7648a8b", k.nums=1 numa=true prevnuma=1 device numa=1
+(v=5) I0422 02:15:42.349712 1 score.go:137] CardInsufficientMemory pod="llm/deepseek-5996b8569d-kgwgx" node="node2" device="GPU-c12ea36b-23c1-400a-a18f-df9d25b956f0" device index=0 device total memory=45912 device used memory=41645 request memory=20480
+(v=5) I0422 02:15:42.349712 1 score.go:137] CardInsufficientMemory pod="llm/deepseek-5996b8569d-kgwgx" node="node1" device="GPU-4972a185-507a-4e0b-82b4-ef5af46fa229" device index=0 device total memory=45912 device used memory=39645 request memory=20480
+(v=5) I0422 02:15:42.349712 1 score.go:137] CardInsufficientMemory pod="llm/deepseek-5996b8569d-kgwgx" node="node3" device="GPU-0fs12a5-e98b-a25b-s454-cf5c855d1448" device index=0 device total memory=45912 device used memory=27645 request memory=20480
+(v=5) I0422 02:15:42.349712 1 score.go:137] CardInsufficientMemory pod="llm/deepseek-5996b8569d-kgwgx" node="node4" device="GPU-f5fa8195-324f-47be-a514-c3f856b4fef2" device index=0 device total memory=45912 device used memory=36145 request memory=20480
+(v=5) I0422 02:15:42.349712 1 score.go:137] CardInsufficientMemory pod="llm/deepseek-5996b8569d-kgwgx" node="node1" device="GPU-9e83b1fc-a3a6-4b7b-920a-66c344f0955e" device index=0 device total memory=45912 device used memory=37645 request memory=20480
+(v=5) I0422 02:15:42.349712 1 score.go:137] CardInsufficientMemory pod="llm/deepseek-5996b8569d-kgwgx" node="node2" device="GPU-9518dec4-e81e-44b5-973f-567be552fd4c" device index=0 device total memory=45912 device used memory=37645 request memory=20480
+(v=5) I0422 02:15:42.349712 1 score.go:137] CardInsufficientMemory pod="llm/deepseek-5996b8569d-kgwgx" node="node4" device="GPU-acaecde2-cfed-4240-8d86-3155a7648a8b" device index=0 device total memory=45912 device used memory=37645 request memory=20480
+(v=5) I0422 02:15:42.349712 1 score.go:142] CardInsufficientCore pod="llm/deepseek-5996b8569d-kgwgx" node="node3" device="GPU-f4c45e36-2860-4050-99bd-17a437c3e53c" device index=4 device total core=100 device used core=90 request cores=20
+(v=5) I0422 02:15:42.349712 1 score.go:142] CardInsufficientCore pod="llm/deepseek-5996b8569d-kgwgx" node="node2" device="GPU-477b6dba-1cb6-4de4-9aa8-14ec4cf4db8a" device index=0 device total core=100 device used core=85 request cores=20
+(v=5) I0422 02:15:42.349712 1 score.go:142] CardInsufficientCore pod="llm/deepseek-5996b8569d-kgwgx" node="node1" device="GPU-05ba43a8-1eb5-4645-8bdd-7fab4276ed87" device index=7 device total core=100 device used core=95 request cores=20
+(v=5) I0422 02:15:42.349712 1 score.go:142] CardInsufficientCore pod="llm/deepseek-5996b8569d-kgwgx" node="node1" device="GPU-41ee1864-ebdc-4846-a304-f49ca3105f9d" device index=3 device total core=100 device used core=89 request cores=20
+(v=5) I0422 02:15:42.349712 1 score.go:148] ExclusiveDeviceAllocateConflict pod="llm/deepseek-5996b8569d-kgwgx" node="node1" device index=0 used=9
+(v=4) I0422 02:15:42.349712 1 score.go:289] NodeUnfitPod pod="llm/deepseek-5996b8569d-kgwgx" node="node1" reason="2/8 NumaNotFit, 3/8 CardInsufficientMemory, 2/8 CardInsufficientCore, 1/8 ExclusiveDeviceAllocateConflict"
+(v=5) I0422 02:15:42.349712 1 score.go:113] CardUuidMismatch pod="llm/deepseek-5996b8569d-kgwgx" node="node2" device=6,
+(v=5) I0422 02:15:42.349712 1 score.go:137] CardInsufficientMemory pod="llm/deepseek-5996b8569d-kgwgx" node="node2" device="GPU-f4c45e36-2860-4050-99bd-17a437c3e53c" device index=0 device total memory=45912 device used memory=37645 request memory=20480
+(v=4) I0422 02:15:42.349712 1 score.go:300] NodeFitPod pod="llm/deepseek-5996b8569d-kgwgx" node="node3" score="0.65"
+(v=5) I0422 02:15:42.349712 1 score.go:142] CardInsufficientCore pod="llm/deepseek-5996b8569d-kgwgx" node="node4" device="GPU-c86aa2dc-456d-4a76-966e-8c09c16139b8" device index=7 device total core=100 device used core=95 request cores=20
+(v=4) I0422 02:15:42.349712 1 score.go:300] NodeFitPod pod="llm/deepseek-5996b8569d-kgwgx" node="node4" score="0.26"
+(v=5) I0422 02:15:42.349712 1 score.go:142] CardInsufficientCore pod="llm/deepseek-5996b8569d-kgwgx" node="node2" device="GPU-41ee1864-ebdc-4846-a304-f49ca3105f9d" device index=7 device total core=100 device used core=95 request cores=20
+(v=5) I0422 02:15:42.349712 1 score.go:148] ExclusiveDeviceAllocateConflict pod="llm/deepseek-5996b8569d-kgwgx" node="node2" device index=0 used=4
+(v=4) I0422 02:15:42.349712 1 score.go:289] NodeUnfitPod pod="llm/deepseek-5996b8569d-kgwgx" node="node2" reason="4/8 CardUuidMismatch, 3/8 CardInsufficientMemory, 1/8 ExclusiveDeviceAllocateConflict"
+(v=4) I0422 02:15:42.349712 1 score.go:300] NodeFitPod pod="llm/deepseek-5996b8569d-kgwgx" node="node5" score="0.85"
+```
\ No newline at end of file
diff --git a/example.yaml b/example.yaml
new file mode 100644
index 000000000..a1ad2ec83
--- /dev/null
+++ b/example.yaml
@@ -0,0 +1,59 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ labels:
+ kubernetes.io/metadata.name: gpu-test-workloads
+ pod-security.kubernetes.io/enforce: privileged
+ name: gpu-test-workloads
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: cuda-sample-vector-add
+ namespace: gpu-test-workloads
+ labels:
+ app: cuda-sample-vector-add
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: cuda-sample-vector-add
+ template:
+ metadata:
+ labels:
+ app: cuda-sample-vector-add
+ spec:
+ containers:
+ - name: cuda-sample-vector-add
+ image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda11.7.1-ubuntu20.04
+ command:
+ - /bin/bash
+ - '-c'
+ - '--'
+ args:
+ - while true; do /cuda-samples/vectorAdd; done
+ resources:
+ limits:
+ nvidia.com/gpu: 1 # declare how many physical GPUs the pod needs
+ nvidia.com/gpumem: 3000 # Each vGPU contains 3000M device memory （Optional,Integer）
+ terminationMessagePath: /dev/termination-log
+ terminationMessagePolicy: File
+ imagePullPolicy: IfNotPresent
+ restartPolicy: Always
+ terminationGracePeriodSeconds: 30
+ dnsPolicy: ClusterFirst
+ hostPID: true
+ securityContext: {}
+ schedulerName: default-scheduler
+ tolerations:
+ - key: nvidia.com/gpu
+ operator: Exists
+ effect: NoSchedule
+ priorityClassName: system-cluster-critical
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 25%
+ maxSurge: 25%
+ revisionHistoryLimit: 10
+ progressDeadlineSeconds: 600
\ No newline at end of file
diff --git a/examples/ascend/job-310P.yaml b/examples/ascend/job-310P.yaml
new file mode 100644
index 000000000..e84431a25
--- /dev/null
+++ b/examples/ascend/job-310P.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend310p-job
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend310P: 1 # requesting 1 NPU
+ huawei.com/Ascend310P-memory: 2000 # requesting 2000m device memory
\ No newline at end of file
diff --git a/examples/ascend/job-910A.yaml b/examples/ascend/job-910A.yaml
new file mode 100644
index 000000000..7f5950173
--- /dev/null
+++ b/examples/ascend/job-910A.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910a-job
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910A: 1 # requesting 1 NPU
+ huawei.com/Ascend910A-memory: 2000 # requesting 2000m device memory
\ No newline at end of file
diff --git a/examples/ascend/job-910B2.yaml b/examples/ascend/job-910B2.yaml
new file mode 100644
index 000000000..24687bcb1
--- /dev/null
+++ b/examples/ascend/job-910B2.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910b2-job
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910B2: 1 # requesting 1 NPU
+ huawei.com/Ascend910B2-memory: 2000 # requesting 2000m device memory
\ No newline at end of file
diff --git a/examples/ascend/job-910B3.yaml b/examples/ascend/job-910B3.yaml
new file mode 100644
index 000000000..bcdcd9c6a
--- /dev/null
+++ b/examples/ascend/job-910B3.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910b-job
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910B: 1 # requesting 1 NPU
+ huawei.com/Ascend910B-memory: 2000 # requesting 2000m device memory
\ No newline at end of file
diff --git a/examples/ascend/job-910B4.yaml b/examples/ascend/job-910B4.yaml
new file mode 100644
index 000000000..09d81634b
--- /dev/null
+++ b/examples/ascend/job-910B4.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ascend910b4-job
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ascendhub.huawei.com/public-ascendhub/ascend-mindspore:23.0.RC3-centos7
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ huawei.com/Ascend910B4: 1 # requesting 1 NPU
+ huawei.com/Ascend910B4-memory: 2000 # requesting 2000m device memory
\ No newline at end of file
diff --git a/examples/awsneuron/allocate_core.yaml b/examples/awsneuron/allocate_core.yaml
new file mode 100644
index 000000000..c6ce09d1e
--- /dev/null
+++ b/examples/awsneuron/allocate_core.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: npod
+spec:
+ restartPolicy: Never
+ containers:
+ - name: npod
+ command: ["sleep","infinity"]
+ image: public.ecr.aws/neuron/pytorch-inference-neuron:1.13.1-neuron-py310-sdk2.20.2-ubuntu20.04
+ resources:
+ limits:
+ cpu: "4"
+ memory: 4Gi
+ aws.amazon.com/neuroncore: 1
+ requests:
+ cpu: "1"
+ memory: 1Gi
\ No newline at end of file
diff --git a/examples/awsneuron/allocate_dev.yaml b/examples/awsneuron/allocate_dev.yaml
new file mode 100644
index 000000000..866cc264f
--- /dev/null
+++ b/examples/awsneuron/allocate_dev.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: nuropod
+spec:
+ restartPolicy: Never
+ containers:
+ - name: nuropod
+ command: ["sleep","infinity"]
+ image: public.ecr.aws/neuron/pytorch-inference-neuron:1.13.1-neuron-py310-sdk2.20.2-ubuntu20.04
+ resources:
+ limits:
+ cpu: "4"
+ memory: 4Gi
+ aws.amazon.com/neuron: 1
+ requests:
+ cpu: "1"
+ memory: 1Gi
\ No newline at end of file
diff --git a/examples/awsneuron/multi_containers.yaml b/examples/awsneuron/multi_containers.yaml
new file mode 100644
index 000000000..543803328
--- /dev/null
+++ b/examples/awsneuron/multi_containers.yaml
@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: npod
+spec:
+ restartPolicy: Never
+ containers:
+ - name: npod1
+ command: ["sleep","infinity"]
+ image: public.ecr.aws/neuron/pytorch-inference-neuron:1.13.1-neuron-py310-sdk2.20.2-ubuntu20.04
+ resources:
+ limits:
+ cpu: "4"
+ memory: 4Gi
+ aws.amazon.com/neuroncore: 1
+ requests:
+ cpu: "1"
+ - name: npod
+ command: ["sleep","infinity"]
+ image: public.ecr.aws/neuron/pytorch-inference-neuron:1.13.1-neuron-py310-sdk2.20.2-ubuntu20.04
+ resources:
+ limits:
+ cpu: "4"
+ memory: 4Gi
+ aws.amazon.com/neuroncore: 1
+ requests:
+ cpu: "1"
+ memory: 1Gi
\ No newline at end of file
diff --git a/examples/enflame/default_use.yaml b/examples/enflame/default_use.yaml
new file mode 100644
index 000000000..07e382bdc
--- /dev/null
+++ b/examples/enflame/default_use.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gcushare-pod-2
+ namespace: kube-system
+spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: pod-gcu-example1
+ image: ubuntu:18.04
+ imagePullPolicy: IfNotPresent
+ command:
+ - sleep
+ args:
+ - '100000'
+ resources:
+ limits:
+ enflame.com/vgcu: 1
+ enflame.com/vgcu-percentage: 22
\ No newline at end of file
diff --git a/examples/enflame/use_exclusive.yaml b/examples/enflame/use_exclusive.yaml
new file mode 100644
index 000000000..a44b25ffb
--- /dev/null
+++ b/examples/enflame/use_exclusive.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gcushare-pod-4
+ namespace: kube-system
+spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: pod-gcu-example3
+ image: ubuntu:18.04
+ imagePullPolicy: IfNotPresent
+ command:
+ - sleep
+ args:
+ - '100000'
+ resources:
+ limits:
+ enflame.com/vgcu: 1
\ No newline at end of file
diff --git a/examples/hygon/default_use.yaml b/examples/hygon/default_use.yaml
new file mode 100644
index 000000000..ea911fa86
--- /dev/null
+++ b/examples/hygon/default_use.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: alexnet-tf-gpu-pod-mem
+ labels:
+ purpose: demo-tf-amdgpu
+spec:
+ containers:
+ - name: alexnet-tf-gpu-container
+ image: pytorch:resnet50
+ workingDir: /root
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ hygon.com/dcunum: 1 # requesting a GPU
+ hygon.com/dcumem: 2000 # each dcu require 2000 MiB device memory
+ hygon.com/dcucores: 60 # each dcu use 60% of total compute cores
diff --git a/examples/hygon/specify_card_type_not_use.yaml b/examples/hygon/specify_card_type_not_use.yaml
new file mode 100644
index 000000000..7e2628b0a
--- /dev/null
+++ b/examples/hygon/specify_card_type_not_use.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: alexnet-tf-gpu-pod-mem
+ annotations:
+ hygon.com/nouse-dcutype: "Z100L" # Specify the card type for this job, use comma to seperate, will not launch job on non-specified card
+ #In this example, we don't want this container to run on Z100L
+ purpose: demo-tf-amdgpu
+spec:
+ containers:
+ - name: alexnet-tf-gpu-container
+ image: pytorch:resnet50
+ workingDir: /root
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ hygon.com/dcunum: 1 # requesting a GPU
+ hygon.com/dcumem: 2000
+ hygon.com/dcucores: 60
diff --git a/examples/hygon/specify_card_type_to_use.yaml b/examples/hygon/specify_card_type_to_use.yaml
new file mode 100644
index 000000000..b7bd877be
--- /dev/null
+++ b/examples/hygon/specify_card_type_to_use.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: alexnet-tf-gpu-pod-mem
+ annotations:
+ hygon.com/use-dcutype: "Z100" # Specify the card type for this job, use comma to seperate, will not launch job on non-specified card
+ #In this example, we want to run this job on Z100
+ labels:
+ purpose: demo-tf-amdgpu
+spec:
+ containers:
+ - name: alexnet-tf-gpu-container
+ image: pytorch:resnet50
+ workingDir: /root
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ hygon.com/dcunum: 1 # requesting a GPU
+ hygon.com/dcumem: 2000
+ hygon.com/dcucores: 60
diff --git a/examples/iluvatar/default_use.yaml b/examples/iluvatar/default_use.yaml
new file mode 100644
index 000000000..89ecfc719
--- /dev/null
+++ b/examples/iluvatar/default_use.yaml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: poddemo
+spec:
+ restartPolicy: Never
+ containers:
+ - name: poddemo
+ image: harbor.4pd.io/vgpu/corex_transformers@sha256:36a01ec452e6ee63c7aa08bfa1fa16d469ad19cc1e6000cf120ada83e4ceec1e
+ command:
+ - bash
+ args:
+ - -c
+ - |
+ set -ex
+ echo "export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH">> /root/.bashrc
+ cp -f /usr/local/iluvatar/lib64/libcuda.* /usr/local/corex/lib64/
+ cp -f /usr/local/iluvatar/lib64/libixml.* /usr/local/corex/lib64/
+ source /root/.bashrc
+ sleep 360000
+ resources:
+ requests:
+ iluvatar.ai/vgpu: 1
+ iluvatar.ai/vcuda-core: 50
+ iluvatar.ai/vcuda-memory: 64
+ limits:
+ iluvatar.ai/vgpu: 1
+ iluvatar.ai/vcuda-core: 50
+ iluvatar.ai/vcuda-memory: 64
\ No newline at end of file
diff --git a/examples/iluvatar/multi-containers.yaml b/examples/iluvatar/multi-containers.yaml
new file mode 100644
index 000000000..49d0c6e00
--- /dev/null
+++ b/examples/iluvatar/multi-containers.yaml
@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: poddemo
+spec:
+ restartPolicy: Never
+ containers:
+ - name: poddemo
+ image: harbor.4pd.io/vgpu/corex_transformers@sha256:36a01ec452e6ee63c7aa08bfa1fa16d469ad19cc1e6000cf120ada83e4ceec1e
+ command:
+ - bash
+ args:
+ - -c
+ - |
+ set -ex
+ echo "export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH">> /root/.bashrc
+ cp -f /usr/local/iluvatar/lib64/libcuda.* /usr/local/corex/lib64/
+ cp -f /usr/local/iluvatar/lib64/libixml.* /usr/local/corex/lib64/
+ source /root/.bashrc
+ sleep 360000
+ resources:
+ requests:
+ iluvatar.ai/vgpu: 1
+ iluvatar.ai/vcuda-core: 50
+ iluvatar.ai/vcuda-memory: 64
+ limits:
+ iluvatar.ai/vgpu: 1
+ iluvatar.ai/vcuda-core: 50
+ iluvatar.ai/vcuda-memory: 64
+ - name: poddemo1
+ image: harbor.4pd.io/vgpu/corex_transformers@sha256:36a01ec452e6ee63c7aa08bfa1fa16d469ad19cc1e6000cf120ada83e4ceec1e
+ command:
+ - bash
+ args:
+ - -c
+ - |
+ set -ex
+ echo "export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH">> /root/.bashrc
+ cp -f /usr/local/iluvatar/lib64/libcuda.* /usr/local/corex/lib64/
+ cp -f /usr/local/iluvatar/lib64/libixml.* /usr/local/corex/lib64/
+ source /root/.bashrc
+ sleep 360000
+ resources:
+ requests:
+ iluvatar.ai/vgpu: 1
+ iluvatar.ai/vcuda-core: 50
+ iluvatar.ai/vcuda-memory: 64
+ limits:
+ iluvatar.ai/vgpu: 1
+ iluvatar.ai/vcuda-core: 50
+ iluvatar.ai/vcuda-memory: 64
\ No newline at end of file
diff --git a/examples/iluvatar/multi-devices.yaml b/examples/iluvatar/multi-devices.yaml
new file mode 100644
index 000000000..2287835e0
--- /dev/null
+++ b/examples/iluvatar/multi-devices.yaml
@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: poddemo
+spec:
+ restartPolicy: Never
+ containers:
+ - name: poddemo
+ image: harbor.4pd.io/vgpu/corex_transformers@sha256:36a01ec452e6ee63c7aa08bfa1fa16d469ad19cc1e6000cf120ada83e4ceec1e
+ command:
+ - bash
+ args:
+ - -c
+ - |
+ set -ex
+ echo "export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH">> /root/.bashrc
+ cp -f /usr/local/iluvatar/lib64/libcuda.* /usr/local/corex/lib64/
+ cp -f /usr/local/iluvatar/lib64/libixml.* /usr/local/corex/lib64/
+ source /root/.bashrc
+ sleep 360000
+ resources:
+ requests:
+ iluvatar.ai/vgpu: 2
+ limits:
+ iluvatar.ai/vgpu: 2
\ No newline at end of file
diff --git a/examples/metax/gpu/binpack.yaml b/examples/metax/gpu/binpack.yaml
new file mode 100644
index 000000000..4fa709d77
--- /dev/null
+++ b/examples/metax/gpu/binpack.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+ annotations:
+ hami.io/node-scheduler-policy: "binpack" # when this parameter is set to binpack, the scheduler will try to minimize the topology loss.
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/gpu: 1 # requesting 1 vGPUs
\ No newline at end of file
diff --git a/examples/metax/gpu/default_use.yaml b/examples/metax/gpu/default_use.yaml
new file mode 100644
index 000000000..66e20fc19
--- /dev/null
+++ b/examples/metax/gpu/default_use.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/gpu: 1 # requesting 1 vGPUs
\ No newline at end of file
diff --git a/examples/metax/gpu/spread.yaml b/examples/metax/gpu/spread.yaml
new file mode 100644
index 000000000..d386d8b54
--- /dev/null
+++ b/examples/metax/gpu/spread.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+ annotations:
+ hami.io/node-scheduler-policy: "spread" # when this parameter is set to spread, the scheduler will try to find the best topology for this task.
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/gpu: 1 # requesting 1 vGPUs
\ No newline at end of file
diff --git a/examples/metax/sgpu/allocate_exclusive.yaml b/examples/metax/sgpu/allocate_exclusive.yaml
new file mode 100644
index 000000000..f6a7ff352
--- /dev/null
+++ b/examples/metax/sgpu/allocate_exclusive.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/sgpu: 1 # requesting 1 exclusive GPU
\ No newline at end of file
diff --git a/examples/metax/sgpu/allocate_specific_gpu.yaml b/examples/metax/sgpu/allocate_specific_gpu.yaml
new file mode 100644
index 000000000..447d44754
--- /dev/null
+++ b/examples/metax/sgpu/allocate_specific_gpu.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ metax-tech.com/use-gpuuuid: "GPU-36beae85-c835-6b14-6ab2-02671837a59c" # allocate specific gpu
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/sgpu: 1 # requesting 1 GPU
+ metax-tech.com/vcore: 60 # each GPU use 60% of total compute cores
+ metax-tech.com/vmemory: 4 # each GPU require 4 GiB device memory
\ No newline at end of file
diff --git a/examples/metax/sgpu/allocate_specific_qos.yaml b/examples/metax/sgpu/allocate_specific_qos.yaml
new file mode 100644
index 000000000..606e00c69
--- /dev/null
+++ b/examples/metax/sgpu/allocate_specific_qos.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ metax-tech.com/sgpu-qos-policy: "best-effort" # allocate specific qos sgpu
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/sgpu: 1 # requesting 1 GPU
+ metax-tech.com/vcore: 60 # each GPU use 60% of total compute cores
+ metax-tech.com/vmemory: 4 # each GPU require 4 GiB device memory
\ No newline at end of file
diff --git a/examples/metax/sgpu/allocate_vmemory_MiB.yaml b/examples/metax/sgpu/allocate_vmemory_MiB.yaml
new file mode 100644
index 000000000..98b0cdd4c
--- /dev/null
+++ b/examples/metax/sgpu/allocate_vmemory_MiB.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/sgpu: 1 # requesting 1 GPU
+ metax-tech.com/vcore: 60 # each GPU use 60% of total compute cores
+ metax-tech.com/vmemory: 2048Mi # each GPU require 2048 MiB device memory
\ No newline at end of file
diff --git a/examples/metax/sgpu/default_use.yaml b/examples/metax/sgpu/default_use.yaml
new file mode 100644
index 000000000..3da4fee37
--- /dev/null
+++ b/examples/metax/sgpu/default_use.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/sgpu: 1 # requesting 1 GPU
+ metax-tech.com/vcore: 60 # each GPU use 60% of total compute cores
+ metax-tech.com/vmemory: 4 # each GPU require 4 GiB device memory
\ No newline at end of file
diff --git a/examples/metax/sgpu/enable_topology_aware.yaml b/examples/metax/sgpu/enable_topology_aware.yaml
new file mode 100644
index 000000000..5855f66e7
--- /dev/null
+++ b/examples/metax/sgpu/enable_topology_aware.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ metax-tech.com/sgpu-topology-aware: "true" # enable topology aware scheduling
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/sgpu: 4 # requesting 4 exclusive GPU
\ No newline at end of file
diff --git a/examples/metax/sgpu/multi-containers.yaml b/examples/metax/sgpu/multi-containers.yaml
new file mode 100644
index 000000000..2c091523f
--- /dev/null
+++ b/examples/metax/sgpu/multi-containers.yaml
@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+spec:
+ containers:
+ - name: ubuntu-container-1
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/sgpu: 1 # requesting 1 GPU
+ metax-tech.com/vcore: 60 # each GPU use 60% of total compute cores
+ metax-tech.com/vmemory: 4 # each GPU require 4 GiB device memory
+ - name: ubuntu-container-2
+ image: ubuntu:22.04
+ imagePullPolicy: IfNotPresent
+ command: ["sleep","infinity"]
+ resources:
+ limits:
+ metax-tech.com/sgpu: 1 # requesting 1 GPU
+ metax-tech.com/vcore: 30 # each GPU use 30% of total compute cores
+ metax-tech.com/vmemory: 8 # each GPU require 8 GiB device memory
\ No newline at end of file
diff --git a/examples/mlu/allocate_whole.yaml b/examples/mlu/allocate_whole.yaml
new file mode 100644
index 000000000..40936cbb5
--- /dev/null
+++ b/examples/mlu/allocate_whole.yaml
@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: binpack-1
+ labels:
+ app: binpack-1
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: binpack-1
+ template:
+ metadata:
+ labels:
+ app: binpack-1
+ spec:
+ containers:
+ - name: c-1
+ image: ubuntu:18.04
+ command: ["sleep"]
+ args: ["100000"]
+ resources:
+ limits:
+ cambricon.com/vmlu: "1" #allocates a whole MLU
\ No newline at end of file
diff --git a/examples/mlu/default_use.yaml b/examples/mlu/default_use.yaml
new file mode 100644
index 000000000..8501c5bdb
--- /dev/null
+++ b/examples/mlu/default_use.yaml
@@ -0,0 +1,26 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: binpack-1
+ labels:
+ app: binpack-1
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: binpack-1
+ template:
+ metadata:
+ labels:
+ app: binpack-1
+ spec:
+ containers:
+ - name: c-1
+ image: ubuntu:18.04
+ command: ["sleep"]
+ args: ["100000"]
+ resources:
+ limits:
+ cambricon.com/vmlu: "1"
+ cambricon.com/mlu370.smlu.vmemory: "20"
+ cambricon.com/mlu370.smlu.vcore: "10"
\ No newline at end of file
diff --git a/examples/mthreads/default_use.yaml b/examples/mthreads/default_use.yaml
new file mode 100644
index 000000000..8b2714c83
--- /dev/null
+++ b/examples/mthreads/default_use.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpushare-pod-default
+spec:
+ restartPolicy: OnFailure
+ containers:
+ - image: core.harbor.zlidc.mthreads.com:30003/mt-ai/lm-qy2:v17-mpc
+ imagePullPolicy: IfNotPresent
+ name: gpushare-pod-1
+ command: ["sleep"]
+ args: ["100000"]
+ resources:
+ limits:
+ mthreads.com/vgpu: 1
+ mthreads.com/sgpu-memory: 32
+ mthreads.com/sgpu-core: 8
\ No newline at end of file
diff --git a/examples/mthreads/multi_cards.yaml b/examples/mthreads/multi_cards.yaml
new file mode 100644
index 000000000..e30eda1aa
--- /dev/null
+++ b/examples/mthreads/multi_cards.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpushare-pod-multi-cards
+spec:
+ restartPolicy: OnFailure
+ containers:
+ - image: core.harbor.zlidc.mthreads.com:30003/mt-ai/lm-qy2:v17-mpc
+ imagePullPolicy: IfNotPresent
+ name: gpushare-pod-1
+ command: ["sleep"]
+ args: ["100000"]
+ resources:
+ limits:
+ mthreads.com/vgpu: 2
\ No newline at end of file
diff --git a/examples/mthreads/use_exclusive.yaml b/examples/mthreads/use_exclusive.yaml
new file mode 100644
index 000000000..d1687fdc4
--- /dev/null
+++ b/examples/mthreads/use_exclusive.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpushare-pod-exclusive
+spec:
+ restartPolicy: OnFailure
+ containers:
+ - image: core.harbor.zlidc.mthreads.com:30003/mt-ai/lm-qy2:v17-mpc
+ imagePullPolicy: IfNotPresent
+ name: gpushare-pod-1
+ command: ["sleep"]
+ args: ["100000"]
+ resources:
+ limits:
+ mthreads.com/vgpu: 1
\ No newline at end of file
diff --git a/examples/nvidia/default_use.yaml b/examples/nvidia/default_use.yaml
new file mode 100644
index 000000000..62b274e29
--- /dev/null
+++ b/examples/nvidia/default_use.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:22.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 1 # declare how many physical GPUs the pod needs
+ nvidia.com/gpumem: 3000 # identifies 3000M GPU memory each physical GPU allocates to the pod （Optional,Integer）
+ nvidia.com/gpucores: 30 # identifies 30% GPU GPU core each physical GPU allocates to the pod （Optional,Integer)
diff --git a/examples/nvidia/default_use_legacy.yaml b/examples/nvidia/default_use_legacy.yaml
new file mode 100644
index 000000000..8336e23e0
--- /dev/null
+++ b/examples/nvidia/default_use_legacy.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # declare how many physical GPUs the pod needs
diff --git a/examples/nvidia/dynamic_mig_example.yaml b/examples/nvidia/dynamic_mig_example.yaml
new file mode 100644
index 000000000..b1e728cac
--- /dev/null
+++ b/examples/nvidia/dynamic_mig_example.yaml
@@ -0,0 +1,19 @@
+## This example will allocate 2g.10gb * 2 for A100-40GB-PCIE device
+## or 1g.10gb * 2 for A100-80GB-XSM device.
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ nvidia.com/vgpu-mode: "mig"
+ hami.io/gpu-scheduler-policy: "binpack" #(Optional)
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2
+ nvidia.com/gpumem: 8000
+
diff --git a/examples/nvidia/example.yaml b/examples/nvidia/example.yaml
new file mode 100644
index 000000000..c78df5078
--- /dev/null
+++ b/examples/nvidia/example.yaml
@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # declare how many physical GPUs the pod needs
+ #nvidia.com/gpumem: 3000 # identifies 3000M GPU memory each physical GPU allocates to the pod
+ nvidia.com/gpumem-percentage: 50 # identifies 50% GPU memory each physical GPU allocates to the pod. Can not be used with nvidia.com/gpumem
+ #nvidia.com/gpucores: 90 # identifies 90% GPU GPU core each physical GPU allocates to the pod
+ #nvidia.com/priority: 0 # we only have two priority class, 0(high) and 1(low), default: 1
+ #The utilization of high priority task won't be limited to resourceCores unless sharing GPU node with other high priority tasks.
+ #The utilization of low priority task won't be limited to resourceCores if no other tasks sharing its GPU.
+ - name: ubuntu-container0
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ - name: ubuntu-container1
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # declare how many physical GPUs the pod needs
+ nvidia.com/gpumem: 2000 # identifies 2000M GPU memory each physical GPU allocates to the pod （Optional,Integer）
+ #nvidia.com/gpucores: 90 # identifies 90% GPU GPU core each physical GPU allocates to the pod
+
diff --git a/examples/nvidia/mig_example.yaml b/examples/nvidia/mig_example.yaml
new file mode 100644
index 000000000..82fb2e386
--- /dev/null
+++ b/examples/nvidia/mig_example.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/mig-3g.20gb: 1 # requesting 1 vGPUs
diff --git a/examples/nvidia/specify_card_type_not_use.yaml b/examples/nvidia/specify_card_type_not_use.yaml
new file mode 100644
index 000000000..caa1ba03e
--- /dev/null
+++ b/examples/nvidia/specify_card_type_not_use.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ # You can run command: kubectl get node $node -o jsonpath='{.metadata.annotations.hami\.io/node-nvidia-register}' to get registered gpu info
+ # The full GPU type name is like NVIDIA-NVIDIA A100, while the short name is like A100
+ nvidia.com/nouse-gputype: "1080,2080" # Specify the blacklist card type for this job, use comma to seperate, will not launch job on specified card
+ # In this example, we don't want our job to run on 1080(include 1080Ti) or 2080(include 2080Ti) type of card.
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # declare how many physical GPUs the pod needs
diff --git a/examples/nvidia/specify_card_type_to_use.yaml b/examples/nvidia/specify_card_type_to_use.yaml
new file mode 100644
index 000000000..b7d688597
--- /dev/null
+++ b/examples/nvidia/specify_card_type_to_use.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ # You can run command: kubectl get node $node -o jsonpath='{.metadata.annotations.hami\.io/node-nvidia-register}' to get registered gpu info
+ # The full GPU type name is like NVIDIA-NVIDIA A100, while the short name is like A100
+ nvidia.com/use-gputype: "A100,V100" # Specify the card type for this job, use comma to seperate, will launch job on specified card
+ # In this example, we want to run this job on A100 or V100
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # declare how many physical GPUs the pod needs
diff --git a/examples/nvidia/specify_index_not_use.yaml b/examples/nvidia/specify_index_not_use.yaml
new file mode 100644
index 000000000..c2be4c15a
--- /dev/null
+++ b/examples/nvidia/specify_index_not_use.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ # You can run command: kubectl get node $node -o jsonpath='{.metadata.annotations.hami\.io/node-nvidia-register}' to get gpu-type
+ # index is like 0
+ nvidia.com/nouse-gpuindexes: "0" # Specify the blacklist card indexes for this job, use comma to separate, will not launch job on specified cards
+ # In this job, we don't want our job to run on index 0.
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # declare how many physical GPUs the pod needs
\ No newline at end of file
diff --git a/examples/nvidia/specify_index_to_use.yaml b/examples/nvidia/specify_index_to_use.yaml
new file mode 100644
index 000000000..1bae72804
--- /dev/null
+++ b/examples/nvidia/specify_index_to_use.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ # You can run command: kubectl get node $node -o jsonpath='{.metadata.annotations.hami\.io/node-nvidia-register}' to get gpu-type
+ # Index is like 0
+ nvidia.com/use-gpuindexes: "0,1" # Specify the card indexes for this job, separated by commas. The job will run on the specified cards
+ # In this example, we want to run this job on index 0 or index 1
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 1 # declare how many physical GPUs the pod needs
diff --git a/examples/nvidia/specify_scheduling_policy.yaml b/examples/nvidia/specify_scheduling_policy.yaml
new file mode 100644
index 000000000..608812d88
--- /dev/null
+++ b/examples/nvidia/specify_scheduling_policy.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ hami.io/node-scheduler-policy: "spread" # when this parameter is set to spread, the scheduler will try to allocate the pod to different GPU nodes for execution.
+ hami.io/gpu-scheduler-policy: "binpack" # when this parameter is set to binpack, the scheduler will try to allocate the pod to the same GPU card for execution.
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 1 # declare how many physical GPUs the pod needs
diff --git a/examples/nvidia/specify_uuid_not_use.yaml b/examples/nvidia/specify_uuid_not_use.yaml
new file mode 100644
index 000000000..137e3d246
--- /dev/null
+++ b/examples/nvidia/specify_uuid_not_use.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ # You can run command: kubectl get node $node -o jsonpath='{.metadata.annotations.hami\.io/node-nvidia-register}' to get gpu-type
+ # UUID is like GPU-03f69c50-207a-2038-9b45-23cac89cb67d
+ nvidia.com/nouse-gpuuuid: "GPU-03f69c50-207a-2038-9b45-23cac89cb67d" # Specify the blacklist card UUIDs for this job, use comma to seperate, will not launch job on specified cards
+ # In this job, we don't want our job to run on GPU-03f69c50-207a-2038-9b45-23cac89cb67d.
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # declare how many physical GPUs the pod needs
\ No newline at end of file
diff --git a/examples/nvidia/specify_uuid_to_use.yaml b/examples/nvidia/specify_uuid_to_use.yaml
new file mode 100644
index 000000000..f52fbc901
--- /dev/null
+++ b/examples/nvidia/specify_uuid_to_use.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+ annotations:
+ # You can run command: kubectl get node $node -o jsonpath='{.metadata.annotations.hami\.io/node-nvidia-register}' to get gpu-type
+ # UUID is like GPU-03f69c50-207a-2038-9b45-23cac89cb67d
+ nvidia.com/use-gpuuuid: "GPU-03f69c50-207a-2038-9b45-23cac89cb67d,GPU-03f69c50-207a-2038-9b45-23cac89cb67e" # Specify the card UUIDs for this job, separated by commas. The job will run on the specified cards
+ # In this example, we want to run this job on GPU-03f69c50-207a-2038-9b45-23cac89cb67d or GPU-03f69c50-207a-2038-9b45-23cac89cb67e
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 1 # declare how many physical GPUs the pod needs
diff --git a/examples/nvidia/use_as_normal.yaml b/examples/nvidia/use_as_normal.yaml
new file mode 100644
index 000000000..1fbf5e10d
--- /dev/null
+++ b/examples/nvidia/use_as_normal.yaml
@@ -0,0 +1,26 @@
+# Gpu-pod1 and gpu-pod2 will NOT share the same GPU
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 1 # declare how many physical GPUs the pod needs
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod2
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 1 # declare how many physical GPUs the pod needs
\ No newline at end of file
diff --git a/examples/nvidia/use_exclusive_card.yaml b/examples/nvidia/use_exclusive_card.yaml
new file mode 100644
index 000000000..79171eadb
--- /dev/null
+++ b/examples/nvidia/use_exclusive_card.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # declare how many physical GPUs the pod needs
+ nvidia.com/gpumem-percentage: 100 # identifies 100% GPU memory each physical GPU allocates to the pod （Optional,Integer）
+ nvidia.com/gpucores: 100 # identifies 100% GPU GPU core each physical GPU allocates to the pod（Optional,Integer)
diff --git a/examples/nvidia/use_memory_fraction.yaml b/examples/nvidia/use_memory_fraction.yaml
new file mode 100644
index 000000000..18480a095
--- /dev/null
+++ b/examples/nvidia/use_memory_fraction.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 2 # declare how many physical GPUs the pod needs
+ nvidia.com/gpumem-percentage: 50 # identifies 50% GPU memory each physical GPU allocates to the pod （Optional,Integer）
+ nvidia.com/gpucores: 30 # identifies 30% GPU GPU core each physical GPU allocates to the pod （Optional,Integer)
diff --git a/examples/nvidia/use_sharing_card.yaml b/examples/nvidia/use_sharing_card.yaml
new file mode 100644
index 000000000..df61f81e0
--- /dev/null
+++ b/examples/nvidia/use_sharing_card.yaml
@@ -0,0 +1,30 @@
+# Gpu-pod1 and gpu-pod2 could share the same GPU
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod1
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 1 # declare how many physical GPUs the pod needs
+ nvidia.com/gpumem-percentage: 40 # identifies 40% GPU memory each physical GPU allocates to the pod （Optional,Integer)
+ nvidia.com/gpucores: 60 # identifies 60% GPU GPU core each physical GPU allocates to the pod （Optional,Integer)
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod2
+spec:
+ containers:
+ - name: ubuntu-container
+ image: ubuntu:18.04
+ command: ["bash", "-c", "sleep 86400"]
+ resources:
+ limits:
+ nvidia.com/gpu: 1 # declare how many physical GPUs the pod needs
+ nvidia.com/gpumem-percentage: 60 # identifies 60% GPU memory each physical GPU allocates to the pod （Optional,Integer)
+ nvidia.com/gpucores: 40 # identifies 40% GPU GPU core each physical GPU allocates to the pod （Optional,Integer)
diff --git a/go.mod b/go.mod
index 773a12346..bca7f5a4a 100644
--- a/go.mod
+++ b/go.mod
@@ -1,52 +1,126 @@
-module 4pd.io/k8s-vgpu
+module github.com/Project-HAMi/HAMi
-go 1.16
+go 1.24.6
require (
- 4pd.io/k8s-vgpu/pkg/api v0.0.0
- github.com/NVIDIA/go-gpuallocator v0.2.1
- github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20210624153948-4902944b3b52
- github.com/fsnotify/fsnotify v1.4.9
+ github.com/NVIDIA/go-gpuallocator v0.6.0
+ github.com/NVIDIA/go-nvlib v0.7.4
+ github.com/NVIDIA/go-nvml v0.12.9-0
+ github.com/NVIDIA/k8s-device-plugin v0.17.3
+ github.com/NVIDIA/nvidia-container-toolkit v1.18.0-rc.2
+ github.com/ccoveille/go-safecast v1.6.1
+ github.com/fsnotify/fsnotify v1.9.0
+ github.com/google/uuid v1.6.0
+ github.com/imdario/mergo v0.3.16
github.com/julienschmidt/httprouter v1.3.0
- github.com/spf13/cobra v1.1.3
- github.com/spf13/jwalterweatherman v1.1.0 // indirect
- github.com/spf13/viper v1.7.0
- golang.org/x/net v0.0.0-20210428140749-89ef3d95e781
- google.golang.org/grpc v1.39.0
- gotest.tools/v3 v3.0.3
- k8s.io/api v0.21.2
- k8s.io/apimachinery v0.21.2
- k8s.io/client-go v0.21.2
- k8s.io/klog/v2 v2.9.0
- k8s.io/kube-scheduler v0.21.2
- k8s.io/kubelet v0.21.2
- sigs.k8s.io/controller-runtime v0.9.3
+ github.com/onsi/ginkgo/v2 v2.23.4
+ github.com/onsi/gomega v1.38.0
+ github.com/opencontainers/runtime-spec v1.2.1
+ github.com/prometheus/client_golang v1.23.0
+ github.com/sirupsen/logrus v1.9.3
+ github.com/spf13/cobra v1.9.1
+ github.com/spf13/pflag v1.0.7
+ github.com/stretchr/testify v1.10.0
+ github.com/urfave/cli/v2 v2.27.7
+ golang.org/x/net v0.43.0
+ golang.org/x/term v0.34.0
+ golang.org/x/tools v0.36.0
+ google.golang.org/grpc v1.75.0
+ google.golang.org/protobuf v1.36.8
+ gopkg.in/yaml.v2 v2.4.0
+ gopkg.in/yaml.v3 v3.0.1
+ gotest.tools/v3 v3.5.2
+ k8s.io/api v0.33.0
+ k8s.io/apimachinery v0.33.0
+ k8s.io/client-go v0.33.0
+ k8s.io/klog/v2 v2.130.1
+ k8s.io/kube-scheduler v0.28.3
+ k8s.io/kubelet v0.31.3
+ sigs.k8s.io/controller-runtime v0.21.0
+ tags.cncf.io/container-device-interface v1.0.1
+)
+
+require (
+ github.com/beorn7/perks v1.0.1 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
+ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+ github.com/emicklei/go-restful/v3 v3.11.3 // indirect
+ github.com/evanphx/json-patch v5.9.0+incompatible // indirect
+ github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+ github.com/go-logr/logr v1.4.3 // indirect
+ github.com/go-openapi/jsonpointer v0.21.0 // indirect
+ github.com/go-openapi/jsonreference v0.20.4 // indirect
+ github.com/go-openapi/swag v0.23.0 // indirect
+ github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+ github.com/golang/protobuf v1.5.4 // indirect
+ github.com/google/gnostic-models v0.6.9 // indirect
+ github.com/google/go-cmp v0.7.0 // indirect
+ github.com/google/gofuzz v1.2.0 // indirect
+ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
+ github.com/inconshreveable/mousetrap v1.1.0 // indirect
+ github.com/josharian/intern v1.0.0 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
+ github.com/mailru/easyjson v0.7.7 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
+ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+ github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
+ github.com/pkg/errors v0.9.1 // indirect
+ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+ github.com/prometheus/client_model v0.6.2 // indirect
+ github.com/prometheus/common v0.65.0 // indirect
+ github.com/prometheus/procfs v0.16.1 // indirect
+ github.com/russross/blackfriday/v2 v2.1.0 // indirect
+ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+ github.com/x448/float16 v0.8.4 // indirect
+ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+ go.uber.org/automaxprocs v1.6.0 // indirect
+ golang.org/x/mod v0.27.0 // indirect
+ golang.org/x/oauth2 v0.30.0 // indirect
+ golang.org/x/sync v0.16.0 // indirect
+ golang.org/x/sys v0.35.0 // indirect
+ golang.org/x/text v0.28.0 // indirect
+ golang.org/x/time v0.9.0 // indirect
+ gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
+ gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+ gopkg.in/inf.v0 v0.9.1 // indirect
+ k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
+ k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
+ sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+ sigs.k8s.io/randfill v1.0.0 // indirect
+ sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
+ sigs.k8s.io/yaml v1.4.0 // indirect
+ tags.cncf.io/container-device-interface/specs-go v1.0.0 // indirect
)
replace (
- 4pd.io/k8s-vgpu/pkg/api => ./pkg/api
- k8s.io/api => k8s.io/api v0.21.2
- k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.2
- k8s.io/apimachinery => k8s.io/apimachinery v0.21.2
- k8s.io/apiserver => k8s.io/apiserver v0.21.2
- k8s.io/cli-runtime => k8s.io/cli-runtime v0.21.2
- k8s.io/client-go => k8s.io/client-go v0.21.2
- k8s.io/cloud-provider => k8s.io/cloud-provider v0.21.2
- k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.21.2
- k8s.io/code-generator => k8s.io/code-generator v0.21.2
- k8s.io/component-base => k8s.io/component-base v0.21.2
- k8s.io/component-helpers => k8s.io/component-helpers v0.21.2
- k8s.io/controller-manager => k8s.io/controller-manager v0.21.2
- k8s.io/cri-api => k8s.io/cri-api v0.21.2
- k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.21.2
- k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.21.2
- k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.21.2
- k8s.io/kube-proxy => k8s.io/kube-proxy v0.21.2
- k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.21.2
- k8s.io/kubectl => k8s.io/kubectl v0.21.2
- k8s.io/kubelet => k8s.io/kubelet v0.21.2
- k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.21.2
- k8s.io/metrics => k8s.io/metrics v0.21.2
- k8s.io/mount-utils => k8s.io/mount-utils v0.21.2
- k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.21.2
+ github.com/Project-HAMi/HAMi/pkg/api => ./pkg/api
+ github.com/Project-HAMi/HAMi/pkg/device-plugin => ./pkg/device-plugin
+ github.com/Project-HAMi/HAMi/test/utils => ./test/utils
+ k8s.io/api => k8s.io/api v0.31.10
+ k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.31.10
+ k8s.io/apimachinery => k8s.io/apimachinery v0.31.10
+ k8s.io/apiserver => k8s.io/apiserver v0.31.10
+ k8s.io/cli-runtime => k8s.io/cli-runtime v0.31.10
+ k8s.io/client-go => k8s.io/client-go v0.31.10
+ k8s.io/cloud-provider => k8s.io/cloud-provider v0.31.10
+ k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.31.10
+ k8s.io/code-generator => k8s.io/code-generator v0.31.10
+ k8s.io/component-base => k8s.io/component-base v0.31.10
+ k8s.io/component-helpers => k8s.io/component-helpers v0.31.10
+ k8s.io/cri-api => k8s.io/cri-api v0.31.10
+ k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.31.10
+ k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.31.10
+ k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.31.10
+ k8s.io/kube-proxy => k8s.io/kube-proxy v0.31.10
+ k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.31.10
+ k8s.io/kubectl => k8s.io/kubectl v0.31.10
+ k8s.io/kubelet => k8s.io/kubelet v0.31.10
+ k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.31.10
+ k8s.io/metrics => k8s.io/metrics v0.31.10
+ k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.31.10
)
diff --git a/go.sum b/go.sum
index 80833e95a..9e7eccec4 100644
--- a/go.sum
+++ b/go.sum
@@ -1,785 +1,295 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
-github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
-github.com/NVIDIA/go-gpuallocator v0.2.1 h1:8EodCWEPjDl5Q+2ONB2EdwFJpcGnmACmlgLB8RkP/W0=
-github.com/NVIDIA/go-gpuallocator v0.2.1/go.mod h1:+2ke2/CGym+5xSxHfkIFi9Oof1Kj0KMSXgQDpk39ikk=
-github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20210624153948-4902944b3b52 h1:GaJrAt0sWaBLk4hB6juBCYlhi9Otng/EpgD4QLqp/T4=
-github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20210624153948-4902944b3b52/go.mod h1:oKPJa5eOTkWvlT4/Y4D8Nds44Fzmww5HUK+xwO+DwTA=
-github.com/NVIDIA/gpu-monitoring-tools/bindings/go/dcgm v0.0.0-20210325210537-29b4f1784f18/go.mod h1:8qXwltEzU3idjUcVpMOv3FNgxxbDeXZPGMLyc/khWiY=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/NVIDIA/go-gpuallocator v0.6.0 h1:2PA2swx59gJYREPkZNTGtyCP6Pnz3WEgnYsXlRkyvkk=
+github.com/NVIDIA/go-gpuallocator v0.6.0/go.mod h1:c+Yspg+/QxWOmoSQeuI48Z/7nS+mMPtxyj1NYUTwewY=
+github.com/NVIDIA/go-nvlib v0.7.4 h1:qnXK8qhm45YfxalhZ76XwKdAMmxz1GIgzE0e/Hhhshs=
+github.com/NVIDIA/go-nvlib v0.7.4/go.mod h1:i95Je7GinMy/+BDs++DAdbPmT2TubjNP8i8joC7DD7I=
+github.com/NVIDIA/go-nvml v0.12.9-0 h1:e344UK8ZkeMeeLkdQtRhmXRxNf+u532LDZPGMtkdus0=
+github.com/NVIDIA/go-nvml v0.12.9-0/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
+github.com/NVIDIA/k8s-device-plugin v0.17.3 h1:s6fN2/WqF0e6dxMfcjMXrN38fStLHkZJ3+9ErDY6DRw=
+github.com/NVIDIA/k8s-device-plugin v0.17.3/go.mod h1:pRr/ZiwAqgP3XGiQOcA2rSM0jbSDpQEPnnNyxwDFz1w=
+github.com/NVIDIA/nvidia-container-toolkit v1.18.0-rc.2 h1:vjEJpEpiGhGqca+JK9p0RqmCIBZPnOpPyT1nPj4nsEM=
+github.com/NVIDIA/nvidia-container-toolkit v1.18.0-rc.2/go.mod h1:z0KTZNkSOiQx8u4SEo1iq0aldqdnwruLHPXHdMd3B+k=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/ccoveille/go-safecast v1.6.1 h1:Nb9WMDR8PqhnKCVs2sCB+OqhohwO5qaXtCviZkIff5Q=
+github.com/ccoveille/go-safecast v1.6.1/go.mod h1:QqwNjxQ7DAqY0C721OIO9InMk9zCwcsO7tnRuHytad8=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
+github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
-github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs=
-github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM=
-github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
-github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
-github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
-github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emicklei/go-restful/v3 v3.11.3 h1:yagOQz/38xJmcNeZJtrUcKjkHRltIaIFXKWeG1SkWGE=
+github.com/emicklei/go-restful/v3 v3.11.3/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls=
+github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
+github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
+github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
+github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
+github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU=
+github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4=
+github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
+github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw=
+github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
-github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
-github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
-github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
-github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
+github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
-github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
-github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
+github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
+github.com/onsi/gomega v1.38.0 h1:c/WX+w8SLAinvuKKQFh77WEucCnPk4j2OTUr7lt7BeY=
+github.com/onsi/gomega v1.38.0/go.mod h1:OcXcwId0b9QsE7Y49u+BTrL4IdKOBOKnD6VQNTJEB6o=
+github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww=
+github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI=
+github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
+github.com/opencontainers/selinux v1.11.1 h1:nHFvthhM0qY8/m+vfhJylliSshm8G1jJ2jDMcgULaH8=
+github.com/opencontainers/selinux v1.11.1/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
-github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
-github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
-github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
-github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
-github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
-github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
-github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
-github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
+github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
+github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc=
+github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
+github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
+github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M=
+github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
+github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
-go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
-go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
+go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
+go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
+go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
+go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
+go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
+go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=
+go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY=
+go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
+go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis=
+go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4=
+go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
+go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
+go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
+go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
+go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
+go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
+golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
+golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
+golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
+golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4=
+golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs=
-golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
+golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
+golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
+golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
-gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8=
-google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI=
-google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
+gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4=
+google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM=
+google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
+google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
+google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A=
+google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
+google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
+google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
+gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
-gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
-gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y=
-k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU=
-k8s.io/apiextensions-apiserver v0.21.2 h1:+exKMRep4pDrphEafRvpEi79wTnCFMqKf8LBtlA3yrE=
-k8s.io/apiextensions-apiserver v0.21.2/go.mod h1:+Axoz5/l3AYpGLlhJDfcVQzCerVYq3K3CvDMvw6X1RA=
-k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc=
-k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM=
-k8s.io/apiserver v0.21.2/go.mod h1:lN4yBoGyiNT7SC1dmNk0ue6a5Wi6O3SWOIw91TsucQw=
-k8s.io/client-go v0.21.2 h1:Q1j4L/iMN4pTw6Y4DWppBoUxgKO8LbffEMVEV00MUp0=
-k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA=
-k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U=
-k8s.io/component-base v0.21.2 h1:EsnmFFoJ86cEywC0DoIkAUiEV6fjgauNugiw1lmIjs4=
-k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
-k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
-k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
-k8s.io/kube-scheduler v0.21.2 h1:mdNXiuxKX2WGhHHvXyrlE3PwIfIdqiS27LqAq4m87OY=
-k8s.io/kube-scheduler v0.21.2/go.mod h1:uMnMNvgw2EAoujObL1tuJ5+tvj2Pnv3k7i3X069crrs=
-k8s.io/kubelet v0.21.2 h1:n6PHxrm0FBlAGi7f3hs3CrNqVr+x3ssfrbb0aKqsBzo=
-k8s.io/kubelet v0.21.2/go.mod h1:1EqOUgp3BqvMXuZZRIlPDNkpgT5MfbJrpEnS4Gxn/mo=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210527160623-6fdb442a123b h1:MSqsVQ3pZvPGTqCjptfimO2WjG7A9un2zcpiHkA6M/s=
-k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/controller-runtime v0.9.3 h1:n075bHQ1wb8hpX7C27pNrqsb0fj8mcfCQfNX+oKTbYE=
-sigs.k8s.io/controller-runtime v0.9.3/go.mod h1:TxzMCHyEUpaeuOiZx/bIdc2T81vfs/aKdvJt9wuu0zk=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
+gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
+k8s.io/api v0.31.10 h1:hR39mlD3fxAMVotfj1aAEUOZhNMf+pL/XpL2zKvfLMk=
+k8s.io/api v0.31.10/go.mod h1:UwhlGlhYzRQuDudTdvUZ6bZZAKp0Zs82m+qEw/BZxCU=
+k8s.io/apimachinery v0.31.10 h1:fKQxHMu8IFRsC5wsiA7ySL9Z/dw9LOmVs3cifAx1cXk=
+k8s.io/apimachinery v0.31.10/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/client-go v0.31.10 h1:2WvGOFKKggxmx6kB6DP1NjdvLPyI6z+CtDWcQsyHpTI=
+k8s.io/client-go v0.31.10/go.mod h1:zRlFekIgyvhAEb8osZ6ar1//EqqGgW9C/j5jGVFNMXI=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
+k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8=
+k8s.io/kube-scheduler v0.31.10 h1:SyoyqMaysiQcdhaaJgkrl63P2goCzxrc/ABuPGkiXrQ=
+k8s.io/kube-scheduler v0.31.10/go.mod h1:HRMWIEt+o6bD2zgshffPuh9V+WbldFT7hXguSg75R2U=
+k8s.io/kubelet v0.31.10 h1:KIJ5PT0aOddRq9JwJ8xyqmifTmepaWWeAgbqawZmTa8=
+k8s.io/kubelet v0.31.10/go.mod h1:O5T/+1GQvKDLD2A4dJTjHPyUTB4qkcBcNqO0xo45Eso=
+k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro=
+k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8=
+sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM=
+sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=
+sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=
+sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc=
+sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
+tags.cncf.io/container-device-interface v1.0.1 h1:KqQDr4vIlxwfYh0Ed/uJGVgX+CHAkahrgabg6Q8GYxc=
+tags.cncf.io/container-device-interface v1.0.1/go.mod h1:JojJIOeW3hNbcnOH2q0NrWNha/JuHoDZcmYxAZwb2i0=
+tags.cncf.io/container-device-interface/specs-go v1.0.0 h1:8gLw29hH1ZQP9K1YtAzpvkHCjjyIxHZYzBAvlQ+0vD8=
+tags.cncf.io/container-device-interface/specs-go v1.0.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ=
diff --git a/hack/.import-aliases b/hack/.import-aliases
new file mode 100644
index 000000000..bfb1e8537
--- /dev/null
+++ b/hack/.import-aliases
@@ -0,0 +1,51 @@
+{
+ "k8s.io/api/admissionregistration/v1": "admissionregistrationv1",
+ "k8s.io/api/admissionregistration/v1beta1": "admissionregistrationv1beta1",
+ "k8s.io/api/admission/v1beta1": "admissionv1beta1",
+ "k8s.io/api/admission/v1": "admissionv1",
+ "k8s.io/api/apps/v1": "appsv1",
+ "k8s.io/api/apps/v1beta1": "appsv1beta1",
+ "k8s.io/api/apps/v1beta2": "appsv1beta2",
+ "k8s.io/api/authentication/v1": "authenticationv1",
+ "k8s.io/api/authentication/v1beta1": "authenticationv1beta1",
+ "k8s.io/api/authorization/v1": "authorizationv1",
+ "k8s.io/api/authorization/v1beta1": "authorizationv1beta1",
+ "k8s.io/api/autoscaling/v1": "autoscalingv1",
+ "k8s.io/api/autoscaling/v2": "autoscalingv2",
+ "k8s.io/api/batch/v1": "batchv1",
+ "k8s.io/api/batch/v1beta1": "batchv1beta1",
+ "k8s.io/api/certificates/v1beta1": "certificatesv1beta1",
+ "k8s.io/api/coordination/v1": "coordinationv1",
+ "k8s.io/api/coordination/v1beta1": "coordinationv1beta1",
+ "k8s.io/api/core/v1": "corev1",
+ "k8s.io/api/discovery/v1": "discoveryv1",
+ "k8s.io/api/events/v1": "eventsv1",
+ "k8s.io/api/events/v1beta1": "eventsv1beta1",
+ "k8s.io/api/extensions/v1beta1": "extensionsv1beta1",
+ "k8s.io/api/imagepolicy/v1alpha1": "imagepolicyv1alpha1",
+ "k8s.io/api/networking/v1": "networkingv1",
+ "k8s.io/api/networking/v1beta1": "networkingv1beta1",
+ "k8s.io/api/node/v1alpha1": "nodev1alpha1",
+ "k8s.io/api/node/v1beta1": "nodev1beta1",
+ "k8s.io/api/node/v1": "nodev1",
+ "k8s.io/api/policy/v1": "policyv1",
+ "k8s.io/api/policy/v1beta1": "policyv1beta1",
+ "k8s.io/api/rbac/v1": "rbacv1",
+ "k8s.io/api/rbac/v1alpha1": "rbacv1alpha1",
+ "k8s.io/api/rbac/v1beta1": "rbacv1beta1",
+ "k8s.io/api/scheduling/v1": "schedulingv1",
+ "k8s.io/api/scheduling/v1alpha1": "schedulingv1alpha1",
+ "k8s.io/api/scheduling/v1beta1": "schedulingv1beta1",
+ "k8s.io/api/storage/v1": "storagev1",
+ "k8s.io/api/storage/v1alpha1": "storagev1alpha1",
+ "k8s.io/api/storage/v1beta1": "storagev1beta1",
+ "k8s.io/apimachinery/pkg/api/errors": "apierrors",
+ "k8s.io/apimachinery/pkg/apis/meta/v1": "metav1",
+ "k8s.io/kubelet/apis/stats/v1alpha1": "kubeletstatsv1alpha1",
+ "k8s.io/kubelet/pkg/apis/deviceplugin/v1alpha": "kubeletdevicepluginv1alpha",
+ "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1": "kubeletdevicepluginv1beta1",
+ "k8s.io/kubelet/pkg/apis/pluginregistration/v1": "kubeletpluginregistrationv1",
+ "k8s.io/kubelet/pkg/apis/pluginregistration/v1alpha1": "kubeletpluginregistrationv1alpha1",
+ "k8s.io/kubelet/pkg/apis/pluginregistration/v1beta1": "kubeletpluginregistrationv1beta1",
+ "k8s.io/kubelet/pkg/apis/podresources/v1alpha1": "kubeletpodresourcesv1alpha1"
+}
diff --git a/hack/boilerplate/boilerplate.go.txt b/hack/boilerplate/boilerplate.go.txt
new file mode 100644
index 000000000..b33968119
--- /dev/null
+++ b/hack/boilerplate/boilerplate.go.txt
@@ -0,0 +1,16 @@
+/*
+Copyright The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
diff --git a/hack/build.sh b/hack/build.sh
index b5881d427..1a54b5805 100755
--- a/hack/build.sh
+++ b/hack/build.sh
@@ -1,6 +1,6 @@
#!/bin/bash
#
-# Copyright © 2021 peizhaoyou
+# Copyright © 2024 HAMi Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -21,8 +21,12 @@ set -e
export SHORT_VERSION
export COMMIT_CODE
export VERSION="${SHORT_VERSION}-${COMMIT_CODE}"
+export LATEST_VERSION="latest"
+export GOLANG_IMAGE="golang:1.24.6-bullseye"
+export NVIDIA_IMAGE="nvidia/cuda:12.2.0-devel-ubuntu20.04"
+export DEST_DIR="/usr/local"
-IMAGE=${IMAGE-"m7-ieg-pico-test01:5000/k8s-vgpu"}
+IMAGE=${IMAGE-"projecthami/hami"}
function go_build() {
[[ -z "$J" ]] && J=$(nproc | awk '{print int(($0 + 1)/ 2)}')
@@ -30,15 +34,17 @@ function go_build() {
}
function docker_build() {
- docker build --build-arg VERSION="${VERSION}" -t "${IMAGE}:${VERSION}" -f docker/Dockerfile .
+ docker build --build-arg VERSION="${VERSION}" --build-arg GOLANG_IMAGE=${GOLANG_IMAGE} --build-arg NVIDIA_IMAGE=${NVIDIA_IMAGE} --build-arg DEST_DIR=${DEST_DIR} -t "${IMAGE}:${VERSION}" -f docker/Dockerfile .
docker tag "${IMAGE}:${VERSION}" "${IMAGE}:${SHORT_VERSION}"
+ docker tag "${IMAGE}:${VERSION}" "${IMAGE}:${LATEST_VERSION}"
}
function docker_push() {
- docker push "${IMAGE}:${VERSION}"
+ #docker push "${IMAGE}:${VERSION}"
docker push "${IMAGE}:${SHORT_VERSION}"
+ docker push "${IMAGE}:${LATEST_VERSION}"
}
go_build
docker_build
-docker_push
\ No newline at end of file
+docker_push
diff --git a/hack/deploy-helm.sh b/hack/deploy-helm.sh
new file mode 100755
index 000000000..3aa55bfb8
--- /dev/null
+++ b/hack/deploy-helm.sh
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# Copyright 2024 The HAMi Authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit # Exit immediately if a command exits with a non-zero status
+set -o nounset # Exit if an unset variable is referenced
+set -o pipefail # Exit if any command in a pipeline fails
+set -x # Enable debug mode
+
+# Default values
+E2E_TYPE=${1:-"pullrequest"}
+KUBE_CONF=${2:-"${HOME}/.kube/config"} # Default to ~/.kube/config
+HELM_VER=${3:-"v2.4.1"}
+HELM_NAME=${4:-"hami-charts"}
+HELM_REPO=${5:-"https://project-hami.github.io/HAMi/"}
+TARGET_NS=${6:-"hami-system"}
+HAMI_ALIAS="hami"
+HELM_SOURCE=""
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+cd "${REPO_ROOT}"
+
+source "${REPO_ROOT}"/hack/util.sh
+
+# Install Helm if not already installed.
+echo -n "Preparing: 'helm' existence check - "
+if util::cmd_exist helm; then
+ echo "passed"
+else
+ echo "installing helm"
+ util::install_helm
+fi
+
+# Set Helm Chart source based on E2E_TYPE.
+echo "E2E Type is: ${E2E_TYPE}"
+
+if [ "${E2E_TYPE}" == "pullrequest" ]; then
+ # Ensure the charts directory exists and contains a .tgz file
+ if [ -d "charts" ] && [ -n "$(ls charts/*.tgz 2>/dev/null)" ]; then
+ HELM_SOURCE=$(ls charts/*.tgz | head -n 1) # Use the first .tgz file found
+ echo "Using local chart: ${HELM_SOURCE}"
+ else
+ echo "Error: No .tgz file found in the charts directory."
+ exit 1
+ fi
+elif [ "${E2E_TYPE}" == "release" ]; then
+ HELM_SOURCE="${HELM_NAME}/${HAMI_ALIAS}"
+ echo "Using remote chart: ${HELM_SOURCE}"
+else
+ echo "Invalid E2E Type: ${E2E_TYPE}"
+ exit 1
+fi
+
+# Fix kubeconfig file permissions.
+chmod 600 "${KUBE_CONF}"
+
+# Add Helm repository.
+echo "Adding Helm repository..."
+if ! helm repo add "${HELM_NAME}" "${HELM_REPO}" --force-update --kubeconfig "${KUBE_CONF}" --insecure-skip-tls-verify; then
+ echo "Error: Failed to add Helm repository. Please check the repository URL and network connectivity."
+ exit 1
+fi
+
+# Update Helm repositories.
+echo "Updating Helm repositories..."
+if ! helm repo update --kubeconfig "${KUBE_CONF}"; then
+ echo "Error: Failed to update Helm repositories. Please check your Helm configuration."
+ exit 1
+fi
+
+# Deploy or upgrade Helm Chart.
+echo "Deploying/Upgrading HAMI Helm Chart..."
+echo "Helm Source: ${HELM_SOURCE}"
+echo "Helm Version: ${HELM_VER}"
+echo "Namespace: ${TARGET_NS}"
+echo "Kubeconfig: ${KUBE_CONF}"
+
+if ! helm --debug upgrade --install --create-namespace --cleanup-on-fail \
+ "${HAMI_ALIAS}" "${HELM_SOURCE}" -n "${TARGET_NS}" \
+ --set devicePlugin.passDeviceSpecsEnabled=false \
+ --version "${HELM_VER}" --set global.imageTag="${HELM_VER}" --wait --timeout 10m --kubeconfig "${KUBE_CONF}"; then
+ echo "Error: Failed to deploy/upgrade Helm Chart. Please check the Helm logs above for more details."
+ exit 1
+fi
+
+# Check Pod status.
+echo "Checking Pod status..."
+kubectl --kubeconfig "${KUBE_CONF}" get po -n "${TARGET_NS}"
+
+if ! util::check_pods_status "${KUBE_CONF}" ; then
+ echo "Error: Pods are not running correctly."
+ exit 1
+fi
+
+echo "HAMI Helm Chart deployed successfully."
\ No newline at end of file
diff --git a/hack/e2e-test-setup.sh b/hack/e2e-test-setup.sh
new file mode 100755
index 000000000..e555ca42c
--- /dev/null
+++ b/hack/e2e-test-setup.sh
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+# Copyright 2024 The HAMi Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+set -x
+
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+cd "${REPO_ROOT}"
+
+source "${REPO_ROOT}"/hack/util.sh
+
+function install_govc() {
+ local govc_version="v0.37.3"
+ local govc_tar_url="https://github.com/vmware/govmomi/releases/download/${govc_version}/govc_Linux_x86_64.tar.gz"
+
+ wget -q $govc_tar_url || { echo "Failed to download govc"; exit 1; }
+ tar -zxvf govc_Linux_x86_64.tar.gz
+ mv govc /usr/local/bin/
+ govc version
+}
+
+function govc_poweron_vm() {
+ local vm_name=${1:-""}
+ local vm_ip=${2:-""}
+ if [[ -z "$vm_name" ]]; then
+ echo "Error: VM name is required"
+ return 1
+ fi
+
+ govc vm.power -on "$vm_name"
+ echo -e "\033[35m === $vm_name: power turned on === \033[0m"
+ until [[ $(govc vm.info "$vm_name" | grep -c poweredOn) -eq 1 ]]; do
+ sleep 5
+ done
+
+ util::wait_ip_reachable "$vm_ip"
+}
+
+function govc_poweroff_vm() {
+ local vm_name=${1:-""}
+ if [[ -z "$vm_name" ]]; then
+ echo "Error: VM name is required"
+ return 1
+ fi
+
+ if [[ $(govc vm.info "$vm_name" | grep -c poweredOn) -eq 1 ]]; then
+ govc vm.power -off -force "$vm_name"
+ echo -e "\033[35m === $vm_name has been down === \033[0m"
+ fi
+}
+
+function govc_restore_vm_snapshot() {
+ local vm_name=${1:-""}
+ local vm_snapshot_name=${2:-""}
+
+ govc snapshot.revert -vm "$vm_name" "$vm_snapshot_name"
+ echo -e "\033[35m === $vm_name reverted to snapshot: $(govc snapshot.tree -vm "$vm_name" -C -D -i -d) === \033[0m"
+}
+
+function setup_gpu_test_env() {
+ export GOVC_INSECURE=1
+ export vm_ip=$VSPHERE_GPU_VM_IP
+ export vm_name=$VSPHERE_GPU_VM_NAME
+ export vm_snapshot_name=$VSPHERE_GPU_VM_NAME_SNAPSHOT
+
+ echo -n "Preparing: 'govc' existence check - "
+ if util::cmd_exist govc; then
+ echo "passed"
+ else
+ echo "installing govc"
+ install_govc
+ fi
+
+ govc_poweroff_vm "$vm_name"
+ govc_restore_vm_snapshot "$vm_name" "$vm_snapshot_name"
+ govc_poweron_vm "$vm_name" "$vm_ip"
+}
+
+
+setup_gpu_test_env
diff --git a/hack/e2e-test.sh b/hack/e2e-test.sh
new file mode 100755
index 000000000..063a614db
--- /dev/null
+++ b/hack/e2e-test.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Copyright 2024 The HAMi Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+set -x
+
+E2E_TYPE=${1:-"pullrequest"}
+KUBE_CONF=${2:-""}
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${REPO_ROOT}"/hack/util.sh
+
+if util::cmd_exist ginkgo; then
+ echo "Using ginkgo version:"
+ ginkgo version
+else
+ go install github.com/onsi/ginkgo/v2/ginkgo
+ go get github.com/onsi/gomega/...
+ ginkgo version
+fi
+
+
+if [ -z "${KUBE_CONF}" ]; then
+ echo "Error: KUBE_CONF environment variable is not set."
+ return 1
+fi
+
+# Run e2e
+if [ "${E2E_TYPE}" == "pullrequest" ] || [ "${E2E_TYPE}" == "release" ]; then
+ ginkgo -v -r --fail-fast ./test/e2e/ --kubeconfig="${KUBE_CONF}"
+ if [ $? -ne 0 ]; then
+ echo "Error: ginkgo command failed."
+ return 1
+ fi
+else
+ echo "Invalid E2E Type: ${E2E_TYPE}"
+ return 1
+fi
diff --git a/hack/kubeconfig-demo.yaml b/hack/kubeconfig-demo.yaml
new file mode 100644
index 000000000..f41fa7bdc
--- /dev/null
+++ b/hack/kubeconfig-demo.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+clusters:
+- cluster:
+ server: http://localhost:8080
+ name: local-server
+contexts:
+- context:
+ cluster: local-server
+ namespace: the-right-prefix
+ user: myself
+ name: default-context
+current-context: default-context
+kind: Config
+preferences: {}
+users:
+- name: myself
+ user:
+ password: secret
+ username: admin
diff --git a/hack/tools/preferredimports/preferredimports.go b/hack/tools/preferredimports/preferredimports.go
new file mode 100644
index 000000000..1ce9b0175
--- /dev/null
+++ b/hack/tools/preferredimports/preferredimports.go
@@ -0,0 +1,271 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This code is directly lifted from the Kubernetes codebase in order to avoid relying on the k8s.io/kubernetes package.
+// For reference: https://github.com/kubernetes/kubernetes/blob/release-1.22/cmd/preferredimports/preferredimports.go
+
+// verify that all the imports have our preferred alias(es).
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "go/ast"
+ "go/build"
+ "go/format"
+ "go/parser"
+ "go/token"
+ "log"
+ "os"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strings"
+
+ "golang.org/x/term"
+)
+
+var (
+ importAliases = flag.String("import-aliases", "hack/.import-aliases", "json file with import aliases")
+ confirm = flag.Bool("confirm", false, "update file with the preferred aliases for imports")
+ includePathRegex = flag.String("include-path", "(test/e2e/|test/e2e_node)", "only files with paths matching this regex is touched")
+ excludePathRegex = flag.String("exclude-path", "(testing)", "files with paths matching this regex is ignored")
+ isTerminal = term.IsTerminal(int(os.Stdout.Fd()))
+ logPrefix = ""
+ aliases map[string]string
+)
+
+type analyzer struct {
+ fset *token.FileSet // positions are relative to fset
+ ctx build.Context
+ failed bool
+ donePaths map[string]any
+}
+
+func newAnalyzer() *analyzer {
+ ctx := build.Default
+ ctx.CgoEnabled = true
+
+ a := &analyzer{
+ fset: token.NewFileSet(),
+ ctx: ctx,
+ donePaths: make(map[string]any),
+ }
+
+ return a
+}
+
+// collect extracts test metadata from a file.
+func (a *analyzer) collect(dir string) {
+ if _, ok := a.donePaths[dir]; ok {
+ return
+ }
+ a.donePaths[dir] = nil
+
+ // Create the AST by parsing src.
+ fs, err := parser.ParseDir(a.fset, dir, nil, parser.AllErrors|parser.ParseComments)
+
+ if err != nil {
+ fmt.Fprintln(os.Stderr, "ERROR(syntax)", logPrefix, err)
+ a.failed = true
+ return
+ }
+
+ for _, p := range fs {
+ // returns first error, but a.handleError deals with it
+ files := a.filterFiles(p.Files)
+ for _, file := range files {
+ replacements := make(map[string]string)
+ pathToFile := a.fset.File(file.Pos()).Name()
+ for _, imp := range file.Imports {
+ importPath := strings.ReplaceAll(imp.Path.Value, "\"", "")
+ pathSegments := strings.Split(importPath, "/")
+ importName := pathSegments[len(pathSegments)-1]
+ if imp.Name != nil {
+ importName = imp.Name.Name
+ }
+ if alias, ok := aliases[importPath]; ok {
+ if alias != importName {
+ if !*confirm {
+ fmt.Fprintf(os.Stderr, "%sERROR wrong alias for import \"%s\" should be %s in file %s\n", logPrefix, importPath, alias, pathToFile)
+ a.failed = true
+ }
+ replacements[importName] = alias
+ if imp.Name != nil {
+ imp.Name.Name = alias
+ } else {
+ imp.Name = ast.NewIdent(alias)
+ }
+ }
+ }
+ }
+
+ if len(replacements) > 0 {
+ if *confirm {
+ fmt.Printf("%sReplacing imports with aliases in file %s\n", logPrefix, pathToFile)
+ for key, value := range replacements {
+ renameImportUsages(file, key, value)
+ }
+ ast.SortImports(a.fset, file)
+ var buffer bytes.Buffer
+ if err = format.Node(&buffer, a.fset, file); err != nil {
+ panic(fmt.Sprintf("Error formatting ast node after rewriting import.\n%s\n", err.Error()))
+ }
+
+ fileInfo, err := os.Stat(pathToFile)
+ if err != nil {
+ panic(fmt.Sprintf("Error stat'ing file: %s\n%s\n", pathToFile, err.Error()))
+ }
+
+ err = os.WriteFile(pathToFile, buffer.Bytes(), fileInfo.Mode())
+ if err != nil {
+ panic(fmt.Sprintf("Error writing file: %s\n%s\n", pathToFile, err.Error()))
+ }
+ }
+ }
+ }
+ }
+}
+
+func renameImportUsages(f *ast.File, old, new string) {
+ // use this to avoid renaming the package declaration, eg:
+ // given: package foo; import foo "bar"; foo.Baz, rename foo->qux
+ // yield: package foo; import qux "bar"; qux.Baz
+ var pkg *ast.Ident
+
+ // Rename top-level old to new, both unresolved names
+ // (probably defined in another file) and names that resolve
+ // to a declaration we renamed.
+ ast.Inspect(f, func(node ast.Node) bool {
+ if node == nil {
+ return false
+ }
+ switch id := node.(type) {
+ case *ast.File:
+ pkg = id.Name
+ case *ast.Ident:
+ if pkg != nil && id == pkg {
+ return false
+ }
+ if id.Name == old {
+ id.Name = new
+ }
+ }
+ return true
+ })
+}
+
+func (a *analyzer) filterFiles(fs map[string]*ast.File) []*ast.File {
+ var files []*ast.File
+ for _, f := range fs {
+ files = append(files, f)
+ }
+ return files
+}
+
+type collector struct {
+ dirs []string
+ includePathRegex *regexp.Regexp
+ excludePathRegex *regexp.Regexp
+}
+
+// handlePath walks the filesystem recursively, collecting directories,
+// ignoring some unneeded directories (hidden/vendored) that are handled
+// specially later.
+func (c *collector) handlePath(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ // Ignore hidden directories (.git, .cache, etc)
+ if len(path) > 1 && path[0] == '.' ||
+ // Staging code is symlinked from vendor/k8s.io, and uses import
+ // paths as if it were inside of vendor/. It fails typechecking
+ // inside of staging/, but works when typechecked as part of vendor/.
+ path == "staging" ||
+ // OS-specific vendor code tends to be imported by OS-specific
+ // packages. We recursively typecheck imported vendored packages for
+ // each OS, but don't typecheck everything for every OS.
+ path == "vendor" ||
+ path == "_output" ||
+ // This is a weird one. /testdata/ is *mostly* ignored by Go,
+ // and this translates to kubernetes/vendor not working.
+ // edit/record.go doesn't compile without gopkg.in/yaml.v2
+ // in $GOSRC/$GOROOT (both typecheck and the shell script).
+ path == "pkg/kubectl/cmd/testdata/edit" {
+ return filepath.SkipDir
+ }
+ if c.includePathRegex.MatchString(path) && !c.excludePathRegex.MatchString(path) {
+ c.dirs = append(c.dirs, path)
+ }
+ }
+ return nil
+}
+
+func main() {
+ flag.Parse()
+ args := flag.Args()
+
+ if len(args) == 0 {
+ args = append(args, ".")
+ }
+
+ includePathRegex, err := regexp.Compile(*includePathRegex)
+ if err != nil {
+ log.Fatalf("Error compiling regex: %v", err)
+ }
+ excludePathRegex, err := regexp.Compile(*excludePathRegex)
+ if err != nil {
+ log.Fatalf("Error compiling regex: %v", err)
+ }
+ c := collector{includePathRegex: includePathRegex, excludePathRegex: excludePathRegex}
+ for _, arg := range args {
+ err := filepath.Walk(arg, c.handlePath)
+ if err != nil {
+ log.Fatalf("Error walking: %v", err)
+ }
+ }
+ sort.Strings(c.dirs)
+
+ if len(*importAliases) > 0 {
+ bytes, err := os.ReadFile(*importAliases)
+ if err != nil {
+ log.Fatalf("Error reading import aliases: %v", err)
+ }
+ err = json.Unmarshal(bytes, &aliases)
+ if err != nil {
+ log.Fatalf("Error loading aliases: %v", err)
+ }
+ }
+ if isTerminal {
+ logPrefix = "\r" // clear status bar when printing
+ }
+ fmt.Println("checking-imports: ")
+
+ a := newAnalyzer()
+ for _, dir := range c.dirs {
+ if isTerminal {
+ fmt.Printf("\r\033[0m %-80s\n", dir)
+ }
+ a.collect(dir)
+ }
+ fmt.Println()
+ if a.failed {
+ os.Exit(1)
+ }
+}
diff --git a/hack/tools/tools.go b/hack/tools/tools.go
new file mode 100644
index 000000000..5a4909822
--- /dev/null
+++ b/hack/tools/tools.go
@@ -0,0 +1,23 @@
+//go:build tools
+
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tools
+
+import (
+ _ "golang.org/x/tools/cmd/goimports"
+)
diff --git a/hack/unit-test.sh b/hack/unit-test.sh
new file mode 100755
index 000000000..bf9c3219c
--- /dev/null
+++ b/hack/unit-test.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Copyright 2024 The HAMi Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+set -x
+
+# init kubeconfig env
+kubeconfig_path="${HOME}/.kube"
+kubeconfig_file="${kubeconfig_path}/config"
+kubeconfig_demo="./hack/kubeconfig-demo.yaml"
+
+echo "kubeconfig: ${kubeconfig_file}"
+
+if [ ! -f "$kubeconfig_file" ]; then
+ echo "Generate fake kubeconfig"
+ if [ ! -d "${kubeconfig_path}" ]; then
+ trap 'rm -rf "$kubeconfig_path"' EXIT
+ mkdir -p "${kubeconfig_path}"
+ cp ${kubeconfig_demo} "${kubeconfig_file}"
+ else
+ trap 'rm -f "$kubeconfig_file"' EXIT
+ cp ${kubeconfig_demo} "${kubeconfig_file}"
+ fi
+else
+ echo "Use local kubeconfig"
+fi
+
+mkdir -p ./_output/coverage/
+mergeF="./_output/coverage/merge.out"
+rm -f ${mergeF}
+cov_file="./_output/coverage/coverage_pkg.txt"
+go test $(go list ./pkg/... | grep -v ./pkg/device-plugin/...) -short --race -count=1 -covermode=atomic -coverprofile=${cov_file}
+cat $cov_file | grep -v mode: | grep -v pkg/version | grep -v fake | grep -v main.go >>${mergeF}
+#merge them
+echo "mode: atomic" >coverage.out
+cat ${mergeF} >>./_output/coverage/coverage.out
+go tool cover -func=coverage.out
diff --git a/hack/update-generated-api.sh b/hack/update-generated-api.sh
index f455acc85..c7b11cf23 100755
--- a/hack/update-generated-api.sh
+++ b/hack/update-generated-api.sh
@@ -1,6 +1,6 @@
#!/bin/bash
#
-# Copyright © 2021 peizhaoyou
+# Copyright © 2024 HAMi Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/hack/util.sh b/hack/util.sh
new file mode 100755
index 000000000..7f4a0969b
--- /dev/null
+++ b/hack/util.sh
@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+# Copyright 2024 The HAMi Authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit # Exit immediately if a command exits with a non-zero status
+set -o nounset # Exit if an unset variable is referenced
+set -o pipefail # Exit if any command in a pipeline fails
+
+MIN_Go_VERSION=1.21.0
+
+# Check if a command exists.
+function util::cmd_exist {
+ command -v "${1}" >/dev/null 2>&1
+}
+
+# Verify Go version.
+function util::verify_go_version {
+ local go_version
+ IFS=" " read -ra go_version <<<"$(GOFLAGS='' go version)"
+ if [[ "${go_version[2]}" == "devel" ]]; then
+ return 0
+ fi
+ util::vercomp "${go_version[2]#go}" "${MIN_Go_VERSION}"
+ if [[ $? -eq 2 ]]; then
+ echo "Detected go version: ${go_version[*]}."
+ echo "Requires ${MIN_Go_VERSION} or greater."
+ echo "Please install ${MIN_Go_VERSION} or later."
+ exit 1
+ fi
+}
+
+# Version comparison function.
+function util::vercomp {
+ if [[ $1 == $2 ]]; then
+ return 0
+ fi
+ local IFS=.
+ local i ver1=($1) ver2=($2)
+ for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do
+ ver1[i]=0
+ done
+ for ((i=0; i<${#ver1[@]}; i++)); do
+ if [[ -z ${ver2[i]} ]]; then
+ ver2[i]=0
+ fi
+ if ((10#${ver1[i]} > 10#${ver2[i]})); then
+ return 1
+ fi
+ if ((10#${ver1[i]} < 10#${ver2[i]})); then
+ return 2
+ fi
+ done
+ return 0
+}
+
+# Install Helm.
+function util::install_helm {
+ if util::cmd_exist helm; then
+ echo "Helm is already installed."
+ return 0
+ fi
+ echo "Installing Helm..."
+ curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+}
+
+# Execute a command and capture output.
+function util::exec_cmd {
+ if [ $# -eq 0 ]; then
+ echo "[Error] No command specified for util::exec_cmd()..."
+ exit 2
+ fi
+ local tmpLog=$(mktemp)
+ set +e
+ "$@" &> "$tmpLog"
+ local ret=$?
+ if [ $ret -ne 0 ]; then
+ echo "[Error] Failed to execute command: $*"
+ echo "Detail logs:"
+ cat "$tmpLog"
+ rm -f "$tmpLog"
+ exit $ret
+ fi
+ echo "Command executed successfully: $*"
+ rm -f "$tmpLog"
+ set -e
+}
+
+# Check if an IP is reachable.
+function util::wait_ip_reachable {
+ local vm_ip=${1:-""}
+ local loop_time=${2:-10}
+ local sleep_time=${3:-60}
+ echo "Waiting for IP $vm_ip to be reachable..."
+ for ((i=1; i<=loop_time; i++)); do
+ if ping -c 1 -W 2 "$vm_ip" &>/dev/null; then
+ echo "IP $vm_ip is reachable."
+ return 0
+ fi
+ echo "Attempt $i/$loop_time: IP $vm_ip not reachable. Retrying in $sleep_time seconds..."
+ sleep "$sleep_time"
+ done
+ echo "Error: IP $vm_ip not reachable after $loop_time attempts."
+ exit 1
+}
+
+# Check Pod status in a namespace.
+function util::check_pods_status {
+ local kubeconfig=${1:-""}
+ local namespace=${2:-""}
+ local retries=${3:-10}
+ local interval=${4:-30}
+
+ local attempt=0
+ local unhealthy_pods
+
+ while (( attempt < retries )); do
+ echo "Checking Pod status (Attempt $(( attempt + 1 ))/$retries)..."
+
+ # Checking unhealthy pods in namespaces，ignore the Running & Succeeded status
+ if [[ -z "$namespace" ]]; then
+ unhealthy_pods=$(kubectl get po -A --kubeconfig "$kubeconfig" --no-headers --ignore-not-found | awk '!/Running|Succeeded|Completed/ {print $2}')
+ else
+ unhealthy_pods=$(kubectl get po -n "$namespace" --kubeconfig "$kubeconfig" --no-headers --ignore-not-found | awk '!/Running|Succeeded|Completed/ {print $1}')
+ fi
+
+ if [[ -z "$unhealthy_pods" ]]; then
+ echo "PASS: All Pods are in Running or Succeeded state."
+ return 0
+ fi
+
+ echo "Found unhealthy pods:"
+ echo "$unhealthy_pods"
+
+ if (( attempt < retries - 1 )); then
+ echo "Retrying pod check in ${interval}s..."
+ sleep "$interval"
+ fi
+
+ (( attempt++ ))
+ done
+
+ if [[ -n "$unhealthy_pods" ]]; then
+ echo "Found unhealthy pods in namespace $namespace:"
+ echo "$unhealthy_pods"
+
+ for pod in $unhealthy_pods; do
+ echo "Describing pod: $pod"
+ kubectl describe po "$pod" -n "$namespace" --kubeconfig "$kubeconfig"
+
+ echo "Fetching logs for pod: $pod"
+ kubectl logs "$pod" -n "$namespace" --kubeconfig "$kubeconfig"
+ echo "---------------------------------------------------"
+ done
+
+ return 1
+ fi
+}
diff --git a/hack/verify-all.sh b/hack/verify-all.sh
new file mode 100755
index 000000000..76be88d70
--- /dev/null
+++ b/hack/verify-all.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Copyright 2024 The HAMi Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+
+# Show progress
+set -x
+
+# Orders are determined by two factors:
+# (1) Less Execution time item should be executed first.
+# (2) More likely to fail item should be executed first.
+
+bash "$REPO_ROOT/hack/verify-staticcheck.sh"
+
+bash "$REPO_ROOT/hack/verify-license.sh"
+
+bash "$REPO_ROOT/hack/verify-import-aliases.sh"
diff --git a/hack/verify-chart-version.sh b/hack/verify-chart-version.sh
new file mode 100755
index 000000000..7c6ff0068
--- /dev/null
+++ b/hack/verify-chart-version.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Copyright 2024 The HAMi Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+cd "${REPO_ROOT}"
+
+source "${REPO_ROOT}"/hack/util.sh
+
+# install helm
+echo -n "Preparing: 'helm' existence check - "
+if util::cmd_exist helm; then
+ echo "passed"
+else
+ echo "installing helm"
+ util::install_helm
+fi
+
+APP_VERSION=$(helm show chart ./charts/hami | grep '^appVersion' |grep -E '[0-9].*.[0-9]' | awk -F ':' '{print $2}' | tr -d ' ')
+VERSION=$(helm show chart ./charts/hami | grep '^version' |grep -E '[0-9].*.[0-9]' | awk -F ':' '{print $2}' | tr -d ' ')
+
+if [[ ${APP_VERSION} != ${VERSION} ]]; then
+ echo "AppVersion of HAMi is ${APP_VERSION}, but version is ${VERSION}!"
+ exit 1
+fi
+
+echo "Both appVersion and version is ${APP_VERSION}."
+
diff --git a/hack/verify-import-aliases.sh b/hack/verify-import-aliases.sh
new file mode 100755
index 000000000..3b8a8f705
--- /dev/null
+++ b/hack/verify-import-aliases.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Copyright 2024 The HAMi Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+cd "${SCRIPT_ROOT}"
+ROOT_PATH=$(pwd)
+
+IMPORT_ALIASES_PATH="${ROOT_PATH}/hack/.import-aliases"
+INCLUDE_PATH="(${ROOT_PATH}/cmd|${ROOT_PATH}/pkg)"
+
+ret=0
+# We can't directly install preferredimports by `go install` due to the go.mod issue:
+# go install k8s.io/kubernetes/cmd/preferredimports@v1.21.3: k8s.io/kubernetes@v1.21.3
+# The go.mod file for the module providing named packages contains one or
+# more replace directives. It must not contain directives that would cause
+# it to be interpreted differently than if it were the main module.
+go run "${ROOT_PATH}/hack/tools/preferredimports/preferredimports.go" -import-aliases "${IMPORT_ALIASES_PATH}" -include-path "${INCLUDE_PATH}" "${ROOT_PATH}" || ret=$?
+if [[ $ret -ne 0 ]]; then
+ echo "!!! Please see hack/.import-aliases for the preferred aliases for imports." >&2
+ exit 1
+fi
+echo "Passed import-aliases verification."
diff --git a/hack/verify-license.sh b/hack/verify-license.sh
new file mode 100755
index 000000000..a28cc234e
--- /dev/null
+++ b/hack/verify-license.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Copyright 2024 The HAMi Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -ex
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+cd "${REPO_ROOT}"
+
+if [[ "$(which addlicense)" == "" ]]; then
+ go install github.com/google/addlicense@v1.1.1
+fi
+ADDLICENSE_BIN=$(which addlicense)
+
+# verify presence of license headers and exit with non-zero code if missing
+missing_license_header_files="$($ADDLICENSE_BIN \
+ -check \
+ -ignore "benchmarks/**" \
+ -ignore "charts/**" \
+ -ignore "docs/**" \
+ -ignore "docker/**" \
+ -ignore "examples/**" \
+ -ignore "lib/**" \
+ -ignore "libvgpu/**" \
+ -ignore "third_party/**" \
+ -ignore "vendor/**" \
+ -ignore "_output/**" \
+ -ignore ".github/**" \
+ -ignore "**/*.md" \
+ -ignore "**/*.yaml" \
+ -ignore "**/*.yml" \
+ -ignore "**/*.json" \
+ -ignore ".idea/**" \
+ .)" || true
+
+if [[ "$missing_license_header_files" ]]; then
+ echo "Files with no license header detected:"
+ echo "$missing_license_header_files"
+ echo "Please add all missing license headers."
+ exit 1
+fi
+
+echo "Congratulations! All files have passed license header check."
diff --git a/hack/verify-staticcheck.sh b/hack/verify-staticcheck.sh
new file mode 100755
index 000000000..a7345d78f
--- /dev/null
+++ b/hack/verify-staticcheck.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Copyright 2024 The HAMi Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+GOLANGCI_LINT_VER="v2.1.1"
+
+cd "${REPO_ROOT}"
+source "hack/util.sh"
+
+if util::cmd_exist golangci-lint; then
+ echo "Using golangci-lint version:"
+ golangci-lint version
+else
+ echo "Installing golangci-lint ${GOLANGCI_LINT_VER}"
+ # https://golangci-lint.run/usage/install/#other-ci
+ curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin ${GOLANGCI_LINT_VER}
+fi
+
+if golangci-lint run; then
+ echo 'Congratulations! All Go source files have passed staticcheck.'
+else
+ echo # print one empty line, separate from warning messages.
+ echo 'Please review the above warnings.'
+ echo 'If the above warnings do not make sense, feel free to file an issue.'
+ exit 1
+fi
diff --git a/imgs/arch.png b/imgs/arch.png
new file mode 100644
index 000000000..f8bcb6c32
Binary files /dev/null and b/imgs/arch.png differ
diff --git a/imgs/benchmark.png b/imgs/benchmark.png
new file mode 100644
index 000000000..3de68fd38
Binary files /dev/null and b/imgs/benchmark.png differ
diff --git a/imgs/benchmark_inf.png b/imgs/benchmark_inf.png
new file mode 100644
index 000000000..ec52cb56d
Binary files /dev/null and b/imgs/benchmark_inf.png differ
diff --git a/imgs/benchmark_train.png b/imgs/benchmark_train.png
new file mode 100644
index 000000000..78eaa9235
Binary files /dev/null and b/imgs/benchmark_train.png differ
diff --git a/imgs/cncf-logo.png b/imgs/cncf-logo.png
new file mode 100644
index 000000000..fc25a4660
Binary files /dev/null and b/imgs/cncf-logo.png differ
diff --git a/imgs/example.png b/imgs/example.png
new file mode 100644
index 000000000..a46f9874a
Binary files /dev/null and b/imgs/example.png differ
diff --git a/imgs/hami-arch.png b/imgs/hami-arch.png
new file mode 100644
index 000000000..f8bcb6c32
Binary files /dev/null and b/imgs/hami-arch.png differ
diff --git a/imgs/hami-arch.pptx b/imgs/hami-arch.pptx
new file mode 100644
index 000000000..790668c39
Binary files /dev/null and b/imgs/hami-arch.pptx differ
diff --git a/imgs/hami-graph-color.png b/imgs/hami-graph-color.png
new file mode 100644
index 000000000..514a585c3
Binary files /dev/null and b/imgs/hami-graph-color.png differ
diff --git a/imgs/hami-horizontal-colordark.png b/imgs/hami-horizontal-colordark.png
new file mode 100644
index 000000000..4e9f35830
Binary files /dev/null and b/imgs/hami-horizontal-colordark.png differ
diff --git a/imgs/hami-vgpu-metrics-dashboard.png b/imgs/hami-vgpu-metrics-dashboard.png
new file mode 100644
index 000000000..0f322f7eb
Binary files /dev/null and b/imgs/hami-vgpu-metrics-dashboard.png differ
diff --git a/imgs/hard_limit.jpg b/imgs/hard_limit.jpg
new file mode 100644
index 000000000..554bfbb79
Binary files /dev/null and b/imgs/hard_limit.jpg differ
diff --git a/imgs/metax_binpack.png b/imgs/metax_binpack.png
new file mode 100644
index 000000000..c40c6ab34
Binary files /dev/null and b/imgs/metax_binpack.png differ
diff --git a/imgs/metax_spread.png b/imgs/metax_spread.png
new file mode 100644
index 000000000..d75ecb0ba
Binary files /dev/null and b/imgs/metax_spread.png differ
diff --git a/imgs/metax_topo.png b/imgs/metax_topo.png
new file mode 100644
index 000000000..1b8ba03d9
Binary files /dev/null and b/imgs/metax_topo.png differ
diff --git a/imgs/release-process.png b/imgs/release-process.png
new file mode 100644
index 000000000..8b1a36c28
Binary files /dev/null and b/imgs/release-process.png differ
diff --git a/lib/libvgpu.so b/lib/libvgpu.so
deleted file mode 100755
index 0c436fe92..000000000
Binary files a/lib/libvgpu.so and /dev/null differ
diff --git a/lib/ld.so.preload b/lib/nvidia/ld.so.preload
similarity index 100%
rename from lib/ld.so.preload
rename to lib/nvidia/ld.so.preload
diff --git a/libvgpu b/libvgpu
new file mode 160000
index 000000000..ecbf908b2
--- /dev/null
+++ b/libvgpu
@@ -0,0 +1 @@
+Subproject commit ecbf908b2881b7ff8b240d74382ab676900098f5
diff --git a/pkg/api/device_register.pb.go b/pkg/api/device_register.pb.go
deleted file mode 100644
index 2cd673df5..000000000
--- a/pkg/api/device_register.pb.go
+++ /dev/null
@@ -1,1512 +0,0 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: pkg/api/device_register.proto
-
-package api
-
-import (
- context "context"
- fmt "fmt"
- proto "github.com/golang/protobuf/proto"
- grpc "google.golang.org/grpc"
- codes "google.golang.org/grpc/codes"
- status "google.golang.org/grpc/status"
- io "io"
- math "math"
- math_bits "math/bits"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
-
-type DeviceInfo struct {
- Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
- Count int32 `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"`
- Health bool `protobuf:"varint,3,opt,name=health,proto3" json:"health,omitempty"`
- XXX_NoUnkeyedLiteral struct{} `json:"-"`
- XXX_unrecognized []byte `json:"-"`
- XXX_sizecache int32 `json:"-"`
-}
-
-func (m *DeviceInfo) Reset() { *m = DeviceInfo{} }
-func (m *DeviceInfo) String() string { return proto.CompactTextString(m) }
-func (*DeviceInfo) ProtoMessage() {}
-func (*DeviceInfo) Descriptor() ([]byte, []int) {
- return fileDescriptor_f726eb77a5b37099, []int{0}
-}
-func (m *DeviceInfo) XXX_Unmarshal(b []byte) error {
- return m.Unmarshal(b)
-}
-func (m *DeviceInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
- if deterministic {
- return xxx_messageInfo_DeviceInfo.Marshal(b, m, deterministic)
- } else {
- b = b[:cap(b)]
- n, err := m.MarshalToSizedBuffer(b)
- if err != nil {
- return nil, err
- }
- return b[:n], nil
- }
-}
-func (m *DeviceInfo) XXX_Merge(src proto.Message) {
- xxx_messageInfo_DeviceInfo.Merge(m, src)
-}
-func (m *DeviceInfo) XXX_Size() int {
- return m.Size()
-}
-func (m *DeviceInfo) XXX_DiscardUnknown() {
- xxx_messageInfo_DeviceInfo.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_DeviceInfo proto.InternalMessageInfo
-
-func (m *DeviceInfo) GetId() string {
- if m != nil {
- return m.Id
- }
- return ""
-}
-
-func (m *DeviceInfo) GetCount() int32 {
- if m != nil {
- return m.Count
- }
- return 0
-}
-
-func (m *DeviceInfo) GetHealth() bool {
- if m != nil {
- return m.Health
- }
- return false
-}
-
-type RegisterRequest struct {
- Node string `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"`
- Devices []*DeviceInfo `protobuf:"bytes,2,rep,name=devices,proto3" json:"devices,omitempty"`
- XXX_NoUnkeyedLiteral struct{} `json:"-"`
- XXX_unrecognized []byte `json:"-"`
- XXX_sizecache int32 `json:"-"`
-}
-
-func (m *RegisterRequest) Reset() { *m = RegisterRequest{} }
-func (m *RegisterRequest) String() string { return proto.CompactTextString(m) }
-func (*RegisterRequest) ProtoMessage() {}
-func (*RegisterRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_f726eb77a5b37099, []int{1}
-}
-func (m *RegisterRequest) XXX_Unmarshal(b []byte) error {
- return m.Unmarshal(b)
-}
-func (m *RegisterRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
- if deterministic {
- return xxx_messageInfo_RegisterRequest.Marshal(b, m, deterministic)
- } else {
- b = b[:cap(b)]
- n, err := m.MarshalToSizedBuffer(b)
- if err != nil {
- return nil, err
- }
- return b[:n], nil
- }
-}
-func (m *RegisterRequest) XXX_Merge(src proto.Message) {
- xxx_messageInfo_RegisterRequest.Merge(m, src)
-}
-func (m *RegisterRequest) XXX_Size() int {
- return m.Size()
-}
-func (m *RegisterRequest) XXX_DiscardUnknown() {
- xxx_messageInfo_RegisterRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_RegisterRequest proto.InternalMessageInfo
-
-func (m *RegisterRequest) GetNode() string {
- if m != nil {
- return m.Node
- }
- return ""
-}
-
-func (m *RegisterRequest) GetDevices() []*DeviceInfo {
- if m != nil {
- return m.Devices
- }
- return nil
-}
-
-type RegisterReply struct {
- XXX_NoUnkeyedLiteral struct{} `json:"-"`
- XXX_unrecognized []byte `json:"-"`
- XXX_sizecache int32 `json:"-"`
-}
-
-func (m *RegisterReply) Reset() { *m = RegisterReply{} }
-func (m *RegisterReply) String() string { return proto.CompactTextString(m) }
-func (*RegisterReply) ProtoMessage() {}
-func (*RegisterReply) Descriptor() ([]byte, []int) {
- return fileDescriptor_f726eb77a5b37099, []int{2}
-}
-func (m *RegisterReply) XXX_Unmarshal(b []byte) error {
- return m.Unmarshal(b)
-}
-func (m *RegisterReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
- if deterministic {
- return xxx_messageInfo_RegisterReply.Marshal(b, m, deterministic)
- } else {
- b = b[:cap(b)]
- n, err := m.MarshalToSizedBuffer(b)
- if err != nil {
- return nil, err
- }
- return b[:n], nil
- }
-}
-func (m *RegisterReply) XXX_Merge(src proto.Message) {
- xxx_messageInfo_RegisterReply.Merge(m, src)
-}
-func (m *RegisterReply) XXX_Size() int {
- return m.Size()
-}
-func (m *RegisterReply) XXX_DiscardUnknown() {
- xxx_messageInfo_RegisterReply.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_RegisterReply proto.InternalMessageInfo
-
-type GetContainerRequest struct {
- Uuid string `protobuf:"bytes,1,opt,name=uuid,proto3" json:"uuid,omitempty"`
- XXX_NoUnkeyedLiteral struct{} `json:"-"`
- XXX_unrecognized []byte `json:"-"`
- XXX_sizecache int32 `json:"-"`
-}
-
-func (m *GetContainerRequest) Reset() { *m = GetContainerRequest{} }
-func (m *GetContainerRequest) String() string { return proto.CompactTextString(m) }
-func (*GetContainerRequest) ProtoMessage() {}
-func (*GetContainerRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_f726eb77a5b37099, []int{3}
-}
-func (m *GetContainerRequest) XXX_Unmarshal(b []byte) error {
- return m.Unmarshal(b)
-}
-func (m *GetContainerRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
- if deterministic {
- return xxx_messageInfo_GetContainerRequest.Marshal(b, m, deterministic)
- } else {
- b = b[:cap(b)]
- n, err := m.MarshalToSizedBuffer(b)
- if err != nil {
- return nil, err
- }
- return b[:n], nil
- }
-}
-func (m *GetContainerRequest) XXX_Merge(src proto.Message) {
- xxx_messageInfo_GetContainerRequest.Merge(m, src)
-}
-func (m *GetContainerRequest) XXX_Size() int {
- return m.Size()
-}
-func (m *GetContainerRequest) XXX_DiscardUnknown() {
- xxx_messageInfo_GetContainerRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_GetContainerRequest proto.InternalMessageInfo
-
-func (m *GetContainerRequest) GetUuid() string {
- if m != nil {
- return m.Uuid
- }
- return ""
-}
-
-type GetContainerReply struct {
- DevList []string `protobuf:"bytes,1,rep,name=devList,proto3" json:"devList,omitempty"`
- PodUID string `protobuf:"bytes,3,opt,name=podUID,proto3" json:"podUID,omitempty"`
- CtrName string `protobuf:"bytes,4,opt,name=ctrName,proto3" json:"ctrName,omitempty"`
- PodNamespace string `protobuf:"bytes,5,opt,name=podNamespace,proto3" json:"podNamespace,omitempty"`
- PodName string `protobuf:"bytes,6,opt,name=podName,proto3" json:"podName,omitempty"`
- XXX_NoUnkeyedLiteral struct{} `json:"-"`
- XXX_unrecognized []byte `json:"-"`
- XXX_sizecache int32 `json:"-"`
-}
-
-func (m *GetContainerReply) Reset() { *m = GetContainerReply{} }
-func (m *GetContainerReply) String() string { return proto.CompactTextString(m) }
-func (*GetContainerReply) ProtoMessage() {}
-func (*GetContainerReply) Descriptor() ([]byte, []int) {
- return fileDescriptor_f726eb77a5b37099, []int{4}
-}
-func (m *GetContainerReply) XXX_Unmarshal(b []byte) error {
- return m.Unmarshal(b)
-}
-func (m *GetContainerReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
- if deterministic {
- return xxx_messageInfo_GetContainerReply.Marshal(b, m, deterministic)
- } else {
- b = b[:cap(b)]
- n, err := m.MarshalToSizedBuffer(b)
- if err != nil {
- return nil, err
- }
- return b[:n], nil
- }
-}
-func (m *GetContainerReply) XXX_Merge(src proto.Message) {
- xxx_messageInfo_GetContainerReply.Merge(m, src)
-}
-func (m *GetContainerReply) XXX_Size() int {
- return m.Size()
-}
-func (m *GetContainerReply) XXX_DiscardUnknown() {
- xxx_messageInfo_GetContainerReply.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_GetContainerReply proto.InternalMessageInfo
-
-func (m *GetContainerReply) GetDevList() []string {
- if m != nil {
- return m.DevList
- }
- return nil
-}
-
-func (m *GetContainerReply) GetPodUID() string {
- if m != nil {
- return m.PodUID
- }
- return ""
-}
-
-func (m *GetContainerReply) GetCtrName() string {
- if m != nil {
- return m.CtrName
- }
- return ""
-}
-
-func (m *GetContainerReply) GetPodNamespace() string {
- if m != nil {
- return m.PodNamespace
- }
- return ""
-}
-
-func (m *GetContainerReply) GetPodName() string {
- if m != nil {
- return m.PodName
- }
- return ""
-}
-
-func init() {
- proto.RegisterType((*DeviceInfo)(nil), "api.DeviceInfo")
- proto.RegisterType((*RegisterRequest)(nil), "api.RegisterRequest")
- proto.RegisterType((*RegisterReply)(nil), "api.RegisterReply")
- proto.RegisterType((*GetContainerRequest)(nil), "api.GetContainerRequest")
- proto.RegisterType((*GetContainerReply)(nil), "api.GetContainerReply")
-}
-
-func init() { proto.RegisterFile("pkg/api/device_register.proto", fileDescriptor_f726eb77a5b37099) }
-
-var fileDescriptor_f726eb77a5b37099 = []byte{
- // 357 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0xcf, 0x4a, 0xeb, 0x40,
- 0x14, 0xc6, 0xef, 0x24, 0x6d, 0x6f, 0x7b, 0x6c, 0xad, 0x8e, 0xa5, 0x0c, 0x05, 0x43, 0xc8, 0x2a,
- 0xdd, 0xb4, 0x50, 0xc1, 0xb5, 0x68, 0x41, 0x2a, 0x22, 0x32, 0xe2, 0x5a, 0x62, 0x32, 0xb6, 0x83,
- 0x31, 0x33, 0x26, 0x13, 0x21, 0x8f, 0xe0, 0x1b, 0xb8, 0xf1, 0x7d, 0x5c, 0xfa, 0x08, 0x52, 0x5f,
- 0x44, 0x32, 0x49, 0xac, 0x2d, 0x5d, 0xe5, 0xfc, 0xe6, 0x3b, 0xff, 0xbe, 0x43, 0xe0, 0x50, 0x3e,
- 0xce, 0xc7, 0x9e, 0xe4, 0xe3, 0x80, 0xbd, 0x70, 0x9f, 0xdd, 0xc5, 0x6c, 0xce, 0x13, 0xc5, 0xe2,
- 0x91, 0x8c, 0x85, 0x12, 0xd8, 0xf4, 0x24, 0x77, 0x2e, 0x00, 0xa6, 0x5a, 0x9d, 0x45, 0x0f, 0x02,
- 0xef, 0x82, 0xc1, 0x03, 0x82, 0x6c, 0xe4, 0xb6, 0xa8, 0xc1, 0x03, 0xdc, 0x83, 0xba, 0x2f, 0xd2,
- 0x48, 0x11, 0xc3, 0x46, 0x6e, 0x9d, 0x16, 0x80, 0xfb, 0xd0, 0x58, 0x30, 0x2f, 0x54, 0x0b, 0x62,
- 0xda, 0xc8, 0x6d, 0xd2, 0x92, 0x9c, 0x6b, 0xe8, 0xd2, 0x72, 0x04, 0x65, 0xcf, 0x29, 0x4b, 0x14,
- 0xc6, 0x50, 0x8b, 0x44, 0xc0, 0xca, 0x96, 0x3a, 0xc6, 0x43, 0xf8, 0x5f, 0x2c, 0x94, 0x10, 0xc3,
- 0x36, 0xdd, 0x9d, 0x49, 0x77, 0xe4, 0x49, 0x3e, 0x5a, 0xad, 0x41, 0x2b, 0xdd, 0xe9, 0x42, 0x67,
- 0xd5, 0x51, 0x86, 0x99, 0x33, 0x84, 0x83, 0x73, 0xa6, 0xce, 0x44, 0xa4, 0x3c, 0x1e, 0xad, 0x8d,
- 0x49, 0xd3, 0xdf, 0xcd, 0x75, 0xec, 0xbc, 0x23, 0xd8, 0x5f, 0xcf, 0x95, 0x61, 0x86, 0x89, 0x1e,
- 0x7e, 0xc9, 0x13, 0x45, 0x90, 0x6d, 0xba, 0x2d, 0x5a, 0x61, 0xee, 0x4a, 0x8a, 0xe0, 0x76, 0x36,
- 0xd5, 0xae, 0x5a, 0xb4, 0xa4, 0xbc, 0xc2, 0x57, 0xf1, 0x95, 0xf7, 0xc4, 0x48, 0x4d, 0x0b, 0x15,
- 0x62, 0x07, 0xda, 0x52, 0x04, 0x79, 0x98, 0x48, 0xcf, 0x67, 0xa4, 0xae, 0xe5, 0xb5, 0xb7, 0xbc,
- 0xba, 0x64, 0xd2, 0x28, 0xaa, 0x4b, 0x9c, 0xbc, 0x22, 0xe8, 0x14, 0x9e, 0x6f, 0x58, 0x9c, 0x7f,
- 0xf0, 0x31, 0x34, 0x2b, 0xb7, 0xb8, 0xa7, 0x6f, 0xb2, 0x71, 0xce, 0x01, 0xde, 0x78, 0x95, 0x61,
- 0xe6, 0x22, 0x7c, 0x02, 0xed, 0xbf, 0x46, 0x31, 0xd1, 0x59, 0x5b, 0xee, 0x34, 0xe8, 0x6f, 0x51,
- 0x64, 0x98, 0x9d, 0xee, 0x7d, 0x2c, 0x2d, 0xf4, 0xb9, 0xb4, 0xd0, 0xd7, 0xd2, 0x42, 0x6f, 0xdf,
- 0xd6, 0xbf, 0xfb, 0x86, 0xfe, 0x47, 0x8e, 0x7e, 0x02, 0x00, 0x00, 0xff, 0xff, 0x5d, 0xc5, 0x41,
- 0x30, 0x44, 0x02, 0x00, 0x00,
-}
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConn
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion4
-
-// DeviceServiceClient is the client API for DeviceService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type DeviceServiceClient interface {
- Register(ctx context.Context, opts ...grpc.CallOption) (DeviceService_RegisterClient, error)
- GetContainer(ctx context.Context, in *GetContainerRequest, opts ...grpc.CallOption) (*GetContainerReply, error)
-}
-
-type deviceServiceClient struct {
- cc *grpc.ClientConn
-}
-
-func NewDeviceServiceClient(cc *grpc.ClientConn) DeviceServiceClient {
- return &deviceServiceClient{cc}
-}
-
-func (c *deviceServiceClient) Register(ctx context.Context, opts ...grpc.CallOption) (DeviceService_RegisterClient, error) {
- stream, err := c.cc.NewStream(ctx, &_DeviceService_serviceDesc.Streams[0], "/api.DeviceService/Register", opts...)
- if err != nil {
- return nil, err
- }
- x := &deviceServiceRegisterClient{stream}
- return x, nil
-}
-
-type DeviceService_RegisterClient interface {
- Send(*RegisterRequest) error
- CloseAndRecv() (*RegisterReply, error)
- grpc.ClientStream
-}
-
-type deviceServiceRegisterClient struct {
- grpc.ClientStream
-}
-
-func (x *deviceServiceRegisterClient) Send(m *RegisterRequest) error {
- return x.ClientStream.SendMsg(m)
-}
-
-func (x *deviceServiceRegisterClient) CloseAndRecv() (*RegisterReply, error) {
- if err := x.ClientStream.CloseSend(); err != nil {
- return nil, err
- }
- m := new(RegisterReply)
- if err := x.ClientStream.RecvMsg(m); err != nil {
- return nil, err
- }
- return m, nil
-}
-
-func (c *deviceServiceClient) GetContainer(ctx context.Context, in *GetContainerRequest, opts ...grpc.CallOption) (*GetContainerReply, error) {
- out := new(GetContainerReply)
- err := c.cc.Invoke(ctx, "/api.DeviceService/GetContainer", in, out, opts...)
- if err != nil {
- return nil, err
- }
- return out, nil
-}
-
-// DeviceServiceServer is the server API for DeviceService service.
-type DeviceServiceServer interface {
- Register(DeviceService_RegisterServer) error
- GetContainer(context.Context, *GetContainerRequest) (*GetContainerReply, error)
-}
-
-// UnimplementedDeviceServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedDeviceServiceServer struct {
-}
-
-func (*UnimplementedDeviceServiceServer) Register(srv DeviceService_RegisterServer) error {
- return status.Errorf(codes.Unimplemented, "method Register not implemented")
-}
-func (*UnimplementedDeviceServiceServer) GetContainer(ctx context.Context, req *GetContainerRequest) (*GetContainerReply, error) {
- return nil, status.Errorf(codes.Unimplemented, "method GetContainer not implemented")
-}
-
-func RegisterDeviceServiceServer(s *grpc.Server, srv DeviceServiceServer) {
- s.RegisterService(&_DeviceService_serviceDesc, srv)
-}
-
-func _DeviceService_Register_Handler(srv interface{}, stream grpc.ServerStream) error {
- return srv.(DeviceServiceServer).Register(&deviceServiceRegisterServer{stream})
-}
-
-type DeviceService_RegisterServer interface {
- SendAndClose(*RegisterReply) error
- Recv() (*RegisterRequest, error)
- grpc.ServerStream
-}
-
-type deviceServiceRegisterServer struct {
- grpc.ServerStream
-}
-
-func (x *deviceServiceRegisterServer) SendAndClose(m *RegisterReply) error {
- return x.ServerStream.SendMsg(m)
-}
-
-func (x *deviceServiceRegisterServer) Recv() (*RegisterRequest, error) {
- m := new(RegisterRequest)
- if err := x.ServerStream.RecvMsg(m); err != nil {
- return nil, err
- }
- return m, nil
-}
-
-func _DeviceService_GetContainer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
- in := new(GetContainerRequest)
- if err := dec(in); err != nil {
- return nil, err
- }
- if interceptor == nil {
- return srv.(DeviceServiceServer).GetContainer(ctx, in)
- }
- info := &grpc.UnaryServerInfo{
- Server: srv,
- FullMethod: "/api.DeviceService/GetContainer",
- }
- handler := func(ctx context.Context, req interface{}) (interface{}, error) {
- return srv.(DeviceServiceServer).GetContainer(ctx, req.(*GetContainerRequest))
- }
- return interceptor(ctx, in, info, handler)
-}
-
-var _DeviceService_serviceDesc = grpc.ServiceDesc{
- ServiceName: "api.DeviceService",
- HandlerType: (*DeviceServiceServer)(nil),
- Methods: []grpc.MethodDesc{
- {
- MethodName: "GetContainer",
- Handler: _DeviceService_GetContainer_Handler,
- },
- },
- Streams: []grpc.StreamDesc{
- {
- StreamName: "Register",
- Handler: _DeviceService_Register_Handler,
- ClientStreams: true,
- },
- },
- Metadata: "pkg/api/device_register.proto",
-}
-
-func (m *DeviceInfo) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalToSizedBuffer(dAtA[:size])
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *DeviceInfo) MarshalTo(dAtA []byte) (int, error) {
- size := m.Size()
- return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *DeviceInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) {
- i := len(dAtA)
- _ = i
- var l int
- _ = l
- if m.XXX_unrecognized != nil {
- i -= len(m.XXX_unrecognized)
- copy(dAtA[i:], m.XXX_unrecognized)
- }
- if m.Health {
- i--
- if m.Health {
- dAtA[i] = 1
- } else {
- dAtA[i] = 0
- }
- i--
- dAtA[i] = 0x18
- }
- if m.Count != 0 {
- i = encodeVarintDeviceRegister(dAtA, i, uint64(m.Count))
- i--
- dAtA[i] = 0x10
- }
- if len(m.Id) > 0 {
- i -= len(m.Id)
- copy(dAtA[i:], m.Id)
- i = encodeVarintDeviceRegister(dAtA, i, uint64(len(m.Id)))
- i--
- dAtA[i] = 0xa
- }
- return len(dAtA) - i, nil
-}
-
-func (m *RegisterRequest) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalToSizedBuffer(dAtA[:size])
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *RegisterRequest) MarshalTo(dAtA []byte) (int, error) {
- size := m.Size()
- return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *RegisterRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
- i := len(dAtA)
- _ = i
- var l int
- _ = l
- if m.XXX_unrecognized != nil {
- i -= len(m.XXX_unrecognized)
- copy(dAtA[i:], m.XXX_unrecognized)
- }
- if len(m.Devices) > 0 {
- for iNdEx := len(m.Devices) - 1; iNdEx >= 0; iNdEx-- {
- {
- size, err := m.Devices[iNdEx].MarshalToSizedBuffer(dAtA[:i])
- if err != nil {
- return 0, err
- }
- i -= size
- i = encodeVarintDeviceRegister(dAtA, i, uint64(size))
- }
- i--
- dAtA[i] = 0x12
- }
- }
- if len(m.Node) > 0 {
- i -= len(m.Node)
- copy(dAtA[i:], m.Node)
- i = encodeVarintDeviceRegister(dAtA, i, uint64(len(m.Node)))
- i--
- dAtA[i] = 0xa
- }
- return len(dAtA) - i, nil
-}
-
-func (m *RegisterReply) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalToSizedBuffer(dAtA[:size])
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *RegisterReply) MarshalTo(dAtA []byte) (int, error) {
- size := m.Size()
- return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *RegisterReply) MarshalToSizedBuffer(dAtA []byte) (int, error) {
- i := len(dAtA)
- _ = i
- var l int
- _ = l
- if m.XXX_unrecognized != nil {
- i -= len(m.XXX_unrecognized)
- copy(dAtA[i:], m.XXX_unrecognized)
- }
- return len(dAtA) - i, nil
-}
-
-func (m *GetContainerRequest) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalToSizedBuffer(dAtA[:size])
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *GetContainerRequest) MarshalTo(dAtA []byte) (int, error) {
- size := m.Size()
- return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *GetContainerRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
- i := len(dAtA)
- _ = i
- var l int
- _ = l
- if m.XXX_unrecognized != nil {
- i -= len(m.XXX_unrecognized)
- copy(dAtA[i:], m.XXX_unrecognized)
- }
- if len(m.Uuid) > 0 {
- i -= len(m.Uuid)
- copy(dAtA[i:], m.Uuid)
- i = encodeVarintDeviceRegister(dAtA, i, uint64(len(m.Uuid)))
- i--
- dAtA[i] = 0xa
- }
- return len(dAtA) - i, nil
-}
-
-func (m *GetContainerReply) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalToSizedBuffer(dAtA[:size])
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *GetContainerReply) MarshalTo(dAtA []byte) (int, error) {
- size := m.Size()
- return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *GetContainerReply) MarshalToSizedBuffer(dAtA []byte) (int, error) {
- i := len(dAtA)
- _ = i
- var l int
- _ = l
- if m.XXX_unrecognized != nil {
- i -= len(m.XXX_unrecognized)
- copy(dAtA[i:], m.XXX_unrecognized)
- }
- if len(m.PodName) > 0 {
- i -= len(m.PodName)
- copy(dAtA[i:], m.PodName)
- i = encodeVarintDeviceRegister(dAtA, i, uint64(len(m.PodName)))
- i--
- dAtA[i] = 0x32
- }
- if len(m.PodNamespace) > 0 {
- i -= len(m.PodNamespace)
- copy(dAtA[i:], m.PodNamespace)
- i = encodeVarintDeviceRegister(dAtA, i, uint64(len(m.PodNamespace)))
- i--
- dAtA[i] = 0x2a
- }
- if len(m.CtrName) > 0 {
- i -= len(m.CtrName)
- copy(dAtA[i:], m.CtrName)
- i = encodeVarintDeviceRegister(dAtA, i, uint64(len(m.CtrName)))
- i--
- dAtA[i] = 0x22
- }
- if len(m.PodUID) > 0 {
- i -= len(m.PodUID)
- copy(dAtA[i:], m.PodUID)
- i = encodeVarintDeviceRegister(dAtA, i, uint64(len(m.PodUID)))
- i--
- dAtA[i] = 0x1a
- }
- if len(m.DevList) > 0 {
- for iNdEx := len(m.DevList) - 1; iNdEx >= 0; iNdEx-- {
- i -= len(m.DevList[iNdEx])
- copy(dAtA[i:], m.DevList[iNdEx])
- i = encodeVarintDeviceRegister(dAtA, i, uint64(len(m.DevList[iNdEx])))
- i--
- dAtA[i] = 0xa
- }
- }
- return len(dAtA) - i, nil
-}
-
-func encodeVarintDeviceRegister(dAtA []byte, offset int, v uint64) int {
- offset -= sovDeviceRegister(v)
- base := offset
- for v >= 1<<7 {
- dAtA[offset] = uint8(v&0x7f | 0x80)
- v >>= 7
- offset++
- }
- dAtA[offset] = uint8(v)
- return base
-}
-func (m *DeviceInfo) Size() (n int) {
- if m == nil {
- return 0
- }
- var l int
- _ = l
- l = len(m.Id)
- if l > 0 {
- n += 1 + l + sovDeviceRegister(uint64(l))
- }
- if m.Count != 0 {
- n += 1 + sovDeviceRegister(uint64(m.Count))
- }
- if m.Health {
- n += 2
- }
- if m.XXX_unrecognized != nil {
- n += len(m.XXX_unrecognized)
- }
- return n
-}
-
-func (m *RegisterRequest) Size() (n int) {
- if m == nil {
- return 0
- }
- var l int
- _ = l
- l = len(m.Node)
- if l > 0 {
- n += 1 + l + sovDeviceRegister(uint64(l))
- }
- if len(m.Devices) > 0 {
- for _, e := range m.Devices {
- l = e.Size()
- n += 1 + l + sovDeviceRegister(uint64(l))
- }
- }
- if m.XXX_unrecognized != nil {
- n += len(m.XXX_unrecognized)
- }
- return n
-}
-
-func (m *RegisterReply) Size() (n int) {
- if m == nil {
- return 0
- }
- var l int
- _ = l
- if m.XXX_unrecognized != nil {
- n += len(m.XXX_unrecognized)
- }
- return n
-}
-
-func (m *GetContainerRequest) Size() (n int) {
- if m == nil {
- return 0
- }
- var l int
- _ = l
- l = len(m.Uuid)
- if l > 0 {
- n += 1 + l + sovDeviceRegister(uint64(l))
- }
- if m.XXX_unrecognized != nil {
- n += len(m.XXX_unrecognized)
- }
- return n
-}
-
-func (m *GetContainerReply) Size() (n int) {
- if m == nil {
- return 0
- }
- var l int
- _ = l
- if len(m.DevList) > 0 {
- for _, s := range m.DevList {
- l = len(s)
- n += 1 + l + sovDeviceRegister(uint64(l))
- }
- }
- l = len(m.PodUID)
- if l > 0 {
- n += 1 + l + sovDeviceRegister(uint64(l))
- }
- l = len(m.CtrName)
- if l > 0 {
- n += 1 + l + sovDeviceRegister(uint64(l))
- }
- l = len(m.PodNamespace)
- if l > 0 {
- n += 1 + l + sovDeviceRegister(uint64(l))
- }
- l = len(m.PodName)
- if l > 0 {
- n += 1 + l + sovDeviceRegister(uint64(l))
- }
- if m.XXX_unrecognized != nil {
- n += len(m.XXX_unrecognized)
- }
- return n
-}
-
-func sovDeviceRegister(x uint64) (n int) {
- return (math_bits.Len64(x|1) + 6) / 7
-}
-func sozDeviceRegister(x uint64) (n int) {
- return sovDeviceRegister(uint64((x << 1) ^ uint64((int64(x) >> 63))))
-}
-func (m *DeviceInfo) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: DeviceInfo: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: DeviceInfo: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Id", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.Id = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- case 2:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType)
- }
- m.Count = 0
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- m.Count |= int32(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- case 3:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field Health", wireType)
- }
- var v int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- v |= int(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- m.Health = bool(v != 0)
- default:
- iNdEx = preIndex
- skippy, err := skipDeviceRegister(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if (skippy < 0) || (iNdEx+skippy) < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *RegisterRequest) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: RegisterRequest: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: RegisterRequest: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Node", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.Node = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- case 2:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Devices", wireType)
- }
- var msglen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- msglen |= int(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if msglen < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- postIndex := iNdEx + msglen
- if postIndex < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.Devices = append(m.Devices, &DeviceInfo{})
- if err := m.Devices[len(m.Devices)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
- return err
- }
- iNdEx = postIndex
- default:
- iNdEx = preIndex
- skippy, err := skipDeviceRegister(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if (skippy < 0) || (iNdEx+skippy) < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *RegisterReply) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: RegisterReply: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: RegisterReply: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- default:
- iNdEx = preIndex
- skippy, err := skipDeviceRegister(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if (skippy < 0) || (iNdEx+skippy) < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *GetContainerRequest) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: GetContainerRequest: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: GetContainerRequest: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Uuid", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.Uuid = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- default:
- iNdEx = preIndex
- skippy, err := skipDeviceRegister(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if (skippy < 0) || (iNdEx+skippy) < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *GetContainerReply) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: GetContainerReply: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: GetContainerReply: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field DevList", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.DevList = append(m.DevList, string(dAtA[iNdEx:postIndex]))
- iNdEx = postIndex
- case 3:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field PodUID", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.PodUID = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- case 4:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field CtrName", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.CtrName = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- case 5:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field PodNamespace", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.PodNamespace = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- case 6:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field PodName", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.PodName = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- default:
- iNdEx = preIndex
- skippy, err := skipDeviceRegister(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if (skippy < 0) || (iNdEx+skippy) < 0 {
- return ErrInvalidLengthDeviceRegister
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func skipDeviceRegister(dAtA []byte) (n int, err error) {
- l := len(dAtA)
- iNdEx := 0
- depth := 0
- for iNdEx < l {
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return 0, ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return 0, io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- wireType := int(wire & 0x7)
- switch wireType {
- case 0:
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return 0, ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return 0, io.ErrUnexpectedEOF
- }
- iNdEx++
- if dAtA[iNdEx-1] < 0x80 {
- break
- }
- }
- case 1:
- iNdEx += 8
- case 2:
- var length int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return 0, ErrIntOverflowDeviceRegister
- }
- if iNdEx >= l {
- return 0, io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- length |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if length < 0 {
- return 0, ErrInvalidLengthDeviceRegister
- }
- iNdEx += length
- case 3:
- depth++
- case 4:
- if depth == 0 {
- return 0, ErrUnexpectedEndOfGroupDeviceRegister
- }
- depth--
- case 5:
- iNdEx += 4
- default:
- return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
- }
- if iNdEx < 0 {
- return 0, ErrInvalidLengthDeviceRegister
- }
- if depth == 0 {
- return iNdEx, nil
- }
- }
- return 0, io.ErrUnexpectedEOF
-}
-
-var (
- ErrInvalidLengthDeviceRegister = fmt.Errorf("proto: negative length found during unmarshaling")
- ErrIntOverflowDeviceRegister = fmt.Errorf("proto: integer overflow")
- ErrUnexpectedEndOfGroupDeviceRegister = fmt.Errorf("proto: unexpected end of group")
-)
diff --git a/pkg/api/device_register.proto b/pkg/api/device_register.proto
deleted file mode 100644
index 5846183a4..000000000
--- a/pkg/api/device_register.proto
+++ /dev/null
@@ -1,34 +0,0 @@
-syntax = "proto3";
-package api;
-
-service DeviceService {
- rpc Register (stream RegisterRequest) returns (RegisterReply);
-
- rpc GetContainer(GetContainerRequest) returns (GetContainerReply);
-}
-
-message DeviceInfo {
- string id = 1;
- int32 count = 2;
- bool health = 3;
-}
-
-message RegisterRequest {
- string node = 1;
- repeated DeviceInfo devices = 2;
-}
-
-message RegisterReply {
-}
-
-message GetContainerRequest {
- string uuid = 1;
-}
-
-message GetContainerReply {
- repeated string devList = 1;
- string podUID = 3;
- string ctrName = 4;
- string podNamespace = 5;
- string podName = 6;
-}
\ No newline at end of file
diff --git a/pkg/api/go.mod b/pkg/api/go.mod
deleted file mode 100644
index 0bd493ad8..000000000
--- a/pkg/api/go.mod
+++ /dev/null
@@ -1,8 +0,0 @@
-module 4pd.io/k8s-vgpu/pkg/api
-
-go 1.16
-
-require (
- github.com/golang/protobuf v1.5.2
- google.golang.org/grpc v1.39.0
-)
diff --git a/pkg/api/go.sum b/pkg/api/go.sum
deleted file mode 100644
index f3cfe9f9d..000000000
--- a/pkg/api/go.sum
+++ /dev/null
@@ -1,114 +0,0 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI=
-google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/pkg/api/runtime.pb.go b/pkg/api/runtime.pb.go
deleted file mode 100644
index f46b44e9a..000000000
--- a/pkg/api/runtime.pb.go
+++ /dev/null
@@ -1,915 +0,0 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: pkg/api/runtime.proto
-
-package api
-
-import (
- context "context"
- fmt "fmt"
- proto "github.com/golang/protobuf/proto"
- grpc "google.golang.org/grpc"
- codes "google.golang.org/grpc/codes"
- status "google.golang.org/grpc/status"
- io "io"
- math "math"
- math_bits "math/bits"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
-
-type GetDeviceRequest struct {
- CtrUUID string `protobuf:"bytes,1,opt,name=ctrUUID,proto3" json:"ctrUUID,omitempty"`
- XXX_NoUnkeyedLiteral struct{} `json:"-"`
- XXX_unrecognized []byte `json:"-"`
- XXX_sizecache int32 `json:"-"`
-}
-
-func (m *GetDeviceRequest) Reset() { *m = GetDeviceRequest{} }
-func (m *GetDeviceRequest) String() string { return proto.CompactTextString(m) }
-func (*GetDeviceRequest) ProtoMessage() {}
-func (*GetDeviceRequest) Descriptor() ([]byte, []int) {
- return fileDescriptor_1e1a7998f4db04c8, []int{0}
-}
-func (m *GetDeviceRequest) XXX_Unmarshal(b []byte) error {
- return m.Unmarshal(b)
-}
-func (m *GetDeviceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
- if deterministic {
- return xxx_messageInfo_GetDeviceRequest.Marshal(b, m, deterministic)
- } else {
- b = b[:cap(b)]
- n, err := m.MarshalToSizedBuffer(b)
- if err != nil {
- return nil, err
- }
- return b[:n], nil
- }
-}
-func (m *GetDeviceRequest) XXX_Merge(src proto.Message) {
- xxx_messageInfo_GetDeviceRequest.Merge(m, src)
-}
-func (m *GetDeviceRequest) XXX_Size() int {
- return m.Size()
-}
-func (m *GetDeviceRequest) XXX_DiscardUnknown() {
- xxx_messageInfo_GetDeviceRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_GetDeviceRequest proto.InternalMessageInfo
-
-func (m *GetDeviceRequest) GetCtrUUID() string {
- if m != nil {
- return m.CtrUUID
- }
- return ""
-}
-
-type GetDeviceReply struct {
- Envs map[string]string `protobuf:"bytes,1,rep,name=envs,proto3" json:"envs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
- PodUID string `protobuf:"bytes,3,opt,name=podUID,proto3" json:"podUID,omitempty"`
- CtrName string `protobuf:"bytes,4,opt,name=ctrName,proto3" json:"ctrName,omitempty"`
- PodNamespace string `protobuf:"bytes,5,opt,name=podNamespace,proto3" json:"podNamespace,omitempty"`
- PodName string `protobuf:"bytes,6,opt,name=podName,proto3" json:"podName,omitempty"`
- XXX_NoUnkeyedLiteral struct{} `json:"-"`
- XXX_unrecognized []byte `json:"-"`
- XXX_sizecache int32 `json:"-"`
-}
-
-func (m *GetDeviceReply) Reset() { *m = GetDeviceReply{} }
-func (m *GetDeviceReply) String() string { return proto.CompactTextString(m) }
-func (*GetDeviceReply) ProtoMessage() {}
-func (*GetDeviceReply) Descriptor() ([]byte, []int) {
- return fileDescriptor_1e1a7998f4db04c8, []int{1}
-}
-func (m *GetDeviceReply) XXX_Unmarshal(b []byte) error {
- return m.Unmarshal(b)
-}
-func (m *GetDeviceReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
- if deterministic {
- return xxx_messageInfo_GetDeviceReply.Marshal(b, m, deterministic)
- } else {
- b = b[:cap(b)]
- n, err := m.MarshalToSizedBuffer(b)
- if err != nil {
- return nil, err
- }
- return b[:n], nil
- }
-}
-func (m *GetDeviceReply) XXX_Merge(src proto.Message) {
- xxx_messageInfo_GetDeviceReply.Merge(m, src)
-}
-func (m *GetDeviceReply) XXX_Size() int {
- return m.Size()
-}
-func (m *GetDeviceReply) XXX_DiscardUnknown() {
- xxx_messageInfo_GetDeviceReply.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_GetDeviceReply proto.InternalMessageInfo
-
-func (m *GetDeviceReply) GetEnvs() map[string]string {
- if m != nil {
- return m.Envs
- }
- return nil
-}
-
-func (m *GetDeviceReply) GetPodUID() string {
- if m != nil {
- return m.PodUID
- }
- return ""
-}
-
-func (m *GetDeviceReply) GetCtrName() string {
- if m != nil {
- return m.CtrName
- }
- return ""
-}
-
-func (m *GetDeviceReply) GetPodNamespace() string {
- if m != nil {
- return m.PodNamespace
- }
- return ""
-}
-
-func (m *GetDeviceReply) GetPodName() string {
- if m != nil {
- return m.PodName
- }
- return ""
-}
-
-func init() {
- proto.RegisterType((*GetDeviceRequest)(nil), "api.GetDeviceRequest")
- proto.RegisterType((*GetDeviceReply)(nil), "api.GetDeviceReply")
- proto.RegisterMapType((map[string]string)(nil), "api.GetDeviceReply.EnvsEntry")
-}
-
-func init() { proto.RegisterFile("pkg/api/runtime.proto", fileDescriptor_1e1a7998f4db04c8) }
-
-var fileDescriptor_1e1a7998f4db04c8 = []byte{
- // 290 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x2d, 0xc8, 0x4e, 0xd7,
- 0x4f, 0x2c, 0xc8, 0xd4, 0x2f, 0x2a, 0xcd, 0x2b, 0xc9, 0xcc, 0x4d, 0xd5, 0x2b, 0x28, 0xca, 0x2f,
- 0xc9, 0x17, 0x62, 0x4e, 0x2c, 0xc8, 0x54, 0xd2, 0xe1, 0x12, 0x70, 0x4f, 0x2d, 0x71, 0x49, 0x2d,
- 0xcb, 0x4c, 0x4e, 0x0d, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x92, 0xe0, 0x62, 0x4f, 0x2e,
- 0x29, 0x0a, 0x0d, 0xf5, 0x74, 0x91, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x82, 0x71, 0x95, 0xde,
- 0x30, 0x72, 0xf1, 0x21, 0x29, 0x2f, 0xc8, 0xa9, 0x14, 0x32, 0xe4, 0x62, 0x49, 0xcd, 0x2b, 0x2b,
- 0x96, 0x60, 0x54, 0x60, 0xd6, 0xe0, 0x36, 0x92, 0xd5, 0x4b, 0x2c, 0xc8, 0xd4, 0x43, 0x55, 0xa2,
- 0xe7, 0x9a, 0x57, 0x56, 0xec, 0x9a, 0x57, 0x52, 0x54, 0x19, 0x04, 0x56, 0x2a, 0x24, 0xc6, 0xc5,
- 0x56, 0x90, 0x9f, 0x02, 0x32, 0x9e, 0x19, 0x6c, 0x3c, 0x94, 0x07, 0xb5, 0xd7, 0x2f, 0x31, 0x37,
- 0x55, 0x82, 0x05, 0x6e, 0x2f, 0x88, 0x2b, 0xa4, 0xc4, 0xc5, 0x53, 0x90, 0x9f, 0x02, 0x62, 0x16,
- 0x17, 0x24, 0x26, 0xa7, 0x4a, 0xb0, 0x82, 0xa5, 0x51, 0xc4, 0x40, 0xba, 0xa1, 0x7c, 0x09, 0x36,
- 0x88, 0x6e, 0x28, 0x57, 0xca, 0x9c, 0x8b, 0x13, 0xee, 0x04, 0x21, 0x01, 0x2e, 0xe6, 0xec, 0xd4,
- 0x4a, 0xa8, 0xc7, 0x40, 0x4c, 0x21, 0x11, 0x2e, 0xd6, 0xb2, 0xc4, 0x9c, 0xd2, 0x54, 0x09, 0x26,
- 0xb0, 0x18, 0x84, 0x63, 0xc5, 0x64, 0xc1, 0x68, 0xe4, 0xcf, 0x25, 0x14, 0xe6, 0x1e, 0x10, 0x1a,
- 0x04, 0x09, 0xb6, 0xe0, 0xd4, 0x22, 0x90, 0x9f, 0x84, 0x2c, 0xb9, 0x38, 0xe1, 0x1e, 0x14, 0x12,
- 0x45, 0xf7, 0x30, 0x38, 0x08, 0xa5, 0x84, 0xb1, 0x84, 0x83, 0x12, 0x83, 0x93, 0xc0, 0x89, 0x47,
- 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe3, 0xb1, 0x1c, 0x43, 0x12,
- 0x1b, 0x38, 0x2e, 0x8c, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xee, 0xa1, 0xc2, 0xa4, 0xa4, 0x01,
- 0x00, 0x00,
-}
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConn
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion4
-
-// VGPURuntimeServiceClient is the client API for VGPURuntimeService service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type VGPURuntimeServiceClient interface {
- GetDevice(ctx context.Context, in *GetDeviceRequest, opts ...grpc.CallOption) (*GetDeviceReply, error)
-}
-
-type vGPURuntimeServiceClient struct {
- cc *grpc.ClientConn
-}
-
-func NewVGPURuntimeServiceClient(cc *grpc.ClientConn) VGPURuntimeServiceClient {
- return &vGPURuntimeServiceClient{cc}
-}
-
-func (c *vGPURuntimeServiceClient) GetDevice(ctx context.Context, in *GetDeviceRequest, opts ...grpc.CallOption) (*GetDeviceReply, error) {
- out := new(GetDeviceReply)
- err := c.cc.Invoke(ctx, "/api.VGPURuntimeService/GetDevice", in, out, opts...)
- if err != nil {
- return nil, err
- }
- return out, nil
-}
-
-// VGPURuntimeServiceServer is the server API for VGPURuntimeService service.
-type VGPURuntimeServiceServer interface {
- GetDevice(context.Context, *GetDeviceRequest) (*GetDeviceReply, error)
-}
-
-// UnimplementedVGPURuntimeServiceServer can be embedded to have forward compatible implementations.
-type UnimplementedVGPURuntimeServiceServer struct {
-}
-
-func (*UnimplementedVGPURuntimeServiceServer) GetDevice(ctx context.Context, req *GetDeviceRequest) (*GetDeviceReply, error) {
- return nil, status.Errorf(codes.Unimplemented, "method GetDevice not implemented")
-}
-
-func RegisterVGPURuntimeServiceServer(s *grpc.Server, srv VGPURuntimeServiceServer) {
- s.RegisterService(&_VGPURuntimeService_serviceDesc, srv)
-}
-
-func _VGPURuntimeService_GetDevice_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
- in := new(GetDeviceRequest)
- if err := dec(in); err != nil {
- return nil, err
- }
- if interceptor == nil {
- return srv.(VGPURuntimeServiceServer).GetDevice(ctx, in)
- }
- info := &grpc.UnaryServerInfo{
- Server: srv,
- FullMethod: "/api.VGPURuntimeService/GetDevice",
- }
- handler := func(ctx context.Context, req interface{}) (interface{}, error) {
- return srv.(VGPURuntimeServiceServer).GetDevice(ctx, req.(*GetDeviceRequest))
- }
- return interceptor(ctx, in, info, handler)
-}
-
-var _VGPURuntimeService_serviceDesc = grpc.ServiceDesc{
- ServiceName: "api.VGPURuntimeService",
- HandlerType: (*VGPURuntimeServiceServer)(nil),
- Methods: []grpc.MethodDesc{
- {
- MethodName: "GetDevice",
- Handler: _VGPURuntimeService_GetDevice_Handler,
- },
- },
- Streams: []grpc.StreamDesc{},
- Metadata: "pkg/api/runtime.proto",
-}
-
-func (m *GetDeviceRequest) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalToSizedBuffer(dAtA[:size])
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *GetDeviceRequest) MarshalTo(dAtA []byte) (int, error) {
- size := m.Size()
- return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *GetDeviceRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
- i := len(dAtA)
- _ = i
- var l int
- _ = l
- if m.XXX_unrecognized != nil {
- i -= len(m.XXX_unrecognized)
- copy(dAtA[i:], m.XXX_unrecognized)
- }
- if len(m.CtrUUID) > 0 {
- i -= len(m.CtrUUID)
- copy(dAtA[i:], m.CtrUUID)
- i = encodeVarintRuntime(dAtA, i, uint64(len(m.CtrUUID)))
- i--
- dAtA[i] = 0xa
- }
- return len(dAtA) - i, nil
-}
-
-func (m *GetDeviceReply) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalToSizedBuffer(dAtA[:size])
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *GetDeviceReply) MarshalTo(dAtA []byte) (int, error) {
- size := m.Size()
- return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *GetDeviceReply) MarshalToSizedBuffer(dAtA []byte) (int, error) {
- i := len(dAtA)
- _ = i
- var l int
- _ = l
- if m.XXX_unrecognized != nil {
- i -= len(m.XXX_unrecognized)
- copy(dAtA[i:], m.XXX_unrecognized)
- }
- if len(m.PodName) > 0 {
- i -= len(m.PodName)
- copy(dAtA[i:], m.PodName)
- i = encodeVarintRuntime(dAtA, i, uint64(len(m.PodName)))
- i--
- dAtA[i] = 0x32
- }
- if len(m.PodNamespace) > 0 {
- i -= len(m.PodNamespace)
- copy(dAtA[i:], m.PodNamespace)
- i = encodeVarintRuntime(dAtA, i, uint64(len(m.PodNamespace)))
- i--
- dAtA[i] = 0x2a
- }
- if len(m.CtrName) > 0 {
- i -= len(m.CtrName)
- copy(dAtA[i:], m.CtrName)
- i = encodeVarintRuntime(dAtA, i, uint64(len(m.CtrName)))
- i--
- dAtA[i] = 0x22
- }
- if len(m.PodUID) > 0 {
- i -= len(m.PodUID)
- copy(dAtA[i:], m.PodUID)
- i = encodeVarintRuntime(dAtA, i, uint64(len(m.PodUID)))
- i--
- dAtA[i] = 0x1a
- }
- if len(m.Envs) > 0 {
- for k := range m.Envs {
- v := m.Envs[k]
- baseI := i
- i -= len(v)
- copy(dAtA[i:], v)
- i = encodeVarintRuntime(dAtA, i, uint64(len(v)))
- i--
- dAtA[i] = 0x12
- i -= len(k)
- copy(dAtA[i:], k)
- i = encodeVarintRuntime(dAtA, i, uint64(len(k)))
- i--
- dAtA[i] = 0xa
- i = encodeVarintRuntime(dAtA, i, uint64(baseI-i))
- i--
- dAtA[i] = 0xa
- }
- }
- return len(dAtA) - i, nil
-}
-
-func encodeVarintRuntime(dAtA []byte, offset int, v uint64) int {
- offset -= sovRuntime(v)
- base := offset
- for v >= 1<<7 {
- dAtA[offset] = uint8(v&0x7f | 0x80)
- v >>= 7
- offset++
- }
- dAtA[offset] = uint8(v)
- return base
-}
-func (m *GetDeviceRequest) Size() (n int) {
- if m == nil {
- return 0
- }
- var l int
- _ = l
- l = len(m.CtrUUID)
- if l > 0 {
- n += 1 + l + sovRuntime(uint64(l))
- }
- if m.XXX_unrecognized != nil {
- n += len(m.XXX_unrecognized)
- }
- return n
-}
-
-func (m *GetDeviceReply) Size() (n int) {
- if m == nil {
- return 0
- }
- var l int
- _ = l
- if len(m.Envs) > 0 {
- for k, v := range m.Envs {
- _ = k
- _ = v
- mapEntrySize := 1 + len(k) + sovRuntime(uint64(len(k))) + 1 + len(v) + sovRuntime(uint64(len(v)))
- n += mapEntrySize + 1 + sovRuntime(uint64(mapEntrySize))
- }
- }
- l = len(m.PodUID)
- if l > 0 {
- n += 1 + l + sovRuntime(uint64(l))
- }
- l = len(m.CtrName)
- if l > 0 {
- n += 1 + l + sovRuntime(uint64(l))
- }
- l = len(m.PodNamespace)
- if l > 0 {
- n += 1 + l + sovRuntime(uint64(l))
- }
- l = len(m.PodName)
- if l > 0 {
- n += 1 + l + sovRuntime(uint64(l))
- }
- if m.XXX_unrecognized != nil {
- n += len(m.XXX_unrecognized)
- }
- return n
-}
-
-func sovRuntime(x uint64) (n int) {
- return (math_bits.Len64(x|1) + 6) / 7
-}
-func sozRuntime(x uint64) (n int) {
- return sovRuntime(uint64((x << 1) ^ uint64((int64(x) >> 63))))
-}
-func (m *GetDeviceRequest) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: GetDeviceRequest: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: GetDeviceRequest: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field CtrUUID", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthRuntime
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthRuntime
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.CtrUUID = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- default:
- iNdEx = preIndex
- skippy, err := skipRuntime(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if (skippy < 0) || (iNdEx+skippy) < 0 {
- return ErrInvalidLengthRuntime
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *GetDeviceReply) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: GetDeviceReply: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: GetDeviceReply: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Envs", wireType)
- }
- var msglen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- msglen |= int(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if msglen < 0 {
- return ErrInvalidLengthRuntime
- }
- postIndex := iNdEx + msglen
- if postIndex < 0 {
- return ErrInvalidLengthRuntime
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- if m.Envs == nil {
- m.Envs = make(map[string]string)
- }
- var mapkey string
- var mapvalue string
- for iNdEx < postIndex {
- entryPreIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- if fieldNum == 1 {
- var stringLenmapkey uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLenmapkey |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLenmapkey := int(stringLenmapkey)
- if intStringLenmapkey < 0 {
- return ErrInvalidLengthRuntime
- }
- postStringIndexmapkey := iNdEx + intStringLenmapkey
- if postStringIndexmapkey < 0 {
- return ErrInvalidLengthRuntime
- }
- if postStringIndexmapkey > l {
- return io.ErrUnexpectedEOF
- }
- mapkey = string(dAtA[iNdEx:postStringIndexmapkey])
- iNdEx = postStringIndexmapkey
- } else if fieldNum == 2 {
- var stringLenmapvalue uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLenmapvalue |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLenmapvalue := int(stringLenmapvalue)
- if intStringLenmapvalue < 0 {
- return ErrInvalidLengthRuntime
- }
- postStringIndexmapvalue := iNdEx + intStringLenmapvalue
- if postStringIndexmapvalue < 0 {
- return ErrInvalidLengthRuntime
- }
- if postStringIndexmapvalue > l {
- return io.ErrUnexpectedEOF
- }
- mapvalue = string(dAtA[iNdEx:postStringIndexmapvalue])
- iNdEx = postStringIndexmapvalue
- } else {
- iNdEx = entryPreIndex
- skippy, err := skipRuntime(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if (skippy < 0) || (iNdEx+skippy) < 0 {
- return ErrInvalidLengthRuntime
- }
- if (iNdEx + skippy) > postIndex {
- return io.ErrUnexpectedEOF
- }
- iNdEx += skippy
- }
- }
- m.Envs[mapkey] = mapvalue
- iNdEx = postIndex
- case 3:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field PodUID", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthRuntime
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthRuntime
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.PodUID = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- case 4:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field CtrName", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthRuntime
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthRuntime
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.CtrName = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- case 5:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field PodNamespace", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthRuntime
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthRuntime
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.PodNamespace = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- case 6:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field PodName", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= uint64(b&0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthRuntime
- }
- postIndex := iNdEx + intStringLen
- if postIndex < 0 {
- return ErrInvalidLengthRuntime
- }
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.PodName = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- default:
- iNdEx = preIndex
- skippy, err := skipRuntime(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if (skippy < 0) || (iNdEx+skippy) < 0 {
- return ErrInvalidLengthRuntime
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func skipRuntime(dAtA []byte) (n int, err error) {
- l := len(dAtA)
- iNdEx := 0
- depth := 0
- for iNdEx < l {
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return 0, ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return 0, io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- wireType := int(wire & 0x7)
- switch wireType {
- case 0:
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return 0, ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return 0, io.ErrUnexpectedEOF
- }
- iNdEx++
- if dAtA[iNdEx-1] < 0x80 {
- break
- }
- }
- case 1:
- iNdEx += 8
- case 2:
- var length int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return 0, ErrIntOverflowRuntime
- }
- if iNdEx >= l {
- return 0, io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- length |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if length < 0 {
- return 0, ErrInvalidLengthRuntime
- }
- iNdEx += length
- case 3:
- depth++
- case 4:
- if depth == 0 {
- return 0, ErrUnexpectedEndOfGroupRuntime
- }
- depth--
- case 5:
- iNdEx += 4
- default:
- return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
- }
- if iNdEx < 0 {
- return 0, ErrInvalidLengthRuntime
- }
- if depth == 0 {
- return iNdEx, nil
- }
- }
- return 0, io.ErrUnexpectedEOF
-}
-
-var (
- ErrInvalidLengthRuntime = fmt.Errorf("proto: negative length found during unmarshaling")
- ErrIntOverflowRuntime = fmt.Errorf("proto: integer overflow")
- ErrUnexpectedEndOfGroupRuntime = fmt.Errorf("proto: unexpected end of group")
-)
diff --git a/pkg/api/runtime.proto b/pkg/api/runtime.proto
deleted file mode 100644
index 2bd89e23d..000000000
--- a/pkg/api/runtime.proto
+++ /dev/null
@@ -1,18 +0,0 @@
-syntax = "proto3";
-package api;
-
-service VGPURuntimeService {
- rpc GetDevice (GetDeviceRequest) returns (GetDeviceReply) {}
-}
-
-message GetDeviceRequest {
- string ctrUUID = 1;
-}
-
-message GetDeviceReply {
- map envs = 1;
- string podUID = 3;
- string ctrName = 4;
- string podNamespace = 5;
- string podName = 6;
-}
diff --git a/pkg/api/types.go b/pkg/api/types.go
deleted file mode 100644
index 8e2e70455..000000000
--- a/pkg/api/types.go
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package api
-
-const (
- ContainerUID = "VGPU_4PD_UUID"
- PluginRuntimeSocket = "VGPU_4PD_SOCKET"
-)
diff --git a/pkg/device-plugin/cache.go b/pkg/device-plugin/cache.go
deleted file mode 100644
index 36ac6fd7b..000000000
--- a/pkg/device-plugin/cache.go
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package device_plugin
-
-import (
- "sync"
-
- pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
-)
-
-type DeviceCache struct {
- GpuDeviceManager
-
- cache []*Device
- stopCh chan interface{}
- unhealthy chan *Device
- notifyCh map[string]chan *Device
- mutex sync.Mutex
-}
-
-func NewDeviceCache() *DeviceCache {
- return &DeviceCache{
- GpuDeviceManager: GpuDeviceManager{true},
- stopCh: make(chan interface{}),
- unhealthy: make(chan *Device),
- notifyCh: make(map[string]chan *Device),
- }
-}
-
-func (d *DeviceCache) AddNotifyChannel(name string, ch chan *Device) {
- d.mutex.Lock()
- defer d.mutex.Unlock()
- d.notifyCh[name] = ch
-}
-
-func (d *DeviceCache) RemoveNotifyChannel(name string) {
- d.mutex.Lock()
- defer d.mutex.Unlock()
- delete(d.notifyCh, name)
-}
-
-func (d *DeviceCache) Start() {
- d.cache = d.Devices()
- go d.CheckHealth(d.stopCh, d.cache, d.unhealthy)
- go d.notify()
-}
-
-func (d *DeviceCache) Stop() {
- close(d.stopCh)
-}
-
-func (d *DeviceCache) GetCache() []*Device {
- return d.cache
-}
-
-func (d *DeviceCache) notify() {
- for {
- select {
- case <-d.stopCh:
- return
- case dev := <-d.unhealthy:
- dev.Health = pluginapi.Unhealthy
- d.mutex.Lock()
- for _, ch := range d.notifyCh {
- ch <- dev
- }
- d.mutex.Unlock()
- }
- }
-}
diff --git a/pkg/device-plugin/config/config.go b/pkg/device-plugin/config/config.go
deleted file mode 100644
index a4f8f3c17..000000000
--- a/pkg/device-plugin/config/config.go
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package config
-
-var (
- DeviceSplitCount uint
- DeviceMemoryScaling float64
- DeviceCoresScaling float64
- SchedulerEndpoint string
- SchedulerTimeout int
- NodeName string
- RuntimeSocketFlag string
-)
diff --git a/pkg/device-plugin/nvidia.go b/pkg/device-plugin/nvidia.go
deleted file mode 100644
index 3f609ae59..000000000
--- a/pkg/device-plugin/nvidia.go
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package device_plugin
-
-import (
- "fmt"
- "log"
- "os"
- "strings"
-
- "github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml"
-
- pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
-)
-
-const (
- envDisableHealthChecks = "DP_DISABLE_HEALTHCHECKS"
- allHealthChecks = "xids"
-)
-
-// Device couples an underlying pluginapi.Device type with its device node paths
-type Device struct {
- pluginapi.Device
- Paths []string
- Index string
- Memory uint64
-}
-
-// ResourceManager provides an interface for listing a set of Devices and checking health on them
-type ResourceManager interface {
- Devices() []*Device
- CheckHealth(stop <-chan interface{}, devices []*Device, unhealthy chan<- *Device)
-}
-
-// GpuDeviceManager implements the ResourceManager interface for full GPU devices
-type GpuDeviceManager struct {
- skipMigEnabledGPUs bool
-}
-
-func check(err error) {
- if err != nil {
- log.Panicln("Fatal:", err)
- }
-}
-
-// NewGpuDeviceManager returns a reference to a new GpuDeviceManager
-func NewGpuDeviceManager(skipMigEnabledGPUs bool) *GpuDeviceManager {
- return &GpuDeviceManager{
- skipMigEnabledGPUs: skipMigEnabledGPUs,
- }
-}
-
-// Devices returns a list of devices from the GpuDeviceManager
-func (g *GpuDeviceManager) Devices() []*Device {
- n, err := nvml.GetDeviceCount()
- check(err)
-
- var devs []*Device
- for i := uint(0); i < n; i++ {
- d, err := nvml.NewDevice(i)
- check(err)
-
- migEnabled, err := d.IsMigEnabled()
- check(err)
-
- if migEnabled && g.skipMigEnabledGPUs {
- continue
- }
-
- devs = append(devs, buildDevice(d, []string{d.Path}, fmt.Sprintf("%v", i)))
- }
-
- return devs
-}
-
-// CheckHealth performs health checks on a set of devices, writing to the 'unhealthy' channel with any unhealthy devices
-func (g *GpuDeviceManager) CheckHealth(stop <-chan interface{}, devices []*Device, unhealthy chan<- *Device) {
- checkHealth(stop, devices, unhealthy)
-}
-
-func buildDevice(d *nvml.Device, paths []string, index string) *Device {
- dev := Device{}
- dev.ID = d.UUID
- dev.Health = pluginapi.Healthy
- dev.Paths = paths
- dev.Index = index
- dev.Memory = *d.Memory
- if d.CPUAffinity != nil {
- dev.Topology = &pluginapi.TopologyInfo{
- Nodes: []*pluginapi.NUMANode{
- {
- ID: int64(*(d.CPUAffinity)),
- },
- },
- }
- }
- return &dev
-}
-
-func checkHealth(stop <-chan interface{}, devices []*Device, unhealthy chan<- *Device) {
- disableHealthChecks := strings.ToLower(os.Getenv(envDisableHealthChecks))
- if disableHealthChecks == "all" {
- disableHealthChecks = allHealthChecks
- }
- if strings.Contains(disableHealthChecks, "xids") {
- return
- }
-
- eventSet := nvml.NewEventSet()
- defer nvml.DeleteEventSet(eventSet)
-
- for _, d := range devices {
- gpu, _, _, err := nvml.ParseMigDeviceUUID(d.ID)
- if err != nil {
- gpu = d.ID
- }
-
- err = nvml.RegisterEventForDevice(eventSet, nvml.XidCriticalError, gpu)
- if err != nil && strings.HasSuffix(err.Error(), "Not Supported") {
- log.Printf("Warning: %s is too old to support healthchecking: %s. Marking it unhealthy.", d.ID, err)
- unhealthy <- d
- continue
- }
- check(err)
- }
-
- for {
- select {
- case <-stop:
- return
- default:
- }
-
- e, err := nvml.WaitForEvent(eventSet, 5000)
- if err != nil && e.Etype != nvml.XidCriticalError {
- continue
- }
-
- // FIXME: formalize the full list and document it.
- // http://docs.nvidia.com/deploy/xid-errors/index.html#topic_4
- // Application errors: the GPU should still be healthy
- if e.Edata == 31 || e.Edata == 43 || e.Edata == 45 {
- continue
- }
-
- if e.UUID == nil || len(*e.UUID) == 0 {
- // All devices are unhealthy
- log.Printf("XidCriticalError: Xid=%d, All devices will go unhealthy.", e.Edata)
- for _, d := range devices {
- unhealthy <- d
- }
- continue
- }
-
- for _, d := range devices {
- // Please see https://github.com/NVIDIA/gpu-monitoring-tools/blob/148415f505c96052cb3b7fdf443b34ac853139ec/bindings/go/nvml/nvml.h#L1424
- // for the rationale why gi and ci can be set as such when the UUID is a full GPU UUID and not a MIG device UUID.
- gpu, gi, ci, err := nvml.ParseMigDeviceUUID(d.ID)
- if err != nil {
- gpu = d.ID
- gi = 0xFFFFFFFF
- ci = 0xFFFFFFFF
- }
-
- if gpu == *e.UUID && gi == *e.GpuInstanceId && ci == *e.ComputeInstanceId {
- log.Printf("XidCriticalError: Xid=%d on Device=%s, the device will go unhealthy.", e.Edata, d.ID)
- unhealthy <- d
- }
- }
- }
-}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/cdi/api.go b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/api.go
new file mode 100644
index 000000000..93c818b0e
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/api.go
@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package cdi
+
+// Interface provides the API to the 'cdi' package
+//
+//go:generate moq -stub -out api_mock.go . Interface
+type Interface interface {
+ CreateSpecFile() error
+ QualifiedName(string, string) string
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/cdi/api_mock.go b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/api_mock.go
new file mode 100644
index 000000000..7c72851c3
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/api_mock.go
@@ -0,0 +1,123 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package cdi
+
+import (
+ "sync"
+)
+
+// Ensure, that InterfaceMock does implement Interface.
+// If this is not the case, regenerate this file with moq.
+var _ Interface = &InterfaceMock{}
+
+// InterfaceMock is a mock implementation of Interface.
+//
+// func TestSomethingThatUsesInterface(t *testing.T) {
+//
+// // make and configure a mocked Interface
+// mockedInterface := &InterfaceMock{
+// CreateSpecFileFunc: func() error {
+// panic("mock out the CreateSpecFile method")
+// },
+// QualifiedNameFunc: func(s1 string, s2 string) string {
+// panic("mock out the QualifiedName method")
+// },
+// }
+//
+// // use mockedInterface in code that requires Interface
+// // and then make assertions.
+//
+// }
+type InterfaceMock struct {
+ // CreateSpecFileFunc mocks the CreateSpecFile method.
+ CreateSpecFileFunc func() error
+
+ // QualifiedNameFunc mocks the QualifiedName method.
+ QualifiedNameFunc func(s1 string, s2 string) string
+
+ // calls tracks calls to the methods.
+ calls struct {
+ // CreateSpecFile holds details about calls to the CreateSpecFile method.
+ CreateSpecFile []struct {
+ }
+ // QualifiedName holds details about calls to the QualifiedName method.
+ QualifiedName []struct {
+ // S1 is the s1 argument value.
+ S1 string
+ // S2 is the s2 argument value.
+ S2 string
+ }
+ }
+ lockCreateSpecFile sync.RWMutex
+ lockQualifiedName sync.RWMutex
+}
+
+// CreateSpecFile calls CreateSpecFileFunc.
+func (mock *InterfaceMock) CreateSpecFile() error {
+ callInfo := struct {
+ }{}
+ mock.lockCreateSpecFile.Lock()
+ mock.calls.CreateSpecFile = append(mock.calls.CreateSpecFile, callInfo)
+ mock.lockCreateSpecFile.Unlock()
+ if mock.CreateSpecFileFunc == nil {
+ var (
+ errOut error
+ )
+ return errOut
+ }
+ return mock.CreateSpecFileFunc()
+}
+
+// CreateSpecFileCalls gets all the calls that were made to CreateSpecFile.
+// Check the length with:
+//
+// len(mockedInterface.CreateSpecFileCalls())
+func (mock *InterfaceMock) CreateSpecFileCalls() []struct {
+} {
+ var calls []struct {
+ }
+ mock.lockCreateSpecFile.RLock()
+ calls = mock.calls.CreateSpecFile
+ mock.lockCreateSpecFile.RUnlock()
+ return calls
+}
+
+// QualifiedName calls QualifiedNameFunc.
+func (mock *InterfaceMock) QualifiedName(s1 string, s2 string) string {
+ callInfo := struct {
+ S1 string
+ S2 string
+ }{
+ S1: s1,
+ S2: s2,
+ }
+ mock.lockQualifiedName.Lock()
+ mock.calls.QualifiedName = append(mock.calls.QualifiedName, callInfo)
+ mock.lockQualifiedName.Unlock()
+ if mock.QualifiedNameFunc == nil {
+ var (
+ sOut string
+ )
+ return sOut
+ }
+ return mock.QualifiedNameFunc(s1, s2)
+}
+
+// QualifiedNameCalls gets all the calls that were made to QualifiedName.
+// Check the length with:
+//
+// len(mockedInterface.QualifiedNameCalls())
+func (mock *InterfaceMock) QualifiedNameCalls() []struct {
+ S1 string
+ S2 string
+} {
+ var calls []struct {
+ S1 string
+ S2 string
+ }
+ mock.lockQualifiedName.RLock()
+ calls = mock.calls.QualifiedName
+ mock.lockQualifiedName.RUnlock()
+ return calls
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/cdi/cdi.go b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/cdi.go
new file mode 100644
index 000000000..d7be071ed
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/cdi.go
@@ -0,0 +1,194 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package cdi
+
+import (
+ "fmt"
+ "path/filepath"
+
+ nvdevice "github.com/NVIDIA/go-nvlib/pkg/nvlib/device"
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
+ roottransform "github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform/root"
+ "github.com/sirupsen/logrus"
+ cdiapi "tags.cncf.io/container-device-interface/pkg/cdi"
+ "tags.cncf.io/container-device-interface/pkg/parser"
+)
+
+const (
+ cdiRoot = "/var/run/cdi"
+)
+
+// cdiHandler creates CDI specs for devices assocatied with the device plugin.
+type cdiHandler struct {
+ logger *logrus.Logger
+ nvml nvml.Interface
+ nvdevice nvdevice.Interface
+ driverRoot string
+ targetDriverRoot string
+ nvidiaCTKPath string
+ cdiRoot string
+ vendor string
+ deviceIDStrategy string
+
+ enabled bool
+ gdsEnabled bool
+ mofedEnabled bool
+
+ cdilibs map[string]nvcdi.Interface
+}
+
+var _ Interface = &cdiHandler{}
+
+// newHandler constructs a new instance of the 'cdi' interface.
+func newHandler(opts ...Option) (Interface, error) {
+ c := &cdiHandler{}
+ for _, opt := range opts {
+ opt(c)
+ }
+
+ if !c.enabled {
+ return &null{}, nil
+ }
+
+ if c.logger == nil {
+ c.logger = logrus.StandardLogger()
+ }
+ if c.nvml == nil {
+ c.nvml = nvml.New()
+ }
+ if c.nvdevice == nil {
+ c.nvdevice = nvdevice.New(c.nvml)
+ }
+ if c.deviceIDStrategy == "" {
+ c.deviceIDStrategy = "uuid"
+ }
+ if c.driverRoot == "" {
+ c.driverRoot = "/"
+ }
+ if c.targetDriverRoot == "" {
+ c.targetDriverRoot = c.driverRoot
+ }
+
+ deviceNamer, err := nvcdi.NewDeviceNamer(c.deviceIDStrategy)
+ if err != nil {
+ return nil, err
+ }
+
+ c.cdilibs = make(map[string]nvcdi.Interface)
+
+ c.cdilibs["gpu"], err = nvcdi.New(
+ nvcdi.WithLogger(c.logger),
+ nvcdi.WithNvmlLib(c.nvml),
+ nvcdi.WithDeviceLib(c.nvdevice),
+ nvcdi.WithNVIDIACTKPath(c.nvidiaCTKPath),
+ nvcdi.WithDriverRoot(c.driverRoot),
+ nvcdi.WithDeviceNamers(deviceNamer),
+ nvcdi.WithVendor(c.vendor),
+ nvcdi.WithClass("gpu"),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("failed to create nvcdi library: %v", err)
+ }
+
+ var additionalModes []string
+ if c.gdsEnabled {
+ additionalModes = append(additionalModes, "gds")
+ }
+ if c.mofedEnabled {
+ additionalModes = append(additionalModes, "mofed")
+ }
+
+ for _, mode := range additionalModes {
+ lib, err := nvcdi.New(
+ nvcdi.WithLogger(c.logger),
+ nvcdi.WithNVIDIACTKPath(c.nvidiaCTKPath),
+ nvcdi.WithDriverRoot(c.driverRoot),
+ nvcdi.WithVendor(c.vendor),
+ nvcdi.WithMode(mode),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("failed to create nvcdi library: %v", err)
+ }
+ c.cdilibs[mode] = lib
+ }
+
+ return c, nil
+}
+
+// CreateSpecFile creates a CDI spec file for the specified devices.
+func (cdi *cdiHandler) CreateSpecFile() error {
+ for class, cdilib := range cdi.cdilibs {
+ cdi.logger.Infof("Generating CDI spec for resource: %s/%s", cdi.vendor, class)
+
+ if class == "gpu" {
+ ret := cdi.nvml.Init()
+ if ret != nvml.SUCCESS {
+ return fmt.Errorf("failed to initialize NVML: %v", ret)
+ }
+ defer cdi.nvml.Shutdown()
+ }
+
+ spec, err := cdilib.GetSpec()
+ if err != nil {
+ return fmt.Errorf("failed to get CDI spec: %v", err)
+ }
+
+ err = roottransform.New(
+ roottransform.WithRoot(cdi.driverRoot),
+ roottransform.WithTargetRoot(cdi.targetDriverRoot),
+ ).Transform(spec.Raw())
+ if err != nil {
+ return fmt.Errorf("failed to transform driver root in CDI spec: %v", err)
+ }
+
+ raw := spec.Raw()
+ specName, err := cdiapi.GenerateNameForSpec(raw)
+ if err != nil {
+ return fmt.Errorf("failed to generate spec name: %v", err)
+ }
+
+ err = spec.Save(filepath.Join(cdiRoot, specName+".json"))
+ if err != nil {
+ return fmt.Errorf("failed to save CDI spec: %v", err)
+ }
+ }
+
+ return nil
+}
+
+// QualifiedName constructs a CDI qualified device name for the specified resources.
+// Note: This assumes that the specified id matches the device name returned by the naming strategy.
+func (cdi *cdiHandler) QualifiedName(class string, id string) string {
+ return parser.QualifiedName(cdi.vendor, class, id)
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/cdi/factory.go b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/factory.go
new file mode 100644
index 000000000..01173c62a
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/factory.go
@@ -0,0 +1,52 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package cdi
+
+import (
+ "github.com/NVIDIA/go-nvlib/pkg/nvlib/info"
+
+ "k8s.io/klog/v2"
+)
+
+// New is a factory method that creates a CDI handler for creating CDI specs.
+func New(opts ...Option) (Interface, error) {
+ infolib := info.New()
+
+ hasNVML, _ := infolib.HasNvml()
+ if !hasNVML {
+ klog.Warning("No valid resources detected, creating a null CDI handler")
+ return NewNullHandler(), nil
+ }
+
+ return newHandler(opts...)
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/cdi/null.go b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/null.go
new file mode 100644
index 000000000..e5a46c73c
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/null.go
@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package cdi
+
+import (
+ "k8s.io/klog/v2"
+)
+
+type null struct{}
+
+var _ Interface = &null{}
+
+// NewNullHandler returns an instance of the 'cdi' interface that can
+// be used when CDI specs are not required.
+func NewNullHandler() Interface {
+ return &null{}
+}
+
+// CreateSpecFile is a no-op for the null handler.
+func (n *null) CreateSpecFile() error {
+ return nil
+}
+
+// QualifiedName is a no-op for the null handler. A error message is logged
+// inidicating this should never be called for the null handler.
+func (n *null) QualifiedName(class string, id string) string {
+ klog.Error("cannot return a qualified CDI device name with the null CDI handler")
+ return ""
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/cdi/options.go b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/options.go
new file mode 100644
index 000000000..491895784
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/options.go
@@ -0,0 +1,103 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package cdi
+
+import (
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+)
+
+// Option defines a function for passing options to the New() call
+type Option func(*cdiHandler)
+
+// WithEnabled provides an Option to set the enabled flag used by the 'cdi' interface
+func WithEnabled(enabled bool) Option {
+ return func(c *cdiHandler) {
+ c.enabled = enabled
+ }
+}
+
+// WithDriverRoot provides an Option to set the driver root used by the 'cdi' interface
+func WithDriverRoot(root string) Option {
+ return func(c *cdiHandler) {
+ c.driverRoot = root
+ }
+}
+
+// WithTargetDriverRoot provides an Option to set the target driver root used by the 'cdi' interface
+func WithTargetDriverRoot(root string) Option {
+ return func(c *cdiHandler) {
+ c.targetDriverRoot = root
+ }
+}
+
+// WithNvidiaCTKPath provides an Option to set the nvidia-ctk path used by the 'cdi' interface
+func WithNvidiaCTKPath(path string) Option {
+ return func(c *cdiHandler) {
+ c.nvidiaCTKPath = path
+ }
+}
+
+// WithNvml provides an Option to set the NVML library used by the 'cdi' interface
+func WithNvml(nvml nvml.Interface) Option {
+ return func(c *cdiHandler) {
+ c.nvml = nvml
+ }
+}
+
+// WithDeviceIDStrategy provides an Option to set the device ID strategy used by the 'cdi' interface
+func WithDeviceIDStrategy(strategy string) Option {
+ return func(c *cdiHandler) {
+ c.deviceIDStrategy = strategy
+ }
+}
+
+// WithVendor provides an Option to set the vendor used by the 'cdi' interface
+func WithVendor(vendor string) Option {
+ return func(c *cdiHandler) {
+ c.vendor = vendor
+ }
+}
+
+// WithGdsEnabled provides and option to set whether a GDS CDI spec should be generated
+func WithGdsEnabled(enabled bool) Option {
+ return func(c *cdiHandler) {
+ c.gdsEnabled = enabled
+ }
+}
+
+// WithMofedEnabled provides and option to set whether a MOFED CDI spec should be generated
+func WithMofedEnabled(enabled bool) Option {
+ return func(c *cdiHandler) {
+ c.mofedEnabled = enabled
+ }
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/info/version.go b/pkg/device-plugin/nvidiadevice/nvinternal/info/version.go
new file mode 100644
index 000000000..503b5ded8
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/info/version.go
@@ -0,0 +1,64 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package info
+
+import "strings"
+
+// version must be set by go build's -X main.version= option in the Makefile.
+var version = "unknown"
+
+// gitCommit will be the hash that the binary was built from
+// and will be populated by the Makefile.
+var gitCommit = ""
+
+// GetVersionParts returns the different version components.
+func GetVersionParts() []string {
+ v := []string{version}
+
+ if gitCommit != "" {
+ v = append(v, "commit: "+gitCommit)
+ }
+
+ return v
+}
+
+// GetVersionString returns the string representation of the version.
+func GetVersionString(more ...string) string {
+ v := append(GetVersionParts(), more...)
+ return strings.Join(v, "\n")
+}
+
+// GetVersion returns the version of the binary.
+func GetVersion() string {
+ return version
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/mig/mig.go b/pkg/device-plugin/nvidiadevice/nvinternal/mig/mig.go
new file mode 100644
index 000000000..cc38b4d25
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/mig/mig.go
@@ -0,0 +1,116 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package mig
+
+import (
+ "bufio"
+ "fmt"
+ "os"
+
+ "k8s.io/klog/v2"
+)
+
+const (
+ nvidiaProcDriverPath = "/proc/driver/nvidia"
+ nvidiaCapabilitiesPath = nvidiaProcDriverPath + "/capabilities"
+
+ nvcapsProcDriverPath = "/proc/driver/nvidia-caps"
+ nvcapsMigMinorsPath = nvcapsProcDriverPath + "/mig-minors"
+ nvcapsDevicePath = "/dev/nvidia-caps"
+)
+
+// GetMigCapabilityDevicePaths returns a mapping of MIG capability path to device node path.
+func GetMigCapabilityDevicePaths() (map[string]string, error) {
+ // Open nvcapsMigMinorsPath for walking.
+ // If the nvcapsMigMinorsPath does not exist, then we are not on a MIG
+ // capable machine, so there is nothing to do.
+ // The format of this file is discussed in:
+ // https://docs.nvidia.com/datacenter/tesla/mig-user-guide/index.html#unique_1576522674
+ minorsFile, err := os.Open(nvcapsMigMinorsPath)
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("error opening MIG minors file: %v", err)
+ }
+ defer minorsFile.Close()
+
+ // Define a function to process each each line of nvcapsMigMinorsPath
+ processLine := func(line string) (string, int, error) {
+ var gpu, gi, ci, migMinor int
+
+ // Look for a CI access file
+ n, _ := fmt.Sscanf(line, "gpu%d/gi%d/ci%d/access %d", &gpu, &gi, &ci, &migMinor)
+ if n == 4 {
+ capPath := fmt.Sprintf(nvidiaCapabilitiesPath+"/gpu%d/mig/gi%d/ci%d/access", gpu, gi, ci)
+ return capPath, migMinor, nil
+ }
+
+ // Look for a GI access file
+ n, _ = fmt.Sscanf(line, "gpu%d/gi%d/access %d", &gpu, &gi, &migMinor)
+ if n == 3 {
+ capPath := fmt.Sprintf(nvidiaCapabilitiesPath+"/gpu%d/mig/gi%d/access", gpu, gi)
+ return capPath, migMinor, nil
+ }
+
+ // Look for the MIG config file
+ n, _ = fmt.Sscanf(line, "config %d", &migMinor)
+ if n == 1 {
+ capPath := fmt.Sprintf(nvidiaCapabilitiesPath + "/mig/config")
+ return capPath, migMinor, nil
+ }
+
+ // Look for the MIG monitor file
+ n, _ = fmt.Sscanf(line, "monitor %d", &migMinor)
+ if n == 1 {
+ capPath := fmt.Sprintf(nvidiaCapabilitiesPath + "/mig/monitor")
+ return capPath, migMinor, nil
+ }
+
+ return "", 0, fmt.Errorf("unparsable line: %v", line)
+ }
+
+ // Walk each line of nvcapsMigMinorsPath and construct a mapping of nvidia
+ // capabilities path to device minor for that capability
+ capsDevicePaths := make(map[string]string)
+ scanner := bufio.NewScanner(minorsFile)
+ for scanner.Scan() {
+ capPath, migMinor, err := processLine(scanner.Text())
+ if err != nil {
+ klog.Errorf("Skipping line in MIG minors file: %v", err)
+ continue
+ }
+ capsDevicePaths[capPath] = fmt.Sprintf(nvcapsDevicePath+"/nvidia-cap%d", migMinor)
+ }
+ return capsDevicePaths, nil
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/api.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/api.go
new file mode 100644
index 000000000..1066c035a
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/api.go
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package plugin
+
+import "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/rm"
+
+// Interface defines the API for the plugin package
+type Interface interface {
+ Devices() rm.Devices
+ Start() error
+ Stop() error
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/lock.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/lock.go
new file mode 100644
index 000000000..25e1d1a0d
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/lock.go
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2024, HAMi. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package plugin
+
+import (
+ "os"
+ "path/filepath"
+
+ "github.com/fsnotify/fsnotify"
+ "k8s.io/klog/v2"
+)
+
+const (
+ MigApplyLockFile = "/tmp/hami/hami-mig-apply.lock"
+)
+
+// CreateMigApplyLockDir creates the lock directory for MIG apply operation
+func CreateMigApplyLockDir() error {
+ return createMigApplyLockDir(MigApplyLockFile)
+}
+func createMigApplyLockDir(file string) error {
+ dir := filepath.Dir(file)
+ _, err := os.Stat(dir)
+ if os.IsNotExist(err) {
+ err := os.MkdirAll(dir, 0755)
+ if err != nil {
+ klog.Errorf("Failed to create MIG apply lock directory: %v", err)
+ return err
+ }
+ return nil
+ }
+ if err != nil {
+ klog.Errorf("Failed to check MIG apply lock directory: %v", err)
+ return err
+ }
+ klog.Info("MIG apply lock directory already exists")
+ return nil
+}
+
+// CreateMigApplyLock creates the lock file for MIG apply operation
+func CreateMigApplyLock() error {
+ return createMigApplyLock(MigApplyLockFile)
+}
+func createMigApplyLock(file string) error {
+ // Check if the lock file already exists
+ if _, err := os.Stat(file); err == nil {
+ klog.Infof("MIG apply lock file already exists: %s", MigApplyLockFile)
+ return nil
+ }
+ _, err := os.Create(file)
+ if err != nil {
+ klog.Errorf("Failed to create MIG apply lock file: %v", err)
+ return err
+ }
+ return nil
+}
+
+// RemoveMigApplyLock removes the lock file for MIG apply operation
+func RemoveMigApplyLock() error {
+ return removeMigApplyLock(MigApplyLockFile)
+}
+
+func removeMigApplyLock(file string) error {
+ err := os.Remove(file)
+ if err != nil && !os.IsNotExist(err) {
+ klog.Errorf("Failed to remove MIG apply lock file: %v", err)
+ return err
+ }
+ return nil
+}
+
+func WatchLockFile() (chan bool, error) {
+ return watchLockFile(MigApplyLockFile)
+}
+
+func watchLockFile(file string) (chan bool, error) {
+ sigChan := make(chan bool, 1)
+ watcher, err := fsnotify.NewWatcher()
+ if err != nil {
+ return nil, err
+ }
+ dir := filepath.Dir(file)
+ if err := watcher.Add(dir); err != nil {
+ watcher.Close()
+ return nil, err
+ }
+
+ go func() {
+ defer watcher.Close()
+ for {
+ select {
+ case event, ok := <-watcher.Events:
+ if !ok {
+ return
+ }
+ if event.Name == file {
+ if event.Has(fsnotify.Create) {
+ select {
+ case sigChan <- true:
+ klog.V(4).Infof("MIG apply lock file detected: %s", event.Name)
+ default:
+ }
+ }
+ if event.Has(fsnotify.Remove) {
+ select {
+ case sigChan <- false:
+ klog.V(4).Infof("MIG apply lock file removed: %s", event.Name)
+ default:
+ }
+ }
+ }
+ case err, ok := <-watcher.Errors:
+ if !ok {
+ return
+ }
+ klog.Errorf("File watch error: %v", err)
+ }
+ }
+ }()
+
+ return sigChan, nil
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/lock_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/lock_test.go
new file mode 100644
index 000000000..9c6c1407d
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/lock_test.go
@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2024, HAMi. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package plugin
+
+import (
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ "k8s.io/klog/v2"
+)
+
+func clearFile(t *testing.T, path string) {
+ err := os.Remove(path)
+ if err != nil && !os.IsNotExist(err) {
+ t.Fatalf("Failed to remove file: %v", err)
+ }
+}
+
+func TestWatchLockFile(t *testing.T) {
+ tmpDir := t.TempDir()
+ testFile := filepath.Join(tmpDir, "test.lock")
+
+ t.Run("FileNotExist", func(t *testing.T) {
+ sigChan, err := watchLockFile(testFile)
+ if err != nil {
+ t.Fatalf("WatchLockFile failed: %v", err)
+ }
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for {
+ select {
+ case <-sigChan:
+ t.Error("Unexpected signal when file not exist")
+ return
+ case <-time.After(100 * time.Millisecond):
+ return
+ }
+ }
+ }()
+ wg.Wait()
+ })
+
+ t.Run("FileCreate", func(t *testing.T) {
+ defer clearFile(t, testFile)
+ sigChan, err := watchLockFile(testFile)
+ if err != nil {
+ t.Fatalf("WatchLockFile failed: %v", err)
+ }
+
+ f, err := os.Create(testFile)
+ if err != nil {
+ t.Fatalf("Create file failed: %v", err)
+ }
+ f.Close()
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for {
+ select {
+ case status := <-sigChan:
+ klog.Infof("Received signal %v", status)
+ return
+ case <-time.After(time.Second):
+ t.Error("Timeout waiting for create signal")
+ return
+ }
+ }
+ }()
+ wg.Wait()
+ })
+
+ t.Run("FileRemove", func(t *testing.T) {
+ sigChan, err := watchLockFile(testFile)
+ if err != nil {
+ t.Fatalf("WatchLockFile failed: %v", err)
+ }
+
+ if _, err := os.Stat(testFile); os.IsNotExist(err) {
+ f, err := os.Create(testFile)
+ if err != nil {
+ t.Fatalf("Create file failed: %v", err)
+ }
+ f.Close()
+ <-sigChan
+ }
+
+ err = os.Remove(testFile)
+ if err != nil {
+ t.Fatalf("Remove file failed: %v", err)
+ }
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for {
+ select {
+ case status := <-sigChan:
+ klog.Infof("Received signal %v", status)
+ return
+ case <-time.After(time.Second):
+ t.Error("Timeout waiting for remove signal")
+ return
+ }
+ }
+ }()
+ wg.Wait()
+ })
+
+ t.Run("FileExistInitially", func(t *testing.T) {
+ defer clearFile(t, testFile)
+ f, err := os.Create(testFile)
+ if err != nil {
+ t.Fatalf("Create file failed: %v", err)
+ }
+ f.Close()
+
+ sigChan, err := watchLockFile(testFile)
+ if err != nil {
+ t.Fatalf("WatchLockFile failed: %v", err)
+ }
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for {
+ select {
+ case <-sigChan:
+ t.Error("Unexpected signal when file exist")
+ return
+ case <-time.After(time.Second):
+ return
+ }
+ }
+ }()
+ wg.Wait()
+ })
+}
+
+func TestCreateAndRemoveMigApplyLock(t *testing.T) {
+
+ t.Run("CreateLock", func(t *testing.T) {
+ err := CreateMigApplyLockDir()
+ if err != nil {
+ t.Errorf("CreateMigApplyLockDir failed: %v", err)
+ }
+ err = CreateMigApplyLock()
+ if err != nil {
+ t.Errorf("CreateMigApplyLock failed: %v", err)
+ }
+
+ if _, err = os.Stat(MigApplyLockFile); os.IsNotExist(err) {
+ t.Error("Lock file was not created")
+ }
+ })
+
+ t.Run("RemoveLock", func(t *testing.T) {
+ defer clearFile(t, filepath.Dir(MigApplyLockFile))
+ err := CreateMigApplyLockDir()
+ if err != nil {
+ t.Errorf("CreateMigApplyLockDir failed: %v", err)
+ }
+ f, err := os.Create(MigApplyLockFile)
+ if err != nil {
+ t.Fatalf("Create file failed: %v", err)
+ }
+ f.Close()
+
+ err = RemoveMigApplyLock()
+ if err != nil {
+ t.Errorf("RemoveMigApplyLock failed: %v", err)
+ }
+
+ if _, err := os.Stat(MigApplyLockFile); !os.IsNotExist(err) {
+ t.Error("Lock file was not removed")
+ }
+ })
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/api.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/api.go
new file mode 100644
index 000000000..59f7e9c71
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/api.go
@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package manager
+
+import "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin"
+
+// Interface defines the API for the plugin manager package
+type Interface interface {
+ GetPlugins() ([]plugin.Interface, error)
+ CreateCDISpecFile() error
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/factory.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/factory.go
new file mode 100644
index 000000000..943833df1
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/factory.go
@@ -0,0 +1,152 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package manager
+
+import (
+ "fmt"
+
+ "github.com/NVIDIA/go-nvlib/pkg/nvlib/info"
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/cdi"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+)
+
+type manager struct {
+ migStrategy string
+ failOnInitError bool
+ nvmllib nvml.Interface
+
+ cdiHandler cdi.Interface
+ cdiEnabled bool
+ config *nvidia.DeviceConfig
+ infolib info.Interface
+}
+
+// New creates a new plugin manager with the supplied options.
+func New(opts ...Option) (Interface, error) {
+ m := &manager{}
+ for _, opt := range opts {
+ opt(m)
+ }
+
+ if m.config == nil {
+ klog.Warning("no config provided, returning a null manager")
+ return &null{}, nil
+ }
+
+ if m.infolib == nil {
+ m.infolib = info.New()
+ }
+ if m.cdiHandler == nil {
+ m.cdiHandler = cdi.NewNullHandler()
+ }
+
+ mode, err := m.resolveMode()
+ if err != nil {
+ return nil, err
+ }
+
+ if mode != "nvml" && m.cdiEnabled {
+ klog.Warning("CDI is not supported; disabling CDI.")
+ m.cdiEnabled = false
+ }
+
+ switch mode {
+ case "nvml":
+ if m.nvmllib == nil {
+ m.nvmllib = nvml.New()
+ }
+ ret := m.nvmllib.Init()
+ if ret != nvml.SUCCESS {
+ klog.Errorf("Failed to initialize NVML: %v.", ret)
+ klog.Errorf("If this is a GPU node, did you set the docker default runtime to `nvidia`?")
+ klog.Errorf("You can check the prerequisites at: https://github.com/NVIDIA/k8s-device-plugin#prerequisites")
+ klog.Errorf("You can learn how to set the runtime at: https://github.com/NVIDIA/k8s-device-plugin#quick-start")
+ klog.Errorf("If this is not a GPU node, you should set up a toleration or nodeSelector to only deploy this plugin on GPU nodes")
+ if m.failOnInitError {
+ return nil, fmt.Errorf("nvml init failed: %v", ret)
+ }
+ klog.Warningf("nvml init failed: %v", ret)
+ return &null{}, nil
+ }
+ defer m.nvmllib.Shutdown()
+
+ return (*nvmlmanager)(m), nil
+ case "tegra":
+ return (*tegramanager)(m), nil
+ case "null":
+ return &null{}, nil
+ }
+
+ return nil, fmt.Errorf("unknown mode: %v", mode)
+}
+
+func (m *manager) resolveMode() (string, error) {
+ // logWithReason logs the output of the has* / is* checks from the info.Interface
+ logWithReason := func(f func() (bool, string), tag string) bool {
+ is, reason := f()
+ if !is {
+ tag = "non-" + tag
+ }
+ klog.Infof("Detected %v platform: %v", tag, reason)
+ return is
+ }
+
+ hasNVML := logWithReason(m.infolib.HasNvml, "NVML")
+ isTegra := logWithReason(m.infolib.IsTegraSystem, "Tegra")
+
+ if !hasNVML && !isTegra {
+ klog.Error("Incompatible platform detected")
+ klog.Error("If this is a GPU node, did you configure the NVIDIA Container Toolkit?")
+ klog.Error("You can check the prerequisites at: https://github.com/NVIDIA/k8s-device-plugin#prerequisites")
+ klog.Error("You can learn how to set the runtime at: https://github.com/NVIDIA/k8s-device-plugin#quick-start")
+ klog.Error("If this is not a GPU node, you should set up a toleration or nodeSelector to only deploy this plugin on GPU nodes")
+ if m.failOnInitError {
+ return "", fmt.Errorf("platform detection failed")
+ }
+ return "null", nil
+ }
+
+ // The NVIDIA container stack does not yet support the use of integrated AND discrete GPUs on the same node.
+ if isTegra {
+ if hasNVML {
+ klog.Warning("Disabling Tegra-based resources on NVML system")
+ return "nvml", nil
+ }
+ return "tegra", nil
+ }
+
+ return "nvml", nil
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/null.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/null.go
new file mode 100644
index 000000000..8cfcae63a
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/null.go
@@ -0,0 +1,49 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package manager
+
+import (
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin"
+)
+
+type null struct{}
+
+// GetPlugins returns an empty set of Plugins for the null manager
+func (m *null) GetPlugins() ([]plugin.Interface, error) {
+ return nil, nil
+}
+
+// CreateCDISpecFile creates the spec is a no-op for the null plugin
+func (m *null) CreateCDISpecFile() error {
+ return nil
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/nvml.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/nvml.go
new file mode 100644
index 000000000..b847ce58d
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/nvml.go
@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package manager
+
+import (
+ "fmt"
+
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin"
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/rm"
+)
+
+type nvmlmanager manager
+
+// GetPlugins returns the plugins associated with the NVML resources available on the node
+func (m *nvmlmanager) GetPlugins() ([]plugin.Interface, error) {
+ sConfig, mode, err := plugin.LoadNvidiaDevicePluginConfig()
+ if err != nil {
+ return nil, fmt.Errorf("failed to load nvidia plugin config: %v", err)
+ }
+
+ rms, err := rm.NewNVMLResourceManagers(m.nvmllib, m.config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to construct NVML resource managers: %v", err)
+ }
+
+ var plugins []plugin.Interface
+ for _, r := range rms {
+ plugins = append(plugins, plugin.NewNvidiaDevicePlugin(m.config, r, m.cdiHandler, m.cdiEnabled, sConfig, mode))
+ }
+ return plugins, nil
+}
+
+// CreateCDISpecFile creates forwards the request to the CDI handler
+func (m *nvmlmanager) CreateCDISpecFile() error {
+ return m.cdiHandler.CreateSpecFile()
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/options.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/options.go
new file mode 100644
index 000000000..6cc24dd51
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/options.go
@@ -0,0 +1,85 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package manager
+
+import (
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/cdi"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+)
+
+// Option is a function that configures a manager
+type Option func(*manager)
+
+// WithCDIEnabled sets whether CDI is enabled for the manager
+func WithCDIEnabled(enabled bool) Option {
+ return func(m *manager) {
+ m.cdiEnabled = enabled
+ }
+}
+
+// WithCDIHandler sets the CDI handler for the manager
+func WithCDIHandler(handler cdi.Interface) Option {
+ return func(m *manager) {
+ m.cdiHandler = handler
+ }
+}
+
+// WithNVML sets the NVML handler for the manager
+func WithNVML(nvmllib nvml.Interface) Option {
+ return func(m *manager) {
+ m.nvmllib = nvmllib
+ }
+}
+
+// WithFailOnInitError sets whether the manager should fail on initialization errors
+func WithFailOnInitError(failOnInitError bool) Option {
+ return func(m *manager) {
+ m.failOnInitError = failOnInitError
+ }
+}
+
+// WithMigStrategy sets the MIG strategy for the manager
+func WithMigStrategy(migStrategy string) Option {
+ return func(m *manager) {
+ m.migStrategy = migStrategy
+ }
+}
+
+// WithConfig sets the config reference for the manager
+func WithConfig(config *nvidia.DeviceConfig) Option {
+ return func(m *manager) {
+ m.config = config
+ }
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/tegra.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/tegra.go
new file mode 100644
index 000000000..9407cff18
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/manager/tegra.go
@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package manager
+
+import (
+ "fmt"
+
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin"
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/rm"
+)
+
+type tegramanager manager
+
+// GetPlugins returns the plugins associated with the NVML resources available on the node
+func (m *tegramanager) GetPlugins() ([]plugin.Interface, error) {
+ sConfig, mode, err := plugin.LoadNvidiaDevicePluginConfig()
+ if err != nil {
+ return nil, fmt.Errorf("failed to load nvidia plugin config: %v", err)
+ }
+
+ rms, err := rm.NewTegraResourceManagers(m.config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to construct NVML resource managers: %v", err)
+ }
+
+ var plugins []plugin.Interface
+ for _, r := range rms {
+ plugins = append(plugins, plugin.NewNvidiaDevicePlugin(m.config, r, m.cdiHandler, m.cdiEnabled, sConfig, mode))
+ }
+ return plugins, nil
+}
+
+// CreateCDISpecFile creates the spec is a no-op for the tegra plugin
+func (m *tegramanager) CreateCDISpecFile() error {
+ return nil
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go
new file mode 100644
index 000000000..3af1509ca
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go
@@ -0,0 +1,259 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package plugin
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/exec"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+func (plugin *NvidiaDevicePlugin) getNumaInformation(idx int) (int, error) {
+ cmd := exec.Command("nvidia-smi", "topo", "-m")
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ return 0, err
+ }
+ klog.V(5).InfoS("nvidia-smi topo -m output", "result", string(out))
+ return parseNvidiaNumaInfo(idx, string(out))
+}
+
+func parseNvidiaNumaInfo(idx int, nvidiaTopoStr string) (int, error) {
+ result := 0
+ numaAffinityColumnIndex := 0
+ for index, val := range strings.Split(nvidiaTopoStr, "\n") {
+ if !strings.Contains(val, "GPU") {
+ continue
+ }
+ // Example: GPU0 X 0-7 N/A N/A
+ // Many values are separated by two tabs, but this actually represents 5 values instead of 7
+ // So add logic to remove multiple tabs
+ words := strings.Split(strings.ReplaceAll(val, "\t\t", "\t"), "\t")
+ klog.V(5).InfoS("parseNumaInfo", "words", words)
+ // get numa affinity column number
+ if index == 0 {
+ for columnIndex, headerVal := range words {
+ // The topology output of a single card is as follows:
+ // GPU0 CPU Affinity NUMA Affinity GPU NUMA ID
+ // GPU0 X 0-7 N/A N/A
+ //Legend: Other content omitted
+
+ // The topology output in the case of multiple cards is as follows:
+ // GPU0 GPU1 CPU Affinity NUMA Affinity
+ // GPU0 X PHB 0-31 N/A
+ // GPU1 PHB X 0-31 N/A
+ // Legend: Other content omitted
+
+ // We need to get the value of the NUMA Affinity column, but their column indexes are inconsistent,
+ // so we need to get the index first and then get the value.
+ if strings.Contains(headerVal, "NUMA Affinity") {
+ // The header is one column less than the actual row.
+ numaAffinityColumnIndex = columnIndex
+ continue
+ }
+ }
+ continue
+ }
+ klog.V(5).InfoS("nvidia-smi topo -m row output", "row output", words, "length", len(words))
+ if strings.Contains(words[0], fmt.Sprint(idx)) {
+ if len(words) <= numaAffinityColumnIndex || words[numaAffinityColumnIndex] == "N/A" {
+ klog.InfoS("current card has not established numa topology", "gpu row info", words, "index", idx)
+ return 0, nil
+ }
+ result, err := strconv.Atoi(words[numaAffinityColumnIndex])
+ if err != nil {
+ return result, err
+ }
+ return result, nil
+ }
+ }
+ return result, nil
+}
+
+func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*util.DeviceInfo {
+ devs := plugin.Devices()
+ defer nvml.Shutdown()
+ klog.V(5).InfoS("getAPIDevices", "devices", devs)
+ if nvret := nvml.Init(); nvret != nvml.SUCCESS {
+ klog.Errorln("nvml Init err: ", nvret)
+ panic(0)
+ }
+ res := make([]*util.DeviceInfo, 0, len(devs))
+ for UUID := range devs {
+ ndev, ret := nvml.DeviceGetHandleByUUID(UUID)
+ if ret != nvml.SUCCESS {
+ klog.Errorln("nvml new device by index error uuid=", UUID, "err=", ret)
+ panic(0)
+ }
+ idx, ret := ndev.GetIndex()
+ if ret != nvml.SUCCESS {
+ klog.Errorln("nvml get index error ret=", ret)
+ panic(0)
+ }
+ memoryTotal := 0
+ memory, ret := ndev.GetMemoryInfo()
+ if ret == nvml.SUCCESS {
+ memoryTotal = int(memory.Total)
+ } else {
+ klog.Error("nvml get memory error ret=", ret)
+ panic(0)
+ }
+ Model, ret := ndev.GetName()
+ if ret != nvml.SUCCESS {
+ klog.Error("nvml get name error ret=", ret)
+ panic(0)
+ }
+
+ registeredmem := int32(memoryTotal / 1024 / 1024)
+ if *plugin.schedulerConfig.DeviceMemoryScaling != 1 {
+ registeredmem = int32(float64(registeredmem) * *plugin.schedulerConfig.DeviceMemoryScaling)
+ }
+ klog.Infoln("MemoryScaling=", plugin.schedulerConfig.DeviceMemoryScaling, "registeredmem=", registeredmem)
+ health := true
+ for _, val := range devs {
+ if strings.Compare(val.ID, UUID) == 0 {
+ // when NVIDIA-Tesla P4, the device info is : ID:GPU-e290caca-2f0c-9582-acab-67a142b61ffa,Health:Healthy,Topology:nil,
+ // it is more reasonable to think of healthy as case-insensitive
+ if strings.EqualFold(val.Health, "healthy") {
+ health = true
+ } else {
+ health = false
+ }
+ break
+ }
+ }
+ numa, err := plugin.getNumaInformation(idx)
+ if err != nil {
+ klog.ErrorS(err, "failed to get numa information", "idx", idx)
+ }
+ res = append(res, &util.DeviceInfo{
+ ID: UUID,
+ Index: uint(idx),
+ Count: int32(*plugin.schedulerConfig.DeviceSplitCount),
+ Devmem: registeredmem,
+ Devcore: int32(*plugin.schedulerConfig.DeviceCoreScaling * 100),
+ Type: fmt.Sprintf("%v-%v", "NVIDIA", Model),
+ Numa: numa,
+ Mode: plugin.operatingMode,
+ Health: health,
+ })
+ klog.Infof("nvml registered device id=%v, memory=%v, type=%v, numa=%v", idx, registeredmem, Model, numa)
+ }
+ return &res
+}
+
+func (plugin *NvidiaDevicePlugin) RegistrInAnnotation() error {
+ devices := plugin.getAPIDevices()
+ klog.InfoS("start working on the devices", "devices", devices)
+ annos := make(map[string]string)
+ node, err := util.GetNode(util.NodeName)
+ if err != nil {
+ klog.Errorln("get node error", err.Error())
+ return err
+ }
+ encodeddevices := util.EncodeNodeDevices(*devices)
+ var data []byte
+ if os.Getenv("ENABLE_TOPOLOGY_SCORE") == "true" {
+ gpuScore, err := nvidia.CalculateGPUScore(util.GetDevicesUUIDList(*devices))
+ if err != nil {
+ klog.ErrorS(err, "calculate gpu topo score error")
+ return err
+ }
+ data, err = json.Marshal(gpuScore)
+ if err != nil {
+ klog.ErrorS(err, "marshal gpu score error.")
+ return err
+ }
+ }
+ klog.V(4).InfoS("patch nvidia topo score to node", "hami.io/node-nvidia-score", string(data))
+ annos[nvidia.HandshakeAnnos] = "Reported " + time.Now().String()
+ annos[nvidia.RegisterAnnos] = encodeddevices
+ if len(data) > 0 {
+ annos[nvidia.RegisterGPUPairScore] = string(data)
+ }
+ klog.Infof("patch node with the following annos %v", fmt.Sprintf("%v", annos))
+ err = util.PatchNodeAnnotations(node, annos)
+
+ if err != nil {
+ klog.Errorln("patch node error", err.Error())
+ }
+ return err
+}
+
+func (plugin *NvidiaDevicePlugin) WatchAndRegister(disableNVML <-chan bool, ackDisableWatchAndRegister chan<- bool) {
+ klog.Info("Starting WatchAndRegister")
+ errorSleepInterval := time.Second * 5
+ successSleepInterval := time.Second * 30
+ var disableWatchAndRegister bool
+ for {
+ select {
+ case disable := <-disableNVML:
+ if disable {
+ // when received disableNVML signal, stop the watch and register all the time
+ klog.Info("Received disableNVML signal, stopping WatchAndRegister")
+ disableWatchAndRegister = true
+ } else {
+ // when received enableNVML signal, start the watch and register again
+ klog.Info("Received enableNVML signal, resuming WatchAndRegister")
+ disableWatchAndRegister = false
+ }
+
+ default:
+ }
+ if disableWatchAndRegister {
+ klog.Info("WatchAndRegister is disabled by disableWatchAndRegister signal, sleep a success interval")
+ ackDisableWatchAndRegister <- true
+ time.Sleep(successSleepInterval)
+ continue
+ }
+ err := plugin.RegistrInAnnotation()
+ if err != nil {
+ klog.Errorf("Failed to register annotation: %v", err)
+ klog.Infof("Retrying in %v seconds...", errorSleepInterval)
+ time.Sleep(errorSleepInterval)
+ } else {
+ klog.Infof("Successfully registered annotation. Next check in %v seconds...", successSleepInterval)
+ time.Sleep(successSleepInterval)
+ }
+ }
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register_test.go
new file mode 100644
index 000000000..935482386
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register_test.go
@@ -0,0 +1,117 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package plugin
+
+import "testing"
+
+func Test_parseNvidiaNumaInfo(t *testing.T) {
+
+ tests := []struct {
+ name string
+ idx int
+ nvidiaTopoStr string
+ want int
+ wantErr bool
+ }{
+ {
+ name: "single Tesla P4 NUMA",
+ idx: 0,
+ nvidiaTopoStr: `GPU0 CPU Affinity NUMA Affinity ...
+ ...`,
+ want: 0,
+ wantErr: false,
+ },
+ {
+ name: "two Tesla P4 NUMA topo with index 0",
+ idx: 0,
+ nvidiaTopoStr: `GPU0 GPU1 CPU Affinity NUMA Affinity ...
+ ...`,
+ want: 0,
+ wantErr: false,
+ },
+ {
+ name: "two Tesla P4 NUMA topo with index 1",
+ idx: 1,
+ nvidiaTopoStr: `GPU0 GPU1 CPU Affinity NUMA Affinity ...
+ ...`,
+ want: 0,
+ wantErr: false,
+ },
+ {
+ name: "NUMA Affinity is empty",
+ idx: 0,
+ nvidiaTopoStr: `GPU0 CPU Affinity NUMA Affinity GPU NUMA ID
+GPU0 X`,
+ want: 0,
+ wantErr: false,
+ },
+ {
+ name: "multi-gpu topo with index 4",
+ idx: 4,
+ nvidiaTopoStr: ` GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 CPU Affinity NUMA Affinity GPU NUMA ID
+GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 0-191 0 N/A
+GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 0-191 0 N/A
+GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 0-191 0 N/A
+GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 0-191 0 N/A
+GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 192-383 1 N/A
+GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 192-383 1 N/A
+GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 192-383 1 N/A
+GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X 192-383 1 N/A
+
+Legend:
+
+ X = Self
+ SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)
+ NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node
+ PHB = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)
+ PXB = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)
+ PIX = Connection traversing at most a single PCIe bridge
+ NV# = Connection traversing a bonded set of # NVLinks`,
+ want: 1,
+ wantErr: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := parseNvidiaNumaInfo(tt.idx, tt.nvidiaTopoStr)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("parseNvidiaNumaInfo() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if got != tt.want {
+ t.Errorf("parseNvidiaNumaInfo() got = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go
new file mode 100644
index 000000000..27bcfa756
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go
@@ -0,0 +1,787 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package plugin
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net"
+ "os"
+ "os/exec"
+ "path"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
+ "github.com/google/uuid"
+ "github.com/imdario/mergo"
+ "golang.org/x/net/context"
+ "google.golang.org/grpc"
+ "k8s.io/apimachinery/pkg/util/yaml"
+ "k8s.io/klog/v2"
+ kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+ cdiapi "tags.cncf.io/container-device-interface/pkg/cdi"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/cdi"
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/rm"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+// Constants for use by the 'volume-mounts' device list strategy
+const (
+ deviceListAsVolumeMountsHostPath = "/dev/null"
+ deviceListAsVolumeMountsContainerPathRoot = "/var/run/nvidia-container-devices"
+ NodeLockNvidia = "hami.io/mutex.lock"
+ ConfigFilePath = "/config/config.json"
+)
+
+var (
+ hostHookPath string
+ ConfigFile *string
+)
+
+func init() {
+ hostHookPath, _ = os.LookupEnv("HOOK_PATH")
+}
+
+// NvidiaDevicePlugin implements the Kubernetes device plugin API
+type NvidiaDevicePlugin struct {
+ rm rm.ResourceManager
+ config *nvidia.DeviceConfig
+ deviceListEnvvar string
+ deviceListStrategies spec.DeviceListStrategies
+ socket string
+ schedulerConfig nvidia.NvidiaConfig
+
+ applyMutex sync.Mutex
+ disableHealthChecks chan bool
+ ackDisableHealthChecks chan bool
+ disableWatchAndRegister chan bool
+ ackDisableWatchAndRegister chan bool
+
+ cdiHandler cdi.Interface
+ cdiEnabled bool
+ cdiAnnotationPrefix string
+
+ operatingMode string
+ migCurrent nvidia.MigPartedSpec
+
+ server *grpc.Server
+ health chan *rm.Device
+ stop chan any
+}
+
+func readFromConfigFile(sConfig *nvidia.NvidiaConfig, path string) (string, error) {
+ jsonbyte, err := os.ReadFile(path)
+ mode := "hami-core"
+ if err != nil {
+ return "", err
+ }
+ var deviceConfigs nvidia.DevicePluginConfigs
+ err = json.Unmarshal(jsonbyte, &deviceConfigs)
+ if err != nil {
+ return "", err
+ }
+ klog.Infof("Device Plugin Configs: %v", fmt.Sprintf("%v", deviceConfigs))
+ for _, val := range deviceConfigs.Nodeconfig {
+ if os.Getenv(util.NodeNameEnvName) == val.Name {
+ klog.Infof("Reading config from file %s", val.Name)
+ if err := mergo.Merge(&sConfig.NodeDefaultConfig, val.NodeDefaultConfig, mergo.WithOverride); err != nil {
+ return "", err
+ }
+ if val.FilterDevice != nil && (len(val.FilterDevice.UUID) > 0 || len(val.FilterDevice.Index) > 0) {
+ nvidia.DevicePluginFilterDevice = val.FilterDevice
+ }
+ if len(val.OperatingMode) > 0 {
+ mode = val.OperatingMode
+ }
+ klog.Infof("FilterDevice: %v", val.FilterDevice)
+ }
+ }
+ return mode, nil
+}
+
+func LoadNvidiaDevicePluginConfig() (*device.Config, string, error) {
+ sConfig, err := device.LoadConfig(*ConfigFile)
+ if err != nil {
+ klog.Fatalf(`failed to load device config file %s: %v`, *ConfigFile, err)
+ }
+ mode, err := readFromConfigFile(&sConfig.NvidiaConfig, ConfigFilePath)
+ if err != nil {
+ klog.Errorf("readFromConfigFile err:%s", err.Error())
+ }
+ return sConfig, mode, nil
+}
+
+// NewNvidiaDevicePlugin returns an initialized NvidiaDevicePlugin
+func NewNvidiaDevicePlugin(config *nvidia.DeviceConfig, resourceManager rm.ResourceManager, cdiHandler cdi.Interface, cdiEnabled bool, sConfig *device.Config, mode string) *NvidiaDevicePlugin {
+ _, name := resourceManager.Resource().Split()
+
+ deviceListStrategies, _ := spec.NewDeviceListStrategies(*config.Flags.Plugin.DeviceListStrategy)
+
+ klog.Infoln("reading config=", config, "resourceName", config.ResourceName, "configfile=", *ConfigFile, "sconfig=", sConfig)
+
+ // Initialize devices with configuration
+ if err := device.InitDevicesWithConfig(sConfig); err != nil {
+ klog.Fatalf("failed to initialize devices: %v", err)
+ }
+ return &NvidiaDevicePlugin{
+ rm: resourceManager,
+ config: config,
+ deviceListEnvvar: "NVIDIA_VISIBLE_DEVICES",
+ deviceListStrategies: deviceListStrategies,
+ applyMutex: sync.Mutex{},
+ disableHealthChecks: nil,
+ ackDisableHealthChecks: nil,
+ disableWatchAndRegister: nil,
+ ackDisableWatchAndRegister: nil,
+ socket: kubeletdevicepluginv1beta1.DevicePluginPath + "nvidia-" + name + ".sock",
+ cdiHandler: cdiHandler,
+ cdiEnabled: cdiEnabled,
+ cdiAnnotationPrefix: *config.Flags.Plugin.CDIAnnotationPrefix,
+ schedulerConfig: sConfig.NvidiaConfig,
+ operatingMode: mode,
+ migCurrent: nvidia.MigPartedSpec{},
+
+ // These will be reinitialized every
+ // time the plugin server is restarted.
+ server: nil,
+ health: nil,
+ stop: nil,
+ }
+}
+
+func (plugin *NvidiaDevicePlugin) initialize() {
+ plugin.server = grpc.NewServer([]grpc.ServerOption{}...)
+ plugin.health = make(chan *rm.Device)
+ plugin.stop = make(chan any)
+ plugin.disableHealthChecks = make(chan bool, 1)
+ plugin.ackDisableHealthChecks = make(chan bool, 1)
+ plugin.disableWatchAndRegister = make(chan bool, 1)
+ plugin.ackDisableWatchAndRegister = make(chan bool, 1)
+}
+
+func (plugin *NvidiaDevicePlugin) cleanup() {
+ close(plugin.stop)
+ plugin.server = nil
+ plugin.health = nil
+ plugin.stop = nil
+ plugin.disableHealthChecks = nil
+ plugin.ackDisableHealthChecks = nil
+ plugin.disableWatchAndRegister = nil
+ plugin.ackDisableWatchAndRegister = nil
+}
+
+// Devices returns the full set of devices associated with the plugin.
+func (plugin *NvidiaDevicePlugin) Devices() rm.Devices {
+ return plugin.rm.Devices()
+}
+
+// Start starts the gRPC server, registers the device plugin with the Kubelet,
+// and starts the device healthchecks.
+func (plugin *NvidiaDevicePlugin) Start() error {
+ plugin.initialize()
+
+ deviceNumbers, err := GetDeviceNums()
+ if err != nil {
+ return err
+ }
+
+ deviceNames, err := GetDeviceNames()
+ if err != nil {
+ return err
+ }
+
+ err = plugin.Serve()
+ if err != nil {
+ klog.Infof("Could not start device plugin for '%s': %s", plugin.rm.Resource(), err)
+ plugin.cleanup()
+ return err
+ }
+ klog.Infof("Starting to serve '%s' on %s", plugin.rm.Resource(), plugin.socket)
+
+ err = plugin.Register()
+ if err != nil {
+ klog.Infof("Could not register device plugin: %s", err)
+ plugin.Stop()
+ return err
+ }
+ klog.Infof("Registered device plugin for '%s' with Kubelet", plugin.rm.Resource())
+ // Prepare the lock file sub directory.Due to the sequence of startup processes, both the device plugin
+ // and the vGPU monitor should attempt to create this directory by default to ensure its creation.
+ err = CreateMigApplyLockDir()
+ if err != nil {
+ klog.Fatalf("CreateMIGLockSubDir failed:%v", err)
+ }
+
+ // If the temporary lock file still exists, it may be a leftover from the last incomplete mig application process.
+ // Delete the temporary lock file to make sure vgpu monitor can start.
+ err = RemoveMigApplyLock()
+ if err != nil {
+ klog.Fatalf("RemoveMigApplyLock failed:%v", err)
+ }
+
+ var deviceSupportMig bool
+ for _, name := range deviceNames {
+ deviceSupportMig = false
+ for _, migTemplate := range plugin.schedulerConfig.MigGeometriesList {
+ if containsModel(name, migTemplate.Models) {
+ deviceSupportMig = true
+ break
+ }
+ }
+ if !deviceSupportMig {
+ break
+ }
+ }
+ if deviceSupportMig {
+ cmd := exec.Command("nvidia-mig-parted", "export")
+ var stdout, stderr bytes.Buffer
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ err := cmd.Run()
+ if err != nil {
+ klog.Fatalf("nvidia-mig-parted failed with %s\n", err)
+ }
+ outStr := stdout.Bytes()
+ yaml.Unmarshal(outStr, &plugin.migCurrent)
+ os.WriteFile("/tmp/migconfig.yaml", outStr, os.ModePerm)
+ if plugin.operatingMode == "mig" {
+ HamiInitMigConfig, err := plugin.processMigConfigs(plugin.migCurrent.MigConfigs, deviceNumbers)
+ if err != nil {
+ klog.Infof("no device in node:%v", err)
+ }
+ plugin.migCurrent.MigConfigs["current"] = HamiInitMigConfig
+ klog.Infoln("Open Mig export", plugin.migCurrent)
+ } else {
+ plugin.migCurrent.MigConfigs = make(map[string]nvidia.MigConfigSpecSlice)
+ configSlice := nvidia.MigConfigSpecSlice{}
+ for i := 0; i < deviceNumbers; i++ {
+ conf := nvidia.MigConfigSpec{MigEnabled: false, Devices: []int32{int32(i)}}
+ configSlice = append(configSlice, conf)
+ }
+ plugin.migCurrent.MigConfigs["current"] = configSlice
+ klog.Infoln("Close Mig export", plugin.migCurrent)
+ }
+ }
+ go func() {
+ err := plugin.rm.CheckHealth(plugin.stop, plugin.health, plugin.disableHealthChecks, plugin.ackDisableHealthChecks)
+ if err != nil {
+ klog.Infof("Failed to start health check: %v; continuing with health checks disabled", err)
+ }
+ }()
+
+ go func() {
+ plugin.WatchAndRegister(plugin.disableWatchAndRegister, plugin.ackDisableWatchAndRegister)
+ }()
+
+ if deviceSupportMig {
+ plugin.ApplyMigTemplate()
+ }
+
+ return nil
+}
+
+// Stop stops the gRPC server.
+func (plugin *NvidiaDevicePlugin) Stop() error {
+ if plugin == nil || plugin.server == nil {
+ return nil
+ }
+ klog.Infof("Stopping to serve '%s' on %s", plugin.rm.Resource(), plugin.socket)
+ plugin.server.Stop()
+ if err := os.Remove(plugin.socket); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ plugin.cleanup()
+ return nil
+}
+
+// Serve starts the gRPC server of the device plugin.
+func (plugin *NvidiaDevicePlugin) Serve() error {
+ os.Remove(plugin.socket)
+ sock, err := net.Listen("unix", plugin.socket)
+ if err != nil {
+ return err
+ }
+
+ kubeletdevicepluginv1beta1.RegisterDevicePluginServer(plugin.server, plugin)
+
+ go func() {
+ lastCrashTime := time.Now()
+ restartCount := 0
+ for {
+ klog.Infof("Starting GRPC server for '%s'", plugin.rm.Resource())
+ err := plugin.server.Serve(sock)
+ if err == nil {
+ break
+ }
+
+ klog.Infof("GRPC server for '%s' crashed with error: %v", plugin.rm.Resource(), err)
+
+ // restart if it has not been too often
+ // i.e. if server has crashed more than 5 times and it didn't last more than one hour each time
+ if restartCount > 5 {
+ // quit
+ klog.Fatalf("GRPC server for '%s' has repeatedly crashed recently. Quitting", plugin.rm.Resource())
+ }
+ timeSinceLastCrash := time.Since(lastCrashTime).Seconds()
+ lastCrashTime = time.Now()
+ if timeSinceLastCrash > 3600 {
+ // it has been one hour since the last crash.. reset the count
+ // to reflect on the frequency
+ restartCount = 1
+ } else {
+ restartCount++
+ }
+ }
+ }()
+
+ // Wait for server to start by launching a blocking connexion
+ conn, err := plugin.dial(plugin.socket, 5*time.Second)
+ if err != nil {
+ return err
+ }
+ conn.Close()
+
+ return nil
+}
+
+// Register registers the device plugin for the given resourceName with Kubelet.
+func (plugin *NvidiaDevicePlugin) Register() error {
+ conn, err := plugin.dial(kubeletdevicepluginv1beta1.KubeletSocket, 5*time.Second)
+ if err != nil {
+ return err
+ }
+ defer conn.Close()
+
+ client := kubeletdevicepluginv1beta1.NewRegistrationClient(conn)
+ reqt := &kubeletdevicepluginv1beta1.RegisterRequest{
+ Version: kubeletdevicepluginv1beta1.Version,
+ Endpoint: path.Base(plugin.socket),
+ ResourceName: string(plugin.rm.Resource()),
+ Options: &kubeletdevicepluginv1beta1.DevicePluginOptions{
+ GetPreferredAllocationAvailable: false,
+ },
+ }
+
+ _, err = client.Register(context.Background(), reqt)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// GetDevicePluginOptions returns the values of the optional settings for this plugin
+func (plugin *NvidiaDevicePlugin) GetDevicePluginOptions(context.Context, *kubeletdevicepluginv1beta1.Empty) (*kubeletdevicepluginv1beta1.DevicePluginOptions, error) {
+ options := &kubeletdevicepluginv1beta1.DevicePluginOptions{
+ GetPreferredAllocationAvailable: false,
+ }
+ return options, nil
+}
+
+// ListAndWatch lists devices and update that list according to the health status
+func (plugin *NvidiaDevicePlugin) ListAndWatch(e *kubeletdevicepluginv1beta1.Empty, s kubeletdevicepluginv1beta1.DevicePlugin_ListAndWatchServer) error {
+ s.Send(&kubeletdevicepluginv1beta1.ListAndWatchResponse{Devices: plugin.apiDevices()})
+
+ for {
+ select {
+ case <-plugin.stop:
+ return nil
+ case d := <-plugin.health:
+ // FIXME: there is no way to recover from the Unhealthy state.
+ d.Health = kubeletdevicepluginv1beta1.Unhealthy
+ klog.Infof("'%s' device marked unhealthy: %s", plugin.rm.Resource(), d.ID)
+ s.Send(&kubeletdevicepluginv1beta1.ListAndWatchResponse{Devices: plugin.apiDevices()})
+ }
+ }
+}
+
+// GetPreferredAllocation returns the preferred allocation from the set of devices specified in the request
+func (plugin *NvidiaDevicePlugin) GetPreferredAllocation(ctx context.Context, r *kubeletdevicepluginv1beta1.PreferredAllocationRequest) (*kubeletdevicepluginv1beta1.PreferredAllocationResponse, error) {
+ response := &kubeletdevicepluginv1beta1.PreferredAllocationResponse{}
+ /*for _, req := range r.ContainerRequests {
+ devices, err := plugin.rm.GetPreferredAllocation(req.AvailableDeviceIDs, req.MustIncludeDeviceIDs, int(req.AllocationSize))
+ if err != nil {
+ return nil, fmt.Errorf("error getting list of preferred allocation devices: %v", err)
+ }
+
+ resp := &kubeletdevicepluginv1beta1.ContainerPreferredAllocationResponse{
+ DeviceIDs: devices,
+ }
+
+ response.ContainerResponses = append(response.ContainerResponses, resp)
+ }*/
+ return response, nil
+}
+
+// Allocate which return list of devices.
+func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *kubeletdevicepluginv1beta1.AllocateRequest) (*kubeletdevicepluginv1beta1.AllocateResponse, error) {
+ klog.InfoS("Allocate", "request", reqs)
+ responses := kubeletdevicepluginv1beta1.AllocateResponse{}
+ nodename := os.Getenv(util.NodeNameEnvName)
+ current, err := util.GetPendingPod(ctx, nodename)
+ if err != nil {
+ //nodelock.ReleaseNodeLock(nodename, NodeLockNvidia, current)
+ return &kubeletdevicepluginv1beta1.AllocateResponse{}, err
+ }
+ klog.Infof("Allocate pod name is %s/%s, annotation is %+v", current.Namespace, current.Name, current.Annotations)
+
+ for idx, req := range reqs.ContainerRequests {
+ // If the devices being allocated are replicas, then (conditionally)
+ // error out if more than one resource is being allocated.
+
+ if strings.Contains(req.DevicesIDs[0], "MIG") {
+ if plugin.config.Sharing.TimeSlicing.FailRequestsGreaterThanOne && rm.AnnotatedIDs(req.DevicesIDs).AnyHasAnnotations() {
+ if len(req.DevicesIDs) > 1 {
+ device.PodAllocationFailed(nodename, current, NodeLockNvidia)
+ return nil, fmt.Errorf("request for '%v: %v' too large: maximum request size for shared resources is 1", plugin.rm.Resource(), len(req.DevicesIDs))
+ }
+ }
+
+ for _, id := range req.DevicesIDs {
+ if !plugin.rm.Devices().Contains(id) {
+ device.PodAllocationFailed(nodename, current, NodeLockNvidia)
+ return nil, fmt.Errorf("invalid allocation request for '%s': unknown device: %s", plugin.rm.Resource(), id)
+ }
+ }
+
+ response, err := plugin.getAllocateResponse(req.DevicesIDs)
+ if err != nil {
+ device.PodAllocationFailed(nodename, current, NodeLockNvidia)
+ return nil, fmt.Errorf("failed to get allocate response: %v", err)
+ }
+ responses.ContainerResponses = append(responses.ContainerResponses, response)
+ } else {
+ currentCtr, devreq, err := GetNextDeviceRequest(nvidia.NvidiaGPUDevice, *current)
+ klog.Infoln("deviceAllocateFromAnnotation=", devreq)
+ if err != nil {
+ device.PodAllocationFailed(nodename, current, NodeLockNvidia)
+ return &kubeletdevicepluginv1beta1.AllocateResponse{}, err
+ }
+ if len(devreq) != len(reqs.ContainerRequests[idx].DevicesIDs) {
+ device.PodAllocationFailed(nodename, current, NodeLockNvidia)
+ return &kubeletdevicepluginv1beta1.AllocateResponse{}, errors.New("device number not matched")
+ }
+ response, err := plugin.getAllocateResponse(plugin.GetContainerDeviceStrArray(devreq))
+ if err != nil {
+ return nil, fmt.Errorf("failed to get allocate response: %v", err)
+ }
+
+ err = EraseNextDeviceTypeFromAnnotation(nvidia.NvidiaGPUDevice, *current)
+ if err != nil {
+ device.PodAllocationFailed(nodename, current, NodeLockNvidia)
+ return &kubeletdevicepluginv1beta1.AllocateResponse{}, err
+ }
+
+ if plugin.operatingMode != "mig" {
+ for i, dev := range devreq {
+ limitKey := fmt.Sprintf("CUDA_DEVICE_MEMORY_LIMIT_%v", i)
+ response.Envs[limitKey] = fmt.Sprintf("%vm", dev.Usedmem)
+ }
+ response.Envs["CUDA_DEVICE_SM_LIMIT"] = fmt.Sprint(devreq[0].Usedcores)
+ response.Envs["CUDA_DEVICE_MEMORY_SHARED_CACHE"] = fmt.Sprintf("%s/vgpu/%v.cache", hostHookPath, uuid.New().String())
+ if *plugin.schedulerConfig.DeviceMemoryScaling > 1 {
+ response.Envs["CUDA_OVERSUBSCRIBE"] = "true"
+ }
+ if *plugin.schedulerConfig.LogLevel != "" {
+ response.Envs["LIBCUDA_LOG_LEVEL"] = string(*plugin.schedulerConfig.LogLevel)
+ }
+ if plugin.schedulerConfig.DisableCoreLimit {
+ response.Envs[util.CoreLimitSwitch] = "disable"
+ }
+ cacheFileHostDirectory := fmt.Sprintf("%s/vgpu/containers/%s_%s", hostHookPath, current.UID, currentCtr.Name)
+ os.RemoveAll(cacheFileHostDirectory)
+
+ os.MkdirAll(cacheFileHostDirectory, 0777)
+ os.Chmod(cacheFileHostDirectory, 0777)
+ os.MkdirAll("/tmp/vgpulock", 0777)
+ os.Chmod("/tmp/vgpulock", 0777)
+ response.Mounts = append(response.Mounts,
+ &kubeletdevicepluginv1beta1.Mount{ContainerPath: fmt.Sprintf("%s/vgpu/libvgpu.so", hostHookPath),
+ HostPath: GetLibPath(),
+ ReadOnly: true},
+ &kubeletdevicepluginv1beta1.Mount{ContainerPath: fmt.Sprintf("%s/vgpu", hostHookPath),
+ HostPath: cacheFileHostDirectory,
+ ReadOnly: false},
+ &kubeletdevicepluginv1beta1.Mount{ContainerPath: "/tmp/vgpulock",
+ HostPath: "/tmp/vgpulock",
+ ReadOnly: false},
+ )
+ found := false
+ for _, val := range currentCtr.Env {
+ if strings.Compare(val.Name, "CUDA_DISABLE_CONTROL") == 0 {
+ // if env existed but is set to false or can not be parsed, ignore
+ t, _ := strconv.ParseBool(val.Value)
+ if !t {
+ continue
+ }
+ // only env existed and set to true, we mark it "found"
+ found = true
+ break
+ }
+ }
+ if !found {
+ response.Mounts = append(response.Mounts, &kubeletdevicepluginv1beta1.Mount{ContainerPath: "/etc/ld.so.preload",
+ HostPath: hostHookPath + "/vgpu/ld.so.preload",
+ ReadOnly: true},
+ )
+ }
+ _, err = os.Stat(fmt.Sprintf("%s/vgpu/license", hostHookPath))
+ if err == nil {
+ response.Mounts = append(response.Mounts, &kubeletdevicepluginv1beta1.Mount{
+ ContainerPath: "/tmp/license",
+ HostPath: fmt.Sprintf("%s/vgpu/license", hostHookPath),
+ ReadOnly: true,
+ })
+ response.Mounts = append(response.Mounts, &kubeletdevicepluginv1beta1.Mount{
+ ContainerPath: "/usr/bin/vgpuvalidator",
+ HostPath: fmt.Sprintf("%s/vgpu/vgpuvalidator", hostHookPath),
+ ReadOnly: true,
+ })
+ }
+ }
+ responses.ContainerResponses = append(responses.ContainerResponses, response)
+ }
+ }
+ klog.Infoln("Allocate Response", responses.ContainerResponses)
+ device.PodAllocationTrySuccess(nodename, nvidia.NvidiaGPUDevice, NodeLockNvidia, current)
+ return &responses, nil
+}
+
+func (plugin *NvidiaDevicePlugin) getAllocateResponse(requestIds []string) (*kubeletdevicepluginv1beta1.ContainerAllocateResponse, error) {
+ deviceIDs := plugin.deviceIDsFromAnnotatedDeviceIDs(requestIds)
+
+ responseID := uuid.New().String()
+ response, err := plugin.getAllocateResponseForCDI(responseID, deviceIDs)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get allocate response for CDI: %v", err)
+ }
+
+ response.Envs = plugin.apiEnvs(plugin.deviceListEnvvar, deviceIDs)
+ //if plugin.deviceListStrategies.Includes(spec.DeviceListStrategyVolumeMounts) || plugin.deviceListStrategies.Includes(spec.DeviceListStrategyEnvvar) {
+ // response.Envs = plugin.apiEnvs(plugin.deviceListEnvvar, deviceIDs)
+ //}
+ /*
+ if plugin.deviceListStrategies.Includes(spec.DeviceListStrategyVolumeMounts) {
+ response.Envs = plugin.apiEnvs(plugin.deviceListEnvvar, []string{deviceListAsVolumeMountsContainerPathRoot})
+ response.Mounts = plugin.apiMounts(deviceIDs)
+ }*/
+ if *plugin.config.Flags.Plugin.PassDeviceSpecs {
+ response.Devices = plugin.apiDeviceSpecs(*plugin.config.Flags.NvidiaDriverRoot, requestIds)
+ }
+ if *plugin.config.Flags.GDSEnabled {
+ response.Envs["NVIDIA_GDS"] = "enabled"
+ }
+ if *plugin.config.Flags.MOFEDEnabled {
+ response.Envs["NVIDIA_MOFED"] = "enabled"
+ }
+
+ return &response, nil
+}
+
+// getAllocateResponseForCDI returns the allocate response for the specified device IDs.
+// This response contains the annotations required to trigger CDI injection in the container engine or nvidia-container-runtime.
+func (plugin *NvidiaDevicePlugin) getAllocateResponseForCDI(responseID string, deviceIDs []string) (kubeletdevicepluginv1beta1.ContainerAllocateResponse, error) {
+ response := kubeletdevicepluginv1beta1.ContainerAllocateResponse{}
+
+ if !plugin.cdiEnabled {
+ return response, nil
+ }
+
+ var devices []string
+ for _, id := range deviceIDs {
+ devices = append(devices, plugin.cdiHandler.QualifiedName("gpu", id))
+ }
+
+ if *plugin.config.Flags.GDSEnabled {
+ devices = append(devices, plugin.cdiHandler.QualifiedName("gds", "all"))
+ }
+ if *plugin.config.Flags.MOFEDEnabled {
+ devices = append(devices, plugin.cdiHandler.QualifiedName("mofed", "all"))
+ }
+
+ if len(devices) == 0 {
+ return response, nil
+ }
+
+ if plugin.deviceListStrategies.Includes(spec.DeviceListStrategyCDIAnnotations) {
+ annotations, err := plugin.getCDIDeviceAnnotations(responseID, devices)
+ if err != nil {
+ return response, err
+ }
+ response.Annotations = annotations
+ }
+
+ return response, nil
+}
+
+func (plugin *NvidiaDevicePlugin) getCDIDeviceAnnotations(id string, devices []string) (map[string]string, error) {
+ annotations, err := cdiapi.UpdateAnnotations(map[string]string{}, "nvidia-device-plugin", id, devices)
+ if err != nil {
+ return nil, fmt.Errorf("failed to add CDI annotations: %v", err)
+ }
+
+ if plugin.cdiAnnotationPrefix == spec.DefaultCDIAnnotationPrefix {
+ return annotations, nil
+ }
+
+ // update annotations if a custom CDI prefix is configured
+ updatedAnnotations := make(map[string]string)
+ for k, v := range annotations {
+ newKey := plugin.cdiAnnotationPrefix + strings.TrimPrefix(k, spec.DefaultCDIAnnotationPrefix)
+ updatedAnnotations[newKey] = v
+ }
+
+ return updatedAnnotations, nil
+}
+
+// PreStartContainer is unimplemented for this plugin
+func (plugin *NvidiaDevicePlugin) PreStartContainer(context.Context, *kubeletdevicepluginv1beta1.PreStartContainerRequest) (*kubeletdevicepluginv1beta1.PreStartContainerResponse, error) {
+ return &kubeletdevicepluginv1beta1.PreStartContainerResponse{}, nil
+}
+
+// dial establishes the gRPC communication with the registered device plugin.
+func (plugin *NvidiaDevicePlugin) dial(unixSocketPath string, timeout time.Duration) (*grpc.ClientConn, error) {
+ c, err := grpc.Dial(unixSocketPath, grpc.WithInsecure(), grpc.WithBlock(),
+ grpc.WithTimeout(timeout),
+ grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
+ return net.DialTimeout("unix", addr, timeout)
+ }),
+ )
+
+ if err != nil {
+ return nil, err
+ }
+
+ return c, nil
+}
+
+func (plugin *NvidiaDevicePlugin) deviceIDsFromAnnotatedDeviceIDs(ids []string) []string {
+ var deviceIDs []string
+ if *plugin.config.Flags.Plugin.DeviceIDStrategy == spec.DeviceIDStrategyUUID {
+ deviceIDs = rm.AnnotatedIDs(ids).GetIDs()
+ }
+ if *plugin.config.Flags.Plugin.DeviceIDStrategy == spec.DeviceIDStrategyIndex {
+ deviceIDs = plugin.rm.Devices().Subset(ids).GetIndices()
+ }
+ return deviceIDs
+}
+
+func (plugin *NvidiaDevicePlugin) apiDevices() []*kubeletdevicepluginv1beta1.Device {
+ return plugin.rm.Devices().GetPluginDevices(*plugin.schedulerConfig.DeviceSplitCount)
+}
+
+func (plugin *NvidiaDevicePlugin) apiEnvs(envvar string, deviceIDs []string) map[string]string {
+ return map[string]string{
+ envvar: strings.Join(deviceIDs, ","),
+ }
+}
+
+func (plugin *NvidiaDevicePlugin) apiDeviceSpecs(driverRoot string, ids []string) []*kubeletdevicepluginv1beta1.DeviceSpec {
+ optional := map[string]bool{
+ "/dev/nvidiactl": true,
+ "/dev/nvidia-uvm": true,
+ "/dev/nvidia-uvm-tools": true,
+ "/dev/nvidia-modeset": true,
+ }
+
+ paths := plugin.rm.GetDevicePaths(ids)
+
+ var specs []*kubeletdevicepluginv1beta1.DeviceSpec
+ for _, p := range paths {
+ if optional[p] {
+ if _, err := os.Stat(p); err != nil {
+ continue
+ }
+ }
+ spec := &kubeletdevicepluginv1beta1.DeviceSpec{
+ ContainerPath: p,
+ HostPath: filepath.Join(driverRoot, p),
+ Permissions: "rw",
+ }
+ specs = append(specs, spec)
+ }
+
+ return specs
+}
+
+func (plugin *NvidiaDevicePlugin) processMigConfigs(migConfigs map[string]nvidia.MigConfigSpecSlice, deviceCount int) (nvidia.MigConfigSpecSlice, error) {
+ if migConfigs == nil {
+ return nil, fmt.Errorf("migConfigs cannot be nil")
+ }
+ if deviceCount <= 0 {
+ return nil, fmt.Errorf("deviceCount must be positive")
+ }
+
+ transformConfigs := func() (nvidia.MigConfigSpecSlice, error) {
+ var result nvidia.MigConfigSpecSlice
+
+ if len(migConfigs["current"]) == 1 && len(migConfigs["current"][0].Devices) == 0 {
+ for i := 0; i < deviceCount; i++ {
+ config := deepCopyMigConfig(migConfigs["current"][0])
+ config.Devices = []int32{int32(i)}
+ result = append(result, config)
+ }
+ return result, nil
+ }
+
+ deviceToConfig := make(map[int32]*nvidia.MigConfigSpec)
+ for i := range migConfigs["current"] {
+ for _, device := range migConfigs["current"][i].Devices {
+ deviceToConfig[device] = &migConfigs["current"][i]
+ }
+ }
+
+ for i := 0; i < deviceCount; i++ {
+ deviceIndex := int32(i)
+ config, exists := deviceToConfig[deviceIndex]
+ if !exists {
+ return nil, fmt.Errorf("device %d does not match any MIG configuration", i)
+ }
+ newConfig := deepCopyMigConfig(*config)
+ newConfig.Devices = []int32{deviceIndex}
+ result = append(result, newConfig)
+
+ }
+ return result, nil
+ }
+
+ return transformConfigs()
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go
new file mode 100644
index 000000000..93838c1c5
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go
@@ -0,0 +1,486 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package plugin
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+
+ v1 "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/cdi"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/stretchr/testify/require"
+ kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+)
+
+func TestCDIAllocateResponse(t *testing.T) {
+ testCases := []struct {
+ description string
+ deviceIds []string
+ deviceListStrategies []string
+ CDIPrefix string
+ CDIEnabled bool
+ GDSEnabled bool
+ MOFEDEnabled bool
+ expectedResponse kubeletdevicepluginv1beta1.ContainerAllocateResponse
+ }{
+ {
+ description: "empty device list has empty response",
+ deviceListStrategies: []string{"cdi-annotations"},
+ CDIPrefix: "cdi.k8s.io/",
+ CDIEnabled: true,
+ },
+ {
+ description: "CDI disabled has empty response",
+ deviceIds: []string{"gpu0"},
+ deviceListStrategies: []string{"cdi-annotations"},
+ CDIPrefix: "cdi.k8s.io/",
+ CDIEnabled: false,
+ },
+ {
+ description: "single device is added to annotations",
+ deviceIds: []string{"gpu0"},
+ deviceListStrategies: []string{"cdi-annotations"},
+ CDIPrefix: "cdi.k8s.io/",
+ CDIEnabled: true,
+ expectedResponse: kubeletdevicepluginv1beta1.ContainerAllocateResponse{
+ Annotations: map[string]string{
+ "cdi.k8s.io/nvidia-device-plugin_uuid": "nvidia.com/gpu=gpu0",
+ },
+ },
+ },
+ {
+ description: "single device is added to annotations with custom prefix",
+ deviceIds: []string{"gpu0"},
+ deviceListStrategies: []string{"cdi-annotations"},
+ CDIPrefix: "custom.cdi.k8s.io/",
+ CDIEnabled: true,
+ expectedResponse: kubeletdevicepluginv1beta1.ContainerAllocateResponse{
+ Annotations: map[string]string{
+ "custom.cdi.k8s.io/nvidia-device-plugin_uuid": "nvidia.com/gpu=gpu0",
+ },
+ },
+ },
+ {
+ description: "multiple devices are added to annotations",
+ deviceIds: []string{"gpu0", "gpu1"},
+ deviceListStrategies: []string{"cdi-annotations"},
+ CDIPrefix: "cdi.k8s.io/",
+ CDIEnabled: true,
+ expectedResponse: kubeletdevicepluginv1beta1.ContainerAllocateResponse{
+ Annotations: map[string]string{
+ "cdi.k8s.io/nvidia-device-plugin_uuid": "nvidia.com/gpu=gpu0,nvidia.com/gpu=gpu1",
+ },
+ },
+ },
+ {
+ description: "multiple devices are added to annotations with custom prefix",
+ deviceIds: []string{"gpu0", "gpu1"},
+ deviceListStrategies: []string{"cdi-annotations"},
+ CDIPrefix: "custom.cdi.k8s.io/",
+ CDIEnabled: true,
+ expectedResponse: kubeletdevicepluginv1beta1.ContainerAllocateResponse{
+ Annotations: map[string]string{
+ "custom.cdi.k8s.io/nvidia-device-plugin_uuid": "nvidia.com/gpu=gpu0,nvidia.com/gpu=gpu1",
+ },
+ },
+ },
+ {
+ description: "mofed devices are selected if configured",
+ deviceListStrategies: []string{"cdi-annotations"},
+ CDIPrefix: "cdi.k8s.io/",
+ CDIEnabled: true,
+ MOFEDEnabled: true,
+ expectedResponse: kubeletdevicepluginv1beta1.ContainerAllocateResponse{
+ Annotations: map[string]string{
+ "cdi.k8s.io/nvidia-device-plugin_uuid": "nvidia.com/mofed=all",
+ },
+ },
+ },
+ {
+ description: "gds devices are selected if configured",
+ deviceListStrategies: []string{"cdi-annotations"},
+ CDIPrefix: "cdi.k8s.io/",
+ CDIEnabled: true,
+ GDSEnabled: true,
+ expectedResponse: kubeletdevicepluginv1beta1.ContainerAllocateResponse{
+ Annotations: map[string]string{
+ "cdi.k8s.io/nvidia-device-plugin_uuid": "nvidia.com/gds=all",
+ },
+ },
+ },
+ {
+ description: "gds and mofed devices are included with device ids",
+ deviceIds: []string{"gpu0"},
+ deviceListStrategies: []string{"cdi-annotations"},
+ CDIPrefix: "cdi.k8s.io/",
+ CDIEnabled: true,
+ GDSEnabled: true,
+ MOFEDEnabled: true,
+ expectedResponse: kubeletdevicepluginv1beta1.ContainerAllocateResponse{
+ Annotations: map[string]string{
+ "cdi.k8s.io/nvidia-device-plugin_uuid": "nvidia.com/gpu=gpu0,nvidia.com/gds=all,nvidia.com/mofed=all",
+ },
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.description, func(t *testing.T) {
+ deviceListStrategies, _ := v1.NewDeviceListStrategies(tc.deviceListStrategies)
+ plugin := NvidiaDevicePlugin{
+ config: &nvidia.DeviceConfig{
+ Config: &v1.Config{
+ Flags: v1.Flags{
+ CommandLineFlags: v1.CommandLineFlags{
+ GDSEnabled: &tc.GDSEnabled,
+ MOFEDEnabled: &tc.MOFEDEnabled,
+ },
+ },
+ },
+ },
+ cdiHandler: &cdi.InterfaceMock{
+ QualifiedNameFunc: func(c string, s string) string {
+ return "nvidia.com/" + c + "=" + s
+ },
+ },
+ cdiEnabled: tc.CDIEnabled,
+ deviceListStrategies: deviceListStrategies,
+ cdiAnnotationPrefix: tc.CDIPrefix,
+ }
+
+ response, err := plugin.getAllocateResponseForCDI("uuid", tc.deviceIds)
+
+ require.Nil(t, err)
+ require.EqualValues(t, &tc.expectedResponse, &response)
+ })
+ }
+}
+
+type MigDeviceConfigs struct {
+ Configs []map[string]int32
+}
+
+func Test_processMigConfigs(t *testing.T) {
+ type testCase struct {
+ name string
+ migConfigs map[string]nvidia.MigConfigSpecSlice
+ deviceCount int
+ expectError bool
+ validate func(t *testing.T, result nvidia.MigConfigSpecSlice)
+ }
+
+ testConfigs := MigDeviceConfigs{
+ Configs: []map[string]int32{
+ {
+ "1g.10gb": 4,
+ "2g.20gb": 1,
+ },
+ {
+ "3g.30gb": 2,
+ },
+ {},
+ },
+ }
+
+ testCases := []testCase{
+ {
+ name: "SingleConfigForAllDevices",
+ migConfigs: map[string]nvidia.MigConfigSpecSlice{
+ "current": {
+ {
+ Devices: []int32{},
+ MigEnabled: true,
+ MigDevices: testConfigs.Configs[1],
+ },
+ },
+ },
+ deviceCount: 3,
+ expectError: false,
+ validate: func(t *testing.T, result nvidia.MigConfigSpecSlice) {
+ if len(result) != 3 {
+ t.Errorf("Expected 3 configs, got %d", len(result))
+ }
+ for i, config := range result {
+ if len(config.Devices) != 1 || config.Devices[0] != int32(i) {
+ t.Errorf("Config for device %d is incorrect: %v", i, config)
+ }
+ if !config.MigEnabled {
+ t.Error("MigEnabled should be true")
+ }
+ if len(config.MigDevices) != 1 || config.MigDevices["3g.30gb"] != 2 {
+ t.Error("MigDevices not preserved correctly")
+ }
+ }
+ },
+ },
+ {
+ name: "MultipleConfigsForSpecificDevicesWithNoEnabled",
+ migConfigs: map[string]nvidia.MigConfigSpecSlice{
+ "current": {
+ {
+ Devices: []int32{0, 1},
+ MigEnabled: true,
+ MigDevices: testConfigs.Configs[0],
+ },
+ {
+ Devices: []int32{2},
+ MigEnabled: false,
+ MigDevices: testConfigs.Configs[1],
+ },
+ },
+ },
+ deviceCount: 3,
+ expectError: false,
+ validate: func(t *testing.T, result nvidia.MigConfigSpecSlice) {
+ if len(result) != 3 {
+ t.Errorf("Expected 3 configs, got %d", len(result))
+ }
+ for i := 0; i < 2; i++ {
+ if len(result[i].Devices) != 1 || result[i].Devices[0] != int32(i) {
+ t.Errorf("Config for device %d is incorrect: %v", i, result[i])
+ }
+ if !result[i].MigEnabled {
+ t.Error("MigEnabled should be true for device", i)
+ }
+ if len(result[i].MigDevices) != 2 || (result[i].MigDevices["1g.10gb"] != 4 || result[i].MigDevices["2g.20gb"] != 1) {
+ t.Error("MigDevices not preserved correctly for device", i)
+ }
+ }
+ if len(result[2].Devices) != 1 || result[2].Devices[0] != 2 {
+ t.Errorf("Config for device 2 is incorrect: %v", result[2])
+ }
+ if result[2].MigEnabled {
+ t.Error("MigEnabled should be false for device 2")
+ }
+ if len(result[2].MigDevices) != 1 || result[2].MigDevices["3g.30gb"] != 2 {
+ t.Error("MigDevices not preserved correctly for device 2")
+ }
+ },
+ },
+ {
+ name: "MultipleConfigsForSpecificDevicesWithAllEnabled",
+ migConfigs: map[string]nvidia.MigConfigSpecSlice{
+ "current": {
+ {
+ Devices: []int32{0, 1},
+ MigEnabled: true,
+ MigDevices: testConfigs.Configs[0],
+ },
+ {
+ Devices: []int32{2},
+ MigEnabled: true,
+ MigDevices: testConfigs.Configs[1],
+ },
+ },
+ },
+ deviceCount: 3,
+ expectError: false,
+ validate: func(t *testing.T, result nvidia.MigConfigSpecSlice) {
+ if len(result) != 3 {
+ t.Errorf("Expected 3 configs, got %d", len(result))
+ }
+ for i := 0; i < 2; i++ {
+ if len(result[i].Devices) != 1 || result[i].Devices[0] != int32(i) {
+ t.Errorf("Config for device %d is incorrect: %v", i, result[i])
+ }
+ if !result[i].MigEnabled {
+ t.Error("MigEnabled should be true for device", i)
+ }
+ if len(result[i].MigDevices) != 2 || (result[i].MigDevices["1g.10gb"] != 4 || result[i].MigDevices["2g.20gb"] != 1) {
+ t.Error("MigDevices not preserved correctly for device", i)
+ }
+ }
+ if len(result[2].Devices) != 1 || result[2].Devices[0] != 2 {
+ t.Errorf("Config for device 2 is incorrect: %v", result[2])
+ }
+ if !result[2].MigEnabled {
+ t.Error("MigEnabled should be false for device 2")
+ }
+ if len(result[2].MigDevices) != 1 || result[2].MigDevices["3g.30gb"] != 2 {
+ t.Error("MigDevices not preserved correctly for device 2")
+ }
+ t.Log(result)
+ },
+ },
+ {
+ name: "DeviceNotMatched",
+ migConfigs: map[string]nvidia.MigConfigSpecSlice{
+ "current": {
+ {
+ Devices: []int32{0, 1},
+ MigEnabled: true,
+ },
+ },
+ },
+ deviceCount: 3,
+ expectError: true,
+ validate: nil,
+ },
+ }
+
+ plugin := NvidiaDevicePlugin{
+ config: &nvidia.DeviceConfig{
+ Config: &v1.Config{
+ Flags: v1.Flags{
+ CommandLineFlags: v1.CommandLineFlags{},
+ },
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ result, err := plugin.processMigConfigs(tc.migConfigs, tc.deviceCount)
+
+ if tc.expectError {
+ if err == nil {
+ t.Error("Expected error but got nil")
+ }
+ t.Log(err)
+ return
+ }
+
+ if err != nil {
+ t.Fatalf("Unexpected error: %v", err)
+ }
+
+ if tc.validate != nil {
+ tc.validate(t, result)
+ }
+ })
+ }
+}
+
+func Test_pathGeneration(t *testing.T) {
+ hostHookPath := "/usr/local/vgpu"
+ uid := "testuid"
+ cname := "testcname"
+ expected := "/usr/local/vgpu/containers/testuid_testcname"
+ result := fmt.Sprintf("%s/containers/%s_%s", hostHookPath, uid, cname)
+
+ if expected != result {
+ t.Errorf("Expected %s, got %s", expected, result)
+ }
+}
+
+func Test_configOverride(t *testing.T) {
+ t.Setenv("NODE_NAME", "testnode")
+ logLevel1 := nvidia.Debugs
+ logLevel2 := nvidia.Infos
+ split1 := uint(2)
+ memScale1 := 1.5
+ coreScale1 := 1.2
+
+ split2 := uint(3)
+ memScale2 := 0.8
+ coreScale2 := 1.4
+
+ config := nvidia.DevicePluginConfigs{
+ Nodeconfig: []struct {
+ nvidia.NodeDefaultConfig `json:",inline"`
+ Name string `json:"name"`
+ OperatingMode string `json:"operatingmode"`
+ Migstrategy string `json:"migstrategy"`
+ FilterDevice *nvidia.FilterDevice `json:"filterdevices"`
+ }{
+ {
+ NodeDefaultConfig: nvidia.NodeDefaultConfig{
+ DeviceSplitCount: &split1,
+ DeviceMemoryScaling: &memScale1,
+ DeviceCoreScaling: &coreScale1,
+ LogLevel: &logLevel1,
+ },
+ Name: "node-1",
+ OperatingMode: "default",
+ Migstrategy: "single",
+ FilterDevice: nil,
+ },
+ {
+ NodeDefaultConfig: nvidia.NodeDefaultConfig{
+ DeviceSplitCount: &split2,
+ DeviceMemoryScaling: &memScale2,
+ DeviceCoreScaling: &coreScale2,
+ LogLevel: &logLevel2,
+ },
+ Name: "testnode",
+ OperatingMode: "custom",
+ Migstrategy: "mixed",
+ FilterDevice: nil,
+ },
+ },
+ }
+
+ bytes, err := json.MarshalIndent(config, "", " ")
+ if err != nil {
+ panic(err)
+ }
+ path := t.TempDir()
+ os.WriteFile(path+"/config.json", bytes, 0644)
+ nvconfig := nvidia.NvidiaConfig{
+ NodeDefaultConfig: nvidia.NodeDefaultConfig{
+ DeviceSplitCount: func() *uint { v := uint(1); return &v }(),
+ DeviceMemoryScaling: func() *float64 { v := 1.0; return &v }(),
+ DeviceCoreScaling: func() *float64 { v := 1.0; return &v }(),
+ LogLevel: func() *nvidia.LibCudaLogLevel { v := nvidia.Error; return &v }(),
+ },
+ ResourceCountName: "nvidia.com/gpu",
+ ResourceMemoryName: "nvidia.com/gpumem",
+ ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
+ ResourceCoreName: "nvidia.com/gpucores",
+ DefaultGPUNum: int32(2),
+ }
+ _, err = readFromConfigFile(&nvconfig, path+"/config.json")
+ if err != nil {
+ t.Fatalf("Unexpected error: %v", err)
+ }
+ expected := nvidia.NvidiaConfig{
+ NodeDefaultConfig: nvidia.NodeDefaultConfig{
+ DeviceSplitCount: func() *uint { v := uint(3); return &v }(),
+ DeviceMemoryScaling: func() *float64 { v := 0.8; return &v }(),
+ DeviceCoreScaling: func() *float64 { v := 1.4; return &v }(),
+ LogLevel: func() *nvidia.LibCudaLogLevel { v := nvidia.Infos; return &v }(),
+ },
+ ResourceCountName: "nvidia.com/gpu",
+ ResourceMemoryName: "nvidia.com/gpumem",
+ ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
+ ResourceCoreName: "nvidia.com/gpucores",
+ DefaultGPUNum: int32(2),
+ }
+ if !reflect.DeepEqual(nvconfig, expected) {
+ t.Errorf("Expected %v, got %v", expected, nvconfig)
+ }
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go
new file mode 100644
index 000000000..f5846c183
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go
@@ -0,0 +1,433 @@
+/*
+ * Copyright (c) 2024, HAMi. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package plugin
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "os"
+ "os/exec"
+ "strconv"
+ "strings"
+
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ "gopkg.in/yaml.v3"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/info"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+// GetLibPath returns the path to the vGPU library.
+func GetLibPath() string {
+ libPath := hostHookPath + "/vgpu/libvgpu.so." + info.GetVersion()
+ if _, err := os.Stat(libPath); os.IsNotExist(err) {
+ libPath = hostHookPath + "/vgpu/libvgpu.so"
+ }
+ return libPath
+}
+
+func GetNextDeviceRequest(dtype string, p corev1.Pod) (corev1.Container, util.ContainerDevices, error) {
+ pdevices, err := util.DecodePodDevices(util.InRequestDevices, p.Annotations)
+ if err != nil {
+ return corev1.Container{}, util.ContainerDevices{}, err
+ }
+ klog.Infof("pod annotation decode vaule is %+v", pdevices)
+ res := util.ContainerDevices{}
+
+ pd, ok := pdevices[dtype]
+ if !ok {
+ return corev1.Container{}, res, errors.New("device request not found")
+ }
+ for ctridx, ctrDevice := range pd {
+ if len(ctrDevice) > 0 {
+ return p.Spec.Containers[ctridx], ctrDevice, nil
+ }
+ }
+ return corev1.Container{}, res, errors.New("device request not found")
+}
+
+func EraseNextDeviceTypeFromAnnotation(dtype string, p corev1.Pod) error {
+ pdevices, err := util.DecodePodDevices(util.InRequestDevices, p.Annotations)
+ if err != nil {
+ return err
+ }
+ res := util.PodSingleDevice{}
+ pd, ok := pdevices[dtype]
+ if !ok {
+ return errors.New("erase device annotation not found")
+ }
+ found := false
+ for _, val := range pd {
+ if found {
+ res = append(res, val)
+ } else {
+ if len(val) > 0 {
+ found = true
+ res = append(res, util.ContainerDevices{})
+ } else {
+ res = append(res, val)
+ }
+ }
+ }
+ klog.Infoln("After erase res=", res)
+ newannos := make(map[string]string)
+ newannos[util.InRequestDevices[dtype]] = util.EncodePodSingleDevice(res)
+ return util.PatchPodAnnotations(&p, newannos)
+}
+
+func GetIndexAndTypeFromUUID(uuid string) (string, int) {
+ defer nvml.Shutdown()
+ if nvret := nvml.Init(); nvret != nvml.SUCCESS {
+ klog.Errorln("nvml Init err: ", nvret)
+ panic(0)
+ }
+ originuuid := strings.Split(uuid, "[")[0]
+ ndev, ret := nvml.DeviceGetHandleByUUID(originuuid)
+ if ret != nvml.SUCCESS {
+ klog.Error("nvml get handlebyuuid error ret=", ret)
+ panic(0)
+ }
+ Model, ret := ndev.GetName()
+ if ret != nvml.SUCCESS {
+ klog.Error("nvml get name error ret=", ret)
+ panic(0)
+ }
+ index, ret := ndev.GetIndex()
+ if ret != nvml.SUCCESS {
+ klog.Error("nvml get index error ret=", ret)
+ panic(0)
+ }
+ return Model, index
+}
+
+func GetMigUUIDFromSmiOutput(output string, uuid string, idx int) string {
+ migmode := false
+ for val := range strings.SplitSeq(output, "\n") {
+ if !strings.Contains(val, "MIG") && strings.Contains(val, uuid) {
+ migmode = true
+ continue
+ }
+ if !strings.Contains(val, "MIG") && !strings.Contains(val, uuid) {
+ migmode = false
+ continue
+ }
+ if !migmode {
+ continue
+ }
+ klog.Infoln("inspecting", val)
+ num := strings.Split(val, "Device")[1]
+ num = strings.Split(num, ":")[0]
+ num = strings.TrimSpace(num)
+ index, err := strconv.Atoi(num)
+ if err != nil {
+ klog.Fatal("atoi failed num=", num)
+ }
+ if index == idx {
+ outputStr := strings.Split(val, ":")[2]
+ outputStr = strings.TrimSpace(outputStr)
+ outputStr = strings.TrimRight(outputStr, ")")
+ return outputStr
+ }
+ }
+ return ""
+}
+
+func GetMigUUIDFromIndex(uuid string, idx int) string {
+ defer nvml.Shutdown()
+ if nvret := nvml.Init(); nvret != nvml.SUCCESS {
+ klog.Errorln("nvml Init err: ", nvret)
+ panic(0)
+ }
+ originuuid := strings.Split(uuid, "[")[0]
+ ndev, ret := nvml.DeviceGetHandleByUUID(originuuid)
+ if ret != nvml.SUCCESS {
+ klog.Error(`nvml get device uuid error ret=`, ret)
+ panic(0)
+ }
+ migdev, ret := nvml.DeviceGetMigDeviceHandleByIndex(ndev, idx)
+ if ret != nvml.SUCCESS {
+ klog.Error("nvml get mig dev error ret=", ret, ",idx=", idx, "using nvidia-smi -L for query")
+ cmd := exec.Command("nvidia-smi", "-L")
+ var stdout, stderr bytes.Buffer
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ err := cmd.Run()
+ if err != nil {
+ klog.Fatalf("nvidia-smi -L failed with %s\n", err)
+ }
+ outStr := stdout.String()
+ uuid := GetMigUUIDFromSmiOutput(outStr, originuuid, idx)
+ return uuid
+ }
+ res, ret := migdev.GetUUID()
+ if ret != nvml.SUCCESS {
+ klog.Error(`nvml get mig uuid error ret=`, ret)
+ panic(0)
+ }
+ return res
+}
+
+func GetMigGpuInstanceIdFromIndex(uuid string, idx int) (int, error) {
+ if nvret := nvml.Init(); nvret != nvml.SUCCESS {
+ klog.Errorln("nvml Init err: ", nvret)
+ return 0, fmt.Errorf("nvml Init err: %s", nvml.ErrorString(nvret))
+ }
+ originuuid := strings.Split(uuid, "[")[0]
+ ndev, ret := nvml.DeviceGetHandleByUUID(originuuid)
+ if ret != nvml.SUCCESS {
+ klog.Error(`nvml get device uuid error ret=`, ret)
+ return 0, fmt.Errorf("nvml get device uuid error ret=%d", ret)
+ }
+ migdev, ret := nvml.DeviceGetMigDeviceHandleByIndex(ndev, idx)
+ if ret != nvml.SUCCESS {
+ klog.Error(`nvml get mig device handle error ret=`, ret)
+ return 0, fmt.Errorf("nvml get mig device handle error ret=%d", ret)
+ }
+ res, ret := migdev.GetGpuInstanceId()
+ if ret != nvml.SUCCESS {
+ klog.Error(`nvml get gpu instance id error ret=`, ret)
+ return 0, fmt.Errorf("nvml get gpu instance id error ret=%d", ret)
+ }
+ return res, nil
+}
+
+func GetDeviceNums() (int, error) {
+ defer nvml.Shutdown()
+ if nvret := nvml.Init(); nvret != nvml.SUCCESS {
+ klog.Errorln("nvml Init err: ", nvret)
+ return 0, fmt.Errorf("nvml Init err: %s", nvml.ErrorString(nvret))
+ }
+ count, ret := nvml.DeviceGetCount()
+ if ret != nvml.SUCCESS {
+ klog.Error(`nvml get count error ret=`, ret)
+ return 0, fmt.Errorf("nvml get count error ret: %s", nvml.ErrorString(ret))
+ }
+ return count, nil
+}
+
+func GetDeviceNames() ([]string, error) {
+ names := []string{}
+ defer nvml.Shutdown()
+ if nvret := nvml.Init(); nvret != nvml.SUCCESS {
+ klog.Errorln("nvml Init err: ", nvret)
+ return names, fmt.Errorf("nvml Init err: %s", nvml.ErrorString(nvret))
+ }
+ count, ret := nvml.DeviceGetCount()
+ if ret != nvml.SUCCESS {
+ klog.Error(`nvml get count error ret=`, ret)
+ return names, fmt.Errorf("nvml get count error ret: %s", nvml.ErrorString(ret))
+ }
+ for i := 0; i < count; i++ {
+ dev, ret := nvml.DeviceGetHandleByIndex(i)
+ if ret != nvml.SUCCESS {
+ klog.Error(`nvml get device error ret=`, ret)
+ return names, fmt.Errorf("nvml get device error ret: %s", nvml.ErrorString(ret))
+ }
+ name, ret := dev.GetName()
+ if ret != nvml.SUCCESS {
+ klog.Error(`nvml get name error ret=`, ret)
+ return names, fmt.Errorf("nvml get name error ret: %s", nvml.ErrorString(ret))
+ }
+ names = append(names, name)
+ }
+ return names, nil
+}
+
+func (nv *NvidiaDevicePlugin) DisableOtherNVMLOperation() {
+ // Create MIG apply lock file
+ if err := CreateMigApplyLock(); err != nil {
+ // If the lock file creation fails, it is highly likely that the mig apply will be failed, so the plugin should terminate.
+ klog.Fatalf("Failed to create MIG apply lock: %v", err)
+ return
+ }
+
+ nv.disableHealthChecks <- true
+ nv.disableWatchAndRegister <- true
+ //wait for disableHealthChecks to be closed,signal must be true or wait forever
+ var ackHealthCheck bool
+ var ackWatchAndRegister bool
+ for {
+ select {
+ case ackDisableHealthChecksSignal := <-nv.ackDisableHealthChecks:
+ if ackDisableHealthChecksSignal {
+ ackHealthCheck = true
+ } else {
+ continue
+ }
+ case ackWatchAndRegisterSignal := <-nv.ackDisableWatchAndRegister:
+ if ackWatchAndRegisterSignal {
+ ackWatchAndRegister = true
+ } else {
+ continue
+ }
+ }
+ if ackHealthCheck && ackWatchAndRegister {
+ break
+ }
+ }
+}
+
+func (nv *NvidiaDevicePlugin) EnableOtherNVMLOperation() {
+ // Remove MIG apply lock file
+ if err := RemoveMigApplyLock(); err != nil {
+ klog.Errorf("Failed to remove MIG apply lock: %v", err)
+ }
+
+ nv.disableHealthChecks <- false
+ nv.disableWatchAndRegister <- false
+}
+
+func (nv *NvidiaDevicePlugin) ApplyMigTemplate() {
+ nv.applyMutex.Lock()
+ nv.DisableOtherNVMLOperation()
+ defer func() {
+ nv.EnableOtherNVMLOperation()
+ nv.applyMutex.Unlock()
+ }()
+ data, err := yaml.Marshal(nv.migCurrent)
+ if err != nil {
+ klog.Error("marshal failed", err.Error())
+ }
+ klog.Infoln("Applying data=", string(data))
+ os.WriteFile("/tmp/migconfig.yaml", data, os.ModePerm)
+ cmd := exec.Command("nvidia-mig-parted", "apply", "-f", "/tmp/migconfig.yaml")
+ var stdout, stderr bytes.Buffer
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ err = cmd.Run()
+ if err != nil {
+ klog.Fatalf("nvidia-mig-parted failed with %s,reason:%s\n", err, stderr.String())
+ }
+ outStr := stdout.String()
+ klog.Infoln("Mig apply", outStr)
+}
+
+func (nv *NvidiaDevicePlugin) GenerateMigTemplate(devtype string, devindex int, val util.ContainerDevice) (int, bool) {
+ needsreset := false
+ position := -1 // Initialize to an invalid position
+
+ for _, migTemplate := range nv.schedulerConfig.MigGeometriesList {
+ if containsModel(devtype, migTemplate.Models) {
+ klog.InfoS("type found", "Type", devtype, "Models", strings.Join(migTemplate.Models, ", "))
+
+ templateIdx, pos, err := util.ExtractMigTemplatesFromUUID(val.UUID)
+ if err != nil {
+ klog.ErrorS(err, "failed to extract template index from UUID", "UUID", val.UUID)
+ return -1, false
+ }
+ position = pos
+
+ if templateIdx < 0 || templateIdx >= len(migTemplate.Geometries) {
+ klog.ErrorS(nil, "invalid template index extracted from UUID", "UUID", val.UUID, "Index", templateIdx)
+ return -1, false
+ }
+
+ v := migTemplate.Geometries[templateIdx]
+
+ for migidx, migpartedDev := range nv.migCurrent.MigConfigs["current"] {
+ if containsDevice(devindex, migpartedDev.Devices) {
+ for _, migTemplateEntry := range v {
+ currentCount, ok := migpartedDev.MigDevices[migTemplateEntry.Name]
+ expectedCount := migTemplateEntry.Count
+
+ if !ok || currentCount != expectedCount {
+ needsreset = true
+ klog.InfoS("updated mig device count", "Template", v)
+ } else {
+ klog.InfoS("incremented mig device count", "TemplateName", migTemplateEntry.Name, "Count", currentCount+1)
+ }
+ }
+
+ if needsreset {
+ for k := range nv.migCurrent.MigConfigs["current"][migidx].MigDevices {
+ delete(nv.migCurrent.MigConfigs["current"][migidx].MigDevices, k)
+ }
+
+ for _, migTemplateEntry := range v {
+ nv.migCurrent.MigConfigs["current"][migidx].MigDevices[migTemplateEntry.Name] = migTemplateEntry.Count
+ nv.migCurrent.MigConfigs["current"][migidx].MigEnabled = true
+ }
+ }
+ break
+ }
+ }
+ break
+ }
+ }
+
+ return position, needsreset
+}
+
+// Helper function to check if a model is in the list of models.
+func containsModel(target string, models []string) bool {
+ for _, model := range models {
+ if strings.Contains(target, model) {
+ return true
+ }
+ }
+ return false
+}
+
+// Helper function to check if a device index is in the list of devices.
+func containsDevice(target int, devices []int32) bool {
+ for _, device := range devices {
+ if int(device) == target {
+ return true
+ }
+ }
+ return false
+}
+
+// Helper function to deepcopy new mig spec
+func deepCopyMigConfig(src nvidia.MigConfigSpec) nvidia.MigConfigSpec {
+ dst := src
+ if src.Devices != nil {
+ dst.Devices = make([]int32, len(src.Devices))
+ copy(dst.Devices, src.Devices)
+ }
+ if src.MigDevices != nil {
+ dst.MigDevices = make(map[string]int32)
+ for k, v := range src.MigDevices {
+ dst.MigDevices[k] = v
+ }
+ }
+ return dst
+}
+
+func (nv *NvidiaDevicePlugin) GetContainerDeviceStrArray(c util.ContainerDevices) []string {
+ tmp := []string{}
+ needsreset := false
+ position := 0
+ for _, val := range c {
+ if !strings.Contains(val.UUID, "[") {
+ tmp = append(tmp, val.UUID)
+ } else {
+ devtype, devindex := GetIndexAndTypeFromUUID(val.UUID)
+ position, needsreset = nv.GenerateMigTemplate(devtype, devindex, val)
+ if needsreset {
+ nv.ApplyMigTemplate()
+ }
+ tmp = append(tmp, GetMigUUIDFromIndex(val.UUID, position))
+ }
+ }
+ klog.V(3).Infoln("mig current=", nv.migCurrent, ":", needsreset, "position=", position, "uuid lists", tmp)
+ return tmp
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util_test.go
new file mode 100644
index 000000000..28a8cdebe
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util_test.go
@@ -0,0 +1,156 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package plugin
+
+import (
+ "testing"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+func TestGenerateMigTemplate(t *testing.T) {
+ sconfig := nvidia.NvidiaConfig{
+ MigGeometriesList: []util.AllowedMigGeometries{
+ {
+ Models: []string{"A30"},
+ Geometries: []util.Geometry{
+ {util.MigTemplate{Name: "1g.6gb", Memory: 6144, Count: 4}},
+ {util.MigTemplate{Name: "2g.12gb", Memory: 12288, Count: 2}},
+ {util.MigTemplate{Name: "4g.24gb", Memory: 24576, Count: 1}},
+ },
+ },
+ {
+ Models: []string{"A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB", "A100-SXM4-40GB"},
+ Geometries: []util.Geometry{
+ {util.MigTemplate{Name: "1g.5gb", Memory: 5120, Count: 7}},
+ {util.MigTemplate{Name: "2g.10gb", Memory: 10240, Count: 3}},
+ {util.MigTemplate{Name: "1g.5gb", Memory: 5120, Count: 1}},
+ {util.MigTemplate{Name: "3g.20gb", Memory: 20480, Count: 2}},
+ {util.MigTemplate{Name: "7g.40gb", Memory: 40960, Count: 1}},
+ },
+ },
+ {
+ Models: []string{"A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"},
+ Geometries: []util.Geometry{
+ {util.MigTemplate{Name: "1g.10gb", Memory: 10240, Count: 7}},
+ {util.MigTemplate{Name: "2g.20gb", Memory: 20480, Count: 3}},
+ {util.MigTemplate{Name: "1g.10gb", Memory: 10240, Count: 1}},
+ {util.MigTemplate{Name: "3g.40gb", Memory: 40960, Count: 2}},
+ {util.MigTemplate{Name: "7g.80gb", Memory: 81920, Count: 1}},
+ },
+ },
+ },
+ }
+
+ plugin := NvidiaDevicePlugin{
+ operatingMode: "mig",
+ schedulerConfig: sconfig,
+ }
+ plugin.migCurrent = nvidia.MigPartedSpec{
+ Version: "v1",
+ MigConfigs: make(map[string]nvidia.MigConfigSpecSlice),
+ }
+ plugin.migCurrent.MigConfigs["current"] = nvidia.MigConfigSpecSlice{
+ nvidia.MigConfigSpec{
+ Devices: []int32{0, 1},
+ MigEnabled: true,
+ MigDevices: make(map[string]int32), // Ensure this map is initialized
+ },
+ }
+
+ testCases := []struct {
+ name string
+ model string
+ deviceIdx int
+ containerDev util.ContainerDevice
+ expectedPos int
+ expectedReset bool
+ expectedMig map[string]int32
+ }{
+ {
+ name: "2g.10gb template",
+ model: "A100-SXM4-40GB",
+ deviceIdx: 0,
+ containerDev: util.ContainerDevice{
+ Idx: 0,
+ UUID: "aaaaabbbb[1-1]",
+ Usedmem: 3000,
+ },
+ expectedPos: 1,
+ expectedReset: true,
+ expectedMig: map[string]int32{
+ "2g.10gb": 3,
+ },
+ },
+ {
+ name: "1g.5gb template",
+ model: "A100-SXM4-40GB",
+ deviceIdx: 0,
+ containerDev: util.ContainerDevice{
+ Idx: 0,
+ UUID: "aaaaabbbb[0-1]",
+ Usedmem: 3000,
+ },
+ expectedPos: 1,
+ expectedReset: true,
+ expectedMig: map[string]int32{
+ "1g.5gb": 7,
+ },
+ },
+ {
+ name: "no reset needed",
+ model: "A100-SXM4-40GB",
+ deviceIdx: 0,
+ containerDev: util.ContainerDevice{
+ Idx: 0,
+ UUID: "aaaaabbbb[0-2]",
+ Usedmem: 3000,
+ },
+ expectedPos: 2,
+ expectedReset: false,
+ expectedMig: map[string]int32{
+ "1g.5gb": 8,
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ pos, needsreset := plugin.GenerateMigTemplate(tc.model, tc.deviceIdx, tc.containerDev)
+
+ // Check if the position matches the expected value
+ if pos != tc.expectedPos {
+ t.Errorf("expected position %d, got %d", tc.expectedPos, pos)
+ }
+
+ // Check if the reset flag matches the expected value
+ if needsreset != tc.expectedReset {
+ t.Errorf("expected reset %v, got %v", tc.expectedReset, needsreset)
+ }
+
+ // Check if the mig devices match the expected values
+ migDevices := plugin.migCurrent.MigConfigs["current"][0].MigDevices
+ for k, v := range tc.expectedMig {
+ actual, ok := migDevices[k]
+ if !ok || actual != v {
+ t.Errorf("expected %s count %d, got %d", k, v, actual)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/allocate.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/allocate.go
new file mode 100644
index 000000000..a75644d59
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/allocate.go
@@ -0,0 +1,137 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "fmt"
+ "sort"
+
+ "github.com/NVIDIA/go-gpuallocator/gpuallocator"
+)
+
+var alignedAllocationPolicy = gpuallocator.NewBestEffortPolicy()
+
+// getPreferredAllocation runs an allocation algorithm over the inputs.
+// The algorithm chosen is based both on the incoming set of available devices and various config settings.
+func (r *resourceManager) getPreferredAllocation(available, required []string, size int) ([]string, error) {
+ // If all of the available devices are full GPUs without replicas, then
+ // calculate an aligned allocation across those devices.
+ if r.Devices().AlignedAllocationSupported() && !AnnotatedIDs(available).AnyHasAnnotations() {
+ return r.alignedAlloc(available, required, size)
+ }
+
+ // Otherwise, distribute them evenly across all replicated GPUs
+ return r.distributedAlloc(available, required, size)
+}
+
+// alignedAlloc shells out to the alignedAllocationPolicy that is set in
+// order to calculate the preferred allocation.
+func (r *resourceManager) alignedAlloc(available, required []string, size int) ([]string, error) {
+ var devices []string
+
+ availableDevices, err := gpuallocator.NewDevicesFrom(available)
+ if err != nil {
+ return nil, fmt.Errorf("unable to retrieve list of available devices: %v", err)
+ }
+
+ requiredDevices, err := gpuallocator.NewDevicesFrom(required)
+ if err != nil {
+ return nil, fmt.Errorf("unable to retrieve list of required devices: %v", err)
+ }
+
+ allocatedDevices := alignedAllocationPolicy.Allocate(availableDevices, requiredDevices, size)
+
+ for _, device := range allocatedDevices {
+ devices = append(devices, device.UUID)
+ }
+
+ return devices, nil
+}
+
+// distributedAlloc returns a list of devices such that any replicated
+// devices are distributed across all replicated GPUs equally. It takes into
+// account already allocated replicas to ensure a proper balance across them.
+func (r *resourceManager) distributedAlloc(available, required []string, size int) ([]string, error) {
+ // Get the set of candidate devices as the difference between available and required.
+ candidates := r.devices.Subset(available).Difference(r.devices.Subset(required)).GetIDs()
+ needed := size - len(required)
+
+ if len(candidates) < needed {
+ return nil, fmt.Errorf("not enough available devices to satisfy allocation")
+ }
+
+ // For each candidate device, build a mapping of (stripped) device ID to
+ // total / available replicas for that device.
+ replicas := make(map[string]*struct{ total, available int })
+ for _, c := range candidates {
+ id := AnnotatedID(c).GetID()
+ if _, exists := replicas[id]; !exists {
+ replicas[id] = &struct{ total, available int }{}
+ }
+ replicas[id].available++
+ }
+ for d := range r.devices {
+ id := AnnotatedID(d).GetID()
+ if _, exists := replicas[id]; !exists {
+ continue
+ }
+ replicas[id].total++
+ }
+
+ // Grab the set of 'needed' devices one-by-one from the candidates list.
+ // Before selecting each candidate, first sort the candidate list using the
+ // replicas map above. After sorting, the first element in the list will
+ // contain the device with the least difference between total and available
+ // replications (based on what's already been allocated). Add this device
+ // to the list of devices to allocate, remove it from the candidate list,
+ // down its available count in the replicas map, and repeat.
+ var devices []string
+ for range needed {
+ sort.Slice(candidates, func(i, j int) bool {
+ iid := AnnotatedID(candidates[i]).GetID()
+ jid := AnnotatedID(candidates[j]).GetID()
+ idiff := replicas[iid].total - replicas[iid].available
+ jdiff := replicas[jid].total - replicas[jid].available
+ return idiff < jdiff
+ })
+ id := AnnotatedID(candidates[0]).GetID()
+ replicas[id].available--
+ devices = append(devices, candidates[0])
+ candidates = candidates[1:]
+ }
+
+ // Add the set of required devices to this list and return it.
+ devices = append(required, devices...)
+
+ return devices, nil
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/device_map.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/device_map.go
new file mode 100644
index 000000000..2ebac15c1
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/device_map.go
@@ -0,0 +1,334 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "fmt"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+
+ "github.com/NVIDIA/go-nvlib/pkg/nvlib/device"
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
+)
+
+type deviceMapBuilder struct {
+ device.Interface
+ config *nvidia.DeviceConfig
+}
+
+// DeviceMap stores a set of devices per resource name.
+type DeviceMap map[spec.ResourceName]Devices
+
+// NewDeviceMap creates a device map for the specified NVML library and config.
+func NewDeviceMap(nvmllib nvml.Interface, config *nvidia.DeviceConfig) (DeviceMap, error) {
+ b := deviceMapBuilder{
+ Interface: device.New(nvmllib),
+ config: config,
+ }
+ return b.build()
+}
+
+// build builds a map of resource names to devices.
+func (b *deviceMapBuilder) build() (DeviceMap, error) {
+ devices, err := b.buildDeviceMapFromConfigResources()
+ if err != nil {
+ return nil, fmt.Errorf("error building device map from config.resources: %v", err)
+ }
+ devices, err = updateDeviceMapWithReplicas(b.config, devices)
+ if err != nil {
+ return nil, fmt.Errorf("error updating device map with replicas from config.sharing.timeSlicing.resources: %v", err)
+ }
+ return devices, nil
+}
+
+// buildDeviceMapFromConfigResources builds a map of resource names to devices from spec.Config.Resources
+func (b *deviceMapBuilder) buildDeviceMapFromConfigResources() (DeviceMap, error) {
+ deviceMap, err := b.buildGPUDeviceMap()
+ if err != nil {
+ return nil, fmt.Errorf("error building GPU device map: %v", err)
+ }
+
+ if *b.config.Flags.MigStrategy == spec.MigStrategyNone {
+ return deviceMap, nil
+ }
+
+ migDeviceMap, err := b.buildMigDeviceMap()
+ if err != nil {
+ return nil, fmt.Errorf("error building MIG device map: %v", err)
+ }
+
+ var requireUniformMIGDevices bool
+ if *b.config.Flags.MigStrategy == spec.MigStrategySingle {
+ requireUniformMIGDevices = true
+ }
+
+ err = b.assertAllMigDevicesAreValid(requireUniformMIGDevices)
+ if err != nil {
+ return nil, fmt.Errorf("invalid MIG configuration: %v", err)
+ }
+
+ if requireUniformMIGDevices && !deviceMap.isEmpty() && !migDeviceMap.isEmpty() {
+ return nil, fmt.Errorf("all devices on the node must be configured with the same migEnabled value")
+ }
+
+ deviceMap.merge(migDeviceMap)
+
+ return deviceMap, nil
+}
+
+// buildGPUDeviceMap builds a map of resource names to GPU devices
+func (b *deviceMapBuilder) buildGPUDeviceMap() (DeviceMap, error) {
+ devices := make(DeviceMap)
+
+ b.VisitDevices(func(i int, gpu device.Device) error {
+ name, ret := gpu.GetName()
+ if ret != nvml.SUCCESS {
+ return fmt.Errorf("error getting product name for GPU: %v", ret)
+ }
+ migEnabled, err := gpu.IsMigEnabled()
+ if err != nil {
+ return fmt.Errorf("error checking if MIG is enabled on GPU: %v", err)
+ }
+ if migEnabled && *b.config.Flags.MigStrategy != spec.MigStrategyNone {
+ return nil
+ }
+ for _, resource := range b.config.Resources.GPUs {
+ if resource.Pattern.Matches(name) {
+ index, info := newGPUDevice(i, gpu)
+ return devices.setEntry(resource.Name, index, info)
+ }
+ }
+ return fmt.Errorf("GPU name '%v' does not match any resource patterns", name)
+ })
+ return devices, nil
+}
+
+// buildMigDeviceMap builds a map of resource names to MIG devices
+func (b *deviceMapBuilder) buildMigDeviceMap() (DeviceMap, error) {
+ devices := make(DeviceMap)
+ err := b.VisitMigDevices(func(i int, d device.Device, j int, mig device.MigDevice) error {
+ migProfile, err := mig.GetProfile()
+ if err != nil {
+ return fmt.Errorf("error getting MIG profile for MIG device at index '(%v, %v)': %v", i, j, err)
+ }
+ for _, resource := range b.config.Resources.MIGs {
+ if resource.Pattern.Matches(migProfile.String()) {
+ index, info := newMigDevice(i, j, mig)
+ return devices.setEntry(resource.Name, index, info)
+ }
+ }
+ return fmt.Errorf("MIG profile '%v' does not match any resource patterns", migProfile)
+ })
+ return devices, err
+}
+
+// assertAllMigDevicesAreValid ensures that each MIG-enabled device has at least one MIG device
+// associated with it.
+func (b *deviceMapBuilder) assertAllMigDevicesAreValid(uniform bool) error {
+ err := b.VisitDevices(func(i int, d device.Device) error {
+ isMigEnabled, err := d.IsMigEnabled()
+ if err != nil {
+ return err
+ }
+ if !isMigEnabled {
+ return nil
+ }
+ migDevices, err := d.GetMigDevices()
+ if err != nil {
+ return err
+ }
+ if len(migDevices) == 0 {
+ i := 0
+ return fmt.Errorf("device %v has an invalid MIG configuration", i)
+ }
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("at least one device with migEnabled=true was not configured correctly: %v", err)
+ }
+
+ if !uniform {
+ return nil
+ }
+
+ var previousAttributes *nvml.DeviceAttributes
+ return b.VisitMigDevices(func(i int, d device.Device, j int, m device.MigDevice) error {
+ attrs, ret := m.GetAttributes()
+ if ret != nvml.SUCCESS {
+ return fmt.Errorf("error getting device attributes: %v", ret)
+ }
+ if previousAttributes == nil {
+ previousAttributes = &attrs
+ } else if attrs != *previousAttributes {
+ return fmt.Errorf("more than one MIG device type present on node")
+ }
+
+ return nil
+ })
+}
+
+// setEntry sets the DeviceMap entry for the specified resource.
+func (d DeviceMap) setEntry(name spec.ResourceName, index string, info deviceInfo) error {
+ dev, err := BuildDevice(index, info)
+ if err != nil {
+ return fmt.Errorf("error building Device: %v", err)
+ }
+ d.insert(name, dev)
+ return nil
+}
+
+// insert adds the specified device to the device map
+func (d DeviceMap) insert(name spec.ResourceName, dev *Device) {
+ if d[name] == nil {
+ d[name] = make(Devices)
+ }
+ d[name][dev.ID] = dev
+}
+
+// merge merges two devices maps
+func (d DeviceMap) merge(o DeviceMap) {
+ for name, devices := range o {
+ for _, device := range devices {
+ d.insert(name, device)
+ }
+ }
+}
+
+// isEmpty checks whether a device map is empty
+func (d DeviceMap) isEmpty() bool {
+ for _, devices := range d {
+ if len(devices) > 0 {
+ return false
+ }
+ }
+ return true
+}
+
+// getIDsOfDevicesToReplicate returns a list of dervice IDs that we want to replicate.
+func (d DeviceMap) getIDsOfDevicesToReplicate(r *spec.ReplicatedResource) ([]string, error) {
+ devices, exists := d[r.Name]
+ if !exists {
+ return nil, nil
+ }
+
+ // If all devices for this resource type are to be replicated.
+ if r.Devices.All {
+ return devices.GetIDs(), nil
+ }
+
+ // If a specific number of devices for this resource type are to be replicated.
+ if r.Devices.Count > 0 {
+ if r.Devices.Count > len(devices) {
+ return nil, fmt.Errorf("requested %d devices to be replicated, but only %d devices available", r.Devices.Count, len(devices))
+ }
+ return devices.GetIDs()[:r.Devices.Count], nil
+ }
+
+ // If a specific set of devices for this resource type are to be replicated.
+ if len(r.Devices.List) > 0 {
+ var ids []string
+ for _, ref := range r.Devices.List {
+ if ref.IsUUID() {
+ d := devices.GetByID(string(ref))
+ if d == nil {
+ return nil, fmt.Errorf("no matching device with UUID: %v", ref)
+ }
+ ids = append(ids, d.ID)
+ }
+ if ref.IsGPUIndex() || ref.IsMigIndex() {
+ d := devices.GetByIndex(string(ref))
+ if d == nil {
+ return nil, fmt.Errorf("no matching device at index: %v", ref)
+ }
+ ids = append(ids, d.ID)
+ }
+ }
+ return ids, nil
+ }
+
+ return nil, fmt.Errorf("unexpected error")
+}
+
+// updateDeviceMapWithReplicas returns an updated map of resource names to devices with replica information from spec.Config.Sharing.TimeSlicing.Resources
+func updateDeviceMapWithReplicas(config *nvidia.DeviceConfig, oDevices DeviceMap) (DeviceMap, error) {
+ devices := make(DeviceMap)
+
+ // Begin by walking config.Sharing.TimeSlicing.Resources and building a map of just the resource names.
+ names := make(map[spec.ResourceName]bool)
+ for _, r := range config.Sharing.TimeSlicing.Resources {
+ names[r.Name] = true
+ }
+
+ // Copy over all devices from oDevices without a resource reference in TimeSlicing.Resources.
+ for r, ds := range oDevices {
+ if !names[r] {
+ devices[r] = ds
+ }
+ }
+
+ // Walk TimeSlicing.Resources and update devices in the device map as appropriate.
+ for _, r := range config.Sharing.TimeSlicing.Resources {
+ // Get the IDs of the devices we want to replicate from oDevices
+ ids, err := oDevices.getIDsOfDevicesToReplicate(&r)
+ if err != nil {
+ return nil, fmt.Errorf("unable to get IDs of devices to replicate for '%v' resource: %v", r.Name, err)
+ }
+ // Skip any resources not matched in oDevices
+ if len(ids) == 0 {
+ continue
+ }
+
+ // Add any devices we don't want replicated directly into the device map.
+ for _, d := range oDevices[r.Name].Difference(oDevices[r.Name].Subset(ids)) {
+ devices.insert(r.Name, d)
+ }
+
+ // Create replicated devices add them to the device map.
+ // Rename the resource for replicated devices as requested.
+ name := r.Name
+ if r.Rename != "" {
+ name = r.Rename
+ }
+ for _, id := range ids {
+ for i := range r.Replicas {
+ annotatedID := string(NewAnnotatedID(id, i))
+ replicatedDevice := *(oDevices[r.Name][id])
+ replicatedDevice.ID = annotatedID
+ devices.insert(name, &replicatedDevice)
+ }
+ }
+ }
+
+ return devices, nil
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/device_map_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/device_map_test.go
new file mode 100644
index 000000000..c65147443
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/device_map_test.go
@@ -0,0 +1,583 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+
+ spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
+ "github.com/stretchr/testify/require"
+ kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+)
+
+func TestDeviceMapInsert(t *testing.T) {
+ device0 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "0"}}
+ device0withIndex := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "0"}, Index: "index"}
+ device1 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "1"}}
+
+ testCases := []struct {
+ description string
+ deviceMap DeviceMap
+ key string
+ value *Device
+ expectedDeviceMap DeviceMap
+ }{
+ {
+ description: "insert into empty map",
+ deviceMap: make(DeviceMap),
+ key: "resource",
+ value: &device0,
+ expectedDeviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ },
+ },
+ },
+ {
+ description: "add to existing resource",
+ deviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ },
+ },
+ key: "resource",
+ value: &device1,
+ expectedDeviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ "1": &device1,
+ },
+ },
+ },
+ {
+ description: "add new resource",
+ deviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ },
+ },
+ key: "resource1",
+ value: &device0,
+ expectedDeviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ },
+ "resource1": Devices{
+ "0": &device0,
+ },
+ },
+ },
+ {
+ description: "overwrite existing device",
+ deviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ },
+ },
+ key: "resource",
+ value: &device0withIndex,
+ expectedDeviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0withIndex,
+ },
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.description, func(t *testing.T) {
+ tc.deviceMap.insert(spec.ResourceName(tc.key), tc.value)
+
+ require.EqualValues(t, tc.expectedDeviceMap, tc.deviceMap)
+ })
+ }
+}
+
+func TestUpdateDeviceMapWithReplicas(t *testing.T) {
+ device0 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "0"}, Index: "0"}
+ device1 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "1"}}
+ device2 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "2"}}
+ device3 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "3"}}
+
+ testCases := []struct {
+ description string
+ config *nvidia.DeviceConfig
+ devices DeviceMap
+ expectedDeviceMap DeviceMap
+ }{
+ {
+ description: "Update device map with replicas",
+ config: &nvidia.DeviceConfig{
+ Config: &spec.Config{
+ Sharing: spec.Sharing{
+ TimeSlicing: spec.ReplicatedResources{
+ Resources: []spec.ReplicatedResource{
+ {
+ Name: "resource1",
+ Replicas: 2,
+ Rename: "replicated-resource1",
+ Devices: spec.ReplicatedDevices{
+ All: true,
+ },
+ },
+ {
+ Name: "resource2",
+ Replicas: 1,
+ Devices: spec.ReplicatedDevices{
+ All: true,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ devices: DeviceMap{
+ "resource1": Devices{
+ "0": &device0,
+ "1": &device1,
+ },
+ "resource2": Devices{
+ "2": &device2,
+ },
+ "resource3": Devices{
+ "3": &device3,
+ },
+ },
+ expectedDeviceMap: DeviceMap{
+ "replicated-resource1": Devices{
+ "0::0": &Device{Device: kubeletdevicepluginv1beta1.Device{ID: "0::0"}, Index: "0"},
+ "0::1": &Device{Device: kubeletdevicepluginv1beta1.Device{ID: "0::1"}, Index: "0"},
+ "1::0": &Device{Device: kubeletdevicepluginv1beta1.Device{ID: "1::0"}},
+ "1::1": &Device{Device: kubeletdevicepluginv1beta1.Device{ID: "1::1"}},
+ },
+ "resource2": Devices{
+ "2::0": &Device{Device: kubeletdevicepluginv1beta1.Device{ID: "2::0"}},
+ },
+ "resource3": Devices{
+ "3": &device3,
+ },
+ },
+ },
+ {
+ description: "Some devices are not replicated",
+ config: &nvidia.DeviceConfig{
+ Config: &spec.Config{
+ Sharing: spec.Sharing{
+ TimeSlicing: spec.ReplicatedResources{
+ Resources: []spec.ReplicatedResource{
+ {
+ Name: "resource1",
+ Replicas: 2,
+ Rename: "replicated-resource1",
+ Devices: spec.ReplicatedDevices{
+ List: []spec.ReplicatedDeviceRef{"0"}, // only replicate index 0
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ devices: DeviceMap{
+ "resource1": Devices{
+ "0": &device0,
+ "1": &device1,
+ },
+ },
+ expectedDeviceMap: DeviceMap{
+ "replicated-resource1": Devices{
+ "0::0": &Device{Device: kubeletdevicepluginv1beta1.Device{ID: "0::0"}, Index: "0"},
+ "0::1": &Device{Device: kubeletdevicepluginv1beta1.Device{ID: "0::1"}, Index: "0"},
+ },
+ "resource1": Devices{
+ "1": &device1,
+ },
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.description, func(t *testing.T) {
+ devices, _ := updateDeviceMapWithReplicas(tc.config, tc.devices)
+ require.EqualValues(t, tc.expectedDeviceMap, devices)
+ })
+ }
+}
+
+func TestDeviceMapMerge(t *testing.T) {
+ device0 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "0"}}
+ device1 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "1"}}
+ device2 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "2"}}
+ device0Updated := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "0"}, Index: "updated"}
+
+ testCases := []struct {
+ description string
+ deviceMap DeviceMap
+ otherDeviceMap DeviceMap
+ expectedDeviceMap DeviceMap
+ }{
+ {
+ description: "merge into empty map",
+ deviceMap: make(DeviceMap),
+ otherDeviceMap: DeviceMap{"resource": Devices{"0": &device0}},
+ expectedDeviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ },
+ },
+ },
+ {
+ description: "merge from empty map",
+ deviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ },
+ },
+ otherDeviceMap: make(DeviceMap),
+ expectedDeviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ },
+ },
+ },
+ {
+ description: "merge with overlapping keys",
+ deviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ },
+ },
+ otherDeviceMap: DeviceMap{
+ "resource": Devices{
+ "1": &device1,
+ },
+ },
+ expectedDeviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ "1": &device1,
+ },
+ },
+ },
+ {
+ description: "merge with device ID conflict (overwrite existing device)",
+ deviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ },
+ },
+ otherDeviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0Updated,
+ },
+ },
+ expectedDeviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0Updated,
+ },
+ },
+ },
+ {
+ description: "merge with new resource",
+ deviceMap: DeviceMap{
+ "resource1": Devices{
+ "0": &device0,
+ },
+ },
+ otherDeviceMap: DeviceMap{
+ "resource2": Devices{
+ "1": &device1,
+ },
+ },
+ expectedDeviceMap: DeviceMap{
+ "resource1": Devices{
+ "0": &device0,
+ },
+ "resource2": Devices{
+ "1": &device1,
+ },
+ },
+ },
+ {
+ description: "merge with multiple devices and resources",
+ deviceMap: DeviceMap{
+ "resource1": Devices{
+ "0": &device0,
+ },
+ },
+ otherDeviceMap: DeviceMap{
+ "resource1": Devices{
+ "1": &device1,
+ },
+ "resource2": Devices{
+ "2": &device2,
+ },
+ },
+ expectedDeviceMap: DeviceMap{
+ "resource1": Devices{
+ "0": &device0,
+ "1": &device1,
+ },
+ "resource2": Devices{
+ "2": &device2,
+ },
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.description, func(t *testing.T) {
+ tc.deviceMap.merge(tc.otherDeviceMap)
+
+ require.EqualValues(t, tc.expectedDeviceMap, tc.deviceMap)
+ })
+ }
+}
+
+func TestDeviceMapIsEmpty(t *testing.T) {
+ device0 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "0"}}
+
+ testCases := []struct {
+ description string
+ deviceMap DeviceMap
+ expected bool
+ }{
+ {
+ description: "empty map",
+ deviceMap: make(DeviceMap),
+ expected: true,
+ },
+ {
+ description: "map with empty resource",
+ deviceMap: DeviceMap{
+ "resource": Devices{},
+ },
+ expected: true,
+ },
+ {
+ description: "map with non-empty resource",
+ deviceMap: DeviceMap{
+ "resource": Devices{
+ "0": &device0,
+ },
+ },
+ expected: false,
+ },
+ {
+ description: "map with multiple empty resources",
+ deviceMap: DeviceMap{
+ "resource1": Devices{},
+ "resource2": Devices{},
+ },
+ expected: true,
+ },
+ {
+ description: "map with multiple resources, one non-empty",
+ deviceMap: DeviceMap{
+ "resource1": Devices{},
+ "resource2": Devices{
+ "0": &device0,
+ },
+ },
+ expected: false,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.description, func(t *testing.T) {
+ actual := tc.deviceMap.isEmpty()
+
+ require.Equal(t, tc.expected, actual)
+ })
+ }
+}
+
+func TestDeviceMapGetIDsOfDevicesToReplicate(t *testing.T) {
+ device0 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "0"}, Index: "0"}
+ device1 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "1"}, Index: "1"}
+ device2 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "2"}, Index: "2"}
+ device3 := Device{Device: kubeletdevicepluginv1beta1.Device{ID: "3"}, Index: "3"}
+
+ deviceMap := DeviceMap{
+ "resource1": Devices{
+ "0": &device0,
+ "1": &device1,
+ "2": &device2,
+ "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76": &device3,
+ },
+ }
+
+ testCases := []struct {
+ description string
+ deviceMap DeviceMap
+ resource *spec.ReplicatedResource
+ expectedIDs []string
+ expectedErr error
+ }{
+ {
+ description: "resource does not exist",
+ deviceMap: deviceMap,
+ resource: &spec.ReplicatedResource{
+ Name: "nonexistent_resource",
+ Devices: spec.ReplicatedDevices{},
+ },
+ expectedIDs: nil,
+ expectedErr: nil,
+ },
+ {
+ description: "replicate all devices",
+ deviceMap: deviceMap,
+ resource: &spec.ReplicatedResource{
+ Name: "resource1",
+ Devices: spec.ReplicatedDevices{
+ All: true,
+ },
+ },
+ expectedIDs: []string{"0", "1", "2", "3"},
+ expectedErr: nil,
+ },
+ {
+ description: "replicate specific count of devices (count exceeds available)",
+ deviceMap: deviceMap,
+ resource: &spec.ReplicatedResource{
+ Name: "resource1",
+ Devices: spec.ReplicatedDevices{
+ Count: 5,
+ },
+ },
+ expectedIDs: nil,
+ expectedErr: fmt.Errorf("requested 5 devices to be replicated, but only 4 devices available"),
+ },
+ {
+ description: "replicate specific devices by ID (valid)",
+ deviceMap: deviceMap,
+ resource: &spec.ReplicatedResource{
+ Name: "resource1",
+ Devices: spec.ReplicatedDevices{
+ List: []spec.ReplicatedDeviceRef{
+ spec.ReplicatedDeviceRef("GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76"), // ref UUID
+ },
+ },
+ },
+ expectedIDs: []string{"3"},
+ expectedErr: nil,
+ },
+ {
+ description: "replicate specific devices by ID (invalid ID)",
+ deviceMap: deviceMap,
+ resource: &spec.ReplicatedResource{
+ Name: "resource1",
+ Devices: spec.ReplicatedDevices{
+ List: []spec.ReplicatedDeviceRef{
+ spec.ReplicatedDeviceRef("GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b77"), // Nonexistent device
+ },
+ },
+ },
+ expectedIDs: nil,
+ expectedErr: fmt.Errorf("no matching device with UUID: GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b77"),
+ },
+ {
+ description: "replicate specific devices by GPU index (valid)",
+ deviceMap: deviceMap,
+ resource: &spec.ReplicatedResource{
+ Name: "resource1",
+ Devices: spec.ReplicatedDevices{
+ List: []spec.ReplicatedDeviceRef{
+ spec.ReplicatedDeviceRef("0"), // Index: "0"
+ spec.ReplicatedDeviceRef("1"), // Index: "1"
+ },
+ },
+ },
+ expectedIDs: []string{"0", "1"},
+ expectedErr: nil,
+ },
+ {
+ description: "replicate specific devices by GPU index (invalid)",
+ deviceMap: deviceMap,
+ resource: &spec.ReplicatedResource{
+ Name: "resource1",
+ Devices: spec.ReplicatedDevices{
+ List: []spec.ReplicatedDeviceRef{
+ spec.ReplicatedDeviceRef("0"), // Index: "0"
+ spec.ReplicatedDeviceRef("4"), // Nonexistent Index
+ },
+ },
+ },
+ expectedIDs: nil,
+ expectedErr: fmt.Errorf("no matching device at index: 4"),
+ },
+ {
+ description: "invalid replicated devices",
+ deviceMap: deviceMap,
+ resource: &spec.ReplicatedResource{
+ Name: "resource1",
+ Devices: spec.ReplicatedDevices{
+ List: []spec.ReplicatedDeviceRef{
+ spec.ReplicatedDeviceRef("invalid_index"), // Invalid gpu
+ },
+ },
+ },
+ expectedIDs: nil,
+ expectedErr: nil,
+ },
+ {
+ description: "unexpected error (no replication criteria provided)",
+ deviceMap: deviceMap,
+ resource: &spec.ReplicatedResource{
+ Name: "resource1",
+ Devices: spec.ReplicatedDevices{},
+ },
+ expectedIDs: nil,
+ expectedErr: fmt.Errorf("unexpected error"),
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.description, func(t *testing.T) {
+ ids, err := tc.deviceMap.getIDsOfDevicesToReplicate(tc.resource)
+
+ if tc.expectedErr != nil {
+ require.Error(t, err)
+ require.EqualError(t, err, tc.expectedErr.Error())
+ } else {
+ require.NoError(t, err)
+ }
+
+ require.ElementsMatch(t, tc.expectedIDs, ids)
+ })
+ }
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices.go
new file mode 100644
index 000000000..19f56325b
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices.go
@@ -0,0 +1,278 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "fmt"
+ "slices"
+ "strconv"
+ "strings"
+
+ kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+)
+
+// Device wraps kubeletdevicepluginv1beta1.Device with extra metadata and functions.
+type Device struct {
+ kubeletdevicepluginv1beta1.Device
+ Paths []string
+ Index string
+}
+
+// deviceInfo defines the information the required to construct a Device
+type deviceInfo interface {
+ GetUUID() (string, error)
+ GetPaths() ([]string, error)
+ GetNumaNode() (bool, int, error)
+}
+
+// Devices wraps a map[string]*Device with some functions.
+type Devices map[string]*Device
+
+// AnnotatedID represents an ID with a replica number embedded in it.
+type AnnotatedID string
+
+// AnnotatedIDs can be used to treat a []string as a []AnnotatedID.
+type AnnotatedIDs []string
+
+// BuildDevice builds an rm.Device with the specified index and deviceInfo
+func BuildDevice(index string, d deviceInfo) (*Device, error) {
+ uuid, err := d.GetUUID()
+ if err != nil {
+ return nil, fmt.Errorf("error getting UUID device: %v", err)
+ }
+
+ paths, err := d.GetPaths()
+ if err != nil {
+ return nil, fmt.Errorf("error getting device paths: %v", err)
+ }
+
+ hasNuma, numa, err := d.GetNumaNode()
+ if err != nil {
+ return nil, fmt.Errorf("error getting device NUMA node: %v", err)
+ }
+
+ dev := Device{}
+ dev.ID = uuid
+ dev.Index = index
+ dev.Paths = paths
+ dev.Health = kubeletdevicepluginv1beta1.Healthy
+ if hasNuma {
+ dev.Topology = &kubeletdevicepluginv1beta1.TopologyInfo{
+ Nodes: []*kubeletdevicepluginv1beta1.NUMANode{
+ {
+ ID: int64(numa),
+ },
+ },
+ }
+ }
+
+ return &dev, nil
+}
+
+// Contains checks if Devices contains devices matching all ids.
+func (ds Devices) Contains(ids ...string) bool {
+ for _, id := range ids {
+ if _, exists := ds[id]; !exists {
+ return false
+ }
+ }
+ return true
+}
+
+// GetByID returns a reference to the device matching the specified ID (nil otherwise).
+func (ds Devices) GetByID(id string) *Device {
+ return ds[id]
+}
+
+// GetByIndex returns a reference to the device matching the specified Index (nil otherwise).
+func (ds Devices) GetByIndex(index string) *Device {
+ for _, d := range ds {
+ if d.Index == index {
+ return d
+ }
+ }
+ return nil
+}
+
+// Subset returns the subset of devices in Devices matching the provided ids.
+// If any id in ids is not in Devices, then the subset that did match will be returned.
+func (ds Devices) Subset(ids []string) Devices {
+ res := make(Devices)
+ for _, id := range ids {
+ if ds.Contains(id) {
+ res[id] = ds[id]
+ }
+ }
+ return res
+}
+
+// Difference returns the set of devices contained in ds but not in ods.
+func (ds Devices) Difference(ods Devices) Devices {
+ res := make(Devices)
+ for id := range ds {
+ if !ods.Contains(id) {
+ res[id] = ds[id]
+ }
+ }
+ return res
+}
+
+// GetIDs returns the ids from all devices in the Devices
+func (ds Devices) GetIDs() []string {
+ var res []string
+ for _, d := range ds {
+ res = append(res, d.ID)
+ }
+ return res
+}
+
+// GetPluginDevices returns the plugin Devices from all devices in the Devices
+func (ds Devices) GetPluginDevices(count uint) []*kubeletdevicepluginv1beta1.Device {
+ var res []*kubeletdevicepluginv1beta1.Device
+
+ if !strings.Contains(ds.GetIDs()[0], "MIG") {
+ for _, dev := range ds {
+ for i := uint(0); i < count; i++ {
+ id := fmt.Sprintf("%v-%v", dev.ID, i)
+ res = append(res, &kubeletdevicepluginv1beta1.Device{
+ ID: id,
+ Health: dev.Health,
+ Topology: nil,
+ })
+ }
+ }
+ } else {
+ for _, d := range ds {
+ res = append(res, &d.Device)
+ }
+
+ }
+
+ return res
+}
+
+// GetIndices returns the Indices from all devices in the Devices
+func (ds Devices) GetIndices() []string {
+ var res []string
+ for _, d := range ds {
+ res = append(res, d.Index)
+ }
+ return res
+}
+
+// GetPaths returns the Paths from all devices in the Devices
+func (ds Devices) GetPaths() []string {
+ var res []string
+ for _, d := range ds {
+ res = append(res, d.Paths...)
+ }
+ return res
+}
+
+// AlignedAllocationSupported checks whether all devices support an alligned allocation
+func (ds Devices) AlignedAllocationSupported() bool {
+ for _, d := range ds {
+ if !d.AlignedAllocationSupported() {
+ return false
+ }
+ }
+ return true
+}
+
+// AlignedAllocationSupported checks whether the device supports an alligned allocation
+func (d Device) AlignedAllocationSupported() bool {
+ if d.IsMigDevice() {
+ return false
+ }
+
+ return !slices.Contains(d.Paths, "/dev/dxg")
+}
+
+// IsMigDevice returns checks whether d is a MIG device or not.
+func (d Device) IsMigDevice() bool {
+ return strings.Contains(d.Index, ":")
+}
+
+// GetUUID returns the UUID for the device from the annotated ID.
+func (d Device) GetUUID() string {
+ return AnnotatedID(d.ID).GetID()
+}
+
+// NewAnnotatedID creates a new AnnotatedID from an ID and a replica number.
+func NewAnnotatedID(id string, replica int) AnnotatedID {
+ return AnnotatedID(fmt.Sprintf("%s::%d", id, replica))
+}
+
+// HasAnnotations checks if an AnnotatedID has any annotations or not.
+func (r AnnotatedID) HasAnnotations() bool {
+ split := strings.SplitN(string(r), "::", 2)
+ if len(split) != 2 {
+ return false
+ }
+ return true
+}
+
+// Split splits a AnnotatedID into its ID and replica number parts.
+func (r AnnotatedID) Split() (string, int) {
+ split := strings.SplitN(string(r), "::", 2)
+ if len(split) != 2 {
+ return string(r), 0
+ }
+ replica, _ := strconv.ParseInt(split[1], 10, 0)
+ return split[0], int(replica)
+}
+
+// GetID returns just the ID part of the replicated ID
+func (r AnnotatedID) GetID() string {
+ id, _ := r.Split()
+ return id
+}
+
+// AnyHasAnnotations checks if any ID has annotations or not.
+func (rs AnnotatedIDs) AnyHasAnnotations() bool {
+ for _, r := range rs {
+ if AnnotatedID(r).HasAnnotations() {
+ return true
+ }
+ }
+ return false
+}
+
+// GetIDs returns just the ID parts of the annotated IDs as a []string
+func (rs AnnotatedIDs) GetIDs() []string {
+ res := make([]string, len(rs))
+ for i, r := range rs {
+ res[i] = AnnotatedID(r).GetID()
+ }
+ return res
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/health.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/health.go
new file mode 100644
index 000000000..9a46fcfdf
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/health.go
@@ -0,0 +1,311 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "fmt"
+ "os"
+ "strconv"
+ "strings"
+
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ "k8s.io/klog/v2"
+
+ safecast "github.com/ccoveille/go-safecast"
+)
+
+const (
+ // envDisableHealthChecks defines the environment variable that is checked to determine whether healthchecks
+ // should be disabled. If this envvar is set to "all" or contains the string "xids", healthchecks are
+ // disabled entirely. If set, the envvar is treated as a comma-separated list of Xids to ignore. Note that
+ // this is in addition to the Application errors that are already ignored.
+ envDisableHealthChecks = "DP_DISABLE_HEALTHCHECKS"
+ allHealthChecks = "xids"
+
+ // maxSuccessiveEventErrorCount sets the number of errors waiting for events before marking all devices as unhealthy.
+ maxSuccessiveEventErrorCount = 3
+)
+
+// CheckHealth performs health checks on a set of devices, writing to the 'unhealthy' channel with any unhealthy devices
+func (r *nvmlResourceManager) checkHealth(stop <-chan any, devices Devices, unhealthy chan<- *Device, disableNVML <-chan bool) error {
+ klog.V(4).Info("Check Health start Running")
+ disableHealthChecks := strings.ToLower(os.Getenv(envDisableHealthChecks))
+ if disableHealthChecks == "all" {
+ disableHealthChecks = allHealthChecks
+ }
+ if strings.Contains(disableHealthChecks, "xids") {
+ return nil
+ }
+
+ ret := r.nvml.Init()
+ if ret != nvml.SUCCESS {
+ if *r.config.Flags.FailOnInitError {
+ return fmt.Errorf("failed to initialize NVML: %v", ret)
+ }
+ return nil
+ }
+ defer func() {
+ ret := r.nvml.Shutdown()
+ if ret != nvml.SUCCESS {
+ klog.Infof("Error shutting down NVML: %v", ret)
+ }
+ }()
+
+ // FIXME: formalize the full list and document it.
+ // http://docs.nvidia.com/deploy/xid-errors/index.html#topic_4
+ // Application errors: the GPU should still be healthy
+ applicationErrorXids := []uint64{
+ 13, // Graphics Engine Exception
+ 31, // GPU memory page fault
+ 43, // GPU stopped processing
+ 45, // Preemptive cleanup, due to previous errors
+ 68, // Video processor exception
+ }
+
+ skippedXids := make(map[uint64]bool)
+ for _, id := range applicationErrorXids {
+ skippedXids[id] = true
+ }
+
+ for _, additionalXid := range getAdditionalXids(disableHealthChecks) {
+ skippedXids[additionalXid] = true
+ }
+
+ eventSet, ret := r.nvml.EventSetCreate()
+ if ret != nvml.SUCCESS {
+ return fmt.Errorf("failed to create event set: %v", ret)
+ }
+ defer eventSet.Free()
+
+ parentToDeviceMap := make(map[string]*Device)
+ deviceIDToGiMap := make(map[string]int)
+ deviceIDToCiMap := make(map[string]int)
+
+ eventMask := uint64(nvml.EventTypeXidCriticalError | nvml.EventTypeDoubleBitEccError | nvml.EventTypeSingleBitEccError)
+ for _, d := range devices {
+ uuid, gi, ci, err := r.getDevicePlacement(d)
+ if err != nil {
+ klog.Warningf("Could not determine device placement for %v: %v; Marking it unhealthy.", d.ID, err)
+ unhealthy <- d
+ continue
+ }
+ deviceIDToGiMap[d.ID] = gi
+ deviceIDToCiMap[d.ID] = ci
+ parentToDeviceMap[uuid] = d
+
+ gpu, ret := r.nvml.DeviceGetHandleByUUID(uuid)
+ if ret != nvml.SUCCESS {
+ klog.Infof("unable to get device handle from UUID: %v; marking it as unhealthy", ret)
+ unhealthy <- d
+ continue
+ }
+
+ supportedEvents, ret := gpu.GetSupportedEventTypes()
+ if ret != nvml.SUCCESS {
+ klog.Infof("Unable to determine the supported events for %v: %v; marking it as unhealthy", d.ID, ret)
+ unhealthy <- d
+ continue
+ }
+
+ ret = gpu.RegisterEvents(eventMask&supportedEvents, eventSet)
+ if ret == nvml.ERROR_NOT_SUPPORTED {
+ klog.Warningf("Device %v is too old to support healthchecking.", d.ID)
+ }
+ if ret != nvml.SUCCESS {
+ klog.Infof("Marking device %v as unhealthy: %v", d.ID, ret)
+ unhealthy <- d
+ }
+ }
+
+ for {
+ select {
+ case <-stop:
+ return nil
+ case signal := <-disableNVML:
+ if signal {
+ klog.Info("Check Health has been received close signal")
+ return fmt.Errorf("close signal received")
+ }
+ default:
+ }
+
+ e, ret := eventSet.Wait(5000)
+ if ret == nvml.ERROR_TIMEOUT {
+ continue
+ }
+ if ret != nvml.SUCCESS {
+ klog.Infof("Error waiting for event: %v; Marking all devices as unhealthy", ret)
+ for _, d := range devices {
+ unhealthy <- d
+ }
+ continue
+ }
+
+ if e.EventType != nvml.EventTypeXidCriticalError {
+ klog.Infof("Skipping non-nvmlEventTypeXidCriticalError event: %+v", e)
+ continue
+ }
+
+ if skippedXids[e.EventData] {
+ klog.Infof("Skipping event %+v", e)
+ continue
+ }
+
+ klog.Infof("Processing event %+v", e)
+ eventUUID, ret := e.Device.GetUUID()
+ if ret != nvml.SUCCESS {
+ // If we cannot reliably determine the device UUID, we mark all devices as unhealthy.
+ klog.Infof("Failed to determine uuid for event %v: %v; Marking all devices as unhealthy.", e, ret)
+ for _, d := range devices {
+ unhealthy <- d
+ }
+ continue
+ }
+
+ d, exists := parentToDeviceMap[eventUUID]
+ if !exists {
+ klog.Infof("Ignoring event for unexpected device: %v", eventUUID)
+ continue
+ }
+
+ if d.IsMigDevice() && e.GpuInstanceId != 0xFFFFFFFF && e.ComputeInstanceId != 0xFFFFFFFF {
+ gi := deviceIDToGiMap[d.ID]
+ ci := deviceIDToCiMap[d.ID]
+ giu32, err := safecast.ToUint32(gi)
+ if err != nil || giu32 != e.GpuInstanceId {
+ continue
+ }
+ ciu32, err := safecast.ToUint32(ci)
+ if err != nil || ciu32 != e.ComputeInstanceId {
+ continue
+ }
+ klog.Infof("Event for mig device %v (gi=%v, ci=%v)", d.ID, gi, ci)
+ }
+
+ klog.Infof("XidCriticalError: Xid=%d on Device=%s; marking device as unhealthy.", e.EventData, d.ID)
+ unhealthy <- d
+ }
+}
+
+// getAdditionalXids returns a list of additional Xids to skip from the specified string.
+// The input is treaded as a comma-separated string and all valid uint64 values are considered as Xid values. Invalid values
+// are ignored.
+func getAdditionalXids(input string) []uint64 {
+ if input == "" {
+ return nil
+ }
+
+ var additionalXids []uint64
+ for additionalXid := range strings.SplitSeq(input, ",") {
+ trimmed := strings.TrimSpace(additionalXid)
+ if trimmed == "" {
+ continue
+ }
+ xid, err := strconv.ParseUint(trimmed, 10, 64)
+ if err != nil {
+ klog.Infof("Ignoring malformed Xid value %v: %v", trimmed, err)
+ continue
+ }
+ additionalXids = append(additionalXids, xid)
+ }
+
+ return additionalXids
+}
+
+// getDevicePlacement returns the placement of the specified device.
+// For a MIG device the placement is defined by the 3-tuple
+// For a full device the returned 3-tuple is the device's uuid and 0xFFFFFFFF for the other two elements.
+func (r *nvmlResourceManager) getDevicePlacement(d *Device) (string, int, int, error) {
+ if !d.IsMigDevice() {
+ return d.GetUUID(), 0xFFFFFFFF, 0xFFFFFFFF, nil
+ }
+ return r.getMigDeviceParts(d)
+}
+
+// getMigDeviceParts returns the parent GI and CI ids of the MIG device.
+func (r *nvmlResourceManager) getMigDeviceParts(d *Device) (string, int, int, error) {
+ if !d.IsMigDevice() {
+ return "", 0, 0, fmt.Errorf("cannot get GI and CI of full device")
+ }
+
+ uuid := d.GetUUID()
+ // For older driver versions, the call to DeviceGetHandleByUUID will fail for MIG devices.
+ mig, ret := r.nvml.DeviceGetHandleByUUID(uuid)
+ if ret == nvml.SUCCESS {
+ parentHandle, ret := mig.GetDeviceHandleFromMigDeviceHandle()
+ if ret != nvml.SUCCESS {
+ return "", 0, 0, fmt.Errorf("failed to get parent device handle: %v", ret)
+ }
+
+ parentUUID, ret := parentHandle.GetUUID()
+ if ret != nvml.SUCCESS {
+ return "", 0, 0, fmt.Errorf("failed to get parent uuid: %v", ret)
+ }
+ gi, ret := mig.GetGpuInstanceId()
+ if ret != nvml.SUCCESS {
+ return "", 0, 0, fmt.Errorf("failed to get GPU Instance ID: %v", ret)
+ }
+
+ ci, ret := mig.GetComputeInstanceId()
+ if ret != nvml.SUCCESS {
+ return "", 0, 0, fmt.Errorf("failed to get Compute Instance ID: %v", ret)
+ }
+ return parentUUID, gi, ci, nil
+ }
+ return parseMigDeviceUUID(uuid)
+}
+
+// parseMigDeviceUUID splits the MIG device UUID into the parent device UUID and ci and gi
+func parseMigDeviceUUID(mig string) (string, int, int, error) {
+ tokens := strings.SplitN(mig, "-", 2)
+ if len(tokens) != 2 || tokens[0] != "MIG" {
+ return "", 0, 0, fmt.Errorf("unable to parse UUID as MIG device")
+ }
+
+ tokens = strings.SplitN(tokens[1], "/", 3)
+ if len(tokens) != 3 || !strings.HasPrefix(tokens[0], "GPU-") {
+ return "", 0, 0, fmt.Errorf("unable to parse UUID as MIG device")
+ }
+
+ gi, err := strconv.ParseInt(tokens[1], 10, 32)
+ if err != nil {
+ return "", 0, 0, fmt.Errorf("unable to parse UUID as MIG device")
+ }
+
+ ci, err := strconv.ParseInt(tokens[2], 10, 32)
+ if err != nil {
+ return "", 0, 0, fmt.Errorf("unable to parse UUID as MIG device")
+ }
+
+ return tokens[0], int(gi), int(ci), nil
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/health_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/health_test.go
new file mode 100644
index 000000000..5818839f9
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/health_test.go
@@ -0,0 +1,100 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetAdditionalXids(t *testing.T) {
+ testCases := []struct {
+ description string
+ input string
+ expected []uint64
+ }{
+ {
+ description: "Empty input",
+ },
+ {
+ description: "Only comma",
+ input: ",",
+ },
+ {
+ description: "Non-integer input",
+ input: "not-an-int",
+ },
+ {
+ description: "Single integer",
+ input: "68",
+ expected: []uint64{68},
+ },
+ {
+ description: "Negative integer",
+ input: "-68",
+ },
+ {
+ description: "Single integer with trailing spaces",
+ input: "68 ",
+ expected: []uint64{68},
+ },
+ {
+ description: "Single integer followed by comma without trailing number",
+ input: "68,",
+ expected: []uint64{68},
+ },
+ {
+ description: "Comma without preceding number followed by single integer",
+ input: ",68",
+ expected: []uint64{68},
+ },
+ {
+ description: "Two comma-separated integers",
+ input: "68,67",
+ expected: []uint64{68, 67},
+ },
+ {
+ description: "Two integers separated by non-integer",
+ input: "68,not-an-int,67",
+ expected: []uint64{68, 67},
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.description, func(t *testing.T) {
+ xids := getAdditionalXids(tc.input)
+ require.EqualValues(t, tc.expected, xids)
+ })
+ }
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/helper.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/helper.go
new file mode 100644
index 000000000..4c3ff3c65
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/helper.go
@@ -0,0 +1,54 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+// int8Slice wraps an []int8 with more functions.
+type int8Slice []int8
+
+// String turns a nil terminated int8Slice into a string
+func (s int8Slice) String() string {
+ var b []byte
+ for _, c := range s {
+ if c == 0 {
+ break
+ }
+ b = append(b, byte(c))
+ }
+ return string(b)
+}
+
+// uintPtr returns a *uint from a uint32
+func uintPtr(c uint32) *uint {
+ i := uint(c)
+ return &i
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_devices.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_devices.go
new file mode 100644
index 000000000..d4bc3f7a1
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_devices.go
@@ -0,0 +1,184 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "bytes"
+ "fmt"
+ "os"
+ "strconv"
+ "strings"
+
+ "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/mig"
+
+ "github.com/NVIDIA/go-nvlib/pkg/nvlib/info"
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+)
+
+const (
+ nvidiaProcDriverPath = "/proc/driver/nvidia"
+ nvidiaCapabilitiesPath = nvidiaProcDriverPath + "/capabilities"
+)
+
+// nvmlDevice wraps an nvml.Device with more functions.
+type nvmlDevice struct {
+ nvml.Device
+}
+
+// nvmlMigDevice allows for specific functions of nvmlDevice to be overridden.
+type nvmlMigDevice nvmlDevice
+
+var _ deviceInfo = (*nvmlDevice)(nil)
+var _ deviceInfo = (*nvmlMigDevice)(nil)
+
+func newGPUDevice(i int, gpu nvml.Device) (string, deviceInfo) {
+ index := fmt.Sprintf("%v", i)
+ isWsl, _ := info.New().HasDXCore()
+ if isWsl {
+ return index, wslDevice{gpu}
+ }
+
+ return index, nvmlDevice{gpu}
+}
+
+func newMigDevice(i int, j int, mig nvml.Device) (string, nvmlMigDevice) {
+ return fmt.Sprintf("%v:%v", i, j), nvmlMigDevice{mig}
+}
+
+// GetUUID returns the UUID of the device
+func (d nvmlDevice) GetUUID() (string, error) {
+ uuid, ret := d.Device.GetUUID()
+ if ret != nvml.SUCCESS {
+ return "", ret
+ }
+ return uuid, nil
+}
+
+// GetUUID returns the UUID of the device
+func (d nvmlMigDevice) GetUUID() (string, error) {
+ return nvmlDevice(d).GetUUID()
+}
+
+// GetPaths returns the paths for a GPU device
+func (d nvmlDevice) GetPaths() ([]string, error) {
+ minor, ret := d.GetMinorNumber()
+ if ret != nvml.SUCCESS {
+ return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
+ }
+ path := fmt.Sprintf("/dev/nvidia%d", minor)
+
+ return []string{path}, nil
+}
+
+// GetPaths returns the paths for a MIG device
+func (d nvmlMigDevice) GetPaths() ([]string, error) {
+ capDevicePaths, err := mig.GetMigCapabilityDevicePaths()
+ if err != nil {
+ return nil, fmt.Errorf("error getting MIG capability device paths: %v", err)
+ }
+
+ gi, ret := d.GetGpuInstanceId()
+ if ret != nvml.SUCCESS {
+ return nil, fmt.Errorf("error getting GPU Instance ID: %v", ret)
+ }
+
+ ci, ret := d.GetComputeInstanceId()
+ if ret != nvml.SUCCESS {
+ return nil, fmt.Errorf("error getting Compute Instance ID: %v", ret)
+ }
+
+ parent, ret := d.GetDeviceHandleFromMigDeviceHandle()
+ if ret != nvml.SUCCESS {
+ return nil, fmt.Errorf("error getting parent device: %v", ret)
+ }
+ minor, ret := parent.GetMinorNumber()
+ if ret != nvml.SUCCESS {
+ return nil, fmt.Errorf("error getting GPU device minor number: %v", ret)
+ }
+ parentPath := fmt.Sprintf("/dev/nvidia%d", minor)
+
+ giCapPath := fmt.Sprintf(nvidiaCapabilitiesPath+"/gpu%d/mig/gi%d/access", minor, gi)
+ if _, exists := capDevicePaths[giCapPath]; !exists {
+ return nil, fmt.Errorf("missing MIG GPU instance capability path: %v", giCapPath)
+ }
+
+ ciCapPath := fmt.Sprintf(nvidiaCapabilitiesPath+"/gpu%d/mig/gi%d/ci%d/access", minor, gi, ci)
+ if _, exists := capDevicePaths[ciCapPath]; !exists {
+ return nil, fmt.Errorf("missing MIG GPU instance capability path: %v", giCapPath)
+ }
+
+ devicePaths := []string{
+ parentPath,
+ capDevicePaths[giCapPath],
+ capDevicePaths[ciCapPath],
+ }
+
+ return devicePaths, nil
+}
+
+// GetNumaNode returns the NUMA node associated with the GPU device
+func (d nvmlDevice) GetNumaNode() (bool, int, error) {
+ pciInfo, ret := d.GetPciInfo()
+ if ret != nvml.SUCCESS {
+ return false, 0, fmt.Errorf("error getting PCI Bus Info of device: %v", ret)
+ }
+
+ // Discard leading zeros.
+ busID := strings.ToLower(strings.TrimPrefix(int8Slice(pciInfo.BusId[:]).String(), "0000"))
+
+ b, err := os.ReadFile(fmt.Sprintf("/sys/bus/pci/devices/%s/numa_node", busID))
+ if err != nil {
+ return false, 0, nil
+ }
+
+ node, err := strconv.Atoi(string(bytes.TrimSpace(b)))
+ if err != nil {
+ return false, 0, fmt.Errorf("eror parsing value for NUMA node: %v", err)
+ }
+
+ if node < 0 {
+ return false, 0, nil
+ }
+
+ return true, node, nil
+}
+
+// GetNumaNode for a MIG device is the NUMA node of the parent device.
+func (d nvmlMigDevice) GetNumaNode() (bool, int, error) {
+ parent, ret := d.GetDeviceHandleFromMigDeviceHandle()
+ if ret != nvml.SUCCESS {
+ return false, 0, fmt.Errorf("error getting parent GPU device from MIG device: %v", ret)
+ }
+
+ return nvmlDevice{parent}.GetNumaNode()
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_devices_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_devices_test.go
new file mode 100644
index 000000000..5fe76b7fe
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_devices_test.go
@@ -0,0 +1,181 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ mock "github.com/NVIDIA/go-nvml/pkg/nvml/mock"
+
+ //"github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/rm"
+ "github.com/stretchr/testify/require"
+)
+
+// Test GetUUID for nvmlDevice
+func TestNvmlDevice_GetUUID(t *testing.T) {
+ testCases := []struct {
+ description string
+ nvmlDevice nvml.Device
+ expectedUUID string
+ expectedError error
+ }{
+ {
+ description: "Successful UUID retrieval",
+ nvmlDevice: &mock.Device{
+ GetUUIDFunc: func() (string, nvml.Return) {
+ return "GPU-12345", nvml.SUCCESS
+ },
+ },
+ expectedUUID: "GPU-12345",
+ expectedError: nil,
+ },
+ {
+ description: "Error retrieving UUID",
+ nvmlDevice: &mock.Device{
+ GetUUIDFunc: func() (string, nvml.Return) {
+ return "GPU-12345", nvml.ERROR_UNKNOWN
+ },
+ },
+ expectedUUID: "",
+ expectedError: nvml.ERROR_UNKNOWN,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.description, func(t *testing.T) {
+ device := nvmlDevice{Device: tc.nvmlDevice}
+ uuid, err := device.GetUUID()
+
+ if tc.expectedError == nil {
+ require.NoError(t, err)
+ } else {
+ require.EqualError(t, err, tc.expectedError.Error())
+ }
+ require.Equal(t, tc.expectedUUID, uuid)
+ })
+ }
+}
+
+func TestNvmlDevice_GetPaths(t *testing.T) {
+ testCases := []struct {
+ description string
+ nvmlDevice nvml.Device
+ expectedPaths []string
+ expectedError error
+ }{
+ {
+ description: "Successful path retrieval",
+ nvmlDevice: &mock.Device{
+ GetMinorNumberFunc: func() (int, nvml.Return) {
+ return 0, nvml.SUCCESS
+ },
+ },
+ expectedPaths: []string{"/dev/nvidia0"},
+ expectedError: nil,
+ },
+ {
+ description: "Error retrieving UUID",
+ nvmlDevice: &mock.Device{
+ GetMinorNumberFunc: func() (int, nvml.Return) {
+ return 0, nvml.ERROR_UNKNOWN
+ },
+ },
+ expectedPaths: nil,
+ expectedError: fmt.Errorf("error getting GPU device minor number: %v", nvml.ERROR_UNKNOWN),
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.description, func(t *testing.T) {
+ device := nvmlDevice{Device: tc.nvmlDevice}
+ paths, err := device.GetPaths()
+
+ if tc.expectedError == nil {
+ require.NoError(t, err)
+ } else {
+ require.Contains(t, err.Error(), nvml.ERROR_UNKNOWN.Error())
+ }
+ require.Equal(t, tc.expectedPaths, paths)
+ })
+ }
+}
+
+func TestNvmlDevice_GetNumaNode(t *testing.T) {
+ testCases := []struct {
+ description string
+ nvmlDevice nvml.Device
+ expectedHasNode bool
+ expectedNode int
+ expectedError error
+ }{
+ {
+ description: "No NUMA node",
+ nvmlDevice: &mock.Device{
+ GetPciInfoFunc: func() (nvml.PciInfo, nvml.Return) {
+ return nvml.PciInfo{BusId: [32]int8{'0', '0', '0', '0', ':', '0', '2', ':', '0', '0', '.', '0', 0, 0, 0, 0}}, nvml.SUCCESS
+ },
+ },
+ expectedHasNode: false,
+ expectedNode: 0,
+ expectedError: nil,
+ },
+ {
+ description: "Error getting PCI info",
+ nvmlDevice: &mock.Device{
+ GetPciInfoFunc: func() (nvml.PciInfo, nvml.Return) {
+ return nvml.PciInfo{}, nvml.ERROR_UNKNOWN
+ },
+ },
+ expectedHasNode: false,
+ expectedNode: 0,
+ expectedError: nvml.ERROR_UNKNOWN,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.description, func(t *testing.T) {
+ device := nvmlDevice{Device: tc.nvmlDevice}
+ hasNode, node, err := device.GetNumaNode()
+
+ if tc.expectedError == nil {
+ require.NoError(t, err)
+ } else {
+ require.Contains(t, err.Error(), tc.expectedError.Error())
+ }
+ require.Equal(t, tc.expectedHasNode, hasNode)
+ require.Equal(t, tc.expectedNode, node)
+ })
+ }
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_manager.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_manager.go
new file mode 100644
index 000000000..be1f2915b
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_manager.go
@@ -0,0 +1,135 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "fmt"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ "k8s.io/klog/v2"
+)
+
+type nvmlResourceManager struct {
+ resourceManager
+ nvml nvml.Interface
+}
+
+var _ ResourceManager = (*nvmlResourceManager)(nil)
+
+// NewNVMLResourceManagers returns a set of ResourceManagers, one for each NVML resource in 'config'.
+func NewNVMLResourceManagers(nvmllib nvml.Interface, config *nvidia.DeviceConfig) ([]ResourceManager, error) {
+ ret := nvmllib.Init()
+ if ret != nvml.SUCCESS {
+ return nil, fmt.Errorf("failed to initialize NVML: %v", ret)
+ }
+ defer func() {
+ ret := nvmllib.Shutdown()
+ if ret != nvml.SUCCESS {
+ klog.Infof("Error shutting down NVML: %v", ret)
+ }
+ }()
+
+ deviceMap, err := NewDeviceMap(nvmllib, config)
+ if err != nil {
+ return nil, fmt.Errorf("error building device map: %v", err)
+ }
+
+ var rms []ResourceManager
+ for resourceName, devices := range deviceMap {
+ if len(devices) == 0 {
+ continue
+ }
+ for key, value := range devices {
+ if nvidia.FilterDeviceToRegister(value.ID, value.Index) {
+ klog.V(5).InfoS("Filtering device", "device", value.ID)
+ delete(devices, key)
+ continue
+ }
+ }
+ r := &nvmlResourceManager{
+ resourceManager: resourceManager{
+ config: config,
+ resource: resourceName,
+ devices: devices,
+ },
+ nvml: nvmllib,
+ }
+ rms = append(rms, r)
+ }
+
+ return rms, nil
+}
+
+// GetPreferredAllocation runs an allocation algorithm over the inputs.
+// The algorithm chosen is based both on the incoming set of available devices and various config settings.
+func (r *nvmlResourceManager) GetPreferredAllocation(available, required []string, size int) ([]string, error) {
+ return r.getPreferredAllocation(available, required, size)
+}
+
+// GetDevicePaths returns the required and optional device nodes for the requested resources
+func (r *nvmlResourceManager) GetDevicePaths(ids []string) []string {
+ paths := []string{
+ "/dev/nvidiactl",
+ "/dev/nvidia-uvm",
+ "/dev/nvidia-uvm-tools",
+ "/dev/nvidia-modeset",
+ }
+
+ for _, p := range r.Devices().Subset(ids).GetPaths() {
+ paths = append(paths, p)
+ }
+
+ return paths
+}
+
+// CheckHealth performs health checks on a set of devices, writing to the 'unhealthy' channel with any unhealthy devices
+func (r *nvmlResourceManager) CheckHealth(stop <-chan any, unhealthy chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error {
+ for {
+ // first check if disableNVML channel signal is pass close into checkHealth function
+ // if signal is pass close, return error "close signal received"
+ err := r.checkHealth(stop, r.devices, unhealthy, disableNVML)
+ if err.Error() == "close signal received" {
+ ackDisableHealthChecks <- true
+ klog.Info("Check Health has been closed")
+ // when disableNVML channel signal is pass restart, continue to restart checkHealth function
+ // when disableNVML channel signal is not pass restart, wait for restart signal
+ <-disableNVML
+ klog.Info("Restarting Check Health")
+ continue
+
+ }
+ return err
+ }
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/rm.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/rm.go
new file mode 100644
index 000000000..17ef58a74
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/rm.go
@@ -0,0 +1,174 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/NVIDIA/go-nvlib/pkg/nvlib/device"
+ "github.com/NVIDIA/go-nvlib/pkg/nvlib/info"
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+ spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "k8s.io/klog/v2"
+)
+
+// resourceManager forms the base type for specific resource manager implementations
+type resourceManager struct {
+ config *nvidia.DeviceConfig
+ resource spec.ResourceName
+ devices Devices
+}
+
+// ResourceManager provides an interface for listing a set of Devices and checking health on them
+type ResourceManager interface {
+ Resource() spec.ResourceName
+ Devices() Devices
+ GetDevicePaths([]string) []string
+ GetPreferredAllocation(available, required []string, size int) ([]string, error)
+ CheckHealth(stop <-chan any, unhealthy chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error
+}
+
+// NewResourceManagers returns a []ResourceManager, one for each resource in 'config'.
+func NewResourceManagers(nvmllib nvml.Interface, config *nvidia.DeviceConfig) ([]ResourceManager, error) {
+ // logWithReason logs the output of the has* / is* checks from the info.Interface
+ logWithReason := func(f func() (bool, string), tag string) bool {
+ is, reason := f()
+ if !is {
+ tag = "non-" + tag
+ }
+ klog.Infof("Detected %v platform: %v", tag, reason)
+ return is
+ }
+
+ infolib := info.New()
+
+ hasNVML := logWithReason(infolib.HasNvml, "NVML")
+ isTegra := logWithReason(infolib.IsTegraSystem, "Tegra")
+
+ if !hasNVML && !isTegra {
+ klog.Error("Incompatible platform detected")
+ klog.Error("If this is a GPU node, did you configure the NVIDIA Container Toolkit?")
+ klog.Error("You can check the prerequisites at: https://github.com/NVIDIA/k8s-device-plugin#prerequisites")
+ klog.Error("You can learn how to set the runtime at: https://github.com/NVIDIA/k8s-device-plugin#quick-start")
+ klog.Error("If this is not a GPU node, you should set up a toleration or nodeSelector to only deploy this plugin on GPU nodes")
+ if *config.Flags.FailOnInitError {
+ return nil, fmt.Errorf("platform detection failed")
+ }
+ return nil, nil
+ }
+
+ // The NVIDIA container stack does not yet support the use of integrated AND discrete GPUs on the same node.
+ if hasNVML && isTegra {
+ klog.Warning("Disabling Tegra-based resources on NVML system")
+ isTegra = false
+ }
+
+ var resourceManagers []ResourceManager
+
+ if hasNVML {
+ nvmlManagers, err := NewNVMLResourceManagers(nvmllib, config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to construct NVML resource managers: %v", err)
+ }
+ resourceManagers = append(resourceManagers, nvmlManagers...)
+ }
+
+ if isTegra {
+ tegraManagers, err := NewTegraResourceManagers(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to construct Tegra resource managers: %v", err)
+ }
+ resourceManagers = append(resourceManagers, tegraManagers...)
+ }
+
+ return resourceManagers, nil
+}
+
+// Resource gets the resource name associated with the ResourceManager
+func (r *resourceManager) Resource() spec.ResourceName {
+ return r.resource
+}
+
+// Resource gets the devices managed by the ResourceManager
+func (r *resourceManager) Devices() Devices {
+ return r.devices
+}
+
+// AddDefaultResourcesToConfig adds default resource matching rules to config.Resources
+func AddDefaultResourcesToConfig(config *nvidia.DeviceConfig) error {
+ //config.Resources.AddGPUResource("*", "gpu")
+ config.Resources.GPUs = append(config.Resources.GPUs, spec.Resource{
+ Pattern: "*",
+ Name: spec.ResourceName(*config.ResourceName),
+ })
+ fmt.Println("config=", config.Resources.GPUs)
+ switch *config.Flags.MigStrategy {
+ case spec.MigStrategySingle:
+ return config.Resources.AddMIGResource("*", "gpu")
+ case spec.MigStrategyMixed:
+ hasNVML, reason := info.New().HasNvml()
+ if !hasNVML {
+ klog.Warningf("mig-strategy=%q is only supported with NVML", spec.MigStrategyMixed)
+ klog.Warningf("NVML not detected: %v", reason)
+ return nil
+ }
+
+ nvmllib := nvml.New()
+ ret := nvmllib.Init()
+ if ret != nvml.SUCCESS {
+ if *config.Flags.FailOnInitError {
+ return fmt.Errorf("failed to initialize NVML: %v", ret)
+ }
+ return nil
+ }
+ defer func() {
+ ret := nvmllib.Shutdown()
+ if ret != nvml.SUCCESS {
+ klog.Errorf("Error shutting down NVML: %v", ret)
+ }
+ }()
+
+ devicelib := device.New(nvmllib)
+ return devicelib.VisitMigProfiles(func(p device.MigProfile) error {
+ profileInfo := p.GetInfo()
+ if profileInfo.C != profileInfo.G {
+ return nil
+ }
+ resourceName := strings.ReplaceAll("mig-"+p.String(), "+", ".")
+ return config.Resources.AddMIGResource(p.String(), resourceName)
+ })
+ }
+ return nil
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_devices.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_devices.go
new file mode 100644
index 000000000..4e824d378
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_devices.go
@@ -0,0 +1,85 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "fmt"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+)
+
+const (
+ tegraDeviceName = "tegra"
+)
+
+// buildTegraDeviceMap creates a DeviceMap for the tegra devices in the sytesm.
+// NOTE: At present only a single tegra device is expected.
+func buildTegraDeviceMap(config *nvidia.DeviceConfig) (DeviceMap, error) {
+ devices := make(DeviceMap)
+
+ name := tegraDeviceName
+ i := 0
+ for _, resource := range config.Resources.GPUs {
+ if resource.Pattern.Matches(name) {
+ index := fmt.Sprintf("%d", i)
+ err := devices.setEntry(resource.Name, index, &tegraDevice{})
+ if err != nil {
+ return nil, err
+ }
+ i++
+ }
+
+ }
+ return devices, nil
+}
+
+type tegraDevice struct{}
+
+var _ deviceInfo = (*tegraDevice)(nil)
+
+// GetUUID returns the UUID of the tegra device.
+// TODO: This is currently hardcoded to `tegra`
+func (d *tegraDevice) GetUUID() (string, error) {
+ return tegraDeviceName, nil
+}
+
+// GetPaths returns the paths for a tegra device.
+// A tegra device does not have paths associated with it.
+func (d *tegraDevice) GetPaths() ([]string, error) {
+ return nil, nil
+}
+
+// GetNumaNode always returns unsupported for a Tegra device
+func (d *tegraDevice) GetNumaNode() (bool, int, error) {
+ return false, -1, nil
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_manager.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_manager.go
new file mode 100644
index 000000000..18e455a7a
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_manager.go
@@ -0,0 +1,92 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+import (
+ "fmt"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+)
+
+type tegraResourceManager struct {
+ resourceManager
+}
+
+var _ ResourceManager = (*tegraResourceManager)(nil)
+
+// NewTegraResourceManagers returns a set of ResourceManagers for tegra resources
+func NewTegraResourceManagers(config *nvidia.DeviceConfig) ([]ResourceManager, error) {
+ deviceMap, err := buildTegraDeviceMap(config)
+ if err != nil {
+ return nil, fmt.Errorf("error building Tegra device map: %v", err)
+ }
+
+ deviceMap, err = updateDeviceMapWithReplicas(config, deviceMap)
+ if err != nil {
+ return nil, fmt.Errorf("error updating device map with replicas from config.sharing.timeSlicing.resources: %v", err)
+ }
+
+ var rms []ResourceManager
+ for resourceName, devices := range deviceMap {
+ if len(devices) == 0 {
+ continue
+ }
+ r := &tegraResourceManager{
+ resourceManager: resourceManager{
+ config: config,
+ resource: resourceName,
+ devices: devices,
+ },
+ }
+ if len(devices) != 0 {
+ rms = append(rms, r)
+ }
+ }
+
+ return rms, nil
+}
+
+// GetPreferredAllocation returns a standard allocation for the Tegra resource manager.
+func (r *tegraResourceManager) GetPreferredAllocation(available, required []string, size int) ([]string, error) {
+ return r.distributedAlloc(available, required, size)
+}
+
+// GetDevicePaths returns an empty slice for the tegraResourceManager
+func (r *tegraResourceManager) GetDevicePaths(ids []string) []string {
+ return nil
+}
+
+// CheckHealth is disabled for the tegraResourceManager
+func (r *tegraResourceManager) CheckHealth(stop <-chan any, unhealthy chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error {
+ return nil
+}
diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/wsl_devices.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/wsl_devices.go
new file mode 100644
index 000000000..7696a3038
--- /dev/null
+++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/wsl_devices.go
@@ -0,0 +1,52 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The HAMi Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to NVIDIA CORPORATION under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. NVIDIA CORPORATION licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright The HAMi Authors. See
+ * GitHub history for details.
+ */
+
+package rm
+
+type wslDevice nvmlDevice
+
+var _ deviceInfo = (*wslDevice)(nil)
+
+// GetUUID returns the UUID of the device
+func (d wslDevice) GetUUID() (string, error) {
+ return nvmlDevice(d).GetUUID()
+}
+
+// GetPaths returns the paths for a tegra device.
+func (d wslDevice) GetPaths() ([]string, error) {
+ return []string{"/dev/dxg"}, nil
+}
+
+// GetNumaNode returns the NUMA node associated with the GPU device
+func (d wslDevice) GetNumaNode() (bool, int, error) {
+ return nvmlDevice(d).GetNumaNode()
+}
diff --git a/pkg/device-plugin/plugin.go b/pkg/device-plugin/plugin.go
deleted file mode 100644
index 9378bee07..000000000
--- a/pkg/device-plugin/plugin.go
+++ /dev/null
@@ -1,433 +0,0 @@
-/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package device_plugin
-
-import (
- "4pd.io/k8s-vgpu/pkg/api"
- "4pd.io/k8s-vgpu/pkg/device-plugin/config"
- "fmt"
- "k8s.io/apimachinery/pkg/util/uuid"
- "log"
- "net"
- "os"
- "path"
- "strconv"
- "time"
-
- "github.com/NVIDIA/go-gpuallocator/gpuallocator"
- "golang.org/x/net/context"
- "google.golang.org/grpc"
- pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
-)
-
-// Constants to represent the various device list strategies
-const (
- DeviceListStrategyEnvvar = "envvar"
- DeviceListStrategyVolumeMounts = "volume-mounts"
-)
-
-// Constants to represent the various device id strategies
-const (
- DeviceIDStrategyUUID = "uuid"
- DeviceIDStrategyIndex = "index"
-)
-
-// Constants for use by the 'volume-mounts' device list strategy
-const (
- deviceListAsVolumeMountsHostPath = "/dev/null"
- deviceListAsVolumeMountsContainerPathRoot = "/var/run/nvidia-container-devices"
-)
-
-// NvidiaDevicePlugin implements the Kubernetes device plugin API
-type NvidiaDevicePlugin struct {
- //ResourceManager
- //resourceManager *ResourceManager
- deviceCache *DeviceCache
- resourceName string
- //deviceListEnvvar string
- allocatePolicy gpuallocator.Policy
- socket string
-
- server *grpc.Server
- //cachedDevices []*Device
- health chan *Device
- stop chan interface{}
- //changed chan struct{}
- //devRegister *DeviceRegister
- //podManager *PodManager
-}
-
-// NewNvidiaDevicePlugin returns an initialized NvidiaDevicePlugin
-func NewNvidiaDevicePlugin(resourceName string, deviceCache *DeviceCache, allocatePolicy gpuallocator.Policy, socket string) *NvidiaDevicePlugin {
- return &NvidiaDevicePlugin{
- deviceCache: deviceCache,
- resourceName: resourceName,
- allocatePolicy: allocatePolicy,
- socket: socket,
-
- // These will be reinitialized every
- // time the plugin server is restarted.
- server: nil,
- health: nil,
- stop: nil,
- }
-}
-
-func (m *NvidiaDevicePlugin) initialize() {
- var err error
- m.server = grpc.NewServer([]grpc.ServerOption{}...)
- m.health = make(chan *Device)
- m.stop = make(chan interface{})
- check(err)
-}
-
-func (m *NvidiaDevicePlugin) cleanup() {
- close(m.stop)
- m.server = nil
- m.health = nil
- m.stop = nil
- //m.podManager = nil
-}
-
-// Start starts the gRPC server, registers the device plugin with the Kubelet,
-// and starts the device healthchecks.
-func (m *NvidiaDevicePlugin) Start() error {
- m.initialize()
-
- err := m.Serve()
- if err != nil {
- log.Printf("Could not start device plugin for '%s': %s", m.resourceName, err)
- m.cleanup()
- return err
- }
- log.Printf("Starting to serve '%s' on %s", m.resourceName, m.socket)
-
- err = m.Register()
- if err != nil {
- log.Printf("Could not register device plugin: %s", err)
- m.Stop()
- return err
- }
- log.Printf("Registered device plugin for '%s' with Kubelet", m.resourceName)
-
- m.deviceCache.AddNotifyChannel("plugin", m.health)
- return nil
-}
-
-// Stop stops the gRPC server.
-func (m *NvidiaDevicePlugin) Stop() error {
- if m == nil || m.server == nil {
- return nil
- }
- log.Printf("Stopping to serve '%s' on %s", m.resourceName, m.socket)
- m.deviceCache.RemoveNotifyChannel("plugin")
- m.server.Stop()
- if err := os.Remove(m.socket); err != nil && !os.IsNotExist(err) {
- return err
- }
- m.cleanup()
- return nil
-}
-
-// Serve starts the gRPC server of the device plugin.
-func (m *NvidiaDevicePlugin) Serve() error {
- os.Remove(m.socket)
- sock, err := net.Listen("unix", m.socket)
- if err != nil {
- return err
- }
-
- pluginapi.RegisterDevicePluginServer(m.server, m)
-
- go func() {
- lastCrashTime := time.Now()
- restartCount := 0
- for {
- log.Printf("Starting GRPC server for '%s'", m.resourceName)
- err := m.server.Serve(sock)
- if err == nil {
- break
- }
-
- log.Printf("GRPC server for '%s' crashed with error: %v", m.resourceName, err)
-
- // restart if it has not been too often
- // i.e. if server has crashed more than 5 times and it didn't last more than one hour each time
- if restartCount > 5 {
- // quit
- log.Fatalf("GRPC server for '%s' has repeatedly crashed recently. Quitting", m.resourceName)
- }
- timeSinceLastCrash := time.Since(lastCrashTime).Seconds()
- lastCrashTime = time.Now()
- if timeSinceLastCrash > 3600 {
- // it has been one hour since the last crash.. reset the count
- // to reflect on the frequency
- restartCount = 1
- } else {
- restartCount++
- }
- }
- }()
-
- // Wait for server to start by launching a blocking connexion
- conn, err := m.dial(m.socket, 5*time.Second)
- if err != nil {
- return err
- }
- conn.Close()
-
- return nil
-}
-
-// Register registers the device plugin for the given resourceName with Kubelet.
-func (m *NvidiaDevicePlugin) Register() error {
- conn, err := m.dial(pluginapi.KubeletSocket, 5*time.Second)
- if err != nil {
- return err
- }
- defer conn.Close()
-
- client := pluginapi.NewRegistrationClient(conn)
- reqt := &pluginapi.RegisterRequest{
- Version: pluginapi.Version,
- Endpoint: path.Base(m.socket),
- ResourceName: m.resourceName,
- Options: &pluginapi.DevicePluginOptions{
- GetPreferredAllocationAvailable: false,
- },
- }
-
- _, err = client.Register(context.Background(), reqt)
- if err != nil {
- return err
- }
- return nil
-}
-
-// GetDevicePluginOptions returns the values of the optional settings for this plugin
-func (m *NvidiaDevicePlugin) GetDevicePluginOptions(context.Context, *pluginapi.Empty) (*pluginapi.DevicePluginOptions, error) {
- options := &pluginapi.DevicePluginOptions{
- GetPreferredAllocationAvailable: false,
- }
- return options, nil
-}
-
-// ListAndWatch lists devices and update that list according to the health status
-func (m *NvidiaDevicePlugin) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlugin_ListAndWatchServer) error {
- _ = s.Send(&pluginapi.ListAndWatchResponse{Devices: m.apiDevices()})
- for {
- select {
- case <-m.stop:
- return nil
- case d := <-m.health:
- // FIXME: there is no way to recover from the Unhealthy state.
- //d.Health = pluginapi.Unhealthy
- log.Printf("'%s' device marked unhealthy: %s", m.resourceName, d.ID)
- _ = s.Send(&pluginapi.ListAndWatchResponse{Devices: m.apiDevices()})
- }
- }
-}
-
-// GetPreferredAllocation returns the preferred allocation from the set of devices specified in the request
-func (m *NvidiaDevicePlugin) GetPreferredAllocation(ctx context.Context, r *pluginapi.PreferredAllocationRequest) (*pluginapi.PreferredAllocationResponse, error) {
-
- return &pluginapi.PreferredAllocationResponse{}, nil
-}
-
-// Allocate which return list of devices.
-func (m *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
- ////reqNums := make([]int, 0, len(reqs.ContainerRequests))
- ////for _, req := range reqs.ContainerRequests {
- //// reqNums = append(reqNums, len(req.DevicesIDs))
- ////}
- ////klog.V(3).Infof("allocate for device %v", reqNums)
- //
- ////devRequests, err := m.podManager.getDevices(reqNums)
- //if err != nil {
- // klog.Errorf("get device request error, %v", err)
- // return nil, err
- //}
- //if devRequests == nil {
- // err = fmt.Errorf("get device request empty")
- // klog.Errorf("%v", err)
- // return nil, err
- //}
- responses := pluginapi.AllocateResponse{}
- for _, _ = range reqs.ContainerRequests {
- //reqDeviceIDs := req.DevicesIDs
- //devs, err := m.getDevices(reqDeviceIDs)
- //if err != nil {
- // return nil, err
- //}
-
- response := pluginapi.ContainerAllocateResponse{}
-
- //response.Envs = m.apiEnvs(m.deviceListEnvvar, reqDeviceIDs)
- ////var mapEnvs []string
- //for i, dev := range devs {
- // limitKey := fmt.Sprintf("CUDA_DEVICE_MEMORY_LIMIT_%v", i)
- // response.Envs[limitKey] = fmt.Sprintf("%vm", config.DeviceMemoryScaling*float64(dev.Memory)/float64(config.DeviceSplitCount))
- // //mapEnvs = append(mapEnvs, fmt.Sprintf("%v:%v", i, vd.dev.ID))
- //}
- response.Envs = make(map[string]string)
- //response.Annotations = map[string]string{util.AssignedIDsAnnotations: util.EncodeContainerDevices(reqDeviceIDs)}
- response.Envs["CUDA_DEVICE_SM_LIMIT"] = strconv.Itoa(int(100 * config.DeviceCoresScaling / float64(config.DeviceSplitCount)))
- //response.Envs["NVIDIA_DEVICE_MAP"] = strings.Join(mapEnvs, " ")
- response.Envs["CUDA_DEVICE_MEMORY_SHARED_CACHE"] = fmt.Sprintf("/tmp/%v.cache", uuid.NewUUID())
- if config.DeviceMemoryScaling > 1 {
- response.Envs["CUDA_OVERSUBSCRIBE"] = "true"
- }
- response.Envs[api.PluginRuntimeSocket] = fmt.Sprintf("unix://%v", config.RuntimeSocketFlag)
- response.Mounts = append(response.Mounts,
- &pluginapi.Mount{ContainerPath: "/usr/local/vgpu/libvgpu.so",
- HostPath: "/usr/local/vgpu/libvgpu.so",
- ReadOnly: true},
- &pluginapi.Mount{ContainerPath: "/etc/ld.so.preload",
- HostPath: "/usr/local/vgpu/ld.so.preload",
- ReadOnly: true},
- )
- responses.ContainerResponses = append(responses.ContainerResponses, &response)
- }
- return &responses, nil
-}
-
-// PreStartContainer is unimplemented for this plugin
-func (m *NvidiaDevicePlugin) PreStartContainer(context.Context, *pluginapi.PreStartContainerRequest) (*pluginapi.PreStartContainerResponse, error) {
- return &pluginapi.PreStartContainerResponse{}, nil
-}
-
-// dial establishes the gRPC communication with the registered device plugin.
-func (m *NvidiaDevicePlugin) dial(unixSocketPath string, timeout time.Duration) (*grpc.ClientConn, error) {
- c, err := grpc.Dial(unixSocketPath, grpc.WithInsecure(), grpc.WithBlock(),
- grpc.WithTimeout(timeout),
- grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
- return net.DialTimeout("unix", addr, timeout)
- }),
- )
-
- if err != nil {
- return nil, err
- }
-
- return c, nil
-}
-
-func (m *NvidiaDevicePlugin) Devices() []*Device {
- return m.deviceCache.GetCache()
-}
-
-//func (m *NvidiaDevicePlugin) deviceExists(id string) bool {
-// for _, d := range m.deviceCache.GetCache() {
-// if d.ID == id {
-// return true
-// }
-// }
-// return false
-//}
-//
-//func (m *NvidiaDevicePlugin) getDevices(ids []string) ([]*Device, error) {
-// var res []*Device
-// for _, id := range ids {
-// found := false
-// for _, dev := range m.deviceCache.GetCache() {
-// if id == dev.ID {
-// res = append(res, dev)
-// found = true
-// break
-// }
-// }
-// if !found {
-// return res, fmt.Errorf("device %v not found", id)
-// }
-// }
-// return res, nil
-//}
-
-func (m *NvidiaDevicePlugin) apiDevices() []*pluginapi.Device {
- devices := m.Devices()
- var res []*pluginapi.Device
- for _, dev := range devices {
- for i := uint(0); i < config.DeviceSplitCount; i++ {
- id := fmt.Sprintf("%v-%v", dev.ID, i)
- res = append(res, &pluginapi.Device{
- ID: id,
- Health: dev.Health,
- Topology: nil,
- })
- }
- }
- return res
-}
-
-//func (m *NvidiaDevicePlugin) apiEnvs(envvar string, deviceIDs []string) map[string]string {
-// return map[string]string{
-// envvar: strings.Join(deviceIDs, ","),
-// }
-//}
-//
-//func (m *NvidiaDevicePlugin) apiMounts(deviceIDs []string) []*pluginapi.Mount {
-// var mounts []*pluginapi.Mount
-//
-// for _, id := range deviceIDs {
-// mount := &pluginapi.Mount{
-// HostPath: deviceListAsVolumeMountsHostPath,
-// ContainerPath: filepath.Join(deviceListAsVolumeMountsContainerPathRoot, id),
-// }
-// mounts = append(mounts, mount)
-// }
-//
-// return mounts
-//}
-
-//func (m *NvidiaDevicePlugin) apiDeviceSpecs(driverRoot string, uuids []string) []*pluginapi.DeviceSpec {
-// var specs []*pluginapi.DeviceSpec
-//
-// paths := []string{
-// "/dev/nvidiactl",
-// "/dev/nvidia-uvm",
-// "/dev/nvidia-uvm-tools",
-// "/dev/nvidia-modeset",
-// }
-//
-// for _, p := range paths {
-// if _, err := os.Stat(p); err == nil {
-// spec := &pluginapi.DeviceSpec{
-// ContainerPath: p,
-// HostPath: filepath.Join(driverRoot, p),
-// Permissions: "rw",
-// }
-// specs = append(specs, spec)
-// }
-// }
-//
-// for _, d := range m.deviceCache.GetCache() {
-// for _, id := range uuids {
-// if d.ID == id {
-// for _, p := range d.Paths {
-// spec := &pluginapi.DeviceSpec{
-// ContainerPath: p,
-// HostPath: filepath.Join(driverRoot, p),
-// Permissions: "rw",
-// }
-// specs = append(specs, spec)
-// }
-// }
-// }
-// }
-//
-// return specs
-//}
diff --git a/pkg/device-plugin/register.go b/pkg/device-plugin/register.go
deleted file mode 100644
index 15202b5a8..000000000
--- a/pkg/device-plugin/register.go
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package device_plugin
-
-import (
- "context"
- "fmt"
- "k8s.io/klog/v2"
- "time"
-
- "4pd.io/k8s-vgpu/pkg/api"
- "4pd.io/k8s-vgpu/pkg/device-plugin/config"
- "google.golang.org/grpc"
-)
-
-type DevListFunc func() []*Device
-
-type DeviceRegister struct {
- deviceCache *DeviceCache
- unhealthy chan *Device
- stopCh chan struct{}
-}
-
-func NewDeviceRegister(deviceCache *DeviceCache) *DeviceRegister {
- return &DeviceRegister{
- deviceCache: deviceCache,
- unhealthy: make(chan *Device),
- stopCh: make(chan struct{}),
- }
-}
-
-func (r *DeviceRegister) Start() {
- r.deviceCache.AddNotifyChannel("register", r.unhealthy)
- go r.WatchAndRegister()
-}
-
-func (r *DeviceRegister) Stop() {
- close(r.stopCh)
-}
-
-func (r *DeviceRegister) apiDevices() *[]*api.DeviceInfo {
- devs := r.deviceCache.GetCache()
- res := make([]*api.DeviceInfo, 0, len(devs))
- for _, dev := range devs {
- res = append(res, &api.DeviceInfo{
- Id: dev.ID,
- Count: int32(config.DeviceSplitCount),
- Health: dev.Health == "healthy",
- })
- }
- return &res
-}
-
-func (r *DeviceRegister) Register(ctx context.Context) error {
- conn, err := grpc.DialContext(
- ctx,
- config.SchedulerEndpoint,
- grpc.WithInsecure(),
- grpc.WithBlock(),
- //grpc.WithConnectParams(grpc.ConnectParams{MinConnectTimeout: 3}),
- )
- if err != nil {
- return fmt.Errorf("connect scheduler error, %v", err)
- }
- client := api.NewDeviceServiceClient(conn)
- register, err := client.Register(ctx)
- if err != nil {
- klog.Errorf("register error %v", err)
- err = fmt.Errorf("client register error, %v", err)
- return err
- }
- req := api.RegisterRequest{Node: config.NodeName, Devices: *r.apiDevices()}
- err = register.Send(&req)
- if err != nil {
- klog.Errorf("register send error, %v", err)
- return err
- }
- klog.V(3).Infof("register info %v", req.String())
- closeCh := make(chan struct{})
- go func() {
- reply := api.RegisterReply{}
- err := register.RecvMsg(reply)
- if err != nil {
- klog.Errorf("register recv error, %v", err)
- } else {
- klog.Errorf("register recv closed")
- }
- closeCh <- struct{}{}
- }()
- for {
- select {
- case <-r.unhealthy:
- err = register.Send(&api.RegisterRequest{
- Node: config.NodeName,
- Devices: *r.apiDevices(),
- })
- if err != nil {
- klog.Errorf("register send error, %v", err)
- return err
- }
- klog.V(3).Infof("register info %v", req.String())
- case <-closeCh:
- klog.Infof("register server closed")
- return fmt.Errorf("register server closed")
- case <-r.stopCh:
- return nil
- }
- }
-}
-
-func (r *DeviceRegister) WatchAndRegister() {
- //ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
- //defer cancel()
- ctx := context.Background()
- for {
- err := r.Register(ctx)
- if err != nil {
- klog.Errorf("register error, %v", err)
- time.Sleep(time.Second * 5)
- } else {
- klog.Infof("register stopped")
- break
- }
- }
-}
diff --git a/pkg/device-plugin/runtime.go b/pkg/device-plugin/runtime.go
deleted file mode 100644
index df979f3ce..000000000
--- a/pkg/device-plugin/runtime.go
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package device_plugin
-
-import (
- "context"
- "fmt"
- "strings"
-
- "4pd.io/k8s-vgpu/pkg/api"
- "4pd.io/k8s-vgpu/pkg/device-plugin/config"
- "google.golang.org/grpc"
-)
-
-type VGPURuntimeService struct {
- deviceCache *DeviceCache
-}
-
-func NewVGPURuntimeService(deviceCache *DeviceCache) *VGPURuntimeService {
- return &VGPURuntimeService{deviceCache: deviceCache}
-}
-
-func (s *VGPURuntimeService) GetDevice(ctx context.Context, req *api.GetDeviceRequest) (*api.GetDeviceReply, error) {
- conn, err := grpc.DialContext(
- ctx,
- config.SchedulerEndpoint,
- grpc.WithInsecure(),
- grpc.WithBlock(),
- //grpc.WithConnectParams(grpc.ConnectParams{MinConnectTimeout: 3}),
- )
- if err != nil {
- return nil, fmt.Errorf("connect scheduler error, %v", err)
- }
- client := api.NewDeviceServiceClient(conn)
- sReq := api.GetContainerRequest{Uuid: req.CtrUUID}
- sResp, err := client.GetContainer(ctx, &sReq)
- if err != nil {
- return nil, err
- }
- envs, err := s.containerEnvs(sResp.DevList)
- if err != nil {
- return nil, err
- }
- resp := api.GetDeviceReply{
- Envs: envs,
- PodUID: sResp.PodUID,
- CtrName: sResp.CtrName,
- PodNamespace: sResp.PodNamespace,
- PodName: sResp.PodName,
- }
- return &resp, nil
-}
-
-func (s *VGPURuntimeService) containerEnvs(devIDs []string) (map[string]string, error) {
- envs := make(map[string]string)
- var devs []*Device
- for _, id := range devIDs {
- found := false
- for _, d := range s.deviceCache.GetCache() {
- if id == d.ID {
- found = true
- devs = append(devs, d)
- break
- }
- }
- if !found {
- return nil, fmt.Errorf("device %v not found", id)
- }
- }
-
- envs["NVIDIA_VISIBLE_DEVICES"] = strings.Join(devIDs, ",")
- for i, d := range devs {
- limitKey := fmt.Sprintf("CUDA_DEVICE_MEMORY_LIMIT_%v", i)
- envs[limitKey] = fmt.Sprintf("%vm", config.DeviceMemoryScaling*float64(d.Memory)/float64(config.DeviceSplitCount))
- }
- return envs, nil
-}
diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go
new file mode 100644
index 000000000..abe690416
--- /dev/null
+++ b/pkg/device/ascend/device.go
@@ -0,0 +1,389 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ascend
+
+import (
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "slices"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/common"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/nodelock"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ "k8s.io/klog/v2"
+)
+
+const (
+ NodeLockAscend = "hami.io/mutex.lock"
+)
+
+type Devices struct {
+ config VNPUConfig
+ nodeRegisterAnno string
+ useUUIDAnno string
+ noUseUUIDAnno string
+ handshakeAnno string
+}
+
+type RuntimeInfo struct {
+ UUID string `json:"UUID,omitempty"`
+ Temp string `json:"temp,omitempty"`
+}
+
+var (
+ enableAscend bool
+ configFile string
+)
+
+func (dev *Devices) trimMemory(m int64) (int64, string) {
+ for i := range dev.config.Templates {
+ if m <= dev.config.Templates[i].Memory {
+ return dev.config.Templates[i].Memory, dev.config.Templates[i].Name
+ }
+ }
+ if m <= dev.config.MemoryCapacity {
+ return dev.config.MemoryAllocatable, ""
+ }
+ return 0, ""
+}
+
+func InitDevices(config []VNPUConfig) []*Devices {
+ var devs []*Devices
+ if !enableAscend {
+ return devs
+ }
+ for _, vnpu := range config {
+ commonWord := vnpu.CommonWord
+ dev := &Devices{
+ config: vnpu,
+ nodeRegisterAnno: fmt.Sprintf("hami.io/node-register-%s", commonWord),
+ useUUIDAnno: fmt.Sprintf("hami.io/use-%s-uuid", commonWord),
+ noUseUUIDAnno: fmt.Sprintf("hami.io/no-use-%s-uuid", commonWord),
+ handshakeAnno: fmt.Sprintf("hami.io/node-handshake-%s", commonWord),
+ }
+ sort.Slice(dev.config.Templates, func(i, j int) bool {
+ return dev.config.Templates[i].Memory < dev.config.Templates[j].Memory
+ })
+ util.InRequestDevices[commonWord] = fmt.Sprintf("hami.io/%s-devices-to-allocate", commonWord)
+ util.SupportDevices[commonWord] = fmt.Sprintf("hami.io/%s-devices-allocated", commonWord)
+ util.HandshakeAnnos[commonWord] = dev.handshakeAnno
+ devs = append(devs, dev)
+ klog.Infof("load ascend vnpu config %s: %v", commonWord, dev.config)
+ }
+ return devs
+}
+
+func ParseConfig(fs *flag.FlagSet) {
+ fs.BoolVar(&enableAscend, "enable-ascend", false, "enable ascend device")
+}
+
+func (dev *Devices) CommonWord() string {
+ return dev.config.CommonWord
+}
+
+func (dev *Devices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
+ count, ok := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceName)]
+ if !ok {
+ return false, nil
+ }
+ trimMem := dev.config.MemoryAllocatable
+ memory, ok := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceMemoryName)]
+ if ok {
+ trimMem, _ = dev.trimMemory(memory.Value())
+ if trimMem <= 0 {
+ return false, fmt.Errorf("%s %d is invalid", dev.config.ResourceMemoryName, memory.Value())
+ }
+ }
+ if count.Value() > 1 {
+ if trimMem != dev.config.MemoryAllocatable {
+ return true, errors.New("vNPU nor supported for multiple devices")
+ }
+ }
+ ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceMemoryName)] = resource.MustParse(fmt.Sprint(trimMem))
+ ctr.Resources.Requests[corev1.ResourceName(dev.config.ResourceMemoryName)] = resource.MustParse(fmt.Sprint(trimMem))
+ return true, nil
+}
+
+func (dev *Devices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) {
+ anno, ok := n.Annotations[dev.nodeRegisterAnno]
+ if !ok {
+ return []*util.DeviceInfo{}, fmt.Errorf("annos not found %s", dev.nodeRegisterAnno)
+ }
+ nodeDevices, err := util.UnMarshalNodeDevices(anno)
+ if err != nil {
+ klog.ErrorS(err, "failed to unmarshal node devices", "node", n.Name, "device annotation", anno)
+ return []*util.DeviceInfo{}, err
+ }
+ if len(nodeDevices) == 0 {
+ klog.InfoS("no gpu device found", "node", n.Name, "device annotation", anno)
+ return []*util.DeviceInfo{}, errors.New("no device found on node")
+ }
+ return nodeDevices, nil
+}
+
+func (dev *Devices) PatchAnnotations(pod *corev1.Pod, annoInput *map[string]string, pd util.PodDevices) map[string]string {
+ commonWord := dev.CommonWord()
+ devList, ok := pd[commonWord]
+ if ok && len(devList) > 0 {
+ (*annoInput)[util.InRequestDevices[commonWord]] = util.EncodePodSingleDevice(devList)
+ (*annoInput)[util.SupportDevices[commonWord]] = util.EncodePodSingleDevice(devList)
+ (*annoInput)["predicate-time"] = strconv.FormatInt(time.Now().Unix(), 10)
+ allocateStr := fmt.Sprintf("huawei.com/%s", dev.CommonWord())
+ var rtInfo []RuntimeInfo
+ for _, dp := range devList {
+ for _, val := range dp {
+ _, temp := dev.trimMemory(int64(val.Usedmem))
+ rtInfo = append(rtInfo, RuntimeInfo{
+ UUID: val.UUID,
+ Temp: temp,
+ })
+ }
+ }
+ s, err := json.Marshal(rtInfo)
+ if err != nil {
+ klog.ErrorS(err, "failed to marshal runtime info", "runtime info", rtInfo)
+ }
+ (*annoInput)[allocateStr] = string(s)
+ }
+ return *annoInput
+}
+
+func (dev *Devices) LockNode(n *corev1.Node, p *corev1.Pod) error {
+ found := false
+ for _, val := range p.Spec.Containers {
+ if (dev.GenerateResourceRequests(&val).Nums) > 0 {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil
+ }
+
+ return nodelock.LockNode(n.Name, NodeLockAscend, p)
+}
+
+func (dev *Devices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
+ found := false
+ for _, val := range p.Spec.Containers {
+ if (dev.GenerateResourceRequests(&val).Nums) > 0 {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil
+ }
+
+ return nodelock.ReleaseNodeLock(n.Name, NodeLockAscend, p, false)
+}
+
+func (dev *Devices) NodeCleanUp(nn string) error {
+ return util.MarkAnnotationsToDelete(dev.handshakeAnno, nn)
+}
+
+func (dev *Devices) checkType(annos map[string]string, d util.DeviceUsage, n util.ContainerDeviceRequest) (bool, bool, bool) {
+ if strings.Compare(n.Type, dev.CommonWord()) == 0 {
+ return true, true, false
+ }
+ return false, false, false
+}
+
+func (dev *Devices) checkUUID(annos map[string]string, d util.DeviceUsage) bool {
+ userUUID, ok := annos[dev.useUUIDAnno]
+ if ok {
+ klog.V(5).Infof("check uuid for ascend user uuid [%s], device id is %s", userUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ userUUIDs := strings.Split(userUUID, ",")
+ return slices.Contains(userUUIDs, d.ID)
+ }
+
+ noUserUUID, ok := annos[dev.noUseUUIDAnno]
+ if ok {
+ klog.V(5).Infof("check uuid for ascend not user uuid [%s], device id is %s", noUserUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ noUserUUIDs := strings.Split(noUserUUID, ",")
+ return !slices.Contains(noUserUUIDs, d.ID)
+ }
+ return true
+}
+
+func (dev *Devices) checkIndex(annos map[string]string, d util.DeviceUsage) bool {
+ return true
+}
+
+func (dev *Devices) CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ return util.CheckHealth(devType, n)
+}
+
+func (dev *Devices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest {
+ klog.Infof("Counting %s devices", dev.config.CommonWord)
+ ascendResourceCount := corev1.ResourceName(dev.config.ResourceName)
+ ascendResourceMem := corev1.ResourceName(dev.config.ResourceMemoryName)
+ v, ok := ctr.Resources.Limits[ascendResourceCount]
+ if !ok {
+ v, ok = ctr.Resources.Requests[ascendResourceCount]
+ }
+ if ok {
+ if n, ok := v.AsInt64(); ok {
+ klog.Info("Found AscendDevices devices")
+ memnum := 0
+ mem, ok := ctr.Resources.Limits[ascendResourceMem]
+ if !ok {
+ mem, ok = ctr.Resources.Requests[ascendResourceMem]
+ }
+ if ok {
+ memnums, ok := mem.AsInt64()
+ if ok {
+ m, _ := dev.trimMemory(memnums)
+ memnum = int(m)
+ }
+ }
+ corenum := int32(0)
+
+ mempnum := 0
+ if memnum == 0 {
+ mempnum = 100
+ }
+
+ return util.ContainerDeviceRequest{
+ Nums: int32(n),
+ Type: dev.CommonWord(),
+ Memreq: int32(memnum),
+ MemPercentagereq: int32(mempnum),
+ Coresreq: corenum,
+ }
+ }
+ }
+ return util.ContainerDeviceRequest{}
+}
+
+func (dev *Devices) ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32 {
+ return 0
+}
+
+func (dev *Devices) AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error {
+ n.Used++
+ n.Usedcores += ctr.Usedcores
+ n.Usedmem += ctr.Usedmem
+ return nil
+}
+
+func (npu *Devices) Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string) {
+ k := request
+ originReq := k.Nums
+ prevnuma := -1
+ klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
+ var tmpDevs map[string]util.ContainerDevices
+ tmpDevs = make(map[string]util.ContainerDevices)
+ reason := make(map[string]int)
+ for i := len(devices) - 1; i >= 0; i-- {
+ dev := devices[i]
+ klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
+
+ _, found, numa := npu.checkType(annos, *dev, k)
+ if !found {
+ reason[common.CardTypeMismatch]++
+ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, k.Type)
+ continue
+ }
+ if numa && prevnuma != dev.Numa {
+ if k.Nums != originReq {
+ reason[common.NumaNotFit] += len(tmpDevs)
+ klog.V(5).InfoS(common.NumaNotFit, "pod", klog.KObj(pod), "device", dev.ID, "k.nums", k.Nums, "numa", numa, "prevnuma", prevnuma, "device numa", dev.Numa)
+ }
+ k.Nums = originReq
+ prevnuma = dev.Numa
+ tmpDevs = make(map[string]util.ContainerDevices)
+ }
+ if !npu.checkUUID(annos, *dev) {
+ reason[common.CardUUIDMismatch]++
+ klog.V(5).InfoS(common.CardUUIDMismatch, "pod", klog.KObj(pod), "device", dev.ID, "current device info is:", *dev)
+ continue
+ }
+
+ memreq := int32(0)
+ if dev.Count <= dev.Used {
+ reason[common.CardTimeSlicingExhausted]++
+ klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
+ continue
+ }
+ if k.Coresreq > 100 {
+ klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Coresreq = 100
+ //return false, tmpDevs
+ }
+ if k.Memreq > 0 {
+ memreq = k.Memreq
+ }
+ if k.MemPercentagereq != 101 && k.Memreq == 0 {
+ //This incurs an issue
+ memreq = dev.Totalmem * k.MemPercentagereq / 100
+ }
+ if dev.Totalmem-dev.Usedmem < memreq {
+ reason[common.CardInsufficientMemory]++
+ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq)
+ continue
+ }
+ if dev.Totalcore-dev.Usedcores < k.Coresreq {
+ reason[common.CardInsufficientCore]++
+ klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", k.Coresreq)
+ continue
+ }
+ // Coresreq=100 indicates it want this card exclusively
+ if dev.Totalcore == 100 && k.Coresreq == 100 && dev.Used > 0 {
+ reason[common.ExclusiveDeviceAllocateConflict]++
+ klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
+ continue
+ }
+ // You can't allocate core=0 job to an already full GPU
+ if dev.Totalcore != 0 && dev.Usedcores == dev.Totalcore && k.Coresreq == 0 {
+ reason[common.CardComputeUnitsExhausted]++
+ klog.V(5).InfoS(common.CardComputeUnitsExhausted, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+ if k.Nums > 0 {
+ klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Nums--
+ tmpDevs[k.Type] = append(tmpDevs[k.Type], util.ContainerDevice{
+ Idx: int(dev.Index),
+ UUID: dev.ID,
+ Type: k.Type,
+ Usedmem: memreq,
+ Usedcores: k.Coresreq,
+ })
+ }
+ if k.Nums == 0 {
+ klog.V(4).InfoS("device allocate success", "pod", klog.KObj(pod), "allocate device", tmpDevs)
+ return true, tmpDevs, ""
+ }
+ }
+ if len(tmpDevs) > 0 {
+ reason[common.AllocatedCardsInsufficientRequest] = len(tmpDevs)
+ klog.V(5).InfoS(common.AllocatedCardsInsufficientRequest, "pod", klog.KObj(pod), "request", originReq, "allocated", len(tmpDevs))
+ }
+ return false, tmpDevs, common.GenReason(reason, len(devices))
+}
diff --git a/pkg/device/ascend/device_test.go b/pkg/device/ascend/device_test.go
new file mode 100644
index 000000000..490215ec3
--- /dev/null
+++ b/pkg/device/ascend/device_test.go
@@ -0,0 +1,1453 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ascend
+
+import (
+ "errors"
+ "fmt"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "gopkg.in/yaml.v2"
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+func Test_InitDevices(t *testing.T) {
+ tests := []struct {
+ name string
+ enableAscend bool
+ args []VNPUConfig
+ want []*Devices
+ }{
+ {
+ name: "test with vaild configuration",
+ enableAscend: true,
+ args: []VNPUConfig{
+ {
+ ChipName: "910B",
+ CommonWord: "Ascend910A",
+ ResourceName: "huawei.com/Ascend910A",
+ ResourceMemoryName: "huawei.com/Ascend910A-memory",
+ MemoryAllocatable: int64(32768),
+ MemoryCapacity: int64(32768),
+ AICore: int32(30),
+ Templates: []Template{
+ {
+ Name: "vir02",
+ Memory: int64(2184),
+ AICore: int32(2),
+ }, {
+ Name: "vir04",
+ Memory: int64(4369),
+ AICore: int32(4),
+ }, {
+ Name: "vir08",
+ Memory: int64(8738),
+ AICore: int32(8),
+ }, {
+ Name: "vir16",
+ Memory: int64(17476),
+ AICore: int32(16),
+ },
+ },
+ },
+ },
+ want: []*Devices{
+ {
+ config: VNPUConfig{
+ ChipName: "910B",
+ CommonWord: "Ascend910A",
+ ResourceName: "huawei.com/Ascend910A",
+ ResourceMemoryName: "huawei.com/Ascend910A-memory",
+ MemoryAllocatable: int64(32768),
+ MemoryCapacity: int64(32768),
+ AICore: int32(30),
+ Templates: []Template{
+ {
+ Name: "vir02",
+ Memory: int64(2184),
+ AICore: int32(2),
+ }, {
+ Name: "vir04",
+ Memory: int64(4369),
+ AICore: int32(4),
+ }, {
+ Name: "vir08",
+ Memory: int64(8738),
+ AICore: int32(8),
+ }, {
+ Name: "vir16",
+ Memory: int64(17476),
+ AICore: int32(16),
+ },
+ },
+ },
+ nodeRegisterAnno: "hami.io/node-register-Ascend910A",
+ useUUIDAnno: "hami.io/use-Ascend910A-uuid",
+ noUseUUIDAnno: "hami.io/no-use-Ascend910A-uuid",
+ handshakeAnno: "hami.io/node-handshake-Ascend910A",
+ },
+ },
+ },
+ {
+ name: "enableAscend is false",
+ enableAscend: false,
+ args: []VNPUConfig{
+ {
+ ChipName: "910B",
+ CommonWord: "Ascend910A",
+ ResourceName: "huawei.com/Ascend910A",
+ ResourceMemoryName: "huawei.com/Ascend910A-memory",
+ MemoryAllocatable: int64(32768),
+ MemoryCapacity: int64(32768),
+ AICore: int32(30),
+ Templates: []Template{
+ {
+ Name: "vir02",
+ Memory: int64(2184),
+ AICore: int32(2),
+ }, {
+ Name: "vir04",
+ Memory: int64(4369),
+ AICore: int32(4),
+ }, {
+ Name: "vir08",
+ Memory: int64(8738),
+ AICore: int32(8),
+ }, {
+ Name: "vir16",
+ Memory: int64(17476),
+ AICore: int32(16),
+ },
+ },
+ },
+ },
+ want: []*Devices{},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ enableAscend = test.enableAscend
+ devices := InitDevices(test.args)
+ assert.Equal(t, len(devices), len(test.want), "Expected length of result to match want")
+ if enableAscend {
+ for k, v := range devices {
+ assert.Equal(t, v, devices[k], "load ascend vnpu config %s: %v", devices[k].config.CommonWord, devices[k].config)
+ }
+ assert.Equal(t, "hami.io/Ascend910A-devices-to-allocate", util.InRequestDevices[test.args[0].CommonWord])
+ assert.Equal(t, "hami.io/Ascend910A-devices-allocated", util.SupportDevices[test.args[0].CommonWord])
+ assert.Equal(t, test.want[0].handshakeAnno, util.HandshakeAnnos[test.args[0].CommonWord])
+ }
+ })
+ }
+}
+
+func Test_GetNodeDevices(t *testing.T) {
+ dev := Devices{}
+ tests := []struct {
+ name string
+ args corev1.Node
+ want []*util.DeviceInfo
+ err error
+ }{
+ {
+ name: "exist device",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-01",
+ Annotations: map[string]string{
+ dev.nodeRegisterAnno: "[{\"ID\":\"GPU-0\",\"Count\":4,\"Devmem\":8738,\"Devcore\":8,\"Type\":\"huawei.com/Ascend910\",\"Numa\":0,\"Health\":true}]",
+ },
+ },
+ },
+ want: []*util.DeviceInfo{
+ {
+ ID: "GPU-0",
+ Count: int32(4),
+ Devcore: int32(8),
+ Devmem: int32(8738),
+ Type: "huawei.com/Ascend910",
+ Numa: 0,
+ Health: true,
+ },
+ },
+ err: nil,
+ },
+ {
+ name: "no device",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-02",
+ Annotations: map[string]string{
+ dev.nodeRegisterAnno: "[]",
+ },
+ },
+ },
+ want: []*util.DeviceInfo{},
+ err: errors.New("no device found on node"),
+ },
+ {
+ name: "no annotation",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-03",
+ },
+ },
+ want: []*util.DeviceInfo{},
+ err: fmt.Errorf("annos not found"),
+ },
+ {
+ name: "failed to unmarshal node devices",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-04",
+ Annotations: map[string]string{
+ dev.nodeRegisterAnno: "",
+ },
+ },
+ },
+ want: []*util.DeviceInfo{},
+ err: fmt.Errorf("failed to unmarshal node devices"),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result, err := dev.GetNodeDevices(test.args)
+ if (err != nil) != (test.err != nil) {
+ klog.ErrorS(err, "failed to unmarshal node devices", "node", test.args.Name)
+ }
+ if len(result) != len(test.want) {
+ t.Errorf("GetNodeDevices got %d devices, want %d", len(result), len(test.want))
+ return
+ }
+ if err == nil && len(result) != 0 {
+ for k, v := range test.want {
+ assert.Equal(t, v.Index, result[k].Index)
+ assert.Equal(t, v.ID, result[k].ID)
+ assert.Equal(t, v.Count, result[k].Count)
+ assert.Equal(t, v.Devcore, result[k].Devcore)
+ assert.Equal(t, v.Devmem, result[k].Devmem)
+ assert.Equal(t, v.Type, result[k].Type)
+ assert.Equal(t, v.Numa, result[k].Numa)
+ assert.Equal(t, v.Health, result[k].Health)
+ }
+ }
+ })
+ }
+}
+
+func Test_PatchAnnotations(t *testing.T) {
+ dev := Devices{
+ config: VNPUConfig{
+ CommonWord: "Ascend910A",
+ MemoryCapacity: int64(1024),
+ Templates: []Template{
+ {
+ Name: "vir02",
+ Memory: int64(2184),
+ AICore: int32(2),
+ }, {
+ Name: "vir04",
+ Memory: int64(4369),
+ AICore: int32(4),
+ }, {
+ Name: "vir08",
+ Memory: int64(8738),
+ AICore: int32(8),
+ }, {
+ Name: "vir16",
+ Memory: int64(17476),
+ AICore: int32(16),
+ },
+ },
+ },
+ }
+ tests := []struct {
+ name string
+ args struct {
+ annoinput map[string]string
+ pd util.PodDevices
+ }
+ want map[string]string
+ }{
+ {
+ name: "exist device",
+ args: struct {
+ annoinput map[string]string
+ pd util.PodDevices
+ }{
+ annoinput: map[string]string{},
+ pd: util.PodDevices{
+ dev.config.CommonWord: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ Idx: 0,
+ UUID: "device-0",
+ Type: "Ascend",
+ Usedcores: 1,
+ Usedmem: 8738,
+ },
+ },
+ },
+ },
+ },
+ want: map[string]string{
+ util.InRequestDevices[dev.config.CommonWord]: "device-0,Ascend,8738,1:;",
+ util.SupportDevices[dev.config.CommonWord]: "device-0,Ascend,8738,1:;",
+ "predicate-time": strconv.FormatInt(time.Now().Unix(), 10),
+ "huawei.com/Ascend910A": "[{\"UUID\":\"device-0\",\"temp\":\"vir08\"}]",
+ },
+ },
+ {
+ name: "no device",
+ args: struct {
+ annoinput map[string]string
+ pd util.PodDevices
+ }{
+ annoinput: map[string]string{},
+ pd: util.PodDevices{},
+ },
+ want: map[string]string{},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result := dev.PatchAnnotations(&corev1.Pod{}, &test.args.annoinput, test.args.pd)
+
+ assert.Equal(t, len(test.want), len(result), "Expected length of result to match want")
+ for k, v := range test.want {
+ assert.Equal(t, v, result[k], "pod add annotation key [%s], values is [%s]", k, result[k])
+ }
+ })
+ }
+}
+
+func Test_checkType(t *testing.T) {
+ dev := Devices{
+ config: VNPUConfig{
+ CommonWord: "Ascend910A",
+ },
+ }
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }
+ want bool
+ }{
+ {
+ name: "the same type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: "Ascend910A",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "the different type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: "Ascend910B",
+ },
+ },
+ want: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ _, result, _ := dev.checkType(test.args.annos, test.args.d, test.args.n)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_checkUUID(t *testing.T) {
+ dev := Devices{
+ useUUIDAnno: "hami.io/use-Ascend910A-uuid",
+ noUseUUIDAnno: "hami.io/no-use-Ascend910A-uuid",
+ }
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "don't set GPUUseUUID,GPUNoUseUUID and annotation",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ },
+ want: true,
+ },
+ {
+ name: "set GPUUseUUID,don't set GPUNoUseUUID,annotation and device match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ dev.useUUIDAnno: "test123,111",
+ },
+ d: util.DeviceUsage{
+ ID: "test123",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "don't set GPUUseUUID, set GPUNoUseUUID,annotation and device match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ dev.noUseUUIDAnno: "test123,222",
+ },
+ d: util.DeviceUsage{
+ ID: "test123",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "set GPUUseUUID, don't set GPUNoUseUUID,annotation and device not match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ dev.useUUIDAnno: "test123,222",
+ },
+ d: util.DeviceUsage{
+ ID: "test456",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "don't set GPUUseUUID, set GPUNoUseUUID,annotation and device not match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ dev.noUseUUIDAnno: "test123,222",
+ },
+ d: util.DeviceUsage{
+ ID: "test456",
+ },
+ },
+ want: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result := dev.checkUUID(test.args.annos, test.args.d)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_CheckHealth(t *testing.T) {
+ dev := Devices{}
+ tests := []struct {
+ name string
+ args struct {
+ devType string
+ n corev1.Node
+ }
+ want1 bool
+ want2 bool
+ }{
+ {
+ name: "Requesting state",
+ args: struct {
+ devType string
+ n corev1.Node
+ }{
+ devType: "huawei.com/Ascend910",
+ n: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ util.HandshakeAnnos["huawei.com/Ascend910"]: "Requesting_2128-12-02 00:00:00",
+ },
+ },
+ },
+ },
+ want1: true,
+ want2: false,
+ },
+ {
+ name: "Deleted state",
+ args: struct {
+ devType string
+ n corev1.Node
+ }{
+ devType: "huawei.com/Ascend910",
+ n: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ util.HandshakeAnnos["huawei.com/Ascend910"]: "Deleted",
+ },
+ },
+ },
+ },
+ want1: true,
+ want2: false,
+ },
+ {
+ name: "Unknown state",
+ args: struct {
+ devType string
+ n corev1.Node
+ }{
+ devType: "huawei.com/Ascend910",
+ n: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ util.HandshakeAnnos["huawei.com/Ascend910"]: "Unknown",
+ },
+ },
+ },
+ },
+ want1: true,
+ want2: true,
+ },
+ {
+ name: "Requesting state expired",
+ args: struct {
+ devType string
+ n corev1.Node
+ }{
+ devType: "huawei.com/Ascend910",
+ n: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ util.HandshakeAnnos["huawei.com/Ascend910"]: "Requesting_2024-01-02 00:00:00",
+ },
+ },
+ },
+ },
+ want1: false,
+ want2: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result1, result2 := dev.CheckHealth(test.args.devType, &test.args.n)
+ assert.Equal(t, result1, test.want1)
+ assert.Equal(t, result2, test.want2)
+ })
+ }
+}
+
+func Test_MutateAdmission(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ ctr corev1.Container
+ pod corev1.Pod
+ }
+ want bool
+ }{
+ {
+ name: "no set to resources limits",
+ args: struct {
+ ctr corev1.Container
+ pod corev1.Pod
+ }{
+ ctr: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ },
+ },
+ pod: corev1.Pod{},
+ },
+ want: false,
+ },
+ {
+ name: "resourcename and resourcememoryname set to resources limits",
+ args: struct {
+ ctr corev1.Container
+ pod corev1.Pod
+ }{
+ ctr: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "huawei.com/Ascend910A": resource.MustParse("2"),
+ "huawei.com/Ascend910A-memory": resource.MustParse("8738"),
+ },
+ },
+ },
+ pod: corev1.Pod{},
+ },
+ want: true,
+ },
+ {
+ name: "resourcememoryname is invalid",
+ args: struct {
+ ctr corev1.Container
+ pod corev1.Pod
+ }{
+ ctr: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "huawei.com/Ascend910A": resource.MustParse("2"),
+ "huawei.com/Ascend910A-memory": resource.MustParse("40000"),
+ },
+ },
+ },
+ pod: corev1.Pod{},
+ },
+ want: false,
+ },
+ {
+ name: "resourcememoryname not within the template scope，but smaller than MemoryCapacity",
+ args: struct {
+ ctr corev1.Container
+ pod corev1.Pod
+ }{
+ ctr: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "huawei.com/Ascend910A": resource.MustParse("1"),
+ "huawei.com/Ascend910A-memory": resource.MustParse("20000"),
+ },
+ Requests: corev1.ResourceList{
+ "huawei.com/Ascend910A-memory": resource.MustParse("20000"),
+ },
+ },
+ },
+ pod: corev1.Pod{},
+ },
+ want: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := Devices{
+ config: VNPUConfig{
+ ResourceName: "huawei.com/Ascend910A",
+ ResourceMemoryName: "huawei.com/Ascend910A-memory",
+ MemoryAllocatable: int64(32768),
+ MemoryCapacity: int64(32768),
+ Templates: []Template{
+ {
+ Name: "vir02",
+ Memory: int64(2184),
+ AICore: int32(2),
+ }, {
+ Name: "vir04",
+ Memory: int64(4369),
+ AICore: int32(4),
+ }, {
+ Name: "vir08",
+ Memory: int64(8738),
+ AICore: int32(8),
+ }, {
+ Name: "vir16",
+ Memory: int64(17476),
+ AICore: int32(16),
+ },
+ },
+ },
+ }
+ result, _ := dev.MutateAdmission(&test.args.ctr, &test.args.pod)
+
+ if result != test.want {
+ t.Fatalf("exec MutateAdmission method expect return is %+v, but got is %+v", test.want, result)
+ }
+
+ })
+ }
+}
+
+func Test_GenerateResourceRequests(t *testing.T) {
+ tests := []struct {
+ name string
+ args corev1.Container
+ want util.ContainerDeviceRequest
+ }{
+ {
+ name: "don't set to limits and request",
+ args: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ Requests: corev1.ResourceList{},
+ },
+ },
+ want: util.ContainerDeviceRequest{},
+ },
+ {
+ name: "resourcename and resourcememoryname set to limits and request",
+ args: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "huawei.com/Ascend910A": resource.MustParse("2"),
+ "huawei.com/Ascend910A-memory": resource.MustParse("8738"),
+ },
+ Requests: corev1.ResourceList{
+ "huawei.com/Ascend910A": resource.MustParse("2"),
+ "huawei.com/Ascend910A-memory": resource.MustParse("8738"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(2),
+ Type: "Ascend910A",
+ Memreq: int32(8738),
+ MemPercentagereq: int32(0),
+ Coresreq: int32(0),
+ },
+ },
+ {
+ name: "resourcememoryname don't set to limits and requests",
+ args: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "huawei.com/Ascend910A": resource.MustParse("2"),
+ },
+ Requests: corev1.ResourceList{
+ "huawei.com/Ascend910A": resource.MustParse("2"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(2),
+ Type: "Ascend910A",
+ Memreq: int32(0),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(0),
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := Devices{
+ config: VNPUConfig{
+ CommonWord: "Ascend910A",
+ ResourceName: "huawei.com/Ascend910A",
+ ResourceMemoryName: "huawei.com/Ascend910A-memory",
+ MemoryAllocatable: int64(32768),
+ MemoryCapacity: int64(32768),
+ Templates: []Template{
+ {
+ Name: "vir02",
+ Memory: int64(2184),
+ AICore: int32(2),
+ }, {
+ Name: "vir04",
+ Memory: int64(4369),
+ AICore: int32(4),
+ }, {
+ Name: "vir08",
+ Memory: int64(8738),
+ AICore: int32(8),
+ }, {
+ Name: "vir16",
+ Memory: int64(17476),
+ AICore: int32(16),
+ },
+ },
+ },
+ }
+ result := dev.GenerateResourceRequests(&test.args)
+
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func TestDevices_LockNode(t *testing.T) {
+ tests := []struct {
+ name string
+ node *corev1.Node
+ pod *corev1.Pod
+ expectError bool
+ }{
+ {
+ name: "Test with no containers",
+ node: &corev1.Node{},
+ pod: &corev1.Pod{Spec: corev1.PodSpec{}},
+ expectError: false,
+ },
+ {
+ name: "Test with non-zero resource requests",
+ node: &corev1.Node{},
+ pod: &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{}}}}}},
+ expectError: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &Devices{
+ config: VNPUConfig{
+ CommonWord: "Ascend310P",
+ ResourceName: "huawei.com/Ascend310P",
+ ResourceMemoryName: "huawei.com/Ascend310P-memory",
+ },
+ }
+ err := dev.LockNode(tt.node, tt.pod)
+ if tt.expectError {
+ assert.Equal(t, err != nil, true)
+ } else {
+ assert.NilError(t, err)
+ }
+ })
+ }
+}
+
+func TestDevices_ReleaseNodeLock(t *testing.T) {
+ tests := []struct {
+ name string
+ node *corev1.Node
+ pod *corev1.Pod
+ expectError bool
+ }{
+ {
+ name: "Test with no containers",
+ node: &corev1.Node{},
+ pod: &corev1.Pod{Spec: corev1.PodSpec{}},
+ expectError: false,
+ },
+ {
+ name: "Test with non-zero resource requests",
+ node: &corev1.Node{},
+ pod: &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{}}}}}},
+ expectError: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &Devices{
+ config: VNPUConfig{
+ CommonWord: "Ascend310P",
+ ResourceName: "huawei.com/Ascend310P",
+ ResourceMemoryName: "huawei.com/Ascend310P-memory",
+ },
+ }
+ err := dev.ReleaseNodeLock(tt.node, tt.pod)
+ if tt.expectError {
+ assert.Equal(t, err != nil, true)
+ } else {
+ assert.NilError(t, err)
+ }
+ })
+ }
+}
+
+func TestDevices_Fit(t *testing.T) {
+ configStr := `- chipName: 910B
+ commonWord: Ascend910A
+ resourceName: huawei.com/Ascend910A
+ resourceMemoryName: huawei.com/Ascend910A-memory
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 30
+ templates:
+ - name: vir02
+ memory: 2184
+ aiCore: 2
+ - name: vir04
+ memory: 4369
+ aiCore: 4
+ - name: vir08
+ memory: 8738
+ aiCore: 8
+ - name: vir16
+ memory: 17476
+ aiCore: 16
+- chipName: 910B2
+ commonWord: Ascend910B2
+ resourceName: huawei.com/Ascend910B2
+ resourceMemoryName: huawei.com/Ascend910B2-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 24
+ aiCPU: 6
+ topologyPairs:
+ - 1,2,3,4,5,6,7
+ - 0,2,3,4,5,6,7
+ - 0,1,3,4,5,6,7
+ - 0,1,2,4,5,6,7
+ - 0,1,2,3,5,6,7
+ - 0,1,2,3,4,6,7
+ - 0,1,2,3,4,5,7
+ - 0,1,2,3,4,5,6
+ templates:
+ - name: vir03_1c_8g
+ memory: 8192
+ aiCore: 3
+ aiCPU: 1
+ - name: vir06_1c_16g
+ memory: 16384
+ aiCore: 6
+ aiCPU: 1
+ - name: vir12_3c_32g
+ memory: 32768
+ aiCore: 12
+ aiCPU: 3
+- chipName: 910B3
+ commonWord: Ascend910B
+ resourceName: huawei.com/Ascend910B
+ resourceMemoryName: huawei.com/Ascend910B-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 20
+ aiCPU: 7
+ topologyPairs:
+ - 1,2,3,4,5,6,7
+ - 0,2,3,4,5,6,7
+ - 0,1,3,4,5,6,7
+ - 0,1,2,4,5,6,7
+ - 0,1,2,3,5,6,7
+ - 0,1,2,3,4,6,7
+ - 0,1,2,3,4,5,7
+ - 0,1,2,3,4,5,6
+ templates:
+ - name: vir05_1c_16g
+ memory: 16384
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_32g
+ memory: 32768
+ aiCore: 10
+ aiCPU: 3
+- chipName: 910B4
+ commonWord: Ascend910B4
+ resourceName: huawei.com/Ascend910B4
+ resourceMemoryName: huawei.com/Ascend910B4-memory
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: vir05_1c_8g
+ memory: 8192
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_16g
+ memory: 16384
+ aiCore: 10
+ aiCPU: 3
+- chipName: 910B4-1
+ commonWord: Ascend910B4
+ resourceName: huawei.com/Ascend910B4
+ resourceMemoryName: huawei.com/Ascend910B4-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: vir05_1c_8g
+ memory: 8192
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_16g
+ memory: 16384
+ aiCore: 10
+ aiCPU: 3
+- chipName: 310P3
+ commonWord: Ascend310P
+ resourceName: huawei.com/Ascend310P
+ resourceMemoryName: huawei.com/Ascend310P-memory
+ memoryAllocatable: 21527
+ memoryCapacity: 24576
+ aiCore: 8
+ aiCPU: 7
+ templates:
+ - name: vir01
+ memory: 3072
+ aiCore: 1
+ aiCPU: 1
+ - name: vir02
+ memory: 6144
+ aiCore: 2
+ aiCPU: 2
+ - name: vir04
+ memory: 12288
+ aiCore: 4
+ aiCPU: 4
+`
+
+ var config []VNPUConfig
+ if err := yaml.Unmarshal([]byte(configStr), &config); err != nil {
+ t.Fatalf("failed to unmarshal config: %v", err)
+ }
+ enableAscend = true
+ devs := InitDevices(config)
+
+ tests := []struct {
+ name string
+ devices []*util.DeviceUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ wantFit bool
+ wantLen int
+ wantDevIDs []string
+ wantReason string
+ }{
+ {
+ name: "fit success",
+ devices: []*util.DeviceUsage{
+ {
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ },
+ {
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 64,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-1"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: memory not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientMemory",
+ },
+ {
+ name: "fit fail: core not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1024,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientCore",
+ },
+ {
+ name: "fit fail: type mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: "OtherType",
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTypeMismatch",
+ },
+ {
+ name: "fit fail: user assign use uuid mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{"hami.io/use-Ascend910B2-uuid": "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: user assign no use uuid match",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{"hami.io/no-use-Ascend910B2-uuid": "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: card overused",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 100,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTimeSlicingExhausted",
+ },
+ {
+ name: "fit success: but core limit can't exceed 100",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 120,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: card exclusively",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 100,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 ExclusiveDeviceAllocateConflict",
+ },
+ {
+ name: "fit fail: CardComputeUnitsExhausted",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 0,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardComputeUnitsExhausted",
+ },
+ {
+ name: "fit fail: AllocatedCardsInsufficientRequest",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 20,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 AllocatedCardsInsufficientRequest",
+ },
+ {
+ name: "fit success: memory percentage",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 10,
+ Coresreq: 20,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ }
+
+ for _, dev := range devs {
+ for _, test := range tests {
+ if !strings.Contains(test.name, "type mismatch") {
+ test.request.Type = dev.config.CommonWord
+ }
+ if strings.Contains(test.name, "user assign use uuid mismatch") {
+ test.annos["hami.io/use-"+dev.config.CommonWord+"-uuid"] = "dev-0"
+ }
+ if strings.Contains(test.name, "user assign no use uuid match") {
+ test.annos["hami.io/no-use-"+dev.config.CommonWord+"-uuid"] = "dev-0"
+ }
+ for _, d := range test.devices {
+ d.Type = dev.config.CommonWord
+ }
+
+ t.Run(fmt.Sprintf("%s:%s", dev.config.CommonWord, test.name), func(t *testing.T) {
+ allocated := &util.PodDevices{}
+ fit, result, reason := dev.Fit(test.devices, test.request, test.annos, &corev1.Pod{}, &util.NodeInfo{}, allocated)
+ if fit != test.wantFit {
+ t.Errorf("Fit: got %v, want %v", fit, test.wantFit)
+ }
+ if test.wantFit {
+ if len(result[dev.config.CommonWord]) != test.wantLen {
+ t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[dev.config.CommonWord]))
+ }
+ for idx, id := range test.wantDevIDs {
+ if id != result[dev.config.CommonWord][idx].UUID {
+ t.Errorf("expected device id: %s, got device id %s", id, result[dev.config.CommonWord][idx].UUID)
+ }
+ }
+ }
+
+ if reason != test.wantReason {
+ t.Errorf("expected reason: %s, got reason: %s", test.wantReason, reason)
+ }
+ })
+ }
+ }
+}
+
+func TestDevices_AddResourceUsage(t *testing.T) {
+ tests := []struct {
+ name string
+ deviceUsage *util.DeviceUsage
+ ctr *util.ContainerDevice
+ wantErr bool
+ wantUsage *util.DeviceUsage
+ }{
+ {
+ name: "test add resource usage",
+ deviceUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 0,
+ Usedcores: 15,
+ Usedmem: 2000,
+ },
+ ctr: &util.ContainerDevice{
+ UUID: "dev-0",
+ Usedcores: 50,
+ Usedmem: 1024,
+ },
+ wantUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 1,
+ Usedcores: 65,
+ Usedmem: 3024,
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &Devices{}
+ if err := dev.AddResourceUsage(&corev1.Pod{}, tt.deviceUsage, tt.ctr); (err != nil) != tt.wantErr {
+ t.Errorf("AddResourceUsage() error=%v, wantErr %v", err, tt.wantErr)
+ }
+ if !tt.wantErr {
+ if tt.deviceUsage.Usedcores != tt.wantUsage.Usedcores {
+ t.Errorf("expected used cores: %d, got used cores %d", tt.wantUsage.Usedcores, tt.deviceUsage.Usedcores)
+ }
+ if tt.deviceUsage.Usedmem != tt.wantUsage.Usedmem {
+ t.Errorf("expected used mem: %d, got used mem %d", tt.wantUsage.Usedmem, tt.deviceUsage.Usedmem)
+ }
+ if tt.deviceUsage.Used != tt.wantUsage.Used {
+ t.Errorf("expected used: %d, got used %d", tt.wantUsage.Used, tt.deviceUsage.Used)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/device/ascend/vnpu.go b/pkg/device/ascend/vnpu.go
new file mode 100644
index 000000000..57321f259
--- /dev/null
+++ b/pkg/device/ascend/vnpu.go
@@ -0,0 +1,36 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ascend
+
+type Template struct {
+ Name string `yaml:"name"`
+ Memory int64 `yaml:"memory"`
+ AICore int32 `yaml:"aiCore,omitempty"`
+ AICPU int32 `yaml:"aiCPU,omitempty"`
+}
+
+type VNPUConfig struct {
+ CommonWord string `yaml:"commonWord"`
+ ChipName string `yaml:"chipName"`
+ ResourceName string `yaml:"resourceName"`
+ ResourceMemoryName string `yaml:"resourceMemoryName"`
+ MemoryAllocatable int64 `yaml:"memoryAllocatable"`
+ MemoryCapacity int64 `yaml:"memoryCapacity"`
+ AICore int32 `yaml:"aiCore"`
+ AICPU int32 `yaml:"aiCPU"`
+ Templates []Template `yaml:"templates"`
+}
diff --git a/pkg/device/awsneuron/device.go b/pkg/device/awsneuron/device.go
new file mode 100644
index 000000000..1eac177d8
--- /dev/null
+++ b/pkg/device/awsneuron/device.go
@@ -0,0 +1,464 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awsneuron
+
+import (
+ "flag"
+ "fmt"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/common"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ "k8s.io/klog/v2"
+)
+
+type AWSNeuronDevices struct {
+ resourceCountName string
+ resourceCoreName string
+ coresPerAWSNeuron uint
+ coremask uint
+}
+
+const (
+ AWSNeuronDevice = "AWSNeuron"
+ AWSNeuronCommonWord = "AWSNeuron"
+ AWSNeuronDeviceSelection = "aws.amazon.com/neuron-index"
+ AWSNeuronUseUUID = "aws.amazon.com/use-neuron-uuid"
+ AWSNeuronNoUseUUID = "aws.amazon.com/nouse-neuron-uuid"
+ AWSNeuronAssignedIndex = "AWS_NEURON_IDS"
+ AWSNeuronAssignedNode = "aws.amazon.com/predicate-node"
+ AWSNeuronPredicateTime = "NEURON_ALLOC_TIME"
+ AWSNeuronResourceType = "NEURON_RESOURCE_TYPE"
+ AWSNeuronAllocated = "NEURON_ALLOCATED"
+ AWSUsageInfo = "awsusageinfo"
+ AWSNodeType = "AWSNodeType"
+)
+
+type AWSNeuronConfig struct {
+ ResourceCountName string `yaml:"resourceCountName"`
+ ResourceCoreName string `yaml:"resourceCoreName"`
+}
+
+func InitAWSNeuronDevice(config AWSNeuronConfig) *AWSNeuronDevices {
+ util.SupportDevices[AWSNeuronDevice] = "hami.io/aws-neuron-devices-allocated"
+ return &AWSNeuronDevices{
+ resourceCountName: config.ResourceCountName,
+ resourceCoreName: config.ResourceCoreName,
+ coresPerAWSNeuron: 0,
+ coremask: 0,
+ }
+}
+
+func (dev *AWSNeuronDevices) CommonWord() string {
+ return AWSNeuronCommonWord
+}
+
+func ParseConfig(fs *flag.FlagSet) {
+}
+
+func (dev *AWSNeuronDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
+ _, ok := ctr.Resources.Limits[corev1.ResourceName(dev.resourceCountName)]
+ if !ok {
+ _, ok = ctr.Resources.Limits[corev1.ResourceName(dev.resourceCoreName)]
+ }
+ return ok, nil
+}
+
+func (dev *AWSNeuronDevices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) {
+ nodedevices := []*util.DeviceInfo{}
+ i := 0
+ counts, ok := n.Status.Capacity.Name(corev1.ResourceName(dev.resourceCountName), resource.DecimalSI).AsInt64()
+ if !ok || counts == 0 {
+ return []*util.DeviceInfo{}, fmt.Errorf("device not found %s", dev.resourceCountName)
+ }
+ coresTotal, _ := n.Status.Capacity.Name(corev1.ResourceName(dev.resourceCoreName), resource.DecimalSI).AsInt64()
+ if dev.coresPerAWSNeuron == 0 {
+ dev.coresPerAWSNeuron = uint(coresTotal) / uint(counts)
+ }
+ dev.coremask = 0
+ for i < int(dev.coresPerAWSNeuron) {
+ dev.coremask *= 2
+ dev.coremask++
+ i++
+ }
+ i = 0
+ customInfo := map[string]any{}
+ customInfo[AWSNodeType] = n.Labels["node.kubernetes.io/instance-type"]
+
+ for int64(i) < counts {
+ nodedevices = append(nodedevices, &util.DeviceInfo{
+ Index: uint(i),
+ ID: n.Name + "-" + AWSNeuronDevice + "-" + fmt.Sprint(i),
+ Count: int32(dev.coresPerAWSNeuron),
+ Devmem: 0,
+ Devcore: int32(dev.coremask),
+ Type: AWSNeuronDevice,
+ Numa: 0,
+ Health: true,
+ CustomInfo: customInfo,
+ })
+ i++
+ }
+ i = 0
+ for i < len(nodedevices) {
+ klog.V(4).Infoln("Registered AWS nodedevices:", nodedevices[i])
+ i++
+ }
+ return nodedevices, nil
+}
+
+func (dev *AWSNeuronDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd util.PodDevices) map[string]string {
+ devlist, ok := pd[AWSNeuronDevice]
+ if ok && len(devlist) > 0 {
+ (*annoinput)[util.SupportDevices[AWSNeuronDevice]] = util.EncodePodSingleDevice(devlist)
+ value := ""
+ for ctridx, dp := range devlist {
+ if len(dp) > 0 {
+ for _, val := range dp {
+ devValue, ok := pod.Spec.Containers[ctridx].Resources.Limits[corev1.ResourceName(dev.resourceCountName)]
+ if ok {
+ c, _ := devValue.AsInt64()
+ if c > 0 {
+ value = value + fmt.Sprint(val.Idx) + ","
+ }
+ } else {
+ if (val.Usedcores & 1) != 0 {
+ value = value + fmt.Sprint(dev.coresPerAWSNeuron*uint(val.Idx)) + ","
+ (*annoinput)[AWSNeuronResourceType] = dev.resourceCoreName
+ }
+ if (val.Usedcores & 2) != 0 {
+ value = value + fmt.Sprint(dev.coresPerAWSNeuron*uint(val.Idx)+1) + ","
+ (*annoinput)[AWSNeuronResourceType] = dev.resourceCoreName
+ }
+ }
+ }
+ if len(value) > 0 {
+ // This needs to modify, it has to be core indexes?
+ (*annoinput)[AWSNeuronAssignedIndex] = strings.TrimRight(value, ",")
+
+ tmp := strconv.FormatInt(time.Now().UnixNano(), 10)
+ (*annoinput)[AWSNeuronPredicateTime] = tmp
+ (*annoinput)[AWSNeuronAllocated] = "false"
+ (*annoinput)[AWSNeuronAssignedNode] = (*annoinput)[util.AssignedNodeAnnotations]
+ }
+ }
+ }
+ }
+ klog.V(4).InfoS("annos", "input", (*annoinput))
+ return *annoinput
+}
+
+func (dev *AWSNeuronDevices) LockNode(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *AWSNeuronDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *AWSNeuronDevices) NodeCleanUp(nn string) error {
+ return nil
+}
+
+func (dev *AWSNeuronDevices) checkType(n util.ContainerDeviceRequest) (bool, bool, bool) {
+ if strings.Compare(n.Type, AWSNeuronDevice) == 0 {
+ return true, true, false
+ }
+ return false, false, false
+}
+
+func (dev *AWSNeuronDevices) checkUUID(annos map[string]string, d util.DeviceUsage) bool {
+ userUUID, ok := annos[AWSNeuronUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for AWSNeuron user uuid [%s], device id is %s", userUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ userUUIDs := strings.Split(userUUID, ",")
+ return slices.Contains(userUUIDs, d.ID)
+ }
+
+ noUserUUID, ok := annos[AWSNeuronNoUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for AWSNeuron no-use uuid [%s], device id is %s", noUserUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ noUserUUIDs := strings.Split(noUserUUID, ",")
+ return !slices.Contains(noUserUUIDs, d.ID)
+ }
+ return true
+}
+
+func (dev *AWSNeuronDevices) checkIndex(annos map[string]string, d util.DeviceUsage) bool {
+ return true
+}
+
+func (dev *AWSNeuronDevices) CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ return true, true
+}
+
+func (dev *AWSNeuronDevices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest {
+ klog.Info("Start to count awsNeuron devices for container ", ctr.Name)
+ awsResourceCount := corev1.ResourceName(dev.resourceCountName)
+ awsResourceCores := corev1.ResourceName(dev.resourceCoreName)
+ v, ok := ctr.Resources.Limits[awsResourceCount]
+ if !ok {
+ v, ok = ctr.Resources.Requests[awsResourceCount]
+ }
+ if ok {
+ if n, ok := v.AsInt64(); ok {
+ klog.InfoS("Detected awsNeuron device request",
+ "container", ctr.Name,
+ "deviceCount", n)
+ return util.ContainerDeviceRequest{
+ Nums: int32(n),
+ Type: AWSNeuronDevice,
+ Memreq: 0,
+ MemPercentagereq: 0,
+ Coresreq: int32(dev.coresPerAWSNeuron),
+ }
+ }
+ } else {
+ core, ok := ctr.Resources.Limits[awsResourceCores]
+ if !ok {
+ core, ok = ctr.Resources.Requests[awsResourceCores]
+ }
+ if ok {
+ if n, ok := core.AsInt64(); ok {
+ klog.InfoS("Detected awsNeuron device request",
+ "container", ctr.Name,
+ "deviceCores", n)
+ num := 1
+ if n >= 2 {
+ num = int(n / 2)
+ }
+ corenum := 1
+ if n >= 2 {
+ corenum = 2
+ }
+ return util.ContainerDeviceRequest{
+ Nums: int32(num),
+ Type: AWSNeuronDevice,
+ Memreq: 0,
+ MemPercentagereq: 0,
+ Coresreq: int32(corenum),
+ }
+ }
+ }
+ }
+ return util.ContainerDeviceRequest{}
+}
+
+func (dev *AWSNeuronDevices) ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32 {
+ return 0
+}
+
+func (dev *AWSNeuronDevices) AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error {
+ n.Used++
+ n.Usedcores += ctr.Usedcores
+ n.Usedmem += ctr.Usedmem
+
+ num, ok := n.CustomInfo[AWSUsageInfo]
+ if !ok || num == nil {
+ n.CustomInfo[AWSUsageInfo] = 0
+ }
+ if nValue, ok := n.CustomInfo[AWSUsageInfo].(int); ok {
+ if ctrValue, ok2 := ctr.CustomInfo[AWSUsageInfo].(int); ok2 {
+ n.CustomInfo[AWSUsageInfo] = nValue + ctrValue
+ }
+ }
+ return nil
+}
+
+func countMaskAvailable(mask int32) int32 {
+ tmp := mask
+ ret := int32(0)
+ for tmp > 0 {
+ ret = ret + tmp%2
+ tmp /= 2
+ }
+ return ret
+}
+
+func addCoreUsage(prev map[string]any, require int) map[string]any {
+ res := map[string]any{}
+ count, ok := prev[AWSUsageInfo]
+ if !ok {
+ count = 0
+ }
+ if count == 0 {
+ if require == 2 {
+ res[AWSUsageInfo] = 3
+ return res
+ }
+ if require == 1 {
+ res[AWSUsageInfo] = 1
+ return res
+ }
+ }
+ if countValue, ok := count.(int); ok {
+ res[AWSUsageInfo] = 3 - countValue
+ } else {
+ res[AWSUsageInfo] = 3
+ }
+ return res
+}
+func continuousDeviceAvailable(devices []*util.DeviceUsage, start int, count int) []int {
+ if len(devices) < start+count {
+ return []int{}
+ }
+ res := []int{}
+ iterator := start
+ for iterator < start+count {
+ if devices[iterator].Used > 0 {
+ return []int{}
+ }
+ res = append(res, iterator)
+ iterator++
+ }
+ return res
+}
+
+func graphSelect(devices []*util.DeviceUsage, count int) []int {
+ if len(devices) == 0 || devices[0].CustomInfo == nil || devices[0].CustomInfo[AWSNodeType] == nil {
+ return []int{}
+ }
+ AWSNodetype := ""
+ if nodeType, ok := devices[0].CustomInfo[AWSNodeType].(string); ok {
+ AWSNodetype = nodeType
+ }
+ if strings.Contains(AWSNodetype, "inf") || strings.Contains(AWSNodetype, "Inf") {
+ //Deal with ring
+ start := 0
+ for start < len(devices) {
+ res := continuousDeviceAvailable(devices, start, count)
+ if len(res) > 0 {
+ return res
+ }
+ start += 1
+ }
+ return []int{}
+ }
+ switch count {
+ case 1, 4, 8, 16:
+ {
+ start := 0
+ for start < len(devices) {
+ res := continuousDeviceAvailable(devices, start, count)
+ if len(res) > 0 {
+ return res
+ }
+ start += count
+ }
+ return []int{}
+ }
+ }
+ return []int{}
+}
+
+func (neuron *AWSNeuronDevices) Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeinfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string) {
+ k := request
+ originReq := k.Nums
+ klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
+ tmpDevs := make(map[string]util.ContainerDevices)
+ reason := make(map[string]int)
+ if k.Nums > 1 {
+ alloc := graphSelect(devices, int(request.Nums))
+ if len(alloc) == 0 {
+ reason[common.NumaNotFit]++
+ klog.V(5).InfoS(common.NumaNotFit, "pod", klog.KObj(pod), "device", devices, "request nums", request.Nums, "numa")
+ return false, tmpDevs, common.GenReason(reason, len(reason))
+ }
+ for _, dev := range alloc {
+ for _, val := range devices {
+ if val.Index == uint(dev) {
+ customInfo := addCoreUsage(val.CustomInfo, int(k.Coresreq))
+ tmpDevs[request.Type] = append(tmpDevs[request.Type], util.ContainerDevice{
+ Idx: int(val.Index),
+ UUID: val.ID,
+ Type: request.Type,
+ Usedmem: val.Totalmem,
+ Usedcores: val.Totalcore,
+ CustomInfo: customInfo,
+ })
+ break
+ }
+ }
+ }
+ return true, tmpDevs, ""
+ }
+ for i := len(devices) - 1; i >= 0; i-- {
+ dev := devices[i]
+ _, ok := dev.CustomInfo[AWSUsageInfo]
+ if !ok {
+ dev.CustomInfo[AWSUsageInfo] = int(dev.Usedcores)
+ }
+ klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
+
+ klog.V(3).InfoS("Type check", "device", dev.Type, "req", k.Type, "dev=", dev)
+ if !strings.Contains(dev.Type, k.Type) {
+ reason[common.CardTypeMismatch]++
+ continue
+ }
+
+ _, found, _ := neuron.checkType(k)
+ if !found {
+ reason[common.CardTypeMismatch]++
+ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, k.Type)
+ continue
+ }
+ if !neuron.checkUUID(annos, *dev) {
+ reason[common.CardUUIDMismatch]++
+ klog.V(5).InfoS(common.CardUUIDMismatch, "pod", klog.KObj(pod), "device", dev.ID, "current device info is:", *dev)
+ continue
+ }
+
+ if dev.Count <= dev.Used {
+ reason[common.CardTimeSlicingExhausted]++
+ klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
+ continue
+ }
+
+ if countMaskAvailable(dev.Totalcore)-countMaskAvailable(dev.Usedcores) < k.Coresreq {
+ reason[common.CardInsufficientCore]++
+ klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", k.Coresreq)
+ continue
+ }
+
+ klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
+ customInfo := addCoreUsage(dev.CustomInfo, int(k.Coresreq))
+ usedcores := 0
+ if countValue, ok := customInfo[AWSUsageInfo].(int); ok {
+ usedcores = countValue
+ }
+ tmpDevs[k.Type] = append(tmpDevs[k.Type], util.ContainerDevice{
+ Idx: int(dev.Index),
+ UUID: dev.ID,
+ Type: k.Type,
+ Usedmem: 0,
+ Usedcores: int32(usedcores),
+ CustomInfo: customInfo,
+ })
+ klog.V(4).InfoS("device allocate success", "pod", klog.KObj(pod), "allocate device", tmpDevs)
+ return true, tmpDevs, ""
+ }
+ klog.V(5).InfoS(common.AllocatedCardsInsufficientRequest, "pod", klog.KObj(pod), "request", originReq, "allocated", len(tmpDevs))
+ return false, tmpDevs, common.GenReason(reason, len(devices))
+}
diff --git a/pkg/device/awsneuron/device_test.go b/pkg/device/awsneuron/device_test.go
new file mode 100644
index 000000000..fe7aff060
--- /dev/null
+++ b/pkg/device/awsneuron/device_test.go
@@ -0,0 +1,943 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awsneuron
+
+import (
+ "testing"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "k8s.io/apimachinery/pkg/api/resource"
+)
+
+func Test_MutateAdmission(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }
+ want bool
+ err error
+ }{
+ {
+ name: "set neuron number",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "aws.amazon.com/neuron": *resource.NewQuantity(2, resource.DecimalSI),
+ },
+ },
+ },
+ p: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{},
+ },
+ },
+ want: true,
+ },
+ {
+ name: "set neuron cores",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "aws.amazon.com/neuroncore": *resource.NewQuantity(2, resource.DecimalSI),
+ },
+ },
+ },
+ p: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{},
+ },
+ },
+ want: true,
+ },
+ {
+ name: "no neuron devices",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ },
+ },
+ p: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{},
+ },
+ },
+ want: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ config := AWSNeuronConfig{
+ ResourceCountName: "aws.amazon.com/neuron",
+ ResourceCoreName: "aws.amazon.com/neuroncore",
+ }
+ dev := InitAWSNeuronDevice(config)
+ result, _ := dev.MutateAdmission(test.args.ctr, test.args.p)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_GetNodeDevices(t *testing.T) {
+ tests := []struct {
+ name string
+ args corev1.Node
+ want []*util.DeviceInfo
+ }{
+ {
+ name: "get node device",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "node.kubernetes.io/instance-type": "inf2",
+ },
+ Name: "test",
+ },
+ Status: corev1.NodeStatus{
+ Capacity: corev1.ResourceList{
+ "aws.amazon.com/neuron": *resource.NewQuantity(1, resource.DecimalSI),
+ "aws.amazon.com/neuroncore": *resource.NewQuantity(2, resource.DecimalSI),
+ },
+ },
+ },
+ want: []*util.DeviceInfo{
+ {
+ Index: uint(0),
+ ID: "test-AWSNeuron-0",
+ Count: int32(2),
+ Devmem: int32(0),
+ Devcore: int32(3),
+ Type: AWSNeuronDevice,
+ Numa: 0,
+ Health: true,
+ CustomInfo: map[string]any{"AWSNodeType": string("inf2")},
+ },
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ config := AWSNeuronConfig{
+ ResourceCountName: "aws.amazon.com/neuron",
+ ResourceCoreName: "aws.amazon.com/neuroncore",
+ }
+ dev := InitAWSNeuronDevice(config)
+ result, _ := dev.GetNodeDevices(test.args)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func Test_PatchAnnotations(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annoinput *map[string]string
+ pod corev1.Pod
+ pd util.PodDevices
+ }
+ want map[string]string
+ }{
+ {
+ name: "neuron device",
+ args: struct {
+ annoinput *map[string]string
+ pod corev1.Pod
+ pd util.PodDevices
+ }{
+ annoinput: &map[string]string{},
+ pod: corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "aws.amazon.com/neuron": resource.MustParse("2"),
+ },
+ },
+ },
+ },
+ },
+ },
+ pd: util.PodDevices{
+ AWSNeuronDevice: util.PodSingleDevice{
+ util.ContainerDevices{
+ {
+ Idx: 0,
+ UUID: "test1",
+ Type: AWSNeuronDevice,
+ Usedmem: int32(0),
+ Usedcores: int32(3),
+ CustomInfo: map[string]any{
+ AWSUsageInfo: 3,
+ },
+ },
+ {
+ Idx: 1,
+ UUID: "test2",
+ Type: AWSNeuronDevice,
+ Usedmem: int32(0),
+ Usedcores: int32(3),
+ CustomInfo: map[string]any{
+ AWSUsageInfo: 3,
+ },
+ },
+ },
+ },
+ },
+ },
+ want: map[string]string{
+ util.SupportDevices[AWSNeuronDevice]: "test1,AWSNeuron,0,2:test2,AWSNeuron,0,3;",
+ AWSNeuronAssignedIndex: "0,1",
+ AWSNeuronAssignedNode: "",
+ },
+ },
+ {
+ name: "neuroncore device",
+ args: struct {
+ annoinput *map[string]string
+ pod corev1.Pod
+ pd util.PodDevices
+ }{
+ annoinput: &map[string]string{},
+ pod: corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "aws.amazon.com/neuroncore": resource.MustParse("2"),
+ },
+ },
+ },
+ },
+ },
+ },
+ pd: util.PodDevices{
+ AWSNeuronDevice: util.PodSingleDevice{
+ util.ContainerDevices{
+ {
+ Idx: 0,
+ UUID: "test1",
+ Type: AWSNeuronDevice,
+ Usedmem: int32(0),
+ Usedcores: int32(3),
+ CustomInfo: map[string]any{
+ AWSUsageInfo: 3,
+ },
+ },
+ },
+ },
+ },
+ },
+ want: map[string]string{
+ util.SupportDevices[AWSNeuronDevice]: "test1,AWSNeuron,0,3:;",
+ AWSNeuronAssignedIndex: "0,1",
+ AWSNeuronAssignedNode: "",
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ config := AWSNeuronConfig{
+ ResourceCountName: "aws.amazon.com/neuron",
+ ResourceCoreName: "aws.amazon.com/neuroncore",
+ }
+ dev := InitAWSNeuronDevice(config)
+ dev.coresPerAWSNeuron = 2
+ result := dev.PatchAnnotations(&test.args.pod, test.args.annoinput, test.args.pd)
+ assert.Equal(t, result[dev.CommonWord()], test.want[dev.CommonWord()])
+ assert.Equal(t, result[AWSNeuronAssignedIndex], test.want[AWSNeuronAssignedIndex])
+ assert.Equal(t, result[AWSNeuronAssignedNode], test.want[AWSNeuronAssignedNode])
+ })
+ }
+}
+
+func Test_checkType(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }
+ want1 bool
+ want2 bool
+ want3 bool
+ }{
+ {
+ name: "the same type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: AWSNeuronDevice,
+ },
+ },
+ want1: true,
+ want2: true,
+ want3: false,
+ },
+ {
+ name: "the different type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: "test111",
+ },
+ },
+ want1: false,
+ want2: false,
+ want3: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := AWSNeuronDevices{}
+ result1, result2, result3 := dev.checkType(test.args.n)
+ assert.Equal(t, result1, test.want1)
+ assert.Equal(t, result2, test.want2)
+ assert.Equal(t, result3, test.want3)
+ })
+ }
+}
+
+func Test_checkUUID(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "no annos",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ },
+ want: true,
+ },
+ {
+ name: "use id the same as device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ AWSNeuronUseUUID: "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test1",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "use id the different from device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ AWSNeuronUseUUID: "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test2",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "no use id the same as device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ AWSNeuronNoUseUUID: "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test1",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "no use id the different from device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ AWSNeuronNoUseUUID: "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test2",
+ },
+ },
+ want: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := AWSNeuronDevices{}
+ result := dev.checkUUID(test.args.annos, test.args.d)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_GenerateResourceRequests(t *testing.T) {
+ tests := []struct {
+ name string
+ args *corev1.Container
+ want util.ContainerDeviceRequest
+ }{
+ {
+ name: "allocate neuron device",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "aws.amazon.com/neuron": resource.MustParse("1"),
+ },
+ Requests: corev1.ResourceList{
+ "aws.amazon.com/neuron": resource.MustParse("1"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: AWSNeuronDevice,
+ Memreq: int32(0),
+ MemPercentagereq: int32(0),
+ Coresreq: int32(2),
+ },
+ },
+ {
+ name: "allocate neuron core",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "aws.amazon.com/neuroncore": resource.MustParse("1"),
+ },
+ Requests: corev1.ResourceList{
+ "aws.amazon.com/neuroncore": resource.MustParse("1"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: AWSNeuronDevice,
+ Memreq: int32(0),
+ MemPercentagereq: int32(0),
+ Coresreq: int32(1),
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ config := AWSNeuronConfig{
+ ResourceCountName: "aws.amazon.com/neuron",
+ ResourceCoreName: "aws.amazon.com/neuroncore",
+ }
+ dev := InitAWSNeuronDevice(config)
+ dev.coresPerAWSNeuron = 2
+ result := dev.GenerateResourceRequests(test.args)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func Test_countMaskAvailable(t *testing.T) {
+ tests := []struct {
+ name string
+ args int32
+ want int32
+ }{
+ {
+ name: "test 3",
+ args: int32(3),
+ want: int32(2),
+ },
+ {
+ name: "test 2",
+ args: int32(2),
+ want: int32(1),
+ },
+ {
+ name: "test 1",
+ args: int32(1),
+ want: int32(1),
+ },
+ {
+ name: "test 0",
+ args: int32(0),
+ want: int32(0),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result1 := countMaskAvailable(test.args)
+ assert.DeepEqual(t, result1, test.want)
+ })
+ }
+}
+
+func Test_graphSelect(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ d []*util.DeviceUsage
+ c int
+ }
+ want1 []int
+ }{
+ {
+ name: "full allocate",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0, CustomInfo: map[string]any{
+ AWSNodeType: "inf2",
+ }},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 0},
+ {Index: 3, Used: 0},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 0},
+ {Index: 6, Used: 0},
+ {Index: 7, Used: 0},
+ {Index: 8, Used: 0},
+ {Index: 9, Used: 0},
+ {Index: 10, Used: 0},
+ {Index: 11, Used: 0},
+ {Index: 12, Used: 0},
+ {Index: 13, Used: 0},
+ {Index: 14, Used: 0},
+ {Index: 15, Used: 0},
+ },
+ c: 16,
+ },
+ want1: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
+ },
+ {
+ name: "8 allocate fail",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0, CustomInfo: map[string]any{
+ AWSNodeType: "trn",
+ }},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 0},
+ {Index: 3, Used: 0},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 0},
+ {Index: 6, Used: 0},
+ {Index: 7, Used: 1},
+ {Index: 8, Used: 0},
+ {Index: 9, Used: 0},
+ {Index: 10, Used: 0},
+ {Index: 11, Used: 0},
+ {Index: 12, Used: 0},
+ {Index: 13, Used: 1},
+ {Index: 14, Used: 0},
+ {Index: 15, Used: 0},
+ },
+ c: 8,
+ },
+ want1: []int{},
+ },
+ {
+ name: "8 allocate success",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0, CustomInfo: map[string]any{
+ AWSNodeType: "trn",
+ }},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 0},
+ {Index: 3, Used: 0},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 0},
+ {Index: 6, Used: 0},
+ {Index: 7, Used: 1},
+ {Index: 8, Used: 0},
+ {Index: 9, Used: 0},
+ {Index: 10, Used: 0},
+ {Index: 11, Used: 0},
+ {Index: 12, Used: 0},
+ {Index: 13, Used: 0},
+ {Index: 14, Used: 0},
+ {Index: 15, Used: 0},
+ },
+ c: 8,
+ },
+ want1: []int{8, 9, 10, 11, 12, 13, 14, 15},
+ },
+ {
+ name: "8 allocate",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0, CustomInfo: map[string]any{
+ AWSNodeType: "inf",
+ }},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 0},
+ {Index: 3, Used: 0},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 1},
+ {Index: 6, Used: 0},
+ {Index: 7, Used: 0},
+ {Index: 8, Used: 0},
+ {Index: 9, Used: 0},
+ {Index: 10, Used: 0},
+ {Index: 11, Used: 0},
+ {Index: 12, Used: 0},
+ {Index: 13, Used: 0},
+ {Index: 14, Used: 1},
+ {Index: 15, Used: 0},
+ },
+ c: 8,
+ },
+ want1: []int{6, 7, 8, 9, 10, 11, 12, 13},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result1 := graphSelect(test.args.d, test.args.c)
+ assert.DeepEqual(t, result1, test.want1)
+ })
+ }
+}
+
+func TestDevices_Fit(t *testing.T) {
+ config := AWSNeuronConfig{
+ ResourceCountName: "aws.amazon.com/neuron",
+ ResourceCoreName: "aws.amazon.com/neuroncore",
+ }
+ dev := InitAWSNeuronDevice(config)
+
+ tests := []struct {
+ name string
+ devices []*util.DeviceUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ wantFit bool
+ wantLen int
+ wantDevIDs []string
+ wantReason string
+ }{
+ {
+ name: "fit success",
+ devices: []*util.DeviceUsage{
+ {
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 2,
+ Usedmem: 0,
+ Totalmem: 0,
+ Totalcore: 3,
+ Usedcores: 0,
+ Numa: 0,
+ Type: AWSNeuronDevice,
+ Health: true,
+ CustomInfo: map[string]any{
+ AWSNodeType: "trn",
+ },
+ },
+ {
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 12,
+ Usedmem: 0,
+ Totalmem: 0,
+ Totalcore: 3,
+ Usedcores: 0,
+ Numa: 0,
+ Type: AWSNeuronDevice,
+ Health: true,
+ CustomInfo: map[string]any{
+ AWSNodeType: "trn",
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 0,
+ Coresreq: 2,
+ Type: AWSNeuronDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-1"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: memory not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 2,
+ Usedmem: 0,
+ Totalmem: 0,
+ Totalcore: 3,
+ Usedcores: 0,
+ Numa: 0,
+ Type: AWSNeuronDevice,
+ Health: true,
+ CustomInfo: map[string]any{
+ AWSNodeType: "trn",
+ },
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 0,
+ MemPercentagereq: 0,
+ Coresreq: 2,
+ Type: AWSNeuronDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 NumaNotFit",
+ },
+ {
+ name: "fit fail: core not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 2,
+ Usedmem: 0,
+ Totalmem: 0,
+ Totalcore: 3,
+ Usedcores: 1,
+ Numa: 0,
+ Type: AWSNeuronDevice,
+ Health: true,
+ CustomInfo: map[string]any{
+ AWSNodeType: "trn",
+ },
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 0,
+ Coresreq: 2,
+ Type: AWSNeuronDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientCore",
+ },
+ {
+ name: "fit fail: type mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 2,
+ Usedmem: 0,
+ Totalmem: 0,
+ Totalcore: 3,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ Type: AWSNeuronDevice,
+ CustomInfo: map[string]any{
+ AWSNodeType: "trn",
+ },
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: "OtherType",
+ Memreq: 0,
+ MemPercentagereq: 0,
+ Coresreq: 2,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTypeMismatch",
+ },
+ {
+ name: "fit fail: user assign use uuid mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 2,
+ Usedmem: 0,
+ Totalmem: 0,
+ Totalcore: 3,
+ Usedcores: 0,
+ Numa: 0,
+ Type: AWSNeuronDevice,
+ Health: true,
+ CustomInfo: map[string]any{
+ AWSNodeType: "trn",
+ },
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 0,
+ Coresreq: 2,
+ Type: AWSNeuronDevice,
+ },
+ annos: map[string]string{"aws.amazon.com/use-neuron-uuid": "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: user assign no use uuid match",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 2,
+ Usedmem: 0,
+ Totalmem: 0,
+ Totalcore: 3,
+ Usedcores: 0,
+ Numa: 0,
+ Type: AWSNeuronDevice,
+ Health: true,
+ CustomInfo: map[string]any{
+ AWSNodeType: "trn",
+ },
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 0,
+ Coresreq: 2,
+ Type: AWSNeuronDevice,
+ },
+ annos: map[string]string{"aws.amazon.com/nouse-neuron-uuid": "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: card overused",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 2,
+ Count: 2,
+ Usedmem: 0,
+ Totalmem: 0,
+ Totalcore: 3,
+ Usedcores: 0,
+ Numa: 0,
+ Type: AWSNeuronDevice,
+ Health: true,
+ CustomInfo: map[string]any{
+ AWSNodeType: "trn",
+ },
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 0,
+ Coresreq: 2,
+ Type: AWSNeuronDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTimeSlicingExhausted",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ allocated := &util.PodDevices{}
+ fit, result, reason := dev.Fit(test.devices, test.request, test.annos, &corev1.Pod{}, &util.NodeInfo{}, allocated)
+ if fit != test.wantFit {
+ t.Errorf("Fit: got %v, want %v", fit, test.wantFit)
+ }
+ if test.wantFit {
+ if len(result[AWSNeuronDevice]) != test.wantLen {
+ t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[AWSNeuronDevice]))
+ }
+ for idx, id := range test.wantDevIDs {
+ if id != result[AWSNeuronDevice][idx].UUID {
+ t.Errorf("expected device id: %s, got device id %s", id, result[AWSNeuronDevice][idx].UUID)
+ }
+ }
+ }
+ if reason != test.wantReason {
+ t.Errorf("expected reason: %s, got reason: %s", test.wantReason, reason)
+ }
+ })
+ }
+}
diff --git a/pkg/device/cambricon/device.go b/pkg/device/cambricon/device.go
new file mode 100644
index 000000000..4c060d541
--- /dev/null
+++ b/pkg/device/cambricon/device.go
@@ -0,0 +1,427 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cambricon
+
+import (
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "math/rand"
+ "slices"
+ "strings"
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/common"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/klog/v2"
+)
+
+const (
+ CambriconMLUDevice = "MLU"
+ CambriconMLUCommonWord = "MLU"
+ MluMemSplitLimit = "CAMBRICON_SPLIT_MEMS"
+ MluMemSplitIndex = "CAMBRICON_SPLIT_VISIBLE_DEVICES"
+ MluMemSplitEnable = "CAMBRICON_SPLIT_ENABLE"
+ MLUInUse = "cambricon.com/use-mlutype"
+ MLUNoUse = "cambricon.com/nouse-mlutype"
+ // MLUUseUUID is user can use specify MLU device for set MLU UUID.
+ MLUUseUUID = "cambricon.com/use-gpuuuid"
+ // MLUNoUseUUID is user can not use specify MLU device for set MLU UUID.
+ MLUNoUseUUID = "cambricon.com/nouse-gpuuuid"
+ DsmluLockTime = "cambricon.com/dsmlu.lock"
+ DsmluProfile = "CAMBRICON_DSMLU_PROFILE"
+ DsmluResourceAssigned = "CAMBRICON_DSMLU_ASSIGHED"
+ retry = 5
+)
+
+var (
+ MLUResourceCount string
+ MLUResourceMemory string
+ MLUResourceCores string
+)
+
+type CambriconConfig struct {
+ ResourceCountName string `yaml:"resourceCountName"`
+ ResourceMemoryName string `yaml:"resourceMemoryName"`
+ ResourceCoreName string `yaml:"resourceCoreName"`
+}
+
+type CambriconDevices struct {
+}
+
+func ParseConfig(fs *flag.FlagSet) {
+ fs.StringVar(&MLUResourceCount, "cambricon-mlu-name", "cambricon.com/mlu", "cambricon mlu resource count")
+ fs.StringVar(&MLUResourceMemory, "cambricon-mlu-memory", "cambricon.com/mlu.smlu.vmemory", "cambricon mlu memory resource")
+ fs.StringVar(&MLUResourceCores, "cambricon-mlu-cores", "cambricon.com/mlu.smlu.vcore", "cambricon mlu core resource")
+}
+
+func InitMLUDevice(config CambriconConfig) *CambriconDevices {
+ MLUResourceCount = config.ResourceCountName
+ MLUResourceMemory = config.ResourceMemoryName
+ MLUResourceCores = config.ResourceCoreName
+ util.InRequestDevices[CambriconMLUDevice] = "hami.io/cambricon-mlu-devices-to-allocate"
+ util.SupportDevices[CambriconMLUDevice] = "hami.io/cambricon-mlu-devices-allocated"
+ return &CambriconDevices{}
+}
+
+func (dev *CambriconDevices) CommonWord() string {
+ return CambriconMLUCommonWord
+}
+
+func (dev *CambriconDevices) setNodeLock(node *corev1.Node) error {
+ ctx := context.Background()
+ if _, ok := node.Annotations[DsmluLockTime]; ok {
+ return fmt.Errorf("node %s is locked", node.Name)
+ }
+
+ patchedAnnotation, err := json.Marshal(
+ map[string]any{
+ "metadata": map[string]map[string]string{"annotations": {
+ DsmluLockTime: time.Now().Format(time.RFC3339),
+ }}})
+ if err != nil {
+ klog.ErrorS(err, "Failed to patch node annotation", "node", node.Name)
+ return fmt.Errorf("patch node annotation %v", err)
+ }
+
+ _, err = client.GetClient().CoreV1().Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, patchedAnnotation, metav1.PatchOptions{})
+ for i := 0; i < retry && err != nil; i++ {
+ klog.ErrorS(err, "Failed to patch node annotation", "node", node.Name, "retry", i)
+ time.Sleep(time.Duration(rand.Intn(i+1)) * 10 * time.Millisecond)
+ _, err = client.GetClient().CoreV1().Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, patchedAnnotation, metav1.PatchOptions{})
+ }
+ if err != nil {
+ return fmt.Errorf("setNodeLock exceeds retry count %d", retry)
+ }
+ klog.InfoS("Node lock set", "node", node.Name)
+ return nil
+}
+
+func (dev *CambriconDevices) LockNode(n *corev1.Node, p *corev1.Pod) error {
+ found := false
+ for _, val := range p.Spec.Containers {
+ if (dev.GenerateResourceRequests(&val).Nums) > 0 {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil
+ }
+ if _, ok := n.Annotations[DsmluLockTime]; !ok {
+ return dev.setNodeLock(n)
+ }
+ lockTime, err := time.Parse(time.RFC3339, n.Annotations[DsmluLockTime])
+ if err != nil {
+ return err
+ }
+ if time.Since(lockTime) > time.Minute*2 {
+ klog.InfoS("Node lock expired", "node", n.Name, "lockTime", lockTime)
+ err = dev.ReleaseNodeLock(n, p)
+ if err != nil {
+ klog.ErrorS(err, "Failed to release node lock", "node", n.Name)
+ return err
+ }
+ return dev.setNodeLock(n)
+ }
+ return fmt.Errorf("node %s has been locked within 2 minutes", n.Name)
+}
+
+func (dev *CambriconDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
+ if n.Annotations == nil {
+ return nil
+ }
+ if _, ok := n.Annotations[DsmluLockTime]; !ok {
+ klog.InfoS("Node lock not set", "node", n.Name)
+ return nil
+ }
+
+ newNode := n.DeepCopy()
+ delete(newNode.Annotations, DsmluLockTime)
+ _, err := client.GetClient().CoreV1().Nodes().Update(context.Background(), newNode, metav1.UpdateOptions{})
+ for i := 0; i < retry && err != nil; i++ {
+ klog.ErrorS(err, "Failed to patch node annotation", "node", n.Name, "retry", i)
+ time.Sleep(time.Duration(rand.Intn(i+1)) * 10 * time.Millisecond)
+ _, err = client.GetClient().CoreV1().Nodes().Update(context.Background(), newNode, metav1.UpdateOptions{})
+ }
+ if err != nil {
+ return fmt.Errorf("releaseNodeLock exceeds retry count %d", retry)
+ }
+ delete(n.Annotations, DsmluLockTime)
+ klog.InfoS("Node lock released", "node", n.Name)
+ return nil
+}
+
+func (dev *CambriconDevices) NodeCleanUp(nn string) error {
+ return nil
+}
+
+func (dev *CambriconDevices) CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ return true, true
+}
+
+func (dev *CambriconDevices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) {
+ nodedevices := []*util.DeviceInfo{}
+ i := 0
+ cards, ok := n.Status.Capacity.Name(corev1.ResourceName(MLUResourceCores), resource.DecimalSI).AsInt64()
+ if !ok || cards == 0 {
+ return []*util.DeviceInfo{}, fmt.Errorf("device not found %s", MLUResourceCores)
+ }
+ memoryTotal, _ := n.Status.Capacity.Name(corev1.ResourceName(MLUResourceMemory), resource.DecimalSI).AsInt64()
+ for int64(i)*100 < cards {
+ nodedevices = append(nodedevices, &util.DeviceInfo{
+ Index: uint(i),
+ ID: n.Name + "-cambricon-mlu-" + fmt.Sprint(i),
+ Count: 100,
+ Devmem: int32(memoryTotal * 256 * 100 / cards),
+ Devcore: 100,
+ Type: CambriconMLUDevice,
+ Numa: 0,
+ Health: true,
+ })
+ i++
+ }
+ return nodedevices, nil
+}
+
+func (dev *CambriconDevices) AssertNuma(annos map[string]string) bool {
+ return false
+}
+
+func (dev *CambriconDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
+ _, ok := ctr.Resources.Limits[corev1.ResourceName(MLUResourceCount)]
+ return ok, nil
+}
+
+func (dev *CambriconDevices) checkType(annos map[string]string, d util.DeviceUsage, n util.ContainerDeviceRequest) (bool, bool, bool) {
+ if strings.Compare(n.Type, CambriconMLUDevice) == 0 {
+ return true, true, false
+ }
+ return false, false, false
+}
+
+func (dev *CambriconDevices) checkUUID(annos map[string]string, d util.DeviceUsage) bool {
+ userUUID, ok := annos[MLUUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for mlu user uuid [%s], device id is %s", userUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ userUUIDs := strings.Split(userUUID, ",")
+ return slices.Contains(userUUIDs, d.ID)
+ }
+
+ noUserUUID, ok := annos[MLUNoUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for mlu not user uuid [%s], device id is %s", noUserUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ noUserUUIDs := strings.Split(noUserUUID, ",")
+ return !slices.Contains(noUserUUIDs, d.ID)
+ }
+ return true
+}
+
+func (dev *CambriconDevices) checkIndex(annos map[string]string, d util.DeviceUsage) bool {
+ return true
+}
+
+func (dev *CambriconDevices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest {
+ klog.Info("Start to count mlu devices for container ", ctr.Name)
+ mluResourceCount := corev1.ResourceName(MLUResourceCount)
+ mluResourceMem := corev1.ResourceName(MLUResourceMemory)
+ mluResourceCores := corev1.ResourceName(MLUResourceCores)
+ for idx, val := range ctr.Resources.Limits {
+ klog.Infoln("idx=", idx, "val=", val, ctr.Resources.Limits[mluResourceMem])
+ }
+ v, ok := ctr.Resources.Limits[mluResourceCount]
+ if !ok {
+ v, ok = ctr.Resources.Requests[mluResourceCount]
+ }
+ if ok {
+ if n, ok := v.AsInt64(); ok {
+ klog.Info("Found cambricon devices")
+ memnum := 0
+ mem, ok := ctr.Resources.Limits[mluResourceMem]
+ if !ok {
+ mem, ok = ctr.Resources.Requests[mluResourceMem]
+ }
+ klog.Infoln("mluResourceMem", mem, "ok=", ok, "memoryname=", mluResourceMem)
+ if ok {
+ memnums, ok := mem.AsInt64()
+ klog.Infoln("mluResourceMem", mem, memnums)
+ if ok {
+ memnum = int(memnums) * 256
+ }
+ }
+ corenum := int32(100)
+ core, ok := ctr.Resources.Limits[mluResourceCores]
+ if !ok {
+ core, ok = ctr.Resources.Requests[mluResourceCores]
+ }
+ if ok {
+ corenums, ok := core.AsInt64()
+ if ok {
+ corenum = int32(corenums)
+ }
+ }
+
+ mempnum := 0
+ if memnum == 0 {
+ mempnum = 100
+ }
+
+ return util.ContainerDeviceRequest{
+ Nums: int32(n),
+ Type: CambriconMLUDevice,
+ Memreq: int32(memnum),
+ MemPercentagereq: int32(mempnum),
+ Coresreq: corenum,
+ }
+ }
+ }
+ return util.ContainerDeviceRequest{
+ Nums: 0,
+ }
+}
+
+func (dev *CambriconDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd util.PodDevices) map[string]string {
+ devlist, ok := pd[CambriconMLUDevice]
+ if ok {
+ (*annoinput)[DsmluResourceAssigned] = "false"
+ (*annoinput)[DsmluProfile] = fmt.Sprintf("%d_%d_%d", devlist[0][0].Idx, devlist[0][0].Usedcores, devlist[0][0].Usedmem/256)
+ deviceStr := util.EncodePodSingleDevice(devlist)
+ (*annoinput)[util.InRequestDevices[CambriconMLUDevice]] = deviceStr
+ (*annoinput)[util.SupportDevices[CambriconMLUDevice]] = deviceStr
+ klog.V(5).Infof("pod add notation key [%s], values is [%s]", util.InRequestDevices[CambriconMLUDevice], deviceStr)
+ klog.V(5).Infof("pod add notation key [%s], values is [%s]", util.SupportDevices[CambriconMLUDevice], deviceStr)
+ return *annoinput
+ }
+ return *annoinput
+}
+
+func (dev *CambriconDevices) ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32 {
+ return 0
+}
+
+func (dev *CambriconDevices) AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error {
+ n.Used++
+ n.Usedcores += ctr.Usedcores
+ n.Usedmem += ctr.Usedmem
+ return nil
+}
+
+func (cam *CambriconDevices) Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string) {
+ k := request
+ originReq := k.Nums
+ prevnuma := -1
+ klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
+ var tmpDevs map[string]util.ContainerDevices
+ tmpDevs = make(map[string]util.ContainerDevices)
+ reason := make(map[string]int)
+ for i := len(devices) - 1; i >= 0; i-- {
+ dev := devices[i]
+ klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
+
+ _, found, numa := cam.checkType(annos, *dev, k)
+ if !found {
+ reason[common.CardTypeMismatch]++
+ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, k.Type)
+ continue
+ }
+ if numa && prevnuma != dev.Numa {
+ if k.Nums != originReq {
+ reason[common.NumaNotFit] += len(tmpDevs)
+ klog.V(5).InfoS(common.NumaNotFit, "pod", klog.KObj(pod), "device", dev.ID, "k.nums", k.Nums, "numa", numa, "prevnuma", prevnuma, "device numa", dev.Numa)
+ }
+ k.Nums = originReq
+ prevnuma = dev.Numa
+ tmpDevs = make(map[string]util.ContainerDevices)
+ }
+ if !cam.checkUUID(annos, *dev) {
+ reason[common.CardUUIDMismatch]++
+ klog.V(5).InfoS(common.CardUUIDMismatch, "pod", klog.KObj(pod), "device", dev.ID, "current device info is:", *dev)
+ continue
+ }
+
+ memreq := int32(0)
+ if dev.Count <= dev.Used {
+ reason[common.CardTimeSlicingExhausted]++
+ klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
+ continue
+ }
+ if k.Coresreq > 100 {
+ klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Coresreq = 100
+ //return false, tmpDevs
+ }
+ if k.Memreq > 0 {
+ memreq = k.Memreq
+ }
+ if k.MemPercentagereq != 101 && k.Memreq == 0 {
+ //This incurs an issue
+ memreq = dev.Totalmem * k.MemPercentagereq / 100
+ }
+ if dev.Totalmem-dev.Usedmem < memreq {
+ reason[common.CardInsufficientMemory]++
+ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq)
+ continue
+ }
+ if dev.Totalcore-dev.Usedcores < k.Coresreq {
+ reason[common.CardInsufficientCore]++
+ klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", k.Coresreq)
+ continue
+ }
+ // Coresreq=100 indicates it want this card exclusively
+ if dev.Totalcore == 100 && k.Coresreq == 100 && dev.Used > 0 {
+ reason[common.ExclusiveDeviceAllocateConflict]++
+ klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
+ continue
+ }
+ // You can't allocate core=0 job to an already full GPU
+ if dev.Totalcore != 0 && dev.Usedcores == dev.Totalcore && k.Coresreq == 0 {
+ reason[common.CardComputeUnitsExhausted]++
+ klog.V(5).InfoS(common.CardComputeUnitsExhausted, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+
+ if k.Nums > 0 {
+ klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Nums--
+ tmpDevs[k.Type] = append(tmpDevs[k.Type], util.ContainerDevice{
+ Idx: int(dev.Index),
+ UUID: dev.ID,
+ Type: k.Type,
+ Usedmem: memreq,
+ Usedcores: k.Coresreq,
+ })
+ }
+ if k.Nums == 0 {
+ klog.V(4).InfoS("device allocate success", "pod", klog.KObj(pod), "allocate device", tmpDevs)
+ return true, tmpDevs, ""
+ }
+ }
+ if len(tmpDevs) > 0 {
+ reason[common.AllocatedCardsInsufficientRequest] = len(tmpDevs)
+ klog.V(5).InfoS(common.AllocatedCardsInsufficientRequest, "pod", klog.KObj(pod), "request", originReq, "allocated", len(tmpDevs))
+ }
+ return false, tmpDevs, common.GenReason(reason, len(devices))
+}
diff --git a/pkg/device/cambricon/device_test.go b/pkg/device/cambricon/device_test.go
new file mode 100644
index 000000000..094087fb8
--- /dev/null
+++ b/pkg/device/cambricon/device_test.go
@@ -0,0 +1,1146 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cambricon
+
+import (
+ "context"
+ "flag"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes/fake"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+)
+
+func Test_GetNodeDevices(t *testing.T) {
+ config := CambriconConfig{
+ ResourceMemoryName: "cambricon.com/mlu.smlu.vmemory",
+ ResourceCoreName: "cambricon.com/mlu.smlu.vcore",
+ }
+ InitMLUDevice(config)
+ tests := []struct {
+ name string
+ args corev1.Node
+ want []*util.DeviceInfo
+ }{
+ {
+ name: "test with vaild configuration",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test",
+ },
+ Status: corev1.NodeStatus{
+ Capacity: corev1.ResourceList{
+ "cambricon.com/mlu.smlu.vcore": *resource.NewQuantity(1, resource.DecimalSI),
+ "cambricon.com/mlu.smlu.vmemory": *resource.NewQuantity(1, resource.DecimalSI),
+ },
+ },
+ },
+ want: []*util.DeviceInfo{
+ {
+ Index: 0,
+ ID: "test-cambricon-mlu-0",
+ Count: int32(100),
+ Devmem: int32(25600),
+ Devcore: int32(100),
+ Type: CambriconMLUDevice,
+ Numa: 0,
+ Health: true,
+ },
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := CambriconDevices{}
+ result, err := dev.GetNodeDevices(test.args)
+ if err != nil {
+ assert.NoError(t, err)
+ }
+ for k, v := range test.want {
+ assert.Equal(t, v, result[k])
+ }
+ })
+ }
+}
+
+func Test_MutateAdmission(t *testing.T) {
+ config := CambriconConfig{
+ ResourceMemoryName: "cambricon.com/mlu.smlu.vmemory",
+ ResourceCoreName: "cambricon.com/mlu.smlu.vcore",
+ ResourceCountName: "cambricon.com/mlu",
+ }
+ InitMLUDevice(config)
+ tests := []struct {
+ name string
+ args struct {
+ ctr corev1.Container
+ pod corev1.Pod
+ }
+ want bool
+ err error
+ }{
+ {
+ name: "set to resources limits",
+ args: struct {
+ ctr corev1.Container
+ pod corev1.Pod
+ }{
+ ctr: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "cambricon.com/mlu": resource.MustParse("1"),
+ },
+ },
+ },
+ pod: corev1.Pod{},
+ },
+ want: true,
+ err: nil,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := CambriconDevices{}
+ result, _ := dev.MutateAdmission(&test.args.ctr, &test.args.pod)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_ParseConfig(t *testing.T) {
+ tests := []struct {
+ name string
+ args flag.FlagSet
+ }{
+ {
+ name: "test",
+ args: flag.FlagSet{},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ ParseConfig(&test.args)
+ })
+ }
+}
+
+func Test_checkType(t *testing.T) {
+ dev := CambriconDevices{}
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }
+ want1 bool
+ want2 bool
+ }{
+ {
+ name: "the same type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: dev.CommonWord(),
+ },
+ },
+ want1: true,
+ want2: true,
+ },
+ {
+ name: "the different type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: "TEST",
+ },
+ },
+ want1: false,
+ want2: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result1, result2, _ := dev.checkType(test.args.annos, test.args.d, test.args.n)
+ assert.Equal(t, result1, test.want1)
+ assert.Equal(t, result2, test.want2)
+ })
+ }
+}
+
+func Test_checkUUID(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "don't set UserUUID,NoUserUUID and annotation",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{annos: map[string]string{},
+ d: util.DeviceUsage{},
+ },
+ want: true,
+ },
+ {
+ name: "set UserUUID and annotation, don't set NoUserUUID",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "cambricon.com/use-gpuuuid": "test123,111",
+ },
+ d: util.DeviceUsage{
+ ID: "test123",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "don't set UserUUID, set NoUserUUID and annotation",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "cambricon.com/nouse-gpuuuid": "test123,111",
+ },
+ d: util.DeviceUsage{
+ ID: "test123",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "set UserUUID, don't set NoUserUUID,annotation and device not match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "cambricon.com/nouse-gpuuuid": "test123,111",
+ },
+ d: util.DeviceUsage{
+ ID: "test456",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "don't set UserUUID,set NoUserUUID,annotation and device not match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "cambricon.com/use-gpuuuid": "test123,111",
+ },
+ d: util.DeviceUsage{
+ ID: "test456",
+ },
+ },
+ want: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := CambriconDevices{}
+ result := dev.checkUUID(test.args.annos, test.args.d)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_GenerateResourceRequests(t *testing.T) {
+ tests := []struct {
+ name string
+ args corev1.Container
+ want util.ContainerDeviceRequest
+ }{
+ {
+ name: "don't set to limits and request",
+ args: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ Requests: corev1.ResourceList{},
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: 0,
+ },
+ },
+ {
+ name: "resourcecoresname,resourcecountname and resourcememoryname set to limits and request",
+ args: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "cambricon.com/mlu": resource.MustParse("1"),
+ "cambricon.com/mlu.smlu.vmemory": resource.MustParse("1000"),
+ "cambricon.com/mlu.smlu.vcore": resource.MustParse("2"),
+ },
+ Requests: corev1.ResourceList{
+ "cambricon.com/mlu": resource.MustParse("1"),
+ "cambricon.com/mlu.smlu.vmemory": resource.MustParse("1000"),
+ "cambricon.com/mlu.smlu.vcore": resource.MustParse("2"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: CambriconMLUDevice,
+ Memreq: int32(256000),
+ MemPercentagereq: int32(0),
+ Coresreq: int32(2),
+ },
+ },
+ {
+ name: "resourcememoryname don't set to limits and requests",
+ args: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "cambricon.com/mlu": resource.MustParse("1"),
+ "cambricon.com/mlu.smlu.vmemory": resource.MustParse("1000"),
+ },
+ Requests: corev1.ResourceList{
+ "cambricon.com/mlu": resource.MustParse("1"),
+ "cambricon.com/mlu.smlu.vmemory": resource.MustParse("1000"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: CambriconMLUDevice,
+ Memreq: int32(256000),
+ MemPercentagereq: int32(0),
+ Coresreq: int32(100),
+ },
+ },
+ {
+ name: "resourcecoresname don't set to limits and requests",
+ args: corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "cambricon.com/mlu": resource.MustParse("1"),
+ "cambricon.com/mlu.smlu.vcore": resource.MustParse("2"),
+ },
+ Requests: corev1.ResourceList{
+ "cambricon.com/mlu": resource.MustParse("1"),
+ "cambricon.com/mlu.smlu.vcore": resource.MustParse("2"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: CambriconMLUDevice,
+ Memreq: int32(0),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(2),
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ config := CambriconConfig{
+ ResourceMemoryName: "cambricon.com/mlu.smlu.vmemory",
+ ResourceCoreName: "cambricon.com/mlu.smlu.vcore",
+ ResourceCountName: "cambricon.com/mlu",
+ }
+ InitMLUDevice(config)
+ dev := CambriconDevices{}
+ result := dev.GenerateResourceRequests(&test.args)
+ assert.Equal(t, test.want, result)
+ })
+ }
+}
+
+func Test_PatchAnnotations(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annoinput map[string]string
+ pd util.PodDevices
+ }
+ want map[string]string
+ }{
+ {
+ name: "exist device",
+ args: struct {
+ annoinput map[string]string
+ pd util.PodDevices
+ }{
+ annoinput: map[string]string{},
+ pd: util.PodDevices{
+ CambriconMLUDevice: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ Idx: 0,
+ UUID: "device-0",
+ Type: "MLU",
+ Usedcores: 1,
+ Usedmem: 256000,
+ },
+ },
+ },
+ },
+ },
+ want: map[string]string{
+ "CAMBRICON_DSMLU_ASSIGHED": "false",
+ "CAMBRICON_DSMLU_PROFILE": "0_1_1000",
+ "hami.io/cambricon-mlu-devices-to-allocate": "device-0,MLU,256000,1:;",
+ "hami.io/cambricon-mlu-devices-allocated": "device-0,MLU,256000,1:;",
+ },
+ },
+ {
+ name: "no device",
+ args: struct {
+ annoinput map[string]string
+ pd util.PodDevices
+ }{
+ annoinput: map[string]string{},
+ pd: util.PodDevices{},
+ },
+ want: map[string]string{},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := CambriconDevices{}
+ result := dev.PatchAnnotations(&corev1.Pod{}, &test.args.annoinput, test.args.pd)
+ assert.Equal(t, len(test.want), len(result), "Expected length of result to match want")
+ for k, v := range test.want {
+ assert.Equal(t, v, result[k], "pod add annotation key [%s], values is [%s]", k, result[k])
+ }
+ })
+ }
+}
+
+func Test_setNodeLock(t *testing.T) {
+ tests := []struct {
+ name string
+ node corev1.Node
+ expectErr bool
+ expectMsg string
+ }{
+ {
+ name: "node is locked",
+ node: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-01",
+ Annotations: map[string]string{
+ "cambricon.com/dsmlu.lock": "test123",
+ },
+ },
+ },
+ expectErr: true,
+ expectMsg: "node node-01 is locked",
+ },
+ {
+ name: "set node lock",
+ node: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-02",
+ Annotations: map[string]string{},
+ },
+ },
+ expectErr: false,
+ },
+ }
+
+ client.KubeClient = fake.NewClientset()
+ k8sClient := client.GetClient()
+ if k8sClient != nil {
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ ctx := context.Background()
+
+ defer func() {
+ if tt.node.Name != "" {
+ // Delete the node to clean up
+ err := k8sClient.CoreV1().Nodes().Delete(ctx, tt.node.Name, metav1.DeleteOptions{})
+ if err != nil {
+ t.Errorf("failed to delete node %s: %v", tt.node.Name, err)
+ }
+ }
+ }()
+
+ _, err := k8sClient.CoreV1().Nodes().Create(ctx, &tt.node, metav1.CreateOptions{})
+ if err != nil {
+ t.Fatalf("failed to create node %s: %v", tt.node.Name, err)
+ }
+
+ dev := CambriconDevices{}
+ err = dev.setNodeLock(&tt.node)
+
+ if tt.expectErr {
+ if err == nil {
+ t.Errorf("expected error but got none")
+ } else if !strings.Contains(err.Error(), tt.expectMsg) {
+ t.Errorf("expected error to contain '%s' but got '%s'", tt.expectMsg, err.Error())
+ }
+ } else {
+ if err != nil {
+ t.Errorf("did not expect error but got %v", err)
+ }
+ }
+ })
+ }
+ }
+}
+
+// Setup function to initialize resources for each test case.
+func setupTest(t *testing.T) (*corev1.Node, *corev1.Pod, func(), *fake.Clientset) {
+ ctx := context.Background()
+
+ clientset := fake.NewClientset()
+
+ node := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-node",
+ },
+ Status: corev1.NodeStatus{
+ Capacity: corev1.ResourceList{
+ corev1.ResourceName(MLUResourceCount): resource.MustParse("2"),
+ corev1.ResourceName(MLUResourceMemory): resource.MustParse("4096"),
+ corev1.ResourceName(MLUResourceCores): resource.MustParse("200"),
+ },
+ },
+ }
+ pod := &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{{
+ Name: "test-container",
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ corev1.ResourceName(MLUResourceCount): resource.MustParse("1"),
+ corev1.ResourceName(MLUResourceMemory): resource.MustParse("2048"),
+ corev1.ResourceName(MLUResourceCores): resource.MustParse("100"),
+ },
+ Requests: corev1.ResourceList{
+ corev1.ResourceName(MLUResourceCount): resource.MustParse("1"),
+ corev1.ResourceName(MLUResourceMemory): resource.MustParse("2048"),
+ corev1.ResourceName(MLUResourceCores): resource.MustParse("100"),
+ },
+ },
+ }},
+ },
+ }
+
+ config := CambriconConfig{
+ ResourceCountName: MLUResourceCount,
+ ResourceMemoryName: MLUResourceMemory,
+ ResourceCoreName: MLUResourceCores,
+ }
+ InitMLUDevice(config)
+
+ _, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{})
+ if err != nil {
+ t.Fatalf("Failed to create node: %v", err)
+ }
+
+ return node, pod, func() {
+ clientset.CoreV1().Nodes().Delete(ctx, node.Name, metav1.DeleteOptions{})
+ }, clientset
+}
+
+func Test_LockNode(t *testing.T) {
+ tests := []struct {
+ name string
+ annotations map[string]string
+ wantErr bool
+ }{
+ {
+ name: "node is not locked",
+ annotations: map[string]string{},
+ wantErr: false,
+ },
+ {
+ name: "node is already locked within 2 minutes",
+ annotations: map[string]string{
+ DsmluLockTime: time.Now().Add(-time.Minute).Format(time.RFC3339),
+ },
+ wantErr: true,
+ },
+ {
+ name: "lock time expired (more than 2 minutes)",
+ annotations: map[string]string{
+ DsmluLockTime: time.Now().Add(-time.Hour).Format(time.RFC3339),
+ },
+ wantErr: false,
+ },
+ {
+ name: "invalid lock time format",
+ annotations: map[string]string{
+ DsmluLockTime: "invalid-format",
+ },
+ wantErr: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ node, pod, teardown, clientset := setupTest(t)
+ client.KubeClient = clientset
+ defer teardown()
+
+ // Set up the node with the specified annotations.
+ node.Annotations = tt.annotations
+
+ dev := InitMLUDevice(CambriconConfig{
+ ResourceCountName: MLUResourceCount,
+ ResourceMemoryName: MLUResourceMemory,
+ ResourceCoreName: MLUResourceCores,
+ })
+
+ err := dev.LockNode(node, pod)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("LockNode() error = %v, wantErr %v", err, tt.wantErr)
+ }
+
+ // Optionally check if the node was correctly patched with the lock annotation.
+ if !tt.wantErr {
+ fetchedNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), node.Name, metav1.GetOptions{})
+ if _, ok := fetchedNode.Annotations[DsmluLockTime]; !ok && !tt.wantErr {
+ t.Error("Expected node to be locked but it wasn't")
+ }
+ }
+ })
+ }
+}
+
+func Test_ReleaseNodeLock(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ node corev1.Node
+ pod corev1.Pod
+ }
+ err error
+ }{
+ {
+ name: "no annation",
+ args: struct {
+ node corev1.Node
+ pod corev1.Pod
+ }{
+ node: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-01",
+ },
+ },
+ pod: corev1.Pod{},
+ },
+ err: nil,
+ },
+ {
+ name: "annation no lock value",
+ args: struct {
+ node corev1.Node
+ pod corev1.Pod
+ }{
+ node: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-02",
+ Annotations: map[string]string{
+ "test": "test123",
+ },
+ },
+ },
+ pod: corev1.Pod{},
+ },
+ err: nil,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := CambriconDevices{}
+ result := dev.ReleaseNodeLock(&test.args.node, &test.args.pod)
+ assert.Equal(t, test.err, result)
+ })
+ }
+}
+
+func TestDevices_Fit(t *testing.T) {
+ config := CambriconConfig{
+ ResourceCountName: "cambricon.com/mlu",
+ ResourceMemoryName: "cambricon.com/mlu.smlu.vmemory",
+ ResourceCoreName: "cambricon.com/mlu.smlu.vcore",
+ }
+ dev := InitMLUDevice(config)
+
+ tests := []struct {
+ name string
+ devices []*util.DeviceUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ wantFit bool
+ wantLen int
+ wantDevIDs []string
+ wantReason string
+ }{
+ {
+ name: "fit success",
+ devices: []*util.DeviceUsage{
+ {
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ },
+ {
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 64,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: CambriconMLUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-1"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: memory not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: CambriconMLUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientMemory",
+ },
+ {
+ name: "fit fail: core not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1024,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: CambriconMLUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientCore",
+ },
+ {
+ name: "fit fail: type mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ Type: CambriconMLUDevice,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: "OtherType",
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTypeMismatch",
+ },
+ {
+ name: "fit fail: user assign use uuid mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: CambriconMLUDevice,
+ },
+ annos: map[string]string{"cambricon.com/use-gpuuuid": "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: user assign no use uuid match",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: CambriconMLUDevice,
+ },
+ annos: map[string]string{"cambricon.com/nouse-gpuuuid": "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: card overused",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 100,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: CambriconMLUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTimeSlicingExhausted",
+ },
+ {
+ name: "fit success: but core limit can't exceed 100",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 120,
+ Type: CambriconMLUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: card exclusively",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 100,
+ Type: CambriconMLUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 ExclusiveDeviceAllocateConflict",
+ },
+ {
+ name: "fit fail: CardComputeUnitsExhausted",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 0,
+ Type: CambriconMLUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardComputeUnitsExhausted",
+ },
+ {
+ name: "fit fail: AllocatedCardsInsufficientRequest",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 20,
+ Type: CambriconMLUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 AllocatedCardsInsufficientRequest",
+ },
+ {
+ name: "fit success: memory percentage",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: CambriconMLUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 10,
+ Coresreq: 20,
+ Type: CambriconMLUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ allocated := &util.PodDevices{}
+ fit, result, reason := dev.Fit(test.devices, test.request, test.annos, &corev1.Pod{}, &util.NodeInfo{}, allocated)
+ if fit != test.wantFit {
+ t.Errorf("Fit: got %v, want %v", fit, test.wantFit)
+ }
+ if test.wantFit {
+ if len(result[CambriconMLUDevice]) != test.wantLen {
+ t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[CambriconMLUDevice]))
+ }
+ for idx, id := range test.wantDevIDs {
+ if id != result[CambriconMLUDevice][idx].UUID {
+ t.Errorf("expected device id: %s, got device id %s", id, result[CambriconMLUDevice][idx].UUID)
+ }
+ }
+ }
+
+ if reason != test.wantReason {
+ t.Errorf("expected reason: %s, got reason: %s", test.wantReason, reason)
+ }
+ })
+ }
+}
+
+func TestDevices_AddResourceUsage(t *testing.T) {
+ tests := []struct {
+ name string
+ deviceUsage *util.DeviceUsage
+ ctr *util.ContainerDevice
+ wantErr bool
+ wantUsage *util.DeviceUsage
+ }{
+ {
+ name: "test add resource usage",
+ deviceUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 0,
+ Usedcores: 15,
+ Usedmem: 2000,
+ },
+ ctr: &util.ContainerDevice{
+ UUID: "dev-0",
+ Usedcores: 50,
+ Usedmem: 1024,
+ },
+ wantUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 1,
+ Usedcores: 65,
+ Usedmem: 3024,
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &CambriconDevices{}
+ if err := dev.AddResourceUsage(&corev1.Pod{}, tt.deviceUsage, tt.ctr); (err != nil) != tt.wantErr {
+ t.Errorf("AddResourceUsage() error=%v, wantErr %v", err, tt.wantErr)
+ }
+ if !tt.wantErr {
+ if tt.deviceUsage.Usedcores != tt.wantUsage.Usedcores {
+ t.Errorf("expected used cores: %d, got used cores %d", tt.wantUsage.Usedcores, tt.deviceUsage.Usedcores)
+ }
+ if tt.deviceUsage.Usedmem != tt.wantUsage.Usedmem {
+ t.Errorf("expected used mem: %d, got used mem %d", tt.wantUsage.Usedmem, tt.deviceUsage.Usedmem)
+ }
+ if tt.deviceUsage.Used != tt.wantUsage.Used {
+ t.Errorf("expected used: %d, got used %d", tt.wantUsage.Used, tt.deviceUsage.Used)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/device/common/common.go b/pkg/device/common/common.go
new file mode 100644
index 000000000..363113e82
--- /dev/null
+++ b/pkg/device/common/common.go
@@ -0,0 +1,47 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package common
+
+import (
+ "fmt"
+ "strings"
+)
+
+const (
+ CardTypeMismatch = "CardTypeMismatch"
+ CardUUIDMismatch = "CardUuidMismatch"
+ CardIndexMismatch = "CardIndexMismatch"
+ CardTimeSlicingExhausted = "CardTimeSlicingExhausted"
+ CardComputeUnitsExhausted = "CardComputeUnitsExhausted"
+ CardInsufficientMemory = "CardInsufficientMemory"
+ CardInsufficientCore = "CardInsufficientCore"
+ NumaNotFit = "NumaNotFit"
+ ExclusiveDeviceAllocateConflict = "ExclusiveDeviceAllocateConflict"
+ CardNotFoundCustomFilterRule = "CardNotFoundCustomFilterRule"
+ NodeInsufficientDevice = "NodeInsufficientDevice"
+ AllocatedCardsInsufficientRequest = "AllocatedCardsInsufficientRequest"
+ NodeUnfitPod = "NodeUnfitPod"
+ NodeFitPod = "NodeFitPod"
+)
+
+func GenReason(reasons map[string]int, cards int) string {
+ var reason []string
+ for r, cnt := range reasons {
+ reason = append(reason, fmt.Sprintf("%d/%d %s", cnt, cards, r))
+ }
+ return strings.Join(reason, ", ")
+}
diff --git a/pkg/device/devices.go b/pkg/device/devices.go
new file mode 100644
index 000000000..cd8b8e149
--- /dev/null
+++ b/pkg/device/devices.go
@@ -0,0 +1,467 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package device
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "os"
+ "reflect"
+ "strings"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/ascend"
+ "github.com/Project-HAMi/HAMi/pkg/device/awsneuron"
+ "github.com/Project-HAMi/HAMi/pkg/device/cambricon"
+ "github.com/Project-HAMi/HAMi/pkg/device/enflame"
+ "github.com/Project-HAMi/HAMi/pkg/device/hygon"
+ "github.com/Project-HAMi/HAMi/pkg/device/iluvatar"
+ "github.com/Project-HAMi/HAMi/pkg/device/kunlun"
+ "github.com/Project-HAMi/HAMi/pkg/device/metax"
+ "github.com/Project-HAMi/HAMi/pkg/device/mthreads"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+ "github.com/Project-HAMi/HAMi/pkg/util/nodelock"
+
+ "gopkg.in/yaml.v2"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/klog/v2"
+)
+
+type Devices interface {
+ CommonWord() string
+ MutateAdmission(ctr *corev1.Container, pod *corev1.Pod) (bool, error)
+ CheckHealth(devType string, n *corev1.Node) (bool, bool)
+ NodeCleanUp(nn string) error
+ GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error)
+ LockNode(n *corev1.Node, p *corev1.Pod) error
+ ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error
+ GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest
+ PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd util.PodDevices) map[string]string
+ ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32
+ AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error
+ Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string)
+ // This should not be associated with a specific device object
+ //ParseConfig(fs *flag.FlagSet)
+}
+
+type Config struct {
+ NvidiaConfig nvidia.NvidiaConfig `yaml:"nvidia"`
+ MetaxConfig metax.MetaxConfig `yaml:"metax"`
+ HygonConfig hygon.HygonConfig `yaml:"hygon"`
+ CambriconConfig cambricon.CambriconConfig `yaml:"cambricon"`
+ MthreadsConfig mthreads.MthreadsConfig `yaml:"mthreads"`
+ IluvatarConfig iluvatar.IluvatarConfig `yaml:"iluvatar"`
+ EnflameConfig enflame.EnflameConfig `yaml:"enflame"`
+ KunlunConfig kunlun.KunlunConfig `yaml:"kunlun"`
+ AWSNeuronConfig awsneuron.AWSNeuronConfig `yaml:"awsneuron"`
+ VNPUs []ascend.VNPUConfig `yaml:"vnpus"`
+}
+
+var (
+ HandshakeAnnos = map[string]string{}
+ RegisterAnnos = map[string]string{}
+ devicesMap map[string]Devices
+ DevicesToHandle []string
+ configFile string
+ DebugMode bool
+)
+
+func GetDevices() map[string]Devices {
+ return devicesMap
+}
+
+func InitDevicesWithConfig(config *Config) error {
+ if err := validateConfig(config); err != nil {
+ klog.Errorf("Invalid configuration: %v", err)
+ return err
+ }
+
+ klog.Info("Initializing devices with configuration")
+
+ devicesMap = make(map[string]Devices)
+ DevicesToHandle = []string{}
+ var initErrors []error
+
+ // Helper function to initialize devices and handle errors
+ initializeDevice := func(deviceType string, commonWord string, initFunc func(any) (Devices, error), config any) {
+ klog.Infof("Initializing %s device", commonWord)
+ device, err := initFunc(config)
+ if err != nil {
+ klog.Errorf("Failed to initialize %s device: %v", commonWord, err)
+ initErrors = append(initErrors, fmt.Errorf("%s: %v", commonWord, err))
+ return
+ }
+ devicesMap[deviceType] = device
+ DevicesToHandle = append(DevicesToHandle, commonWord)
+ klog.Infof("%s device initialized successfully", commonWord)
+ }
+
+ // Wrapper for each device's initialization function to include type assertion check
+ deviceInitializers := []struct {
+ deviceType string
+ commonWord string
+ initFunc func(any) (Devices, error)
+ config any
+ }{
+ {nvidia.NvidiaGPUDevice, nvidia.NvidiaGPUCommonWord, func(cfg any) (Devices, error) {
+ nvidiaConfig, ok := cfg.(nvidia.NvidiaConfig)
+ if !ok {
+ return nil, fmt.Errorf("invalid configuration for %s", nvidia.NvidiaGPUCommonWord)
+ }
+ return nvidia.InitNvidiaDevice(nvidiaConfig), nil
+ }, config.NvidiaConfig},
+ {cambricon.CambriconMLUDevice, cambricon.CambriconMLUCommonWord, func(cfg any) (Devices, error) {
+ cambriconConfig, ok := cfg.(cambricon.CambriconConfig)
+ if !ok {
+ return nil, fmt.Errorf("invalid configuration for %s", cambricon.CambriconMLUCommonWord)
+ }
+ return cambricon.InitMLUDevice(cambriconConfig), nil
+ }, config.CambriconConfig},
+ {hygon.HygonDCUDevice, hygon.HygonDCUCommonWord, func(cfg any) (Devices, error) {
+ hygonConfig, ok := cfg.(hygon.HygonConfig)
+ if !ok {
+ return nil, fmt.Errorf("invalid configuration for %s", hygon.HygonDCUCommonWord)
+ }
+ return hygon.InitDCUDevice(hygonConfig), nil
+ }, config.HygonConfig},
+ {iluvatar.IluvatarGPUDevice, iluvatar.IluvatarGPUCommonWord, func(cfg any) (Devices, error) {
+ iluvatarConfig, ok := cfg.(iluvatar.IluvatarConfig)
+ if !ok {
+ return nil, fmt.Errorf("invalid configuration for %s", iluvatar.IluvatarGPUCommonWord)
+ }
+ return iluvatar.InitIluvatarDevice(iluvatarConfig), nil
+ }, config.IluvatarConfig},
+ {enflame.EnflameGPUDevice, enflame.EnflameGPUCommonWord, func(cfg any) (Devices, error) {
+ enflameConfig, ok := cfg.(enflame.EnflameConfig)
+ if !ok {
+ return nil, fmt.Errorf("invalid configuration for %s", enflame.EnflameGPUCommonWord)
+ }
+ return enflame.InitEnflameDevice(enflameConfig), nil
+ }, config.EnflameConfig},
+ {mthreads.MthreadsGPUDevice, mthreads.MthreadsGPUCommonWord, func(cfg any) (Devices, error) {
+ mthreadsConfig, ok := cfg.(mthreads.MthreadsConfig)
+ if !ok {
+ return nil, fmt.Errorf("invalid configuration for %s", mthreads.MthreadsGPUCommonWord)
+ }
+ return mthreads.InitMthreadsDevice(mthreadsConfig), nil
+ }, config.MthreadsConfig},
+ {metax.MetaxGPUDevice, metax.MetaxGPUCommonWord, func(cfg any) (Devices, error) {
+ metaxConfig, ok := cfg.(metax.MetaxConfig)
+ if !ok {
+ return nil, fmt.Errorf("invalid configuration for %s", metax.MetaxGPUCommonWord)
+ }
+ return metax.InitMetaxDevice(metaxConfig), nil
+ }, config.MetaxConfig},
+ {metax.MetaxSGPUDevice, metax.MetaxSGPUCommonWord, func(cfg any) (Devices, error) {
+ metaxConfig, ok := cfg.(metax.MetaxConfig)
+ if !ok {
+ return nil, fmt.Errorf("invalid configuration for %s", metax.MetaxGPUCommonWord)
+ }
+ return metax.InitMetaxSDevice(metaxConfig), nil
+ }, config.MetaxConfig},
+ {kunlun.KunlunGPUDevice, kunlun.KunlunGPUCommonWord, func(cfg any) (Devices, error) {
+ kunlunConfig, ok := cfg.(kunlun.KunlunConfig)
+ if !ok {
+ return nil, fmt.Errorf("invalid configuration for %s", kunlun.KunlunGPUCommonWord)
+ }
+ return kunlun.InitKunlunDevice(kunlunConfig), nil
+ }, config.KunlunConfig},
+ {awsneuron.AWSNeuronDevice, awsneuron.AWSNeuronCommonWord, func(cfg any) (Devices, error) {
+ awsneuronConfig, ok := cfg.(awsneuron.AWSNeuronConfig)
+ if !ok {
+ return nil, fmt.Errorf("invalid configuration for %s", awsneuron.AWSNeuronCommonWord)
+ }
+ return awsneuron.InitAWSNeuronDevice(awsneuronConfig), nil
+ }, config.AWSNeuronConfig},
+ }
+
+ // Initialize all devices using the wrapped functions
+ for _, initializer := range deviceInitializers {
+ initializeDevice(initializer.deviceType, initializer.commonWord, initializer.initFunc, initializer.config)
+ }
+
+ // Initialize Ascend devices
+ for _, dev := range ascend.InitDevices(config.VNPUs) {
+ commonWord := dev.CommonWord()
+ devicesMap[commonWord] = dev
+ DevicesToHandle = append(DevicesToHandle, commonWord)
+ klog.Infof("Ascend device %s initialized", commonWord)
+ }
+
+ if len(initErrors) > 0 {
+ return fmt.Errorf("errors occurred during initialization: %v", initErrors)
+ }
+
+ klog.Info("All devices initialized successfully")
+ return nil
+}
+
+func InitDevices() {
+ if len(devicesMap) > 0 {
+ klog.Info("Devices are already initialized, skipping initialization")
+ return
+ }
+ klog.Infof("Loading device configuration from file: %s", configFile)
+ config, err := LoadConfig(configFile)
+ if err != nil {
+ klog.Fatalf("Failed to load device config file %s: %v", configFile, err)
+ }
+ klog.Infof("Loaded config: %v", config)
+ err = InitDevicesWithConfig(config)
+ if err != nil {
+ klog.Fatalf("Failed to initialize devices: %v", err)
+ }
+}
+
+func InitDefaultDevices() {
+ configMapdata := `
+nvidia:
+ resourceCountName: "nvidia.com/gpu"
+ resourceMemoryName: "nvidia.com/gpumem"
+ resourceMemoryPercentageName: "nvidia.com/gpumem-percentage"
+ resourceCoreName: "nvidia.com/gpucores"
+ resourcePriorityName: "nvidia.com/priority"
+ overwriteEnv: false
+ defaultMemory: 0
+ defaultCores: 0
+ defaultGPUNum: 1
+cambricon:
+ resourceCountName: "cambricon.com/vmlu"
+ resourceMemoryName: "cambricon.com/mlu.smlu.vmemory"
+ resourceCoreName: "cambricon.com/mlu.smlu.vcore"
+hygon:
+ resourceCountName: "hygon.com/dcunum"
+ resourceMemoryName: "hygon.com/dcumem"
+ resourceCoreName: "hygon.com/dcucores"
+metax:
+ resourceCountName: "metax-tech.com/gpu"
+mthreads:
+ resourceCountName: "mthreads.com/vgpu"
+ resourceMemoryName: "mthreads.com/sgpu-memory"
+ resourceCoreName: "mthreads.com/sgpu-core"
+iluvatar:
+ resourceCountName: "iluvatar.ai/vgpu"
+ resourceMemoryName: "iluvatar.ai/vcuda-memory"
+ resourceCoreName: "iluvatar.ai/vcuda-core"
+kunlun:
+ resourceCountName: "kunlunxin.com/xpu"
+awsneuron:
+ resourceCountName: "aws.amazon.com/neuron"
+ resourceCoreName: "aws.amazon.com/neuroncore"
+vnpus:
+ - chipName: "910B"
+ commonWord: "Ascend910A"
+ resourceName: "huawei.com/Ascend910A"
+ resourceMemoryName: "huawei.com/Ascend910A-memory"
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 30
+ templates:
+ - name: "vir02"
+ memory: 2184
+ aiCore: 2
+ - name: "vir04"
+ memory: 4369
+ aiCore: 4
+ - name: "vir08"
+ memory: 8738
+ aiCore: 8
+ - name: "vir16"
+ memory: 17476
+ aiCore: 16
+ - chipName: 910B2
+ commonWord: Ascend910B2
+ resourceName: huawei.com/Ascend910B2
+ resourceMemoryName: huawei.com/Ascend910B2-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 24
+ aiCPU: 6
+ templates:
+ - name: vir03_1c_8g
+ memory: 8192
+ aiCore: 3
+ aiCPU: 1
+ - name: vir06_1c_16g
+ memory: 16384
+ aiCore: 6
+ aiCPU: 1
+ - name: vir12_3c_32g
+ memory: 32768
+ aiCore: 12
+ aiCPU: 3
+ - chipName: "910B3"
+ commonWord: "Ascend910B"
+ resourceName: "huawei.com/Ascend910B"
+ resourceMemoryName: "huawei.com/Ascend910B-memory"
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: "vir05_1c_16g"
+ memory: 16384
+ aiCore: 5
+ aiCPU: 1
+ - name: "vir10_3c_32g"
+ memory: 32768
+ aiCore: 10
+ aiCPU: 3
+ - chipName: 910B4
+ commonWord: Ascend910B4
+ resourceName: huawei.com/Ascend910B4
+ resourceMemoryName: huawei.com/Ascend910B4-memory
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: vir05_1c_8g
+ memory: 8192
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_16g
+ memory: 16384
+ aiCore: 10
+ aiCPU: 3
+ - chipName: "310P3"
+ commonWord: "Ascend310P"
+ resourceName: "huawei.com/Ascend310P"
+ resourceMemoryName: "huawei.com/Ascend310P-memory"
+ memoryAllocatable: 21527
+ memoryCapacity: 24576
+ aiCore: 8
+ aiCPU: 7
+ templates:
+ - name: "vir01"
+ memory: 3072
+ aiCore: 1
+ aiCPU: 1
+ - name: "vir02"
+ memory: 6144
+ aiCore: 2
+ aiCPU: 2
+ - name: "vir04"
+ memory: 12288
+ aiCore: 4
+ aiCPU: 4`
+
+ var yamlData Config
+ err := yaml.Unmarshal([]byte(configMapdata), &yamlData)
+ if err != nil {
+ klog.Fatalf("Failed to unmarshal default config: %v", err)
+ return
+ }
+
+ // Initialize devices with configuration
+ if err := InitDevicesWithConfig(&yamlData); err != nil {
+ klog.Fatalf("Failed to initialize devices with default config: %v", err)
+ }
+}
+
+func PodAllocationTrySuccess(nodeName string, devName string, lockName string, pod *corev1.Pod) {
+ refreshed, err := client.GetClient().CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, metav1.GetOptions{})
+ if err != nil {
+ klog.Errorf("Error getting pod %s/%s: %v", pod.Namespace, pod.Name, err)
+ return
+ }
+ annos := refreshed.Annotations[util.InRequestDevices[devName]]
+ klog.Infof("Trying allocation success: %s", annos)
+ for _, val := range DevicesToHandle {
+ if strings.Contains(annos, val) {
+ return
+ }
+ }
+ klog.Infof("All devices allocate success, releasing lock")
+ PodAllocationSuccess(nodeName, pod, lockName)
+}
+
+func updatePodAnnotationsAndReleaseLock(nodeName string, pod *corev1.Pod, lockName string, deviceBindPhase string) {
+ newAnnos := map[string]string{util.DeviceBindPhase: deviceBindPhase}
+ if err := util.PatchPodAnnotations(pod, newAnnos); err != nil {
+ klog.Errorf("Failed to patch pod annotations for pod %s/%s: %v", pod.Namespace, pod.Name, err)
+ return
+ }
+ if err := nodelock.ReleaseNodeLock(nodeName, lockName, pod, false); err != nil {
+ klog.Errorf("Failed to release node lock for node %s and lock %s: %v", nodeName, lockName, err)
+ }
+}
+
+func PodAllocationSuccess(nodeName string, pod *corev1.Pod, lockName string) {
+ klog.Infof("Pod allocation successful for pod %s/%s on node %s", pod.Namespace, pod.Name, nodeName)
+ updatePodAnnotationsAndReleaseLock(nodeName, pod, lockName, util.DeviceBindSuccess)
+}
+
+func PodAllocationFailed(nodeName string, pod *corev1.Pod, lockName string) {
+ klog.Infof("Pod allocation failed for pod %s/%s on node %s", pod.Namespace, pod.Name, nodeName)
+ updatePodAnnotationsAndReleaseLock(nodeName, pod, lockName, util.DeviceBindFailed)
+}
+
+func GlobalFlagSet() *flag.FlagSet {
+ fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+ ascend.ParseConfig(fs)
+ cambricon.ParseConfig(fs)
+ hygon.ParseConfig(fs)
+ iluvatar.ParseConfig(fs)
+ nvidia.ParseConfig(fs)
+ mthreads.ParseConfig(fs)
+ enflame.ParseConfig(fs)
+ metax.ParseConfig(fs)
+ kunlun.ParseConfig(fs)
+ fs.BoolVar(&DebugMode, "debug", false, "Enable debug mode")
+ fs.StringVar(&configFile, "device-config-file", "", "Path to the device config file")
+ klog.InitFlags(fs)
+ return fs
+}
+
+func LoadConfig(path string) (*Config, error) {
+ klog.Infof("Reading config file from path: %s", path)
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+
+ var yamlData Config
+ if err := yaml.Unmarshal(data, &yamlData); err != nil {
+ return nil, err
+ }
+ klog.Info("Successfully read and parsed config file")
+ return &yamlData, nil
+}
+
+// validateConfig validates the configuration object to ensure it is complete.
+func validateConfig(config *Config) error {
+ var hasAnyConfig bool
+
+ hasAnyConfig = hasAnyConfig || !reflect.DeepEqual(config.NvidiaConfig, nvidia.NvidiaConfig{})
+ hasAnyConfig = hasAnyConfig || !reflect.DeepEqual(config.CambriconConfig, cambricon.CambriconConfig{})
+ hasAnyConfig = hasAnyConfig || !reflect.DeepEqual(config.HygonConfig, hygon.HygonConfig{})
+ hasAnyConfig = hasAnyConfig || !reflect.DeepEqual(config.IluvatarConfig, iluvatar.IluvatarConfig{})
+ hasAnyConfig = hasAnyConfig || !reflect.DeepEqual(config.MthreadsConfig, mthreads.MthreadsConfig{})
+ hasAnyConfig = hasAnyConfig || !reflect.DeepEqual(config.MetaxConfig, metax.MetaxConfig{})
+ hasAnyConfig = hasAnyConfig || !reflect.DeepEqual(config.KunlunConfig, kunlun.KunlunConfig{})
+ hasAnyConfig = hasAnyConfig || !reflect.DeepEqual(config.AWSNeuronConfig, awsneuron.AWSNeuronConfig{})
+ hasAnyConfig = hasAnyConfig || len(config.VNPUs) > 0
+
+ if !hasAnyConfig {
+ return fmt.Errorf("all configurations are empty")
+ }
+ return nil
+}
diff --git a/pkg/device/devices_test.go b/pkg/device/devices_test.go
new file mode 100644
index 000000000..761f9315a
--- /dev/null
+++ b/pkg/device/devices_test.go
@@ -0,0 +1,572 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package device
+
+import (
+ "context"
+ "fmt"
+ "slices"
+ "testing"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/awsneuron"
+ "github.com/Project-HAMi/HAMi/pkg/device/kunlun"
+
+ "gopkg.in/yaml.v2"
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes/fake"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/ascend"
+ "github.com/Project-HAMi/HAMi/pkg/device/cambricon"
+ "github.com/Project-HAMi/HAMi/pkg/device/enflame"
+ "github.com/Project-HAMi/HAMi/pkg/device/hygon"
+ "github.com/Project-HAMi/HAMi/pkg/device/iluvatar"
+ "github.com/Project-HAMi/HAMi/pkg/device/metax"
+ "github.com/Project-HAMi/HAMi/pkg/device/mthreads"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+)
+
+func loadTestConfig() string {
+ return `
+nvidia:
+ resourceCountName: nvidia.com/gpu
+ resourceMemoryName: nvidia.com/gpumem
+ resourceMemoryPercentageName: nvidia.com/gpumem-percentage
+ resourceCoreName: nvidia.com/gpucores
+ resourcePriorityName: nvidia.com/priority
+ overwriteEnv: false
+ defaultMemory: 0
+ defaultCores: 0
+ defaultGPUNum: 1
+cambricon:
+ resourceCountName: cambricon.com/vmlu
+ resourceMemoryName: cambricon.com/mlu.smlu.vmemory
+ resourceCoreName: cambricon.com/mlu.smlu.vcore
+hygon:
+ resourceCountName: hygon.com/dcunum
+ resourceMemoryName: hygon.com/dcumem
+ resourceCoreName: hygon.com/dcucores
+metax:
+ resourceCountName: "metax-tech.com/gpu"
+mthreads:
+ resourceCountName: "mthreads.com/vgpu"
+ resourceMemoryName: "mthreads.com/sgpu-memory"
+ resourceCoreName: "mthreads.com/sgpu-core"
+iluvatar:
+ resourceCountName: iluvatar.ai/vgpu
+ resourceMemoryName: iluvatar.ai/vcuda-memory
+ resourceCoreName: iluvatar.ai/vcuda-core
+kunlun:
+ resourceCountName: "kunlunxin.com/xpu"
+vnpus:
+- chipName: 910B
+ commonWord: Ascend910A
+ resourceName: huawei.com/Ascend910A
+ resourceMemoryName: huawei.com/Ascend910A-memory
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 30
+ templates:
+ - name: vir02
+ memory: 2184
+ aiCore: 2
+ - name: vir04
+ memory: 4369
+ aiCore: 4
+ - name: vir08
+ memory: 8738
+ aiCore: 8
+ - name: vir16
+ memory: 17476
+ aiCore: 16
+- chipName: 910B2
+ commonWord: Ascend910B2
+ resourceName: huawei.com/Ascend910B2
+ resourceMemoryName: huawei.com/Ascend910B2-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 24
+ aiCPU: 6
+ templates:
+ - name: vir03_1c_8g
+ memory: 8192
+ aiCore: 3
+ aiCPU: 1
+ - name: vir06_1c_16g
+ memory: 16384
+ aiCore: 6
+ aiCPU: 1
+ - name: vir12_3c_32g
+ memory: 32768
+ aiCore: 12
+ aiCPU: 3
+- chipName: 910B3
+ commonWord: Ascend910B
+ resourceName: huawei.com/Ascend910B
+ resourceMemoryName: huawei.com/Ascend910B-memory
+ memoryAllocatable: 65536
+ memoryCapacity: 65536
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: vir05_1c_16g
+ memory: 16384
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_32g
+ memory: 32768
+ aiCore: 10
+ aiCPU: 3
+- chipName: 910B4
+ commonWord: Ascend910B4
+ resourceName: huawei.com/Ascend910B4
+ resourceMemoryName: huawei.com/Ascend910B4-memory
+ memoryAllocatable: 32768
+ memoryCapacity: 32768
+ aiCore: 20
+ aiCPU: 7
+ templates:
+ - name: vir05_1c_8g
+ memory: 8192
+ aiCore: 5
+ aiCPU: 1
+ - name: vir10_3c_16g
+ memory: 16384
+ aiCore: 10
+ aiCPU: 3
+- chipName: 310P3
+ commonWord: Ascend310P
+ resourceName: huawei.com/Ascend310P
+ resourceMemoryName: huawei.com/Ascend310P-memory
+ memoryAllocatable: 21527
+ memoryCapacity: 24576
+ aiCore: 8
+ aiCPU: 7
+ templates:
+ - name: vir01
+ memory: 3072
+ aiCore: 1
+ aiCPU: 1
+ - name: vir02
+ memory: 6144
+ aiCore: 2
+ aiCPU: 2
+ - name: vir04
+ memory: 12288
+ aiCore: 4
+ aiCPU: 4`
+}
+
+func Test_LoadConfig(t *testing.T) {
+ var configData Config
+ err := yaml.Unmarshal([]byte(loadTestConfig()), &configData)
+ assert.NilError(t, err)
+
+ dataDrivenTests := []struct {
+ name string
+ expectedConfig any
+ actualConfig any
+ }{
+ {"NVIDIA Config", createNvidiaConfig(), configData.NvidiaConfig},
+ {"Cambricon Config", createCambriconConfig(), configData.CambriconConfig},
+ {"Hygon Config", createHygonConfig(), configData.HygonConfig},
+ {"Iluvatar Config", createIluvatarConfig(), configData.IluvatarConfig},
+ {"Mthreads Config", createMthreadsConfig(), configData.MthreadsConfig},
+ {"Metax Config", createMetaxConfig(), configData.MetaxConfig},
+ {"Kunlun Config", createKunlunConfig(), configData.KunlunConfig},
+ }
+
+ for _, test := range dataDrivenTests {
+ t.Run(test.name, func(t *testing.T) {
+ assert.DeepEqual(t, test.expectedConfig, test.actualConfig)
+ })
+ }
+
+ expectedVNPUs := createVNPUConfigs()
+ assert.DeepEqual(t, configData.VNPUs, expectedVNPUs)
+}
+
+func createNvidiaConfig() nvidia.NvidiaConfig {
+ return nvidia.NvidiaConfig{
+ ResourceCountName: "nvidia.com/gpu",
+ ResourceMemoryName: "nvidia.com/gpumem",
+ ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
+ ResourceCoreName: "nvidia.com/gpucores",
+ ResourcePriority: "nvidia.com/priority",
+ OverwriteEnv: false,
+ DefaultMemory: 0,
+ DefaultCores: 0,
+ DefaultGPUNum: 1,
+ }
+}
+
+func createCambriconConfig() cambricon.CambriconConfig {
+ return cambricon.CambriconConfig{
+ ResourceCountName: "cambricon.com/vmlu",
+ ResourceMemoryName: "cambricon.com/mlu.smlu.vmemory",
+ ResourceCoreName: "cambricon.com/mlu.smlu.vcore",
+ }
+}
+
+func createHygonConfig() hygon.HygonConfig {
+ return hygon.HygonConfig{
+ ResourceCountName: "hygon.com/dcunum",
+ ResourceMemoryName: "hygon.com/dcumem",
+ ResourceCoreName: "hygon.com/dcucores",
+ }
+}
+
+func createIluvatarConfig() iluvatar.IluvatarConfig {
+ return iluvatar.IluvatarConfig{
+ ResourceCountName: "iluvatar.ai/vgpu",
+ ResourceMemoryName: "iluvatar.ai/vcuda-memory",
+ ResourceCoreName: "iluvatar.ai/vcuda-core",
+ }
+}
+
+func createMthreadsConfig() mthreads.MthreadsConfig {
+ return mthreads.MthreadsConfig{
+ ResourceCountName: "mthreads.com/vgpu",
+ ResourceMemoryName: "mthreads.com/sgpu-memory",
+ ResourceCoreName: "mthreads.com/sgpu-core",
+ }
+}
+
+func createMetaxConfig() metax.MetaxConfig {
+ return metax.MetaxConfig{
+ ResourceCountName: "metax-tech.com/gpu",
+ }
+}
+
+func createKunlunConfig() kunlun.KunlunConfig {
+ return kunlun.KunlunConfig{
+ ResourceCountName: "kunlunxin.com/xpu",
+ }
+}
+
+func createVNPUConfigs() []ascend.VNPUConfig {
+ return []ascend.VNPUConfig{
+ {
+ ChipName: "910B",
+ CommonWord: "Ascend910A",
+ ResourceName: "huawei.com/Ascend910A",
+ ResourceMemoryName: "huawei.com/Ascend910A-memory",
+ MemoryAllocatable: 32768,
+ MemoryCapacity: 32768,
+ AICore: 30,
+ Templates: []ascend.Template{
+ {Name: "vir02", Memory: 2184, AICore: 2},
+ {Name: "vir04", Memory: 4369, AICore: 4},
+ {Name: "vir08", Memory: 8738, AICore: 8},
+ {Name: "vir16", Memory: 17476, AICore: 16},
+ },
+ },
+ {
+ ChipName: "910B2",
+ CommonWord: "Ascend910B2",
+ ResourceName: "huawei.com/Ascend910B2",
+ ResourceMemoryName: "huawei.com/Ascend910B2-memory",
+ MemoryAllocatable: 65536,
+ MemoryCapacity: 65536,
+ AICore: 24,
+ AICPU: 6,
+ Templates: []ascend.Template{
+ {Name: "vir03_1c_8g", Memory: 8192, AICore: 3, AICPU: 1},
+ {Name: "vir06_1c_16g", Memory: 16384, AICore: 6, AICPU: 1},
+ {Name: "vir12_3c_32g", Memory: 32768, AICore: 12, AICPU: 3},
+ },
+ },
+ {
+ ChipName: "910B3",
+ CommonWord: "Ascend910B",
+ ResourceName: "huawei.com/Ascend910B",
+ ResourceMemoryName: "huawei.com/Ascend910B-memory",
+ MemoryAllocatable: 65536,
+ MemoryCapacity: 65536,
+ AICore: 20,
+ AICPU: 7,
+ Templates: []ascend.Template{
+ {Name: "vir05_1c_16g", Memory: 16384, AICore: 5, AICPU: 1},
+ {Name: "vir10_3c_32g", Memory: 32768, AICore: 10, AICPU: 3},
+ },
+ },
+ {
+ ChipName: "910B4",
+ CommonWord: "Ascend910B4",
+ ResourceName: "huawei.com/Ascend910B4",
+ ResourceMemoryName: "huawei.com/Ascend910B4-memory",
+ MemoryAllocatable: 32768,
+ MemoryCapacity: 32768,
+ AICore: 20,
+ AICPU: 7,
+ Templates: []ascend.Template{
+ {Name: "vir05_1c_8g", Memory: 8192, AICore: 5, AICPU: 1},
+ {Name: "vir10_3c_16g", Memory: 16384, AICore: 10, AICPU: 3},
+ },
+ },
+ {
+ ChipName: "310P3",
+ CommonWord: "Ascend310P",
+ ResourceName: "huawei.com/Ascend310P",
+ ResourceMemoryName: "huawei.com/Ascend310P-memory",
+ MemoryAllocatable: 21527,
+ MemoryCapacity: 24576,
+ AICore: 8,
+ AICPU: 7,
+ Templates: []ascend.Template{
+ {Name: "vir01", Memory: 3072, AICore: 1, AICPU: 1},
+ {Name: "vir02", Memory: 6144, AICore: 2, AICPU: 2},
+ {Name: "vir04", Memory: 12288, AICore: 4, AICPU: 4},
+ },
+ },
+ }
+}
+
+func setupTest(t *testing.T) (map[string]string, map[string]Devices) {
+ t.Helper()
+
+ configMapData := loadTestConfig()
+ var configData Config
+ err := yaml.Unmarshal([]byte(configMapData), &configData)
+ assert.NilError(t, err)
+
+ err = InitDevicesWithConfig(&configData)
+ assert.NilError(t, err)
+
+ // Expected devices map
+ expectedDevices := map[string]string{
+ nvidia.NvidiaGPUDevice: nvidia.NvidiaGPUCommonWord,
+ cambricon.CambriconMLUDevice: cambricon.CambriconMLUCommonWord,
+ hygon.HygonDCUDevice: hygon.HygonDCUCommonWord,
+ iluvatar.IluvatarGPUDevice: iluvatar.IluvatarGPUCommonWord,
+ mthreads.MthreadsGPUDevice: mthreads.MthreadsGPUCommonWord,
+ metax.MetaxGPUDevice: metax.MetaxGPUCommonWord,
+ metax.MetaxSGPUDevice: metax.MetaxSGPUCommonWord,
+ enflame.EnflameGPUDevice: enflame.EnflameGPUCommonWord,
+ kunlun.KunlunGPUDevice: kunlun.KunlunGPUCommonWord,
+ awsneuron.AWSNeuronDevice: awsneuron.AWSNeuronCommonWord,
+ }
+
+ return expectedDevices, devicesMap
+}
+
+func containsString(slice []string, str string) bool {
+ return slices.Contains(slice, str)
+}
+
+// Test_InitDevicesWithConfig_Success tests the initialization of devices with the provided config.
+func Test_InitDevicesWithConfig_Success(t *testing.T) {
+ expectedDevices, devicesMap := setupTest(t)
+
+ assert.Assert(t, len(devicesMap) > 0, "Expected devicesMap to be populated")
+ assert.Equal(t, len(DevicesToHandle), len(expectedDevices), "Expected DevicesToHandle to contain all devices")
+
+ for deviceType, commonWord := range expectedDevices {
+ assert.Assert(t, devicesMap[deviceType] != nil, fmt.Sprintf("Expected %s device to be initialized", deviceType))
+ assert.Assert(t, containsString(DevicesToHandle, commonWord), fmt.Sprintf("Expected common word %s to be in DevicesToHandle", commonWord))
+ }
+}
+
+// Test_InitDevicesWithConfig_InvalidConfig tests the behavior of InitDevicesWithConfig with invalid configurations.
+func Test_InitDevicesWithConfig_InvalidConfig(t *testing.T) {
+ // Provide an intentionally constructed invalid configuration
+ configData := Config{}
+
+ err := InitDevicesWithConfig(&configData)
+ assert.ErrorContains(t, err, "all configurations are empty", "Expected initialization to fail with 'NvidiaConfig is empty' error")
+
+}
+
+func Test_GetDevices(t *testing.T) {
+ expectedDevices, _ := setupTest(t)
+
+ devices := GetDevices()
+
+ assert.Assert(t, len(devices) > 0, "Expected devicesMap to be populated")
+ assert.Equal(t, len(devices), len(expectedDevices), "Expected devicesMap to contain all initialized devices")
+
+ for deviceType := range expectedDevices {
+ if devices[deviceType] == nil {
+ t.Errorf("Expected %s device to be initialized", deviceType)
+ }
+ }
+}
+
+func Test_InitDefaultDevices(t *testing.T) {
+ InitDefaultDevices()
+ assert.Assert(t, len(devicesMap) > 0, "Expected devicesMap to be populated")
+ assert.Assert(t, len(DevicesToHandle) > 0, "Expected DevicesToHandle to be populated")
+}
+
+func Test_PodAllocationTrySuccess(t *testing.T) {
+ // Initialize fake clientset and pre-load test data
+ client.KubeClient = fake.NewSimpleClientset()
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "testpod",
+ Namespace: "default",
+ Annotations: map[string]string{"test-annotation-key": "test-annotation-value", util.InRequestDevices["NVIDIA"]: "some-value"},
+ },
+ }
+
+ // Add the pod to the fake clientset
+ _, err := client.KubeClient.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{})
+ if err != nil {
+ t.Fatalf("Failed to create test pod: %v", err)
+ }
+
+ nodeName := "test-node"
+ devName := "NVIDIA"
+ lockName := "test-lock"
+
+ // Call the function under test
+ PodAllocationTrySuccess(nodeName, devName, lockName, pod)
+
+ // Refresh the pod state from the fake clientset and check the annotations
+ refreshedPod, err := client.KubeClient.CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("Failed to get refreshed pod: %v", err)
+ }
+
+ annos, ok := refreshedPod.Annotations[util.InRequestDevices[devName]]
+ if !ok || annos == "" {
+ t.Error("Expected annotations to be updated")
+ }
+}
+
+func Test_PodAllocationSuccess(t *testing.T) {
+ // Initialize fake clientset and pre-load test data
+ client.KubeClient = fake.NewSimpleClientset()
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "testpod",
+ Namespace: "default",
+ Annotations: map[string]string{
+ "test-annotation-key": "test-annotation-value",
+ },
+ },
+ }
+
+ // Add the pod to the fake clientset
+ _, err := client.KubeClient.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{})
+ if err != nil {
+ t.Fatalf("Failed to create test pod: %v", err)
+ }
+
+ nodeName := "test-node"
+ lockName := "test-lock"
+
+ // Update pod annotations and release the lock as part of the setup for the test
+ updatePodAnnotationsAndReleaseLock(nodeName, pod, lockName, util.DeviceBindSuccess)
+
+ // Call the function under test
+ PodAllocationSuccess(nodeName, pod, lockName)
+
+ // Refresh the pod state from the fake clientset and check the DeviceBindPhase annotation
+ refreshedPod, err := client.KubeClient.CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("Failed to get refreshed pod: %v", err)
+ }
+
+ annos, ok := refreshedPod.Annotations[util.DeviceBindPhase]
+ if !ok || annos != util.DeviceBindSuccess {
+ t.Errorf("Expected DeviceBindPhase annotation to be '%s', got '%s'", util.DeviceBindSuccess, annos)
+ }
+}
+func Test_PodAllocationFailed(t *testing.T) {
+
+ client.KubeClient = fake.NewSimpleClientset()
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "testpod",
+ Namespace: "default",
+ Annotations: map[string]string{"test-annotation-key": "test-annotation-value"},
+ },
+ }
+
+ // add pod to the fake client
+ _, err := client.KubeClient.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{})
+ if err != nil {
+ t.Fatalf("Failed to create test pod: %v", err)
+ }
+
+ nodeName := "test-node"
+ lockName := "test-lock"
+
+ // simulate a failed pod allocation
+ PodAllocationFailed(nodeName, pod, lockName)
+
+ // retrieve the pod from the fake client
+ refreshedPod, err := client.KubeClient.CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("Failed to get refreshed pod: %v", err)
+ }
+
+ annos, ok := refreshedPod.Annotations[util.DeviceBindPhase]
+ if !ok {
+ t.Error("Expected DeviceBindPhase annotation to be present")
+ } else if annos != util.DeviceBindFailed {
+ t.Errorf("Expected DeviceBindPhase annotation to be '%s', got '%s'", util.DeviceBindFailed, annos)
+ }
+}
+
+func Test_GlobalFlagSet(t *testing.T) {
+ fs := GlobalFlagSet()
+ fs.Parse([]string{"-debug=true", "-device-config-file=test-config-file.yaml"})
+ assert.Assert(t, DebugMode, "Expected DebugMode to be true")
+ assert.Equal(t, configFile, "test-config-file.yaml", "Expected configFile to be test-config-file.yaml")
+}
+
+func Test_validateConfig(t *testing.T) {
+ validConfig := &Config{
+ NvidiaConfig: nvidia.NvidiaConfig{
+ ResourceCountName: "nvidia.com/gpu",
+ ResourceMemoryName: "nvidia.com/gpumem",
+ ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
+ ResourceCoreName: "nvidia.com/gpucores",
+ ResourcePriority: "nvidia.com/priority",
+ OverwriteEnv: false,
+ DefaultMemory: 0,
+ DefaultCores: 0,
+ DefaultGPUNum: 1,
+ },
+ }
+ emptyConfig := &Config{}
+
+ tests := []struct {
+ name string
+ config *Config
+ expectError bool
+ }{
+ {"Valid config", validConfig, false},
+ {"Empty config", emptyConfig, true},
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ err := validateConfig(test.config)
+ if test.expectError {
+ assert.ErrorContains(t, err, "all configurations are empty")
+ } else {
+ assert.NilError(t, err)
+ }
+ })
+ }
+}
diff --git a/pkg/device/enflame/device.go b/pkg/device/enflame/device.go
new file mode 100644
index 000000000..616089372
--- /dev/null
+++ b/pkg/device/enflame/device.go
@@ -0,0 +1,351 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package enflame
+
+import (
+ "flag"
+ "fmt"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/common"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ "k8s.io/klog/v2"
+)
+
+type EnflameDevices struct {
+ factor int
+}
+
+const (
+ EnflameGPUDevice = "Enflame"
+ EnflameGPUCommonWord = "Enflame"
+ // IluvatarUseUUID is user can use specify Iluvatar device for set Iluvatar UUID.
+ EnflameUseUUID = "enflame.com/use-gpuuuid"
+ // IluvatarNoUseUUID is user can not use specify Iluvatar device for set Iluvatar UUID.
+ EnflameNoUseUUID = "enflame.com/nouse-gpuuuid"
+ PodRequestGCUSize = "enflame.com/gcu-request-size"
+ PodAssignedGCUID = "enflame.com/gcu-assigned-id"
+ PodHasAssignedGCU = "enflame.com/gcu-assigned"
+ PodAssignedGCUTime = "enflame.com/gcu-assigned-time"
+ GCUSharedCapacity = "enflame.com/gcu-shared-capacity"
+
+ SharedResourceName = "enflame.com/shared-gcu"
+ CountNoSharedName = "enflame.com/gcu-count"
+)
+
+var (
+ EnflameResourceCount string
+ EnflameResourcePercentage string
+)
+
+type EnflameConfig struct {
+ ResourceCountName string `yaml:"resourceCountName"`
+ ResourcePercentageName string `yaml:"resourcePercentageName"`
+ ResourceMemoryName string `yaml:"resourceMemoryName"`
+ ResourceCoreName string `yaml:"resourceCoreName"`
+}
+
+func InitEnflameDevice(config EnflameConfig) *EnflameDevices {
+ EnflameResourceCount = config.ResourceCountName
+ EnflameResourcePercentage = config.ResourcePercentageName
+ util.SupportDevices[EnflameGPUDevice] = "hami.io/enflame-vgpu-devices-allocated"
+ return &EnflameDevices{
+ factor: 0,
+ }
+}
+
+func (dev *EnflameDevices) CommonWord() string {
+ return EnflameGPUCommonWord
+}
+
+func ParseConfig(fs *flag.FlagSet) {
+ fs.StringVar(&EnflameResourceCount, "enflame-name", "enflame.com/vgcu", "enflame resource count name")
+ fs.StringVar(&EnflameResourcePercentage, "enflame-resource-percentage-name", "enflame.com/vgcu-percentage", "enflame resource percentage name")
+}
+
+func (dev *EnflameDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
+ count, ok := ctr.Resources.Limits[corev1.ResourceName(EnflameResourceCount)]
+ if ok {
+ if count.Value() > 1 {
+ ctr.Resources.Limits[corev1.ResourceName(EnflameResourcePercentage)] = *resource.NewQuantity(int64(100), resource.DecimalSI)
+ ctr.Resources.Limits[corev1.ResourceName(SharedResourceName)] = *resource.NewQuantity(int64(dev.factor*int(count.Value())), resource.DecimalSI)
+ } else {
+ percentageResource, ok := ctr.Resources.Limits[corev1.ResourceName(EnflameResourcePercentage)]
+ percentage := percentageResource.Value()
+ if !ok {
+ percentage = 100
+ }
+ if percentage < 1 {
+ percentage = 1
+ }
+ slice := float64(100) / float64(dev.factor)
+ for i := 0; i < dev.factor; i++ {
+ if slice*float64(i) < float64(percentage) && float64(percentage) <= slice*float64((i+1)) {
+ percentage = int64(slice * float64(i+1))
+ ctr.Resources.Limits[corev1.ResourceName(EnflameResourcePercentage)] = *resource.NewQuantity(percentage, resource.DecimalSI)
+ ctr.Resources.Limits[corev1.ResourceName(SharedResourceName)] = *resource.NewQuantity(int64(i+1), resource.DecimalSI)
+ ctr.Resources.Requests[corev1.ResourceName(EnflameResourcePercentage)] = *resource.NewQuantity(percentage, resource.DecimalSI)
+ ctr.Resources.Requests[corev1.ResourceName(SharedResourceName)] = *resource.NewQuantity(int64(i+1), resource.DecimalSI)
+ break
+ }
+ }
+ }
+ }
+ return ok, nil
+}
+
+func (dev *EnflameDevices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) {
+ nodedevices := []*util.DeviceInfo{}
+ i := 0
+ cards, ok := n.Status.Capacity.Name(corev1.ResourceName(CountNoSharedName), resource.DecimalSI).AsInt64()
+ if !ok || cards == 0 {
+ return []*util.DeviceInfo{}, fmt.Errorf("device not found %s", CountNoSharedName)
+ }
+ shared, _ := n.Status.Capacity.Name(corev1.ResourceName(SharedResourceName), resource.DecimalSI).AsInt64()
+ dev.factor = int(shared / cards)
+ for i < int(cards) {
+ nodedevices = append(nodedevices, &util.DeviceInfo{
+ Index: uint(i),
+ ID: n.Name + "-enflame-" + fmt.Sprint(i),
+ Count: 100,
+ Devmem: 100,
+ Devcore: 100,
+ Type: EnflameGPUDevice,
+ Numa: 0,
+ Health: true,
+ })
+ i++
+ }
+ return nodedevices, nil
+}
+
+func (dev *EnflameDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd util.PodDevices) map[string]string {
+ devlist, ok := pd[EnflameGPUDevice]
+ if ok && len(devlist) > 0 {
+ (*annoinput)[util.SupportDevices[EnflameGPUDevice]] = util.EncodePodSingleDevice(devlist)
+ (*annoinput)[PodHasAssignedGCU] = "false"
+ (*annoinput)[PodAssignedGCUTime] = strconv.FormatInt(time.Now().UnixNano(), 10)
+ annoKey := PodAssignedGCUID
+ value := ""
+ for _, val := range devlist[0] {
+ value = value + fmt.Sprint(val.Idx) + ","
+ }
+ if len(value) > 0 {
+ (*annoinput)[annoKey] = strings.TrimRight(value, ",")
+ }
+ }
+ return *annoinput
+}
+
+func (dev *EnflameDevices) LockNode(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *EnflameDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *EnflameDevices) NodeCleanUp(nn string) error {
+ return nil
+}
+
+func (dev *EnflameDevices) checkType(annos map[string]string, d util.DeviceUsage, n util.ContainerDeviceRequest) (bool, bool, bool) {
+ if strings.Compare(n.Type, EnflameGPUDevice) == 0 {
+ return true, true, false
+ }
+ return false, false, false
+}
+
+func (dev *EnflameDevices) checkUUID(annos map[string]string, d util.DeviceUsage) bool {
+ userUUID, ok := annos[EnflameUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for Iluvatar user uuid [%s], device id is %s", userUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ for uuid := range strings.SplitSeq(userUUID, ",") {
+ if d.ID == uuid {
+ return true
+ }
+ }
+ return false
+ }
+
+ noUserUUID, ok := annos[EnflameNoUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for Iluvatar not user uuid [%s], device id is %s", noUserUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ for uuid := range strings.SplitSeq(noUserUUID, ",") {
+ if d.ID == uuid {
+ return false
+ }
+ }
+ return true
+ }
+ return true
+}
+
+func (dev *EnflameDevices) checkIndex(annos map[string]string, d util.DeviceUsage) bool {
+ return true
+}
+
+func (dev *EnflameDevices) CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ return true, true
+}
+
+func (dev *EnflameDevices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest {
+ klog.Info("Start to count enflame devices for container ", ctr.Name)
+ resourceCount := corev1.ResourceName(EnflameResourceCount)
+ resourcePercentage := corev1.ResourceName(EnflameResourcePercentage)
+ v, ok := ctr.Resources.Limits[resourceCount]
+ if !ok {
+ v, ok = ctr.Resources.Requests[resourceCount]
+ }
+ if ok {
+ if n, ok := v.AsInt64(); ok {
+ klog.Info("Found enflame devices")
+ memnum := 100
+ mem, ok := ctr.Resources.Limits[resourcePercentage]
+ if !ok {
+ mem, ok = ctr.Resources.Requests[resourcePercentage]
+ }
+ if ok {
+ memnum = int(mem.Value())
+ }
+ return util.ContainerDeviceRequest{
+ Nums: int32(n),
+ Type: EnflameGPUDevice,
+ Memreq: int32(memnum),
+ MemPercentagereq: 0,
+ Coresreq: 0,
+ }
+ }
+ }
+ return util.ContainerDeviceRequest{}
+}
+
+func (dev *EnflameDevices) ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32 {
+ return 0
+}
+
+func (dev *EnflameDevices) AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error {
+ n.Used++
+ n.Usedcores += ctr.Usedcores
+ n.Usedmem += ctr.Usedmem
+ return nil
+}
+
+func (enf *EnflameDevices) Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string) {
+ k := request
+ originReq := k.Nums
+ prevnuma := -1
+ klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
+ var tmpDevs map[string]util.ContainerDevices
+ tmpDevs = make(map[string]util.ContainerDevices)
+ reason := make(map[string]int)
+ for i := len(devices) - 1; i >= 0; i-- {
+ dev := devices[i]
+ klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
+
+ _, found, numa := enf.checkType(annos, *dev, k)
+ if !found {
+ reason[common.CardTypeMismatch]++
+ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, k.Type)
+ continue
+ }
+ if numa && prevnuma != dev.Numa {
+ if k.Nums != originReq {
+ reason[common.NumaNotFit] += len(tmpDevs)
+ klog.V(5).InfoS(common.NumaNotFit, "pod", klog.KObj(pod), "device", dev.ID, "k.nums", k.Nums, "numa", numa, "prevnuma", prevnuma, "device numa", dev.Numa)
+ }
+ k.Nums = originReq
+ prevnuma = dev.Numa
+ tmpDevs = make(map[string]util.ContainerDevices)
+ }
+ if !enf.checkUUID(annos, *dev) {
+ reason[common.CardUUIDMismatch]++
+ klog.V(5).InfoS(common.CardUUIDMismatch, "pod", klog.KObj(pod), "device", dev.ID, "current device info is:", *dev)
+ continue
+ }
+
+ memreq := int32(0)
+ if dev.Count <= dev.Used {
+ reason[common.CardTimeSlicingExhausted]++
+ klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
+ continue
+ }
+ if k.Coresreq > 100 {
+ klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Coresreq = 100
+ //return false, tmpDevs
+ }
+ if k.Memreq > 0 {
+ memreq = k.Memreq
+ }
+ if k.MemPercentagereq != 101 && k.Memreq == 0 {
+ //This incurs an issue
+ memreq = dev.Totalmem * k.MemPercentagereq / 100
+ }
+ if dev.Totalmem-dev.Usedmem < memreq {
+ reason[common.CardInsufficientMemory]++
+ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq)
+ continue
+ }
+ if dev.Totalcore-dev.Usedcores < k.Coresreq {
+ reason[common.CardInsufficientCore]++
+ klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", k.Coresreq)
+ continue
+ }
+ // Coresreq=100 indicates it want this card exclusively
+ if dev.Totalcore == 100 && k.Coresreq == 100 && dev.Used > 0 {
+ reason[common.ExclusiveDeviceAllocateConflict]++
+ klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
+ continue
+ }
+ // You can't allocate core=0 job to an already full GPU
+ if dev.Totalcore != 0 && dev.Usedcores == dev.Totalcore && k.Coresreq == 0 {
+ reason[common.CardComputeUnitsExhausted]++
+ klog.V(5).InfoS(common.CardComputeUnitsExhausted, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+
+ if k.Nums > 0 {
+ klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Nums--
+ tmpDevs[k.Type] = append(tmpDevs[k.Type], util.ContainerDevice{
+ Idx: int(dev.Index),
+ UUID: dev.ID,
+ Type: k.Type,
+ Usedmem: memreq,
+ Usedcores: k.Coresreq,
+ })
+ }
+ if k.Nums == 0 {
+ klog.V(4).InfoS("device allocate success", "pod", klog.KObj(pod), "allocate device", tmpDevs)
+ return true, tmpDevs, ""
+ }
+
+ }
+ if len(tmpDevs) > 0 {
+ reason[common.AllocatedCardsInsufficientRequest] = len(tmpDevs)
+ klog.V(5).InfoS(common.AllocatedCardsInsufficientRequest, "pod", klog.KObj(pod), "request", originReq, "allocated", len(tmpDevs))
+ }
+ return false, tmpDevs, common.GenReason(reason, len(devices))
+}
diff --git a/pkg/device/enflame/device_test.go b/pkg/device/enflame/device_test.go
new file mode 100644
index 000000000..69d2678d1
--- /dev/null
+++ b/pkg/device/enflame/device_test.go
@@ -0,0 +1,884 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package enflame
+
+import (
+ "flag"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestGetNodeDevices(t *testing.T) {
+
+ tests := []struct {
+ name string
+ node corev1.Node
+ expected []*util.DeviceInfo
+ err error
+ }{
+ {
+ name: "Test with valid node",
+ node: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test",
+ },
+ Status: corev1.NodeStatus{
+ Capacity: corev1.ResourceList{
+ corev1.ResourceName(CountNoSharedName): *resource.NewQuantity(1, resource.DecimalSI),
+ corev1.ResourceName(SharedResourceName): *resource.NewQuantity(6, resource.DecimalSI),
+ },
+ },
+ },
+ expected: []*util.DeviceInfo{
+ {
+ Index: 0,
+ ID: "test-enflame-0",
+ Count: 100,
+ Devmem: 100,
+ Devcore: 100,
+ Type: EnflameGPUDevice,
+ Numa: 0,
+ Health: true,
+ },
+ },
+ err: nil,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &EnflameDevices{}
+ got, err := dev.GetNodeDevices(tt.node)
+ if (err != nil) != (tt.err != nil) {
+ t.Errorf("GetNodeDevices() error = %v, expected %v", err, tt.err)
+ return
+ }
+
+ if len(got) != len(tt.expected) {
+ t.Errorf("GetNodeDevices() got %d devices, expected %d", len(got), len(tt.expected))
+ return
+ }
+
+ for i, device := range got {
+ if device.Index != tt.expected[i].Index {
+ t.Errorf("Expected index %d, got %d", tt.expected[i].Index, device.Index)
+ }
+ if device.ID != tt.expected[i].ID {
+ t.Errorf("Expected id %s, got %s", tt.expected[i].ID, device.ID)
+ }
+ if device.Devcore != tt.expected[i].Devcore {
+ t.Errorf("Expected devcore %d, got %d", tt.expected[i].Devcore, device.Devcore)
+ }
+ if device.Devmem != tt.expected[i].Devmem {
+ t.Errorf("Expected cevmem %d, got %d", tt.expected[i].Devmem, device.Devmem)
+ }
+ }
+ })
+ }
+}
+
+func TestPatchAnnotations(t *testing.T) {
+ InitEnflameDevice(EnflameConfig{})
+
+ tests := []struct {
+ name string
+ annoInput map[string]string
+ podDevices util.PodDevices
+ expected map[string]string
+ }{
+ {
+ name: "No devices",
+ annoInput: map[string]string{},
+ podDevices: util.PodDevices{},
+ expected: map[string]string{},
+ },
+ {
+ name: "With devices",
+ annoInput: map[string]string{},
+ podDevices: util.PodDevices{
+ EnflameGPUDevice: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ Idx: 0,
+ UUID: "k8s-gpu-enflame-0",
+ Type: "Enflame",
+ },
+ },
+ },
+ },
+ expected: map[string]string{
+ util.SupportDevices[EnflameGPUDevice]: "k8s-gpu-enflame-0,Enflame,0,0:;",
+ PodHasAssignedGCU: "false",
+ PodAssignedGCUTime: strconv.FormatInt(time.Now().UnixNano(), 10),
+ PodAssignedGCUID: "0",
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ annoInputCopy := make(map[string]string)
+ for k, v := range tt.annoInput {
+ annoInputCopy[k] = v
+ }
+
+ dev := &EnflameDevices{}
+ got := dev.PatchAnnotations(&corev1.Pod{}, &annoInputCopy, tt.podDevices)
+
+ if len(got) != len(tt.expected) {
+ t.Errorf("PatchAnnotations() got %d annotations, expected %d", len(got), len(tt.expected))
+ return
+ }
+
+ for k, v := range tt.expected {
+ if k == PodAssignedGCUTime {
+ if len(got[k]) != len(v) {
+ t.Errorf("Expected %s %s, got %s", k, v, got[k])
+ }
+ continue
+ }
+
+ if got[k] != v {
+ t.Errorf("Expected %s %s, got %s", k, v, got[k])
+ }
+ }
+ })
+ }
+}
+
+func Test_MutateAdmission(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }
+ want bool
+ err error
+ }{
+ {
+ name: "EnflameResourceCount and EnflameResourcePercentage set to limits",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "enflame.com/vgcu": *resource.NewQuantity(1, resource.DecimalSI),
+ "enflame.com/vgcu-percentage": *resource.NewQuantity(15, resource.DecimalSI),
+ },
+ Requests: corev1.ResourceList{},
+ },
+ },
+ p: &corev1.Pod{},
+ },
+ want: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ config := EnflameConfig{
+ ResourceCountName: "enflame.com/vgcu",
+ ResourcePercentageName: "enflame.com/vgcu-percentage",
+ }
+ InitEnflameDevice(config)
+ dev := EnflameDevices{
+ factor: 4,
+ }
+ result, _ := dev.MutateAdmission(test.args.ctr, test.args.p)
+ assert.Equal(t, result, test.want)
+ limits := test.args.ctr.Resources.Limits[corev1.ResourceName(EnflameResourcePercentage)]
+ number, _ := limits.AsInt64()
+ assert.Equal(t, number, int64(25))
+ })
+ }
+}
+
+func Test_checkType(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }
+ want1 bool
+ want2 bool
+ want3 bool
+ }{
+ {
+ name: "the same type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: EnflameGPUDevice,
+ },
+ },
+ want1: true,
+ want2: true,
+ want3: false,
+ },
+ {
+ name: "the different type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: "test123",
+ },
+ },
+ want1: false,
+ want2: false,
+ want3: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := EnflameDevices{}
+ result1, result2, result3 := dev.checkType(test.args.annos, test.args.d, test.args.n)
+ assert.Equal(t, result1, test.want1)
+ assert.Equal(t, result2, test.want2)
+ assert.Equal(t, result3, test.want3)
+ })
+ }
+}
+
+func Test_checkUUID(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "useid is same as the device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "enflame.com/use-gpuuuid": "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test1",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "useid is different from the device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "enflame.com/use-gpuuuid": "test2",
+ },
+ d: util.DeviceUsage{
+ ID: "test1",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "no annos",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{
+ ID: "test3",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "nouseid is same as the device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "enflame.com/nouse-gpuuuid": "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test1",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "nouseid is different from the device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "enflame.com/nouse-gpuuuid": "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test2",
+ },
+ },
+ want: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := EnflameDevices{}
+ result := dev.checkUUID(test.args.annos, test.args.d)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_GenerateResourceRequests(t *testing.T) {
+ tests := []struct {
+ name string
+ args *corev1.Container
+ want util.ContainerDeviceRequest
+ }{
+ {
+ name: "all resources set to limits and requests",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "enflame.com/vgcu": resource.MustParse("1"),
+ "enflame.com/vgcu-percentage": resource.MustParse("15"),
+ },
+ Requests: corev1.ResourceList{
+ "enflame.com/vgcu": resource.MustParse("1"),
+ "enflame.com/vgcu-percentage": resource.MustParse("15"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: EnflameGPUDevice,
+ Memreq: int32(15),
+ MemPercentagereq: int32(0),
+ Coresreq: int32(0),
+ },
+ },
+ {
+ name: "all resources don't set to limits and requests",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ Requests: corev1.ResourceList{},
+ },
+ },
+ want: util.ContainerDeviceRequest{},
+ },
+ {
+ name: "resourcemem don't set to limits and requests",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "enflame.com/vgcu": resource.MustParse("1"),
+ },
+ Requests: corev1.ResourceList{
+ "enflame.com/vgcu": resource.MustParse("1"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: EnflameGPUDevice,
+ Memreq: int32(100),
+ MemPercentagereq: int32(0),
+ Coresreq: int32(0),
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := EnflameDevices{factor: 4}
+ fs := flag.FlagSet{}
+ ParseConfig(&fs)
+ result := dev.GenerateResourceRequests(test.args)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func TestDevices_Fit(t *testing.T) {
+ config := EnflameConfig{
+ ResourceCountName: "enflame.com/vgcu",
+ ResourcePercentageName: "enflame.com/vgcu-percentage",
+ }
+ dev := InitEnflameDevice(config)
+
+ tests := []struct {
+ name string
+ devices []*util.DeviceUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ wantFit bool
+ wantLen int
+ wantDevIDs []string
+ wantReason string
+ }{
+ {
+ name: "fit success",
+ devices: []*util.DeviceUsage{
+ {
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ },
+ {
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 64,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: EnflameGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-1"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: memory not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: EnflameGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientMemory",
+ },
+ {
+ name: "fit fail: core not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1024,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: EnflameGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientCore",
+ },
+ {
+ name: "fit fail: type mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ Type: EnflameGPUDevice,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: "OtherType",
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTypeMismatch",
+ },
+ {
+ name: "fit fail: user assign use uuid mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: EnflameGPUDevice,
+ },
+ annos: map[string]string{EnflameUseUUID: "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: user assign no use uuid match",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: EnflameGPUDevice,
+ },
+ annos: map[string]string{EnflameNoUseUUID: "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: card overused",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 100,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: EnflameGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTimeSlicingExhausted",
+ },
+ {
+ name: "fit success: but core limit can't exceed 100",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 120,
+ Type: EnflameGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: card exclusively",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 100,
+ Type: EnflameGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 ExclusiveDeviceAllocateConflict",
+ },
+ {
+ name: "fit fail: CardComputeUnitsExhausted",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 0,
+ Type: EnflameGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardComputeUnitsExhausted",
+ },
+ {
+ name: "fit fail: AllocatedCardsInsufficientRequest",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 20,
+ Type: EnflameGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 AllocatedCardsInsufficientRequest",
+ },
+ {
+ name: "fit success: memory percentage",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: EnflameGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 10,
+ Coresreq: 20,
+ Type: EnflameGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ allocated := &util.PodDevices{}
+ fit, result, reason := dev.Fit(test.devices, test.request, test.annos, &corev1.Pod{}, &util.NodeInfo{}, allocated)
+ if fit != test.wantFit {
+ t.Errorf("Fit: got %v, want %v", fit, test.wantFit)
+ }
+ if test.wantFit {
+ if len(result[EnflameGPUDevice]) != test.wantLen {
+ t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[EnflameGPUDevice]))
+ }
+ for idx, id := range test.wantDevIDs {
+ if id != result[EnflameGPUDevice][idx].UUID {
+ t.Errorf("expected device id: %s, got device id %s", id, result[EnflameGPUDevice][idx].UUID)
+ }
+ }
+ }
+
+ if reason != test.wantReason {
+ t.Errorf("expected reason: %s, got reason: %s", test.wantReason, reason)
+ }
+ })
+ }
+}
+
+func TestDevices_AddResourceUsage(t *testing.T) {
+ tests := []struct {
+ name string
+ deviceUsage *util.DeviceUsage
+ ctr *util.ContainerDevice
+ wantErr bool
+ wantUsage *util.DeviceUsage
+ }{
+ {
+ name: "test add resource usage",
+ deviceUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 0,
+ Usedcores: 15,
+ Usedmem: 2000,
+ },
+ ctr: &util.ContainerDevice{
+ UUID: "dev-0",
+ Usedcores: 50,
+ Usedmem: 1024,
+ },
+ wantUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 1,
+ Usedcores: 65,
+ Usedmem: 3024,
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &EnflameDevices{}
+ if err := dev.AddResourceUsage(&corev1.Pod{}, tt.deviceUsage, tt.ctr); (err != nil) != tt.wantErr {
+ t.Errorf("AddResourceUsage() error=%v, wantErr %v", err, tt.wantErr)
+ }
+ if !tt.wantErr {
+ if tt.deviceUsage.Usedcores != tt.wantUsage.Usedcores {
+ t.Errorf("expected used cores: %d, got used cores %d", tt.wantUsage.Usedcores, tt.deviceUsage.Usedcores)
+ }
+ if tt.deviceUsage.Usedmem != tt.wantUsage.Usedmem {
+ t.Errorf("expected used mem: %d, got used mem %d", tt.wantUsage.Usedmem, tt.deviceUsage.Usedmem)
+ }
+ if tt.deviceUsage.Used != tt.wantUsage.Used {
+ t.Errorf("expected used: %d, got used %d", tt.wantUsage.Used, tt.deviceUsage.Used)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/device/hygon/device.go b/pkg/device/hygon/device.go
new file mode 100644
index 000000000..073f878b9
--- /dev/null
+++ b/pkg/device/hygon/device.go
@@ -0,0 +1,378 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hygon
+
+import (
+ "errors"
+ "flag"
+ "slices"
+ "strings"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/common"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/nodelock"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/klog/v2"
+)
+
+type DCUDevices struct {
+}
+
+const (
+ HandshakeAnnos = "hami.io/node-handshake-dcu"
+ RegisterAnnos = "hami.io/node-dcu-register"
+ HygonDCUDevice = "DCU"
+ HygonDCUCommonWord = "DCU"
+ DCUInUse = "hygon.com/use-dcutype"
+ DCUNoUse = "hygon.com/nouse-dcutype"
+ // DCUUseUUID is user can use specify DCU device for set DCU UUID.
+ DCUUseUUID = "hygon.com/use-gpuuuid"
+ // DCUNoUseUUID is user can not use specify DCU device for set DCU UUID.
+ DCUNoUseUUID = "hygon.com/nouse-gpuuuid"
+
+ // NodeLockDCU should same with device plugin node lock name
+ // there is a bug with nodelock package utils, the key is hard coded as "hami.io/mutex.lock"
+ // so we can only use this value now.
+ NodeLockDCU = "hami.io/mutex.lock"
+)
+
+var (
+ HygonResourceCount string
+ HygonResourceMemory string
+ HygonResourceCores string
+)
+
+type HygonConfig struct {
+ ResourceCountName string `yaml:"resourceCountName"`
+ ResourceMemoryName string `yaml:"resourceMemoryName"`
+ ResourceCoreName string `yaml:"resourceCoreName"`
+}
+
+func InitDCUDevice(config HygonConfig) *DCUDevices {
+ HygonResourceCount = config.ResourceCountName
+ HygonResourceMemory = config.ResourceMemoryName
+ HygonResourceCores = config.ResourceCoreName
+ util.InRequestDevices[HygonDCUDevice] = "hami.io/dcu-devices-to-allocate"
+ util.SupportDevices[HygonDCUDevice] = "hami.io/dcu-devices-allocated"
+ util.HandshakeAnnos[HygonDCUDevice] = HandshakeAnnos
+ return &DCUDevices{}
+}
+
+func (dev *DCUDevices) CommonWord() string {
+ return HygonDCUCommonWord
+}
+
+func ParseConfig(fs *flag.FlagSet) {
+ fs.StringVar(&HygonResourceCount, "dcu-name", "hygon.com/dcunum", "dcu resource count")
+ fs.StringVar(&HygonResourceMemory, "dcu-memory", "hygon.com/dcumem", "dcu memory resource")
+ fs.StringVar(&HygonResourceCores, "dcu-cores", "hygon.com/dcucores", "dcu core resource")
+}
+
+func (dev *DCUDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
+ _, ok := ctr.Resources.Limits[corev1.ResourceName(HygonResourceCount)]
+ return ok, nil
+}
+
+func checkDCUtype(annos map[string]string, cardtype string) bool {
+ if inuse, ok := annos[DCUInUse]; ok {
+ if !strings.Contains(inuse, ",") {
+ if strings.Contains(strings.ToUpper(cardtype), strings.ToUpper(inuse)) {
+ return true
+ }
+ } else {
+ for val := range strings.SplitSeq(inuse, ",") {
+ if strings.Contains(strings.ToUpper(cardtype), strings.ToUpper(val)) {
+ return true
+ }
+ }
+ }
+ return false
+ }
+ if nouse, ok := annos[DCUNoUse]; ok {
+ if !strings.Contains(nouse, ",") {
+ if strings.Contains(strings.ToUpper(cardtype), strings.ToUpper(nouse)) {
+ return false
+ }
+ } else {
+ for val := range strings.SplitSeq(nouse, ",") {
+ if strings.Contains(strings.ToUpper(cardtype), strings.ToUpper(val)) {
+ return false
+ }
+ }
+ }
+ return true
+ }
+ return true
+}
+
+func (dev *DCUDevices) LockNode(n *corev1.Node, p *corev1.Pod) error {
+ found := false
+ for _, val := range p.Spec.Containers {
+ if (dev.GenerateResourceRequests(&val).Nums) > 0 {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil
+ }
+ return nodelock.LockNode(n.Name, NodeLockDCU, p)
+}
+
+func (dev *DCUDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
+ found := false
+ for _, val := range p.Spec.Containers {
+ if (dev.GenerateResourceRequests(&val).Nums) > 0 {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil
+ }
+ return nodelock.ReleaseNodeLock(n.Name, NodeLockDCU, p, false)
+}
+
+func (dev *DCUDevices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) {
+ devEncoded, ok := n.Annotations[RegisterAnnos]
+ if !ok {
+ return []*util.DeviceInfo{}, errors.New("annos not found " + RegisterAnnos)
+ }
+ nodedevices, err := util.DecodeNodeDevices(devEncoded)
+ if err != nil {
+ klog.ErrorS(err, "failed to decode node devices", "node", n.Name, "device annotation", devEncoded)
+ return []*util.DeviceInfo{}, err
+ }
+ if len(nodedevices) == 0 {
+ klog.InfoS("no gpu device found", "node", n.Name, "device annotation", devEncoded)
+ return []*util.DeviceInfo{}, errors.New("no gpu found on node")
+ }
+ devDecoded := util.EncodeNodeDevices(nodedevices)
+ klog.V(5).InfoS("nodes device information", "node", n.Name, "nodedevices", devDecoded)
+ return nodedevices, nil
+}
+
+func (dev *DCUDevices) NodeCleanUp(nn string) error {
+ return util.MarkAnnotationsToDelete(HandshakeAnnos, nn)
+}
+
+func (dev *DCUDevices) CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ return util.CheckHealth(devType, n)
+}
+
+func (dev *DCUDevices) checkType(annos map[string]string, d util.DeviceUsage, n util.ContainerDeviceRequest) (bool, bool, bool) {
+ if strings.Compare(n.Type, HygonDCUDevice) == 0 {
+ return true, checkDCUtype(annos, d.Type), false
+ }
+ return false, false, false
+}
+
+func (dev *DCUDevices) checkUUID(annos map[string]string, d util.DeviceUsage) bool {
+ userUUID, ok := annos[DCUUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for dcu user uuid [%s], device id is %s", userUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ userUUIDs := strings.Split(userUUID, ",")
+ return slices.Contains(userUUIDs, d.ID)
+ }
+
+ noUserUUID, ok := annos[DCUNoUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for dcu not user uuid [%s], device id is %s", noUserUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ noUserUUIDs := strings.Split(noUserUUID, ",")
+ return !slices.Contains(noUserUUIDs, d.ID)
+ }
+ return true
+}
+
+func (dev *DCUDevices) checkIndex(annos map[string]string, d util.DeviceUsage) bool {
+ return true
+}
+
+func (dev *DCUDevices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest {
+ klog.Info("Start to count dcu devices for container ", ctr.Name)
+ dcuResourceCount := corev1.ResourceName(HygonResourceCount)
+ dcuResourceMem := corev1.ResourceName(HygonResourceMemory)
+ dcuResourceCores := corev1.ResourceName(HygonResourceCores)
+ v, ok := ctr.Resources.Limits[dcuResourceCount]
+ if !ok {
+ v, ok = ctr.Resources.Requests[dcuResourceCount]
+ }
+ if ok {
+ if n, ok := v.AsInt64(); ok {
+ klog.Info("Found dcu devices")
+ memnum := 0
+ mem, ok := ctr.Resources.Limits[dcuResourceMem]
+ if !ok {
+ mem, ok = ctr.Resources.Requests[dcuResourceMem]
+ }
+ if ok {
+ memnums, ok := mem.AsInt64()
+ if ok {
+ memnum = int(memnums)
+ }
+ }
+ corenum := int32(100)
+ core, ok := ctr.Resources.Limits[dcuResourceCores]
+ if !ok {
+ core, ok = ctr.Resources.Requests[dcuResourceCores]
+ }
+ if ok {
+ corenums, ok := core.AsInt64()
+ if ok {
+ corenum = int32(corenums)
+ }
+ }
+
+ mempnum := 0
+ if memnum == 0 {
+ mempnum = 100
+ }
+
+ return util.ContainerDeviceRequest{
+ Nums: int32(n),
+ Type: HygonDCUDevice,
+ Memreq: int32(memnum),
+ MemPercentagereq: int32(mempnum),
+ Coresreq: corenum,
+ }
+ }
+ }
+ return util.ContainerDeviceRequest{}
+}
+
+func (dev *DCUDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd util.PodDevices) map[string]string {
+ devlist, ok := pd[HygonDCUDevice]
+ if ok && len(devlist) > 0 {
+ deviceStr := util.EncodePodSingleDevice(devlist)
+ (*annoinput)[util.InRequestDevices[HygonDCUDevice]] = deviceStr
+ (*annoinput)[util.SupportDevices[HygonDCUDevice]] = deviceStr
+ klog.V(5).Infof("pod add notation key [%s], values is [%s]", util.InRequestDevices[HygonDCUDevice], deviceStr)
+ klog.V(5).Infof("pod add notation key [%s], values is [%s]", util.SupportDevices[HygonDCUDevice], deviceStr)
+ }
+ return *annoinput
+}
+
+func (dev *DCUDevices) ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32 {
+ return 0
+}
+
+func (dev *DCUDevices) AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error {
+ n.Used++
+ n.Usedcores += ctr.Usedcores
+ n.Usedmem += ctr.Usedmem
+ return nil
+}
+
+func (dcu *DCUDevices) Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string) {
+ k := request
+ originReq := k.Nums
+ prevnuma := -1
+ klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
+ var tmpDevs map[string]util.ContainerDevices
+ tmpDevs = make(map[string]util.ContainerDevices)
+ reason := make(map[string]int)
+ for i := len(devices) - 1; i >= 0; i-- {
+ dev := devices[i]
+ klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
+
+ _, found, numa := dcu.checkType(annos, *dev, k)
+ if !found {
+ reason[common.CardTypeMismatch]++
+ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, k.Type)
+ continue
+ }
+ if numa && prevnuma != dev.Numa {
+ if k.Nums != originReq {
+ reason[common.NumaNotFit] += len(tmpDevs)
+ klog.V(5).InfoS(common.NumaNotFit, "pod", klog.KObj(pod), "device", dev.ID, "k.nums", k.Nums, "numa", numa, "prevnuma", prevnuma, "device numa", dev.Numa)
+ }
+ k.Nums = originReq
+ prevnuma = dev.Numa
+ tmpDevs = make(map[string]util.ContainerDevices)
+ }
+ if !dcu.checkUUID(annos, *dev) {
+ reason[common.CardUUIDMismatch]++
+ klog.V(5).InfoS(common.CardUUIDMismatch, "pod", klog.KObj(pod), "device", dev.ID, "current device info is:", *dev)
+ continue
+ }
+
+ memreq := int32(0)
+ if dev.Count <= dev.Used {
+ reason[common.CardTimeSlicingExhausted]++
+ klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
+ continue
+ }
+ if k.Coresreq > 100 {
+ klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Coresreq = 100
+ //return false, tmpDevs
+ }
+ if k.Memreq > 0 {
+ memreq = k.Memreq
+ }
+ if k.MemPercentagereq != 101 && k.Memreq == 0 {
+ //This incurs an issue
+ memreq = dev.Totalmem * k.MemPercentagereq / 100
+ }
+ if dev.Totalmem-dev.Usedmem < memreq {
+ reason[common.CardInsufficientMemory]++
+ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq)
+ continue
+ }
+ if dev.Totalcore-dev.Usedcores < k.Coresreq {
+ reason[common.CardInsufficientCore]++
+ klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", k.Coresreq)
+ continue
+ }
+ // Coresreq=100 indicates it want this card exclusively
+ if dev.Totalcore == 100 && k.Coresreq == 100 && dev.Used > 0 {
+ reason[common.ExclusiveDeviceAllocateConflict]++
+ klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
+ continue
+ }
+ // You can't allocate core=0 job to an already full GPU
+ if dev.Totalcore != 0 && dev.Usedcores == dev.Totalcore && k.Coresreq == 0 {
+ reason[common.CardComputeUnitsExhausted]++
+ klog.V(5).InfoS(common.CardComputeUnitsExhausted, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+
+ if k.Nums > 0 {
+ klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Nums--
+ tmpDevs[k.Type] = append(tmpDevs[k.Type], util.ContainerDevice{
+ Idx: int(dev.Index),
+ UUID: dev.ID,
+ Type: k.Type,
+ Usedmem: memreq,
+ Usedcores: k.Coresreq,
+ })
+ }
+ if k.Nums == 0 {
+ klog.V(4).InfoS("device allocate success", "pod", klog.KObj(pod), "allocate device", tmpDevs)
+ return true, tmpDevs, ""
+ }
+ }
+ if len(tmpDevs) > 0 {
+ reason[common.AllocatedCardsInsufficientRequest] = len(tmpDevs)
+ klog.V(5).InfoS(common.AllocatedCardsInsufficientRequest, "pod", klog.KObj(pod), "request", originReq, "allocated", len(tmpDevs))
+ }
+ return false, tmpDevs, common.GenReason(reason, len(devices))
+}
diff --git a/pkg/device/hygon/device_test.go b/pkg/device/hygon/device_test.go
new file mode 100644
index 000000000..8e1bf1352
--- /dev/null
+++ b/pkg/device/hygon/device_test.go
@@ -0,0 +1,1247 @@
+/*cardtype string
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hygon
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "testing"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes/fake"
+ "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+)
+
+func Test_MutateAdmission(t *testing.T) {
+ config := HygonConfig{
+ ResourceMemoryName: "hygon.com/dcumem",
+ ResourceCountName: "hygon.com/dcunum",
+ ResourceCoreName: "hygon.com/dcucores",
+ }
+ InitDCUDevice(config)
+ tests := []struct {
+ name string
+ args struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }
+ want bool
+ err error
+ }{
+ {
+ name: "set to resource limits",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hygon.com/dcunum": resource.MustParse("1"),
+ },
+ },
+ },
+ p: &corev1.Pod{},
+ },
+ want: true,
+ err: nil,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := DCUDevices{}
+ result, err := dev.MutateAdmission(test.args.ctr, test.args.p)
+ if err != test.err {
+ klog.InfoS("set to resource limits failed")
+ }
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_checkDCUtype(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ cardtype string
+ }
+ want bool
+ }{
+ {
+ name: "exist one dcu in use type",
+ args: struct {
+ annos map[string]string
+ cardtype string
+ }{
+ annos: map[string]string{
+ "hygon.com/use-dcutype": "dcu",
+ },
+ cardtype: "dcu",
+ },
+ want: true,
+ },
+ {
+ name: "the different of dcu in use type",
+ args: struct {
+ annos map[string]string
+ cardtype string
+ }{
+ annos: map[string]string{
+ "hygon.com/use-dcutype": "dcu",
+ },
+ cardtype: "test",
+ },
+ want: false,
+ },
+ {
+ name: "no dcu in use annotation",
+ args: struct {
+ annos map[string]string
+ cardtype string
+ }{
+ annos: map[string]string{},
+ cardtype: "dcu",
+ },
+ want: true,
+ },
+ {
+ name: "exist multi dcu in use type",
+ args: struct {
+ annos map[string]string
+ cardtype string
+ }{
+ annos: map[string]string{
+ "hygon.com/use-dcutype": "dcu,test",
+ },
+ cardtype: "dcu",
+ },
+ want: true,
+ },
+ {
+ name: "exist one dcu no use type",
+ args: struct {
+ annos map[string]string
+ cardtype string
+ }{
+ annos: map[string]string{
+ "hygon.com/nouse-dcutype": "dcu",
+ },
+ cardtype: "dcu",
+ },
+ want: false,
+ },
+ {
+ name: "no dcu in use annotation",
+ args: struct {
+ annos map[string]string
+ cardtype string
+ }{
+ annos: map[string]string{},
+ cardtype: "dcu",
+ },
+ want: true,
+ },
+ {
+ name: "exist multi dcu no use type",
+ args: struct {
+ annos map[string]string
+ cardtype string
+ }{
+ annos: map[string]string{
+ "hygon.com/nouse-dcutype": "test,dcu",
+ },
+ cardtype: "test",
+ },
+ want: false,
+ },
+ {
+ name: "the different of dcu no use type",
+ args: struct {
+ annos map[string]string
+ cardtype string
+ }{
+ annos: map[string]string{
+ "hygon.com/nouse-dcutype": "dcu",
+ },
+ cardtype: "test",
+ },
+ want: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result := checkDCUtype(test.args.annos, test.args.cardtype)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_GetNodeDevices(t *testing.T) {
+ dev := DCUDevices{}
+ tests := []struct {
+ name string
+ args corev1.Node
+ want []*util.DeviceInfo
+ err error
+ }{
+ {
+ name: "no annotation",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{},
+ },
+ },
+ want: []*util.DeviceInfo{},
+ err: errors.New("annos not found " + RegisterAnnos),
+ },
+ {
+ name: "exist dcu device",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "hami.io/node-dcu-register": "test-0,1,1024,100,DCU,0,true,0,test:",
+ },
+ },
+ },
+ want: []*util.DeviceInfo{
+ {
+ ID: "test-0",
+ Count: int32(1),
+ Devmem: int32(1024),
+ Devcore: int32(100),
+ Type: dev.CommonWord(),
+ Numa: 0,
+ Health: true,
+ Index: uint(0),
+ Mode: "test",
+ },
+ },
+ err: nil,
+ },
+ {
+ name: "no dcu device",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "hami.io/node-dcu-register": ":",
+ },
+ },
+ },
+ want: []*util.DeviceInfo{},
+ err: errors.New("no gpu found on node"),
+ },
+ {
+ name: "node annotations not decode successfully",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "hami.io/node-dcu-register": "",
+ },
+ },
+ },
+ want: []*util.DeviceInfo{},
+ err: errors.New("node annotations not decode successfully"),
+ },
+ {
+ name: "dcu device length less than 7",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "hami.io/node-dcu-register": "test-0,1,1024,100,DCU:",
+ },
+ },
+ },
+ want: []*util.DeviceInfo{},
+ err: errors.New("node annotations not decode successfully"),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result, err := dev.GetNodeDevices(test.args)
+ if err != nil {
+ klog.Errorf("got %v, want %v", err, test.err)
+ }
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func Test_CheckHealth(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ devType string
+ n *corev1.Node
+ }
+ want1 bool
+ want2 bool
+ }{
+ {
+ name: "Requesting state expired",
+ args: struct {
+ devType string
+ n *corev1.Node
+ }{
+ devType: "hygon.com/dcu",
+ n: &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ util.HandshakeAnnos["hygon.com/dcu"]: "Requesting_2025-01-07 00:00:00",
+ },
+ },
+ },
+ },
+ want1: false,
+ want2: false,
+ },
+ {
+ name: "Deleted state",
+ args: struct {
+ devType string
+ n *corev1.Node
+ }{
+ devType: "hygon.com/dcu",
+ n: &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ util.HandshakeAnnos["hygon.com/dcu"]: "Deleted",
+ },
+ },
+ },
+ },
+ want1: true,
+ want2: false,
+ },
+ {
+ name: "Unknown state",
+ args: struct {
+ devType string
+ n *corev1.Node
+ }{
+ devType: "hygon.com/dcu",
+ n: &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ util.HandshakeAnnos["hygon.com/dcu"]: "Unknown",
+ },
+ },
+ },
+ },
+ want1: true,
+ want2: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := DCUDevices{}
+ result1, result2 := dev.CheckHealth(test.args.devType, test.args.n)
+ assert.Equal(t, result1, test.want1)
+ assert.Equal(t, result2, test.want2)
+ })
+ }
+}
+
+func Test_checkType(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }
+ want1 bool
+ want2 bool
+ want3 bool
+ }{
+ {
+ name: "the same type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{
+ "hygon.com/use-dcutype": "DCU",
+ },
+ d: util.DeviceUsage{
+ Type: "DCU",
+ },
+ n: util.ContainerDeviceRequest{
+ Type: "DCU",
+ },
+ },
+ want1: true,
+ want2: true,
+ want3: false,
+ },
+ {
+ name: "the different type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{
+ "hygon.com/use-dcutype": "DCU",
+ },
+ d: util.DeviceUsage{
+ Type: "DCU",
+ },
+ n: util.ContainerDeviceRequest{
+ Type: "test",
+ },
+ },
+ want1: false,
+ want2: false,
+ want3: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := DCUDevices{}
+ result1, result2, result3 := dev.checkType(test.args.annos, test.args.d, test.args.n)
+ assert.Equal(t, result1, test.want1)
+ assert.Equal(t, result2, test.want2)
+ assert.Equal(t, result3, test.want3)
+ })
+ }
+}
+
+func Test_checkUUID(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "device id the same as the dcu in use uuid",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "hygon.com/use-gpuuuid": "123",
+ },
+ d: util.DeviceUsage{
+ ID: "123",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "device id the different from the dcu in use uuid",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "hygon.com/use-gpuuuid": "123",
+ },
+ d: util.DeviceUsage{
+ ID: "456",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "no dcu in use uuid annos",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{
+ ID: "456",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "device id the same as the dcu no use uuid",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "hygon.com/nouse-gpuuuid": "123",
+ },
+ d: util.DeviceUsage{
+ ID: "123",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "device id the different from the dcu no use uuid",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "hygon.com/nouse-gpuuuid": "123",
+ },
+ d: util.DeviceUsage{
+ ID: "456",
+ },
+ },
+ want: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := DCUDevices{}
+ result := dev.checkUUID(test.args.annos, test.args.d)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_PatchAnnotations(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annoinput *map[string]string
+ pd util.PodDevices
+ }
+ want map[string]string
+ }{
+ {
+ name: "exist device",
+ args: struct {
+ annoinput *map[string]string
+ pd util.PodDevices
+ }{
+ annoinput: &map[string]string{},
+ pd: util.PodDevices{
+ HygonDCUDevice: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ Idx: 1,
+ UUID: "test1",
+ Type: HygonDCUDevice,
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ },
+ },
+ },
+ },
+ },
+ want: map[string]string{
+ util.InRequestDevices[HygonDCUDevice]: "test1,DCU,2048,1:;",
+ util.SupportDevices[HygonDCUDevice]: "test1,DCU,2048,1:;",
+ },
+ },
+ {
+ name: "no device",
+ args: struct {
+ annoinput *map[string]string
+ pd util.PodDevices
+ }{
+ annoinput: &map[string]string{},
+ pd: util.PodDevices{},
+ },
+ want: map[string]string{},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := DCUDevices{}
+ result := dev.PatchAnnotations(&corev1.Pod{}, test.args.annoinput, test.args.pd)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func Test_GenerateResourceRequests(t *testing.T) {
+ tests := []struct {
+ name string
+ args *corev1.Container
+ want util.ContainerDeviceRequest
+ }{
+ {
+ name: "don't set to limits and request",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ Requests: corev1.ResourceList{},
+ },
+ },
+ want: util.ContainerDeviceRequest{},
+ },
+ {
+ name: "dcuResourceCount,dcuesourceMem and dcuResourceCores set to limits and request",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hygon.com/dcunum": resource.MustParse("1"),
+ "hygon.com/dcucores": resource.MustParse("1"),
+ "hygon.com/dcumem": resource.MustParse("1024"),
+ },
+ Requests: corev1.ResourceList{
+ "hygon.com/dcunum": resource.MustParse("1"),
+ "hygon.com/dcucores": resource.MustParse("1"),
+ "hygon.com/dcumem": resource.MustParse("1024"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: HygonDCUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(0),
+ Coresreq: int32(1),
+ },
+ },
+ {
+ name: "dcuResourceMem,dcuResourceCores don't set limit and dcuResourceCount set to limits and request",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hygon.com/dcunum": resource.MustParse("1"),
+ },
+ Requests: corev1.ResourceList{
+ "hygon.com/dcunum": resource.MustParse("1"),
+ "hygon.com/dcucores": resource.MustParse("1"),
+ "hygon.com/dcumem": resource.MustParse("1024"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: HygonDCUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(0),
+ Coresreq: int32(1),
+ },
+ },
+ {
+ name: "dcuResourceMem don't set limit and request,dcuResourceCores don't set limit and dcuResourceCount set to limits and request",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hygon.com/dcunum": resource.MustParse("1"),
+ },
+ Requests: corev1.ResourceList{
+ "hygon.com/dcunum": resource.MustParse("1"),
+ "hygon.com/dcucores": resource.MustParse("1"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: HygonDCUDevice,
+ Memreq: int32(0),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(1),
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := DCUDevices{}
+ fs := flag.FlagSet{}
+ ParseConfig(&fs)
+ result := dev.GenerateResourceRequests(test.args)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func TestDevices_LockNode(t *testing.T) {
+ tests := []struct {
+ name string
+ node *corev1.Node
+ pod *corev1.Pod
+ hasLock bool
+ expectError bool
+ }{
+ {
+ name: "Test with no containers",
+ node: &corev1.Node{},
+ pod: &corev1.Pod{Spec: corev1.PodSpec{}},
+ hasLock: false,
+ expectError: false,
+ },
+ {
+ name: "Test with non-zero resource requests",
+ node: &corev1.Node{},
+ pod: &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{
+ "hygon.com/dcunum": resource.MustParse("1"),
+ }}}}}},
+ hasLock: true,
+ expectError: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // Initialize fake clientset and pre-load test data
+ client.KubeClient = fake.NewSimpleClientset()
+ node := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "testNode",
+ Annotations: map[string]string{"test-annotation-key": "test-annotation-value", util.InRequestDevices["DCU"]: "some-value"},
+ },
+ }
+
+ // Add the node to the fake clientset
+ _, err := client.KubeClient.CoreV1().Nodes().Create(context.Background(), node, metav1.CreateOptions{})
+ if err != nil {
+ t.Fatalf("Failed to create test node: %v", err)
+ }
+
+ dev := InitDCUDevice(HygonConfig{
+ ResourceCountName: "hygon.com/dcunum",
+ ResourceMemoryName: "hygon.com/dcumem",
+ ResourceCoreName: "hygon.com/dcucores",
+ })
+ err = dev.LockNode(node, tt.pod)
+ if tt.expectError {
+ assert.Equal(t, err != nil, true)
+ } else {
+ assert.NilError(t, err)
+ }
+ node, err = client.KubeClient.CoreV1().Nodes().Get(context.Background(), node.Name, metav1.GetOptions{})
+ assert.NilError(t, err)
+ fmt.Println(node.Annotations)
+ _, ok := node.Annotations[NodeLockDCU]
+ assert.Equal(t, ok, tt.hasLock)
+ })
+ }
+}
+
+func TestDevices_ReleaseNodeLock(t *testing.T) {
+ tests := []struct {
+ name string
+ node *corev1.Node
+ pod *corev1.Pod
+ hasLock bool
+ expectError bool
+ }{
+ {
+ name: "Test with no containers",
+ node: &corev1.Node{},
+ pod: &corev1.Pod{Spec: corev1.PodSpec{}},
+ hasLock: true,
+ expectError: false,
+ },
+ {
+ name: "Test with non-zero resource requests",
+ node: &corev1.Node{},
+ pod: &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{
+ "hygon.com/dcunum": resource.MustParse("1"),
+ }}}}}},
+ hasLock: false,
+ expectError: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // Initialize fake clientset and pre-load test data
+ client.KubeClient = fake.NewSimpleClientset()
+ node := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "testNode",
+ Annotations: map[string]string{"test-annotation-key": "test-annotation-value", util.InRequestDevices["DCU"]: "some-value", NodeLockDCU: "lock-values"},
+ },
+ }
+
+ // Add the node to the fake clientset
+ _, err := client.KubeClient.CoreV1().Nodes().Create(context.Background(), node, metav1.CreateOptions{})
+ if err != nil {
+ t.Fatalf("Failed to create test node: %v", err)
+ }
+
+ dev := InitDCUDevice(HygonConfig{
+ ResourceCountName: "hygon.com/dcunum",
+ ResourceMemoryName: "hygon.com/dcumem",
+ ResourceCoreName: "hygon.com/dcucores",
+ })
+ err = dev.ReleaseNodeLock(node, tt.pod)
+ if tt.expectError {
+ assert.Equal(t, err != nil, true)
+ } else {
+ assert.NilError(t, err)
+ }
+ node, err = client.KubeClient.CoreV1().Nodes().Get(context.Background(), node.Name, metav1.GetOptions{})
+ assert.NilError(t, err)
+ fmt.Println(node.Annotations)
+ _, ok := node.Annotations[NodeLockDCU]
+ assert.Equal(t, ok, tt.hasLock)
+ })
+ }
+}
+
+func TestDevices_Fit(t *testing.T) {
+ config := HygonConfig{
+ ResourceCountName: "hygon.com/dcunum",
+ ResourceMemoryName: "hygon.com/dcumem",
+ ResourceCoreName: "hygon.com/dcucores",
+ }
+ dev := InitDCUDevice(config)
+
+ tests := []struct {
+ name string
+ devices []*util.DeviceUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ wantFit bool
+ wantLen int
+ wantDevIDs []string
+ wantReason string
+ }{
+ {
+ name: "fit success",
+ devices: []*util.DeviceUsage{
+ {
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ },
+ {
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 64,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: HygonDCUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-1"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: memory not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: HygonDCUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientMemory",
+ },
+ {
+ name: "fit fail: core not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1024,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: HygonDCUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientCore",
+ },
+ {
+ name: "fit fail: type mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ Type: HygonDCUDevice,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: "OtherType",
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTypeMismatch",
+ },
+ {
+ name: "fit fail: user assign use uuid mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: HygonDCUDevice,
+ },
+ annos: map[string]string{DCUUseUUID: "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: user assign no use uuid match",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: HygonDCUDevice,
+ },
+ annos: map[string]string{DCUNoUseUUID: "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: card overused",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 100,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: HygonDCUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTimeSlicingExhausted",
+ },
+ {
+ name: "fit success: but core limit can't exceed 100",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 120,
+ Type: HygonDCUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: card exclusively",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 100,
+ Type: HygonDCUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 ExclusiveDeviceAllocateConflict",
+ },
+ {
+ name: "fit fail: CardComputeUnitsExhausted",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 0,
+ Type: HygonDCUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardComputeUnitsExhausted",
+ },
+ {
+ name: "fit fail: AllocatedCardsInsufficientRequest",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 20,
+ Type: HygonDCUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 AllocatedCardsInsufficientRequest",
+ },
+ {
+ name: "fit success: memory percentage",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: HygonDCUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 10,
+ Coresreq: 20,
+ Type: HygonDCUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ allocated := &util.PodDevices{}
+ fit, result, reason := dev.Fit(test.devices, test.request, test.annos, &corev1.Pod{}, &util.NodeInfo{}, allocated)
+ if fit != test.wantFit {
+ t.Errorf("Fit: got %v, want %v", fit, test.wantFit)
+ }
+ if test.wantFit {
+ if len(result[HygonDCUDevice]) != test.wantLen {
+ t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[HygonDCUDevice]))
+ }
+ for idx, id := range test.wantDevIDs {
+ if id != result[HygonDCUDevice][idx].UUID {
+ t.Errorf("expected device id: %s, got device id %s", id, result[HygonDCUDevice][idx].UUID)
+ }
+ }
+ }
+
+ if reason != test.wantReason {
+ t.Errorf("expected reason: %s, got reason: %s", test.wantReason, reason)
+ }
+ })
+ }
+}
+
+func TestDevices_AddResourceUsage(t *testing.T) {
+ tests := []struct {
+ name string
+ deviceUsage *util.DeviceUsage
+ ctr *util.ContainerDevice
+ wantErr bool
+ wantUsage *util.DeviceUsage
+ }{
+ {
+ name: "test add resource usage",
+ deviceUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 0,
+ Usedcores: 15,
+ Usedmem: 2000,
+ },
+ ctr: &util.ContainerDevice{
+ UUID: "dev-0",
+ Usedcores: 50,
+ Usedmem: 1024,
+ },
+ wantUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 1,
+ Usedcores: 65,
+ Usedmem: 3024,
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &DCUDevices{}
+ if err := dev.AddResourceUsage(&corev1.Pod{}, tt.deviceUsage, tt.ctr); (err != nil) != tt.wantErr {
+ t.Errorf("AddResourceUsage() error=%v, wantErr %v", err, tt.wantErr)
+ }
+ if !tt.wantErr {
+ if tt.deviceUsage.Usedcores != tt.wantUsage.Usedcores {
+ t.Errorf("expected used cores: %d, got used cores %d", tt.wantUsage.Usedcores, tt.deviceUsage.Usedcores)
+ }
+ if tt.deviceUsage.Usedmem != tt.wantUsage.Usedmem {
+ t.Errorf("expected used mem: %d, got used mem %d", tt.wantUsage.Usedmem, tt.deviceUsage.Usedmem)
+ }
+ if tt.deviceUsage.Used != tt.wantUsage.Used {
+ t.Errorf("expected used: %d, got used %d", tt.wantUsage.Used, tt.deviceUsage.Used)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/device/iluvatar/device.go b/pkg/device/iluvatar/device.go
new file mode 100644
index 000000000..14ce22055
--- /dev/null
+++ b/pkg/device/iluvatar/device.go
@@ -0,0 +1,338 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package iluvatar
+
+import (
+ "flag"
+ "fmt"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/common"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ "k8s.io/klog/v2"
+)
+
+type IluvatarDevices struct {
+}
+
+const (
+ IluvatarGPUDevice = "Iluvatar"
+ IluvatarGPUCommonWord = "Iluvatar"
+ IluvatarDeviceSelection = "iluvatar.ai/predicate-gpu-idx-"
+ // IluvatarUseUUID is user can use specify Iluvatar device for set Iluvatar UUID.
+ IluvatarUseUUID = "iluvatar.ai/use-gpuuuid"
+ // IluvatarNoUseUUID is user can not use specify Iluvatar device for set Iluvatar UUID.
+ IluvatarNoUseUUID = "iluvatar.ai/nouse-gpuuuid"
+)
+
+var (
+ IluvatarResourceCount string
+ IluvatarResourceMemory string
+ IluvatarResourceCores string
+)
+
+type IluvatarConfig struct {
+ ResourceCountName string `yaml:"resourceCountName"`
+ ResourceMemoryName string `yaml:"resourceMemoryName"`
+ ResourceCoreName string `yaml:"resourceCoreName"`
+}
+
+func InitIluvatarDevice(config IluvatarConfig) *IluvatarDevices {
+ IluvatarResourceCount = config.ResourceCountName
+ IluvatarResourceMemory = config.ResourceMemoryName
+ IluvatarResourceCores = config.ResourceCoreName
+ util.InRequestDevices[IluvatarGPUDevice] = "hami.io/iluvatar-vgpu-devices-to-allocate"
+ util.SupportDevices[IluvatarGPUDevice] = "hami.io/iluvatar-vgpu-devices-allocated"
+ return &IluvatarDevices{}
+}
+
+func (dev *IluvatarDevices) CommonWord() string {
+ return IluvatarGPUCommonWord
+}
+
+func ParseConfig(fs *flag.FlagSet) {
+ fs.StringVar(&IluvatarResourceCount, "iluvatar-name", "iluvatar.ai/vgpu", "iluvatar resource count")
+ fs.StringVar(&IluvatarResourceMemory, "iluvatar-memory", "iluvatar.ai/vcuda-memory", "iluvatar memory resource")
+ fs.StringVar(&IluvatarResourceCores, "iluvatar-cores", "iluvatar.ai/vcuda-core", "iluvatar core resource")
+}
+
+func (dev *IluvatarDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
+ count, ok := ctr.Resources.Limits[corev1.ResourceName(IluvatarResourceCount)]
+ if ok {
+ if count.Value() > 1 {
+ ctr.Resources.Limits[corev1.ResourceName(IluvatarResourceCores)] = *resource.NewQuantity(count.Value()*int64(100), resource.DecimalSI)
+ }
+ }
+ return ok, nil
+}
+
+func (dev *IluvatarDevices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) {
+ nodedevices := []*util.DeviceInfo{}
+ i := 0
+ cards, ok := n.Status.Capacity.Name(corev1.ResourceName(IluvatarResourceCores), resource.DecimalSI).AsInt64()
+ if !ok || cards == 0 {
+ return []*util.DeviceInfo{}, fmt.Errorf("device not found %s", IluvatarResourceCores)
+ }
+ memoryTotal, _ := n.Status.Capacity.Name(corev1.ResourceName(IluvatarResourceMemory), resource.DecimalSI).AsInt64()
+ for int64(i)*100 < cards {
+ nodedevices = append(nodedevices, &util.DeviceInfo{
+ Index: uint(i),
+ ID: n.Name + "-iluvatar-" + fmt.Sprint(i),
+ Count: 100,
+ Devmem: int32(memoryTotal * 256 * 100 / cards),
+ Devcore: 100,
+ Type: IluvatarGPUDevice,
+ Numa: 0,
+ Health: true,
+ })
+ i++
+ }
+ return nodedevices, nil
+}
+
+func (dev *IluvatarDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd util.PodDevices) map[string]string {
+ devlist, ok := pd[IluvatarGPUDevice]
+ if ok && len(devlist) > 0 {
+ (*annoinput)[util.InRequestDevices[IluvatarGPUDevice]] = util.EncodePodSingleDevice(devlist)
+ (*annoinput)[util.SupportDevices[IluvatarGPUDevice]] = util.EncodePodSingleDevice(devlist)
+ (*annoinput)["iluvatar.ai/gpu-assigned"] = "false"
+ (*annoinput)["iluvatar.ai/predicate-time"] = strconv.FormatInt(time.Now().UnixNano(), 10)
+ for idx, dp := range devlist {
+ annoKey := IluvatarDeviceSelection + fmt.Sprint(idx)
+ value := ""
+ for _, val := range dp {
+ value = value + fmt.Sprint(val.Idx) + ","
+ }
+ if len(value) > 0 {
+ (*annoinput)[annoKey] = strings.TrimRight(value, ",")
+ }
+ }
+ }
+ return *annoinput
+}
+
+func (dev *IluvatarDevices) LockNode(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *IluvatarDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *IluvatarDevices) NodeCleanUp(nn string) error {
+ return nil
+}
+
+func (dev *IluvatarDevices) checkType(annos map[string]string, d util.DeviceUsage, n util.ContainerDeviceRequest) (bool, bool, bool) {
+ if strings.Compare(n.Type, IluvatarGPUDevice) == 0 {
+ return true, true, false
+ }
+ return false, false, false
+}
+
+func (dev *IluvatarDevices) checkUUID(annos map[string]string, d util.DeviceUsage) bool {
+ userUUID, ok := annos[IluvatarUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for Iluvatar user uuid [%s], device id is %s", userUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ userUUIDs := strings.Split(userUUID, ",")
+ return slices.Contains(userUUIDs, d.ID)
+ }
+
+ noUserUUID, ok := annos[IluvatarNoUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for Iluvatar not user uuid [%s], device id is %s", noUserUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ noUserUUIDs := strings.Split(noUserUUID, ",")
+ return !slices.Contains(noUserUUIDs, d.ID)
+ }
+ return true
+}
+
+func (dev *IluvatarDevices) checkIndex(annos map[string]string, d util.DeviceUsage) bool {
+ return true
+}
+
+func (dev *IluvatarDevices) CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ return true, true
+}
+
+func (dev *IluvatarDevices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest {
+ klog.Info("Start to count iluvatar devices for container ", ctr.Name)
+ iluvatarResourceCount := corev1.ResourceName(IluvatarResourceCount)
+ iluvatarResourceMem := corev1.ResourceName(IluvatarResourceMemory)
+ iluvatarResourceCores := corev1.ResourceName(IluvatarResourceCores)
+ v, ok := ctr.Resources.Limits[iluvatarResourceCount]
+ if !ok {
+ v, ok = ctr.Resources.Requests[iluvatarResourceCount]
+ }
+ if ok {
+ if n, ok := v.AsInt64(); ok {
+ klog.Info("Found iluvatar devices")
+ memnum := 0
+ mem, ok := ctr.Resources.Limits[iluvatarResourceMem]
+ if !ok {
+ mem, ok = ctr.Resources.Requests[iluvatarResourceMem]
+ }
+ if ok {
+ memnums, ok := mem.AsInt64()
+ if ok {
+ memnum = int(memnums) * 256
+ }
+ }
+ corenum := int32(0)
+ core, ok := ctr.Resources.Limits[iluvatarResourceCores]
+ if !ok {
+ core, ok = ctr.Resources.Requests[iluvatarResourceCores]
+ }
+ if ok {
+ corenums, ok := core.AsInt64()
+ if ok {
+ corenum = int32(corenums)
+ }
+ }
+
+ mempnum := 0
+ if memnum == 0 {
+ mempnum = 100
+ }
+
+ return util.ContainerDeviceRequest{
+ Nums: int32(n),
+ Type: IluvatarGPUDevice,
+ Memreq: int32(memnum),
+ MemPercentagereq: int32(mempnum),
+ Coresreq: corenum,
+ }
+ }
+ }
+ return util.ContainerDeviceRequest{}
+}
+
+func (dev *IluvatarDevices) ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32 {
+ return 0
+}
+
+func (dev *IluvatarDevices) AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error {
+ n.Used++
+ n.Usedcores += ctr.Usedcores
+ n.Usedmem += ctr.Usedmem
+ return nil
+}
+
+func (ilu *IluvatarDevices) Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string) {
+ k := request
+ originReq := k.Nums
+ prevnuma := -1
+ klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
+ var tmpDevs map[string]util.ContainerDevices
+ tmpDevs = make(map[string]util.ContainerDevices)
+ reason := make(map[string]int)
+ for i := 0; i < len(devices); i++ {
+ dev := devices[i]
+ klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
+
+ _, found, numa := ilu.checkType(annos, *dev, k)
+ if !found {
+ reason[common.CardTypeMismatch]++
+ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, k.Type)
+ continue
+ }
+ if numa && prevnuma != dev.Numa {
+ if k.Nums != originReq {
+ reason[common.NumaNotFit] += len(tmpDevs)
+ klog.V(5).InfoS(common.NumaNotFit, "pod", klog.KObj(pod), "device", dev.ID, "k.nums", k.Nums, "numa", numa, "prevnuma", prevnuma, "device numa", dev.Numa)
+ }
+ k.Nums = originReq
+ prevnuma = dev.Numa
+ tmpDevs = make(map[string]util.ContainerDevices)
+ }
+ if !ilu.checkUUID(annos, *dev) {
+ reason[common.CardUUIDMismatch]++
+ klog.V(5).InfoS(common.CardUUIDMismatch, "pod", klog.KObj(pod), "device", dev.ID, "current device info is:", *dev)
+ continue
+ }
+
+ memreq := int32(0)
+ if dev.Count <= dev.Used {
+ reason[common.CardTimeSlicingExhausted]++
+ klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
+ continue
+ }
+ if k.Coresreq > 100 {
+ klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Coresreq = 100
+ //return false, tmpDevs
+ }
+ if k.Memreq > 0 {
+ memreq = k.Memreq
+ }
+ if k.MemPercentagereq != 101 && k.Memreq == 0 {
+ //This incurs an issue
+ memreq = dev.Totalmem * k.MemPercentagereq / 100
+ }
+ if dev.Totalmem-dev.Usedmem < memreq {
+ reason[common.CardInsufficientMemory]++
+ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq)
+ continue
+ }
+ if dev.Totalcore-dev.Usedcores < k.Coresreq {
+ reason[common.CardInsufficientCore]++
+ klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", k.Coresreq)
+ continue
+ }
+ // Coresreq=100 indicates it want this card exclusively
+ if dev.Totalcore == 100 && k.Coresreq == 100 && dev.Used > 0 {
+ reason[common.ExclusiveDeviceAllocateConflict]++
+ klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
+ continue
+ }
+ // You can't allocate core=0 job to an already full GPU
+ if dev.Totalcore != 0 && dev.Usedcores == dev.Totalcore && k.Coresreq == 0 {
+ reason[common.CardComputeUnitsExhausted]++
+ klog.V(5).InfoS(common.CardComputeUnitsExhausted, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+ if k.Nums > 0 {
+ klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Nums--
+ tmpDevs[k.Type] = append(tmpDevs[k.Type], util.ContainerDevice{
+ Idx: int(dev.Index),
+ UUID: dev.ID,
+ Type: k.Type,
+ Usedmem: memreq,
+ Usedcores: k.Coresreq,
+ })
+ }
+ if k.Nums == 0 {
+ klog.V(4).InfoS("device allocate success", "pod", klog.KObj(pod), "allocate device", tmpDevs)
+ return true, tmpDevs, ""
+ }
+
+ }
+ if len(tmpDevs) > 0 {
+ reason[common.AllocatedCardsInsufficientRequest] = len(tmpDevs)
+ klog.V(5).InfoS(common.AllocatedCardsInsufficientRequest, "pod", klog.KObj(pod), "request", originReq, "allocated", len(tmpDevs))
+ }
+ return false, tmpDevs, common.GenReason(reason, len(devices))
+}
diff --git a/pkg/device/iluvatar/device_test.go b/pkg/device/iluvatar/device_test.go
new file mode 100644
index 000000000..b4f77e953
--- /dev/null
+++ b/pkg/device/iluvatar/device_test.go
@@ -0,0 +1,607 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package iluvatar
+
+import (
+ "flag"
+ "maps"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestGetNodeDevices(t *testing.T) {
+ IluvatarResourceCores = "iluvatar.ai/MR-V100.vCore"
+ IluvatarResourceMemory = "iluvatar.ai/MR-V100.vMem"
+
+ tests := []struct {
+ name string
+ node corev1.Node
+ expected []*util.DeviceInfo
+ err error
+ }{
+ {
+ name: "Test with valid node",
+ node: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test",
+ },
+ Status: corev1.NodeStatus{
+ Capacity: corev1.ResourceList{
+ corev1.ResourceName(IluvatarResourceCores): *resource.NewQuantity(100, resource.DecimalSI),
+ corev1.ResourceName(IluvatarResourceMemory): *resource.NewQuantity(128, resource.DecimalSI),
+ },
+ },
+ },
+ expected: []*util.DeviceInfo{
+ {
+ Index: 0,
+ ID: "test-iluvatar-0",
+ Count: 100,
+ Devmem: 32768,
+ Devcore: 100,
+ Type: IluvatarGPUDevice,
+ Numa: 0,
+ Health: true,
+ },
+ },
+ err: nil,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &IluvatarDevices{}
+ got, err := dev.GetNodeDevices(tt.node)
+ if (err != nil) != (tt.err != nil) {
+ t.Errorf("GetNodeDevices() error = %v, expected %v", err, tt.err)
+ return
+ }
+
+ if len(got) != len(tt.expected) {
+ t.Errorf("GetNodeDevices() got %d devices, expected %d", len(got), len(tt.expected))
+ return
+ }
+
+ for i, device := range got {
+ if device.Index != tt.expected[i].Index {
+ t.Errorf("Expected index %d, got %d", tt.expected[i].Index, device.Index)
+ }
+ if device.ID != tt.expected[i].ID {
+ t.Errorf("Expected id %s, got %s", tt.expected[i].ID, device.ID)
+ }
+ if device.Devcore != tt.expected[i].Devcore {
+ t.Errorf("Expected devcore %d, got %d", tt.expected[i].Devcore, device.Devcore)
+ }
+ if device.Devmem != tt.expected[i].Devmem {
+ t.Errorf("Expected cevmem %d, got %d", tt.expected[i].Devmem, device.Devmem)
+ }
+ }
+ })
+ }
+}
+
+func TestPatchAnnotations(t *testing.T) {
+ InitIluvatarDevice(IluvatarConfig{})
+
+ tests := []struct {
+ name string
+ annoInput map[string]string
+ podDevices util.PodDevices
+ expected map[string]string
+ }{
+ {
+ name: "No devices",
+ annoInput: map[string]string{},
+ podDevices: util.PodDevices{},
+ expected: map[string]string{},
+ },
+ {
+ name: "With devices",
+ annoInput: map[string]string{},
+ podDevices: util.PodDevices{
+ IluvatarGPUDevice: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ Idx: 0,
+ UUID: "k8s-gpu-iluvatar-0",
+ Type: "Iluvatar",
+ },
+ },
+ },
+ },
+ expected: map[string]string{
+ util.InRequestDevices[IluvatarGPUDevice]: "k8s-gpu-iluvatar-0,Iluvatar,0,0:;",
+ util.SupportDevices[IluvatarGPUDevice]: "k8s-gpu-iluvatar-0,Iluvatar,0,0:;",
+ "iluvatar.ai/gpu-assigned": "false",
+ "iluvatar.ai/predicate-time": strconv.FormatInt(time.Now().UnixNano(), 10),
+ IluvatarDeviceSelection + "0": "0",
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ annoInputCopy := make(map[string]string)
+ maps.Copy(annoInputCopy, tt.annoInput)
+
+ dev := &IluvatarDevices{}
+ got := dev.PatchAnnotations(&corev1.Pod{}, &annoInputCopy, tt.podDevices)
+
+ if len(got) != len(tt.expected) {
+ t.Errorf("PatchAnnotations() got %d annotations, expected %d", len(got), len(tt.expected))
+ return
+ }
+
+ for k, v := range tt.expected {
+ if k == "iluvatar.ai/predicate-time" {
+ if len(got[k]) != len(v) {
+ t.Errorf("Expected %s %s, got %s", k, v, got[k])
+ }
+ continue
+ }
+
+ if got[k] != v {
+ t.Errorf("Expected %s %s, got %s", k, v, got[k])
+ }
+ }
+ })
+ }
+}
+
+func Test_MutateAdmission(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }
+ want bool
+ err error
+ }{
+ {
+ name: "IluvatarResourceCount and IluvatarResourceCores set to limits",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "iluvatar.ai/vgpu": *resource.NewQuantity(2, resource.DecimalSI),
+ "iluvatar.ai/vcuda-core": *resource.NewQuantity(1, resource.DecimalSI),
+ },
+ },
+ },
+ p: &corev1.Pod{},
+ },
+ want: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ config := IluvatarConfig{
+ ResourceCountName: "iluvatar.ai/vgpu",
+ ResourceCoreName: "iluvatar.ai/vcuda-core",
+ ResourceMemoryName: "iluvatar.ai/vcuda-memory",
+ }
+ InitIluvatarDevice(config)
+ dev := IluvatarDevices{}
+ result, _ := dev.MutateAdmission(test.args.ctr, test.args.p)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_checkType(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }
+ want1 bool
+ want2 bool
+ want3 bool
+ }{
+ {
+ name: "the same type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: IluvatarGPUDevice,
+ },
+ },
+ want1: true,
+ want2: true,
+ want3: false,
+ },
+ {
+ name: "the different type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: "test123",
+ },
+ },
+ want1: false,
+ want2: false,
+ want3: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := IluvatarDevices{}
+ result1, result2, result3 := dev.checkType(test.args.annos, test.args.d, test.args.n)
+ assert.Equal(t, result1, test.want1)
+ assert.Equal(t, result2, test.want2)
+ assert.Equal(t, result3, test.want3)
+ })
+ }
+}
+
+func Test_checkUUID(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "useid is same as the device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "iluvatar.ai/use-gpuuuid": "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test1",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "useid is different from the device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "iluvatar.ai/use-gpuuuid": "test2",
+ },
+ d: util.DeviceUsage{
+ ID: "test1",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "no annos",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{
+ ID: "test3",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "nouseid is same as the device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "iluvatar.ai/nouse-gpuuuid": "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test1",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "nouseid is different from the device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ "iluvatar.ai/nouse-gpuuuid": "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test2",
+ },
+ },
+ want: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := IluvatarDevices{}
+ result := dev.checkUUID(test.args.annos, test.args.d)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_GenerateResourceRequests(t *testing.T) {
+ tests := []struct {
+ name string
+ args *corev1.Container
+ want util.ContainerDeviceRequest
+ }{
+ {
+ name: "all resources set to limits and requests",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "iluvatar.ai/vgpu": resource.MustParse("1"),
+ "iluvatar.ai/vcuda-memory": resource.MustParse("1000"),
+ "iluvatar.ai/vcuda-core": resource.MustParse("100"),
+ },
+ Requests: corev1.ResourceList{
+ "iluvatar.ai/vgpu": resource.MustParse("1"),
+ "iluvatar.ai/vcuda-memory": resource.MustParse("1000"),
+ "iluvatar.ai/vcuda-core": resource.MustParse("100"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: IluvatarGPUDevice,
+ Memreq: int32(256000),
+ MemPercentagereq: int32(0),
+ Coresreq: int32(100),
+ },
+ },
+ {
+ name: "all resources don't set to limits and requests",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ Requests: corev1.ResourceList{},
+ },
+ },
+ want: util.ContainerDeviceRequest{},
+ },
+ {
+ name: "resourcemem don't set to limits and requests",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "iluvatar.ai/vgpu": resource.MustParse("1"),
+ },
+ Requests: corev1.ResourceList{
+ "iluvatar.ai/vgpu": resource.MustParse("1"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: IluvatarGPUDevice,
+ Memreq: int32(0),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(0),
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := IluvatarDevices{}
+ fs := flag.FlagSet{}
+ ParseConfig(&fs)
+ result := dev.GenerateResourceRequests(test.args)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func Test_Fit(t *testing.T) {
+ config := IluvatarConfig{
+ ResourceCountName: "iluvatar.ai/vgpu",
+ ResourceCoreName: "iluvatar.ai/MR-V100.vCore",
+ ResourceMemoryName: "iluvatar.ai/MR-V100.vMem",
+ }
+ dev := InitIluvatarDevice(config)
+
+ tests := []struct {
+ name string
+ devices []*util.DeviceUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ wantOK bool
+ wantLen int
+ wantDevIDs []string
+ }{
+ {
+ name: "fit success",
+ devices: []*util.DeviceUsage{
+ {
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: IluvatarGPUDevice,
+ Health: true,
+ },
+ {
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: IluvatarGPUDevice,
+ Health: true,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: IluvatarGPUDevice,
+ Memreq: 64,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantOK: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ },
+ {
+ name: "fit fail: memory not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: IluvatarGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: IluvatarGPUDevice,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantOK: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ },
+ {
+ name: "fit fail: core not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: IluvatarGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: IluvatarGPUDevice,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantOK: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ },
+ {
+ name: "fit fail: type mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: IluvatarGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: "OtherType",
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantOK: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ allocated := &util.PodDevices{}
+ ok, result, _ := dev.Fit(test.devices, test.request, test.annos, &corev1.Pod{}, &util.NodeInfo{}, allocated)
+ if test.wantOK {
+ if len(result[IluvatarGPUDevice]) != test.wantLen {
+ t.Errorf("expected %d, got %d", test.wantLen, len(result[IluvatarGPUDevice]))
+ }
+ for idx, id := range test.wantDevIDs {
+ if id != result[IluvatarGPUDevice][idx].UUID {
+ t.Errorf("expected %s, got %s", id, result[IluvatarGPUDevice][idx].UUID)
+ }
+ }
+ if !ok {
+ t.Errorf("expected ok true, got false")
+ }
+ } else {
+ if ok {
+ t.Errorf("expected ok false, got true")
+ }
+ if len(result[IluvatarGPUDevice]) != test.wantLen {
+ t.Errorf("expected %d, got %d", test.wantLen, len(result[IluvatarGPUDevice]))
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/device/kunlun/device.go b/pkg/device/kunlun/device.go
new file mode 100644
index 000000000..2f827e4ed
--- /dev/null
+++ b/pkg/device/kunlun/device.go
@@ -0,0 +1,481 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kunlun
+
+import (
+ "flag"
+ "fmt"
+ "slices"
+ "sort"
+ "strconv"
+ "strings"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/common"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ "k8s.io/klog/v2"
+)
+
+type KunlunDevices struct {
+}
+
+const (
+ KunlunGPUDevice = "kunlun"
+ KunlunGPUCommonWord = "kunlun"
+ KunlunDeviceSelection = "BAIDU_COM_DEVICE_IDX"
+ KunlunUseUUID = "baidu.com/use-gpuuuid"
+ KunlunNoUseUUID = "baidu.com/nouse-gpuuuid"
+ InterGroupConnection = "0-4,1-5,2-6,3-7"
+ InterGroupConnection2 = "0-1-4-5,2-3-6-7,0-2-4-6,1-3-5-7,0-3-4-7,1-2-5-6"
+ GroupConnection = "0-1,0-2,0-3,1-2,1-3,2-3,4-5,4-6,4-7,5-6,5-7,6-7"
+)
+
+var (
+ KunlunResourceCount string
+)
+
+type KunlunConfig struct {
+ ResourceCountName string `yaml:"resourceCountName"`
+}
+
+func InitKunlunDevice(config KunlunConfig) *KunlunDevices {
+ KunlunResourceCount = config.ResourceCountName
+ util.SupportDevices[KunlunGPUDevice] = "hami.io/kunlun-allocated"
+ return &KunlunDevices{}
+}
+
+func (dev *KunlunDevices) CommonWord() string {
+ return KunlunGPUCommonWord
+}
+
+func ParseConfig(fs *flag.FlagSet) {
+ fs.StringVar(&KunlunResourceCount, "kunlun-name", "kunlunxin.com/xpu", "kunlunxin resource count")
+}
+
+func (dev *KunlunDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
+ _, ok := ctr.Resources.Limits[corev1.ResourceName(KunlunResourceCount)]
+ return ok, nil
+}
+
+func (dev *KunlunDevices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) {
+ nodedevices := []*util.DeviceInfo{}
+ i := 0
+ cards, ok := n.Status.Capacity.Name(corev1.ResourceName(KunlunResourceCount), resource.DecimalSI).AsInt64()
+ if !ok || cards == 0 {
+ return []*util.DeviceInfo{}, fmt.Errorf("device not found %s", KunlunResourceCount)
+ }
+ for int64(i) < cards {
+ nodedevices = append(nodedevices, &util.DeviceInfo{
+ Index: uint(i),
+ ID: n.Name + "-kunlun-" + fmt.Sprint(i),
+ Count: 100,
+ Devmem: 98304,
+ Devcore: 100,
+ Type: KunlunGPUDevice,
+ Numa: 0,
+ Health: true,
+ })
+ if int64(i) >= (cards / 2) {
+ nodedevices[i].Numa = 1
+ }
+ i++
+ }
+ return nodedevices, nil
+}
+
+func (dev *KunlunDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd util.PodDevices) map[string]string {
+ devlist, ok := pd[KunlunGPUDevice]
+ if ok && len(devlist) > 0 {
+ (*annoinput)[util.SupportDevices[KunlunGPUDevice]] = util.EncodePodSingleDevice(devlist)
+ for _, dp := range devlist {
+ annoKey := KunlunDeviceSelection
+ value := ""
+ for _, val := range dp {
+ value = value + fmt.Sprint(val.Idx) + ","
+ }
+ if len(value) > 0 {
+ (*annoinput)[annoKey] = strings.TrimRight(value, ",")
+ }
+ }
+ }
+ return *annoinput
+}
+
+func (dev *KunlunDevices) LockNode(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *KunlunDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *KunlunDevices) NodeCleanUp(nn string) error {
+ return nil
+}
+
+func (dev *KunlunDevices) CheckType(annos map[string]string, d util.DeviceUsage, n util.ContainerDeviceRequest) (bool, bool) {
+ if strings.Compare(n.Type, KunlunGPUDevice) == 0 {
+ return true, false
+ }
+ return false, false
+}
+
+func (dev *KunlunDevices) CheckUUID(annos map[string]string, d util.DeviceUsage) bool {
+ userUUID, ok := annos[KunlunUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for Kunlun user uuid [%s], device id is %s", userUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ userUUIDs := strings.Split(userUUID, ",")
+ return slices.Contains(userUUIDs, d.ID)
+ }
+
+ noUserUUID, ok := annos[KunlunNoUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for Kunlun not user uuid [%s], device id is %s", noUserUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ noUserUUIDs := strings.Split(noUserUUID, ",")
+ return !slices.Contains(noUserUUIDs, d.ID)
+ }
+ return true
+}
+
+func (dev *KunlunDevices) CheckIndex(annos map[string]string, d util.DeviceUsage) bool {
+ return true
+}
+
+func (dev *KunlunDevices) CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ return true, true
+}
+
+func (dev *KunlunDevices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest {
+ klog.Info("Start to count kunlun devices for container ", ctr.Name)
+ kunlunResourceCount := corev1.ResourceName(KunlunResourceCount)
+ v, ok := ctr.Resources.Limits[kunlunResourceCount]
+ if !ok {
+ v, ok = ctr.Resources.Requests[kunlunResourceCount]
+ }
+ if ok {
+ if n, ok := v.AsInt64(); ok {
+ klog.Info("Found kunlunxin devices")
+
+ return util.ContainerDeviceRequest{
+ Nums: int32(n),
+ Type: KunlunGPUDevice,
+ Memreq: 0,
+ MemPercentagereq: 100,
+ Coresreq: 0,
+ }
+ }
+ }
+ return util.ContainerDeviceRequest{}
+}
+
+func addidx(temp []int, value int) []int {
+ for _, val := range temp {
+ if val == value {
+ return temp
+ }
+ }
+ temp = append(temp, value)
+ return temp
+}
+
+func getvalue(t int) int {
+ if t == 4 {
+ return 0
+ }
+ if t == 1 {
+ return 2
+ }
+ return 1
+}
+
+func countbubble(t []int) int {
+ left := 0
+ right := 0
+ for _, val := range t {
+ if val < 4 {
+ left++
+ } else {
+ right++
+ }
+ }
+ if left == 0 && right == 0 {
+ return 1
+ }
+ return getvalue(left) + getvalue(right)
+}
+
+func calcscore(p []int, c []int) float32 {
+ sort.Slice(p, func(i, j int) bool {
+ return i < j
+ })
+ sort.Slice(c, func(i, j int) bool {
+ return i < j
+ })
+ prev := countbubble(p)
+ cur := countbubble(c)
+ klog.V(5).Infoln("Score kunlun num prev=", prev, "cur=", cur)
+ switch cur - prev {
+ case -1:
+ return 3000
+ case 0:
+ return 2000
+ case 1:
+ return 1000
+ case 2:
+ return 0
+ }
+ return 1000
+}
+
+func parseUsage(devices []*util.DeviceUsage) []int {
+ usage := []int{}
+ for _, val := range devices {
+ if val.Used == 0 {
+ usage = append(usage, int(val.Index))
+ }
+ }
+ return usage
+}
+
+func parseInterconnection() [][]int {
+ var interconnection [][]int
+ pairs := strings.Split(InterGroupConnection, ",")
+ for _, pair := range pairs {
+ lw, _ := strconv.Atoi(strings.Split(pair, "-")[0])
+ rw, _ := strconv.Atoi(strings.Split(pair, "-")[1])
+ interconnection = append(interconnection, []int{lw, rw})
+ }
+ pairs = strings.Split(GroupConnection, ",")
+ for _, pair := range pairs {
+ lw, _ := strconv.Atoi(strings.Split(pair, "-")[0])
+ rw, _ := strconv.Atoi(strings.Split(pair, "-")[1])
+ interconnection = append(interconnection, []int{lw, rw})
+ }
+ return interconnection
+}
+
+func parseInterconnection2() [][]int {
+ var interconnection2 [][]int
+ for group := range strings.SplitSeq(InterGroupConnection2, ",") {
+ values := strings.Split(group, "-")
+ connect := make([]int, 4)
+ for i, value := range values {
+ v, _ := strconv.Atoi(value)
+ connect[i] = v
+ }
+ interconnection2 = append(interconnection2, connect)
+ }
+ return interconnection2
+}
+
+func interconnect(devices []*util.DeviceUsage, count int) []int {
+ if count == 2 {
+ for _, val := range devices {
+ if val.Used > 0 {
+ continue
+ }
+ for _, val2 := range devices {
+ if val2.Used > 0 || val2.Index == val.Index {
+ continue
+ }
+ for p := range strings.SplitSeq(InterGroupConnection, ",") {
+ lw, _ := strconv.Atoi(strings.Split(p, "-")[0])
+ rw, _ := strconv.Atoi(strings.Split(p, "-")[1])
+ klog.V(5).InfoS("interconnect", "lw", lw, "rw", rw, "left device", val.Index, "right device", val2.Index)
+ if lw == int(val.Index) && rw == int(val2.Index) || lw == int(val2.Index) && rw == int(val.Index) {
+ return []int{int(val.Index), int(val2.Index)}
+ }
+ }
+ }
+ }
+ }
+ if count == 4 {
+ unused := parseUsage(devices)
+ interconnect2 := parseInterconnection2()
+ if len(unused) == 4 || len(unused) == 5 {
+ for _, c := range interconnect2 {
+ if canMeet(unused, c) {
+ return c
+ }
+ }
+ }
+ if len(unused) == 6 {
+ ret := []int{}
+ for _, c := range interconnect2 {
+ if canMeet(unused, c) {
+ ret = c
+ delta := delta(unused, c)
+ for _, val := range parseInterconnection() {
+ if canMeet(delta, val) {
+ return ret
+ }
+ }
+ }
+ }
+ return ret
+ }
+ }
+ return []int{}
+}
+
+func canMeet(have, want []int) bool {
+ mp := make(map[int]bool)
+ for _, v := range have {
+ mp[v] = true
+ }
+ for _, v := range want {
+ if !mp[v] {
+ return false
+ }
+ }
+ return true
+}
+
+func delta(have, want []int) []int {
+ var ret []int
+ mp := make(map[int]bool)
+ for _, v := range want {
+ mp[v] = true
+ }
+ for _, v := range have {
+ if !mp[v] {
+ ret = append(ret, v)
+ }
+ }
+ return ret
+}
+
+func (dev *KunlunDevices) ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32 {
+ current := []int{}
+ prev := []int{}
+ for _, dev := range previous {
+ if !strings.Contains(dev.Type, KunlunGPUDevice) {
+ return 0
+ }
+ if dev.Used > 0 {
+ prev = addidx(prev, int(dev.Index))
+ }
+ }
+ for _, ctr := range podDevices {
+ for _, val := range ctr {
+ if !strings.Contains(val.Type, KunlunGPUDevice) {
+ return 0
+ }
+ current = addidx(current, val.Idx)
+ }
+ }
+ klog.V(3).Infoln("Score kunlun device previous=", prev, "current=", current)
+ return calcscore(prev, current)
+}
+
+func (dev *KunlunDevices) AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error {
+ n.Used++
+ return nil
+}
+
+func devicepick(devices []*util.DeviceUsage, start int, count int) []int {
+ res := []int{}
+ for t := start; t < 8; t++ {
+ if devices[t].Used == 0 {
+ res = append(res, int(devices[t].Index))
+ if len(res) == count {
+ return res
+ }
+ }
+ }
+ return res
+}
+
+func graghSelect(devices []*util.DeviceUsage, count int) []int {
+ leftwing := 0
+ rightwing := 0
+ for idx, val := range devices {
+ klog.Infoln("graph select val=", *val)
+ if idx < 4 {
+ if val.Used == 0 {
+ leftwing++
+ }
+ } else {
+ if val.Used == 0 {
+ rightwing++
+ }
+ }
+ }
+ oddorder := []int{1, 3, 2, 4}
+ switch count {
+ case 8:
+ {
+ if leftwing+rightwing == count {
+ return []int{0, 1, 2, 3, 4, 5, 6, 7}
+ }
+ return []int{}
+ }
+ case 1, 2, 4:
+ {
+ if leftwing >= count || rightwing >= count {
+ for slots := count; slots <= 4; slots++ {
+ num := slots
+ if count%2 == 1 {
+ num = oddorder[slots-1]
+ }
+ klog.Infoln("slots=", slots, "num=", num, "leftwing=", leftwing, "==", rightwing)
+ if leftwing == num {
+ return devicepick(devices, 0, count)
+ }
+ if rightwing == num {
+ return devicepick(devices, 4, count)
+ }
+ }
+ }
+ return interconnect(devices, count)
+ }
+ }
+ return []int{}
+}
+
+func (kl *KunlunDevices) Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string) {
+ klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", request)
+ tmpDevs := make(map[string]util.ContainerDevices)
+ reason := make(map[string]int)
+
+ alloc := graghSelect(devices, int(request.Nums))
+ if len(alloc) == 0 {
+ reason[common.NumaNotFit]++
+ klog.V(5).InfoS(common.NumaNotFit, "pod", klog.KObj(pod), "device", devices, "request nums", request.Nums, "numa")
+ return false, tmpDevs, common.GenReason(reason, len(reason))
+ }
+
+ for _, dev := range alloc {
+ for _, val := range devices {
+ if val.Index == uint(dev) {
+ tmpDevs[request.Type] = append(tmpDevs[request.Type], util.ContainerDevice{
+ Idx: int(val.Index),
+ UUID: val.ID,
+ Type: request.Type,
+ Usedmem: val.Totalmem,
+ Usedcores: val.Totalcore,
+ })
+ break
+ }
+ }
+ }
+ return true, tmpDevs, ""
+}
diff --git a/pkg/device/kunlun/device_test.go b/pkg/device/kunlun/device_test.go
new file mode 100644
index 000000000..b80d3b7d0
--- /dev/null
+++ b/pkg/device/kunlun/device_test.go
@@ -0,0 +1,322 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kunlun
+
+import (
+ "testing"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+func Test_graphSelect(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ d []*util.DeviceUsage
+ c int
+ }
+ want1 []int
+ }{
+ {
+ name: "full allocate",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 0},
+ {Index: 3, Used: 0},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 0},
+ {Index: 6, Used: 0},
+ {Index: 7, Used: 0},
+ },
+ c: 8,
+ },
+ want1: []int{0, 1, 2, 3, 4, 5, 6, 7},
+ },
+ {
+ name: "full allocate not success",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 0},
+ {Index: 3, Used: 0},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 1},
+ {Index: 6, Used: 0},
+ {Index: 7, Used: 0},
+ },
+ c: 8,
+ },
+ want1: []int{},
+ },
+ {
+ name: "allocate 2 cards",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 0},
+ {Index: 3, Used: 0},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 1},
+ {Index: 6, Used: 0},
+ {Index: 7, Used: 0},
+ },
+ c: 2,
+ },
+ want1: []int{4, 6},
+ },
+ {
+ name: "allocate 1 card",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 0},
+ {Index: 3, Used: 0},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 1},
+ {Index: 6, Used: 0},
+ {Index: 7, Used: 0},
+ },
+ c: 1,
+ },
+ want1: []int{4},
+ },
+ {
+ name: "allocate 1 card",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 0},
+ {Index: 3, Used: 0},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 0},
+ {Index: 6, Used: 1},
+ {Index: 7, Used: 1},
+ },
+ c: 1,
+ },
+ want1: []int{4},
+ },
+ {
+ name: "allocate 2 card according to interconnect",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 1},
+ {Index: 1, Used: 1},
+ {Index: 2, Used: 1},
+ {Index: 3, Used: 0},
+ {Index: 4, Used: 1},
+ {Index: 5, Used: 1},
+ {Index: 6, Used: 1},
+ {Index: 7, Used: 0},
+ },
+ c: 2,
+ },
+ want1: []int{3, 7},
+ },
+ {
+ name: "allocate 4 cards according to interconnect when have 5 cards",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 1},
+ {Index: 3, Used: 1},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 0},
+ {Index: 6, Used: 0},
+ {Index: 7, Used: 1},
+ },
+ c: 4,
+ },
+ want1: []int{0, 1, 4, 5},
+ },
+ {
+ name: "allocate 4 cards according to interconnect when have 6 cards, leave 2 cards unconnected",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 0},
+ {Index: 3, Used: 1},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 0},
+ {Index: 6, Used: 1},
+ {Index: 7, Used: 0},
+ },
+ c: 4,
+ },
+ want1: []int{0, 1, 4, 5},
+ },
+ {
+ name: "allocate 4 cards according to interconnect when have 6 cards, leave 2 cards connected",
+ args: struct {
+ d []*util.DeviceUsage
+ c int
+ }{
+ d: []*util.DeviceUsage{
+ {Index: 0, Used: 0},
+ {Index: 1, Used: 0},
+ {Index: 2, Used: 1},
+ {Index: 3, Used: 0},
+ {Index: 4, Used: 0},
+ {Index: 5, Used: 0},
+ {Index: 6, Used: 1},
+ {Index: 7, Used: 0},
+ },
+ c: 4,
+ },
+ want1: []int{0, 1, 4, 5},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result1 := graghSelect(test.args.d, test.args.c)
+ assert.DeepEqual(t, result1, test.want1)
+ })
+ }
+}
+
+func Test_ScoreNode(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ node *corev1.Node
+ podDevices util.PodSingleDevice
+ usage []*util.DeviceUsage
+ policy string
+ }
+ want float32
+ }{
+ {
+ name: "Scenario 1",
+ args: struct {
+ node *corev1.Node
+ podDevices util.PodSingleDevice
+ usage []*util.DeviceUsage
+ policy string
+ }{
+ node: &corev1.Node{},
+ podDevices: util.PodSingleDevice{
+ util.ContainerDevices{
+ {
+ Idx: int(0),
+ Type: KunlunGPUDevice,
+ },
+ {
+ Idx: int(1),
+ Type: KunlunGPUDevice,
+ },
+ {
+ Idx: int(2),
+ Type: KunlunGPUDevice,
+ },
+ {
+ Idx: int(3),
+ Type: KunlunGPUDevice,
+ },
+ {
+ Idx: int(4),
+ Type: KunlunGPUDevice,
+ },
+ {
+ Idx: int(5),
+ Type: KunlunGPUDevice,
+ },
+ {
+ Idx: int(6),
+ Type: KunlunGPUDevice,
+ },
+ {
+ Idx: int(7),
+ Type: KunlunGPUDevice,
+ },
+ },
+ },
+ usage: []*util.DeviceUsage{
+ {Index: 0, Used: 1, Type: KunlunGPUDevice},
+ {Index: 1, Used: 1, Type: KunlunGPUDevice},
+ {Index: 2, Used: 1, Type: KunlunGPUDevice},
+ {Index: 3, Used: 1, Type: KunlunGPUDevice},
+ },
+ policy: "binpack",
+ },
+ want: float32(3000),
+ },
+ {
+ name: "Scenario 2",
+ args: struct {
+ node *corev1.Node
+ podDevices util.PodSingleDevice
+ usage []*util.DeviceUsage
+ policy string
+ }{
+ node: &corev1.Node{},
+ podDevices: util.PodSingleDevice{
+ util.ContainerDevices{
+ {
+ Idx: int(0),
+ Type: KunlunGPUDevice,
+ },
+ },
+ },
+ usage: []*util.DeviceUsage{},
+ policy: "spread",
+ },
+ want: float32(0),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := KunlunDevices{}
+ result := dev.ScoreNode(test.args.node, test.args.podDevices, test.args.usage, test.args.policy)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
diff --git a/pkg/device/metax/config.go b/pkg/device/metax/config.go
new file mode 100644
index 000000000..aac0f6a10
--- /dev/null
+++ b/pkg/device/metax/config.go
@@ -0,0 +1,41 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metax
+
+import "flag"
+
+type MetaxConfig struct {
+ // GPU
+ ResourceCountName string `yaml:"resourceCountName"`
+
+ // SGPU
+ ResourceVCountName string `yaml:"resourceVCountName"`
+ ResourceVMemoryName string `yaml:"resourceVMemoryName"`
+ ResourceVCoreName string `yaml:"resourceVCoreName"`
+ TopologyAware bool `yaml:"sgpuTopologyAware"`
+}
+
+func ParseConfig(fs *flag.FlagSet) {
+ // GPU
+ fs.StringVar(&MetaxResourceCount, "metax-name", "metax-tech.com/gpu", "metax resource count")
+
+ // SGPU
+ fs.StringVar(&MetaxResourceNameVCount, "metax-vcount", "metax-tech.com/sgpu", "metax vcount name")
+ fs.StringVar(&MetaxResourceNameVCore, "metax-vcore", "metax-tech.com/vcore", "metax vcore name")
+ fs.StringVar(&MetaxResourceNameVMemory, "metax-vmemory", "metax-tech.com/vmemory", "metax vmemory name")
+ fs.BoolVar(&MetaxTopologyAware, "sgpu-topology-aware", false, "sGPU topology aware enable")
+}
diff --git a/pkg/device/metax/device.go b/pkg/device/metax/device.go
new file mode 100644
index 000000000..195751e37
--- /dev/null
+++ b/pkg/device/metax/device.go
@@ -0,0 +1,322 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metax
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/common"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ "k8s.io/klog/v2"
+)
+
+type MetaxDevices struct {
+}
+
+const (
+ MetaxGPUDevice = "Metax-GPU"
+ MetaxGPUCommonWord = "Metax-GPU"
+ MetaxAnnotationLoss = "metax-tech.com/gpu.topology.losses"
+ MetaxAnnotationScore = "metax-tech.com/gpu.topology.scores"
+)
+
+var (
+ MetaxResourceCount string
+)
+
+func InitMetaxDevice(config MetaxConfig) *MetaxDevices {
+ MetaxResourceCount = config.ResourceCountName
+ util.InRequestDevices[MetaxGPUDevice] = "hami.io/metax-gpu-devices-to-allocate"
+ util.SupportDevices[MetaxGPUDevice] = "hami.io/metax-gpu-devices-allocated"
+ return &MetaxDevices{}
+}
+
+func (dev *MetaxDevices) CommonWord() string {
+ return MetaxGPUCommonWord
+}
+
+func (dev *MetaxDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
+ _, ok := ctr.Resources.Limits[corev1.ResourceName(MetaxResourceCount)]
+ return ok, nil
+}
+
+func (dev *MetaxDevices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) {
+ nodedevices := []*util.DeviceInfo{}
+ i := 0
+ count, ok := n.Status.Capacity.Name(corev1.ResourceName(MetaxResourceCount), resource.DecimalSI).AsInt64()
+ if !ok || count == 0 {
+ return []*util.DeviceInfo{}, fmt.Errorf("device not found %s", MetaxResourceCount)
+ }
+ for int64(i) < count {
+ nodedevices = append(nodedevices, &util.DeviceInfo{
+ Index: uint(i),
+ ID: n.Name + "-metax-" + fmt.Sprint(i),
+ Count: 1,
+ Devmem: 65536,
+ Devcore: 100,
+ Type: MetaxGPUDevice,
+ Numa: 0,
+ Health: true,
+ })
+ i++
+ }
+ return nodedevices, nil
+}
+
+func (dev *MetaxDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd util.PodDevices) map[string]string {
+ devlist, ok := pd[MetaxGPUDevice]
+ if ok && len(devlist) > 0 {
+ deviceStr := util.EncodePodSingleDevice(devlist)
+
+ (*annoinput)[util.InRequestDevices[MetaxGPUDevice]] = deviceStr
+ (*annoinput)[util.SupportDevices[MetaxGPUDevice]] = deviceStr
+ klog.Infof("pod add annotation key [%s, %s], values is [%s]",
+ util.InRequestDevices[MetaxGPUDevice], util.SupportDevices[MetaxGPUDevice], deviceStr)
+ }
+
+ return *annoinput
+}
+
+func (dev *MetaxDevices) LockNode(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *MetaxDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *MetaxDevices) NodeCleanUp(nn string) error {
+ return nil
+}
+
+func (dev *MetaxDevices) checkType(annos map[string]string, d util.DeviceUsage, n util.ContainerDeviceRequest) (bool, bool, bool) {
+ if strings.Compare(n.Type, MetaxGPUDevice) == 0 {
+ return true, true, false
+ }
+ return false, false, false
+}
+
+func (dev *MetaxDevices) checkUUID(annos map[string]string, d util.DeviceUsage) bool {
+ return true
+}
+
+func (dev *MetaxDevices) checkIndex(annos map[string]string, d util.DeviceUsage) bool {
+ return true
+}
+
+func (dev *MetaxDevices) CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ count, _ := n.Status.Capacity.Name(corev1.ResourceName(MetaxResourceCount), resource.DecimalSI).AsInt64()
+
+ return count > 0, true
+}
+
+func (dev *MetaxDevices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest {
+ klog.Info("Start to count metax devices for container ", ctr.Name)
+ metaxResourceCount := corev1.ResourceName(MetaxResourceCount)
+ v, ok := ctr.Resources.Limits[metaxResourceCount]
+ if !ok {
+ v, ok = ctr.Resources.Requests[metaxResourceCount]
+ }
+ if ok {
+ if n, ok := v.AsInt64(); ok {
+ klog.Info("Found metax devices")
+ return util.ContainerDeviceRequest{
+ Nums: int32(n),
+ Type: MetaxGPUDevice,
+ Memreq: 0,
+ MemPercentagereq: 100,
+ Coresreq: 100,
+ }
+ }
+ }
+ return util.ContainerDeviceRequest{}
+}
+
+func (dev *MetaxDevices) customFilterRule(allocated *util.PodDevices, request util.ContainerDeviceRequest, toAllocate util.ContainerDevices, device *util.DeviceUsage) bool {
+ for _, ctrs := range (*allocated)[device.Type] {
+ for _, ctrdev := range ctrs {
+ if strings.Compare(ctrdev.UUID, device.ID) != 0 {
+ klog.InfoS("Metax needs all devices on a device", "used", ctrdev.UUID, "allocating", device.ID)
+ return false
+ }
+ }
+ }
+ return true
+}
+
+func parseMetaxAnnos(annos string, index int) float32 {
+ scoreMap := map[int]int{}
+ err := json.Unmarshal([]byte(annos), &scoreMap)
+ if err != nil {
+ klog.Warningf("annos[%s] Unmarshal failed, %v", annos, err)
+ return 0
+ }
+
+ res, ok := scoreMap[index]
+ if !ok {
+ klog.Warningf("scoreMap[%v] not contains [%d]", scoreMap, index)
+ return 0
+ }
+
+ return float32(res)
+}
+
+func (dev *MetaxDevices) ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32 {
+ sum := 0
+ for _, dev := range podDevices {
+ sum += len(dev)
+ }
+
+ res := float32(0)
+ if policy == string(util.NodeSchedulerPolicyBinpack) {
+ lossAnno, ok := node.Annotations[MetaxAnnotationLoss]
+ if ok {
+ // it's preferred to select the node with lower loss
+ loss := parseMetaxAnnos(lossAnno, sum)
+ res = 2000 - loss
+
+ klog.InfoS("Detected annotations", "policy", policy, "key", MetaxAnnotationLoss, "value", lossAnno, "requesting", sum, "extract", loss)
+ }
+ } else if policy == string(util.NodeSchedulerPolicySpread) {
+ scoreAnno, ok := node.Annotations[MetaxAnnotationScore]
+ if ok {
+ // it's preferred to select the node with higher score
+ // But we have to give it a smaller value because of Spread policy
+ score := parseMetaxAnnos(scoreAnno, sum)
+ res = 2000 - score
+
+ klog.InfoS("Detected annotations", "policy", policy, "key", MetaxAnnotationScore, "value", scoreAnno, "requesting", sum, "extract", score)
+ }
+ }
+
+ return res
+}
+
+func (dev *MetaxDevices) AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error {
+ n.Used++
+ n.Usedcores += ctr.Usedcores
+ n.Usedmem += ctr.Usedmem
+ return nil
+}
+
+func (mat *MetaxDevices) Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string) {
+ k := request
+ originReq := k.Nums
+ prevnuma := -1
+ klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
+ var tmpDevs map[string]util.ContainerDevices
+ tmpDevs = make(map[string]util.ContainerDevices)
+ reason := make(map[string]int)
+ for i := len(devices) - 1; i >= 0; i-- {
+ dev := devices[i]
+ klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
+
+ _, found, numa := mat.checkType(annos, *dev, k)
+ if !found {
+ reason[common.CardTypeMismatch]++
+ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, k.Type)
+ continue
+ }
+ if numa && prevnuma != dev.Numa {
+ if k.Nums != originReq {
+ reason[common.NumaNotFit] += len(tmpDevs)
+ klog.V(5).InfoS(common.NumaNotFit, "pod", klog.KObj(pod), "device", dev.ID, "k.nums", k.Nums, "numa", numa, "prevnuma", prevnuma, "device numa", dev.Numa)
+ }
+ k.Nums = originReq
+ prevnuma = dev.Numa
+ tmpDevs = make(map[string]util.ContainerDevices)
+ }
+ if !mat.checkUUID(annos, *dev) {
+ reason[common.CardUUIDMismatch]++
+ klog.V(5).InfoS(common.CardUUIDMismatch, "pod", klog.KObj(pod), "device", dev.ID, "current device info is:", *dev)
+ continue
+ }
+
+ memreq := int32(0)
+ if dev.Count <= dev.Used {
+ reason[common.CardTimeSlicingExhausted]++
+ klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
+ continue
+ }
+ if k.Coresreq > 100 {
+ klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Coresreq = 100
+ //return false, tmpDevs
+ }
+ if k.Memreq > 0 {
+ memreq = k.Memreq
+ }
+ if k.MemPercentagereq != 101 && k.Memreq == 0 {
+ //This incurs an issue
+ memreq = dev.Totalmem * k.MemPercentagereq / 100
+ }
+ if dev.Totalmem-dev.Usedmem < memreq {
+ reason[common.CardInsufficientMemory]++
+ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq)
+ continue
+ }
+ if dev.Totalcore-dev.Usedcores < k.Coresreq {
+ reason[common.CardInsufficientCore]++
+ klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", k.Coresreq)
+ continue
+ }
+ // Coresreq=100 indicates it want this card exclusively
+ if dev.Totalcore == 100 && k.Coresreq == 100 && dev.Used > 0 {
+ reason[common.ExclusiveDeviceAllocateConflict]++
+ klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
+ continue
+ }
+ // You can't allocate core=0 job to an already full GPU
+ if dev.Totalcore != 0 && dev.Usedcores == dev.Totalcore && k.Coresreq == 0 {
+ reason[common.CardComputeUnitsExhausted]++
+ klog.V(5).InfoS(common.CardComputeUnitsExhausted, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+
+ if !mat.customFilterRule(allocated, request, tmpDevs[k.Type], dev) {
+ reason[common.CardNotFoundCustomFilterRule]++
+ klog.V(5).InfoS(common.CardNotFoundCustomFilterRule, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+
+ if k.Nums > 0 {
+ klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Nums--
+ tmpDevs[k.Type] = append(tmpDevs[k.Type], util.ContainerDevice{
+ Idx: int(dev.Index),
+ UUID: dev.ID,
+ Type: k.Type,
+ Usedmem: memreq,
+ Usedcores: k.Coresreq,
+ })
+ }
+ if k.Nums == 0 {
+ klog.V(4).InfoS("device allocate success", "pod", klog.KObj(pod), "allocate device", tmpDevs)
+ return true, tmpDevs, ""
+ }
+ }
+ if len(tmpDevs) > 0 {
+ reason[common.AllocatedCardsInsufficientRequest] = len(tmpDevs)
+ klog.V(5).InfoS(common.AllocatedCardsInsufficientRequest, "pod", klog.KObj(pod), "request", originReq, "allocated", len(tmpDevs))
+ }
+ return false, tmpDevs, common.GenReason(reason, len(devices))
+}
diff --git a/pkg/device/metax/device_test.go b/pkg/device/metax/device_test.go
new file mode 100644
index 000000000..b79c876b7
--- /dev/null
+++ b/pkg/device/metax/device_test.go
@@ -0,0 +1,871 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metax
+
+import (
+ "flag"
+ "testing"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+func TestGetNodeDevices(t *testing.T) {
+ MetaxResourceCount = "metax-tech.com/gpu"
+
+ tests := []struct {
+ name string
+ node corev1.Node
+ expected []*util.DeviceInfo
+ err error
+ }{
+ {
+ name: "Test with valid node",
+ node: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test",
+ },
+ Status: corev1.NodeStatus{
+ Capacity: corev1.ResourceList{
+ corev1.ResourceName(MetaxResourceCount): *resource.NewQuantity(1, resource.DecimalSI),
+ },
+ },
+ },
+ expected: []*util.DeviceInfo{
+ {
+ Index: 0,
+ ID: "test-metax-0",
+ Count: 100,
+ Devmem: 65536,
+ Devcore: 100,
+ Type: MetaxGPUDevice,
+ Numa: 0,
+ Health: true,
+ },
+ },
+ err: nil,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &MetaxDevices{}
+ got, err := dev.GetNodeDevices(tt.node)
+ if (err != nil) != (tt.err != nil) {
+ t.Errorf("GetNodeDevices() error = %v, expected %v", err, tt.err)
+ return
+ }
+
+ if len(got) != len(tt.expected) {
+ t.Errorf("GetNodeDevices() got %d devices, expected %d", len(got), len(tt.expected))
+ return
+ }
+
+ for i, device := range got {
+ if device.Index != tt.expected[i].Index {
+ t.Errorf("Expected index %d, got %d", tt.expected[i].Index, device.Index)
+ }
+ if device.ID != tt.expected[i].ID {
+ t.Errorf("Expected id %s, got %s", tt.expected[i].ID, device.ID)
+ }
+ if device.Devcore != tt.expected[i].Devcore {
+ t.Errorf("Expected devcore %d, got %d", tt.expected[i].Devcore, device.Devcore)
+ }
+ if device.Devmem != tt.expected[i].Devmem {
+ t.Errorf("Expected cevmem %d, got %d", tt.expected[i].Devmem, device.Devmem)
+ }
+ }
+ })
+ }
+}
+
+func TestParseMetaxAnnos(t *testing.T) {
+ tests := []struct {
+ name string
+ index int
+ value float32
+ }{
+ {
+ name: "{\"1\":0,\"2\":110,\"3\":270,\"4\":540,\"5\":580,\"6\":730,\"7\":930,\"8\":1240}",
+ index: 1,
+ value: 0,
+ },
+ {
+ name: "{\"1\":0,\"2\":110,\"3\":270,\"4\":540,\"5\":580,\"6\":730,\"7\":930,\"8\":1240}",
+ index: 2,
+ value: 110,
+ },
+ {
+ name: "{\"1\":0,\"2\":110,\"3\":270,\"4\":540,\"5\":580,\"6\":730,\"7\":930,\"8\":1240}",
+ index: 3,
+ value: 270,
+ },
+ {
+ name: "{\"1\":0,\"2\":110,\"3\":270,\"4\":540,\"5\":580,\"6\":730,\"7\":930,\"8\":1240}",
+ index: 4,
+ value: 540,
+ },
+ {
+ name: "{\"1\":0,\"2\":110,\"3\":270,\"4\":540,\"5\":580,\"6\":730,\"7\":930,\"8\":1240}",
+ index: 5,
+ value: 580,
+ },
+ {
+ name: "{\"1\":0,\"2\":110,\"3\":270,\"4\":540,\"5\":580,\"6\":730,\"7\":930,\"8\":1240}",
+ index: 6,
+ value: 730,
+ },
+ {
+ name: "{\"1\":0,\"2\":110,\"3\":270,\"4\":540,\"5\":580,\"6\":730,\"7\":930,\"8\":1240}",
+ index: 7,
+ value: 930,
+ },
+ {
+ name: "{\"1\":0,\"2\":110,\"3\":270,\"4\":540,\"5\":580,\"6\":730,\"7\":930,\"8\":1240}",
+ index: 8,
+ value: 1240,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ value := parseMetaxAnnos(tt.name, tt.index)
+ if value != tt.value {
+ t.Errorf("Expected index %f, got %f", tt.value, value)
+ }
+ })
+ }
+}
+
+func Test_MutateAdmission(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }
+ want bool
+ }{
+ {
+ name: "set to resources limits",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/gpu": resource.MustParse("1"),
+ },
+ },
+ },
+ p: &corev1.Pod{},
+ },
+ want: true,
+ },
+ {
+ name: "don't set to resources limits",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ },
+ },
+ p: &corev1.Pod{},
+ },
+ want: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ config := MetaxConfig{
+ ResourceCountName: "metax-tech.com/gpu",
+ }
+ InitMetaxDevice(config)
+ dev := MetaxDevices{}
+ result, _ := dev.MutateAdmission(test.args.ctr, test.args.p)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_checkType(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }
+ want1 bool
+ want2 bool
+ want3 bool
+ }{
+ {
+ name: "node type the same as device",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: MetaxGPUDevice,
+ },
+ },
+ want1: true,
+ want2: true,
+ want3: false,
+ },
+ {
+ name: "node type the different from device",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: "test",
+ },
+ },
+ want1: false,
+ want2: false,
+ want3: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := MetaxDevices{}
+ result1, result2, result3 := dev.checkType(test.args.annos, test.args.d, test.args.n)
+ assert.DeepEqual(t, result1, test.want1)
+ assert.DeepEqual(t, result2, test.want2)
+ assert.DeepEqual(t, result3, test.want3)
+ })
+ }
+}
+
+func Test_GenerateResourceRequests(t *testing.T) {
+ tests := []struct {
+ name string
+ args *corev1.Container
+ want util.ContainerDeviceRequest
+ }{
+ {
+ name: "resource set to limit and request",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/gpu": resource.MustParse("1"),
+ },
+ Requests: corev1.ResourceList{
+ "metax-tech.com/gpu": resource.MustParse("1"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: MetaxGPUDevice,
+ Memreq: 0,
+ MemPercentagereq: 100,
+ Coresreq: 100,
+ },
+ },
+ {
+ name: "resource don't set to limit and request",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ Requests: corev1.ResourceList{},
+ },
+ },
+ want: util.ContainerDeviceRequest{},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ fs := flag.FlagSet{}
+ ParseConfig(&fs)
+ dev := MetaxDevices{}
+ result := dev.GenerateResourceRequests(test.args)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func Test_CustomFilterRule(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ allocated *util.PodDevices
+ request util.ContainerDeviceRequest
+ toAllocate util.ContainerDevices
+ device *util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "allocated id is same as device id",
+ args: struct {
+ allocated *util.PodDevices
+ request util.ContainerDeviceRequest
+ toAllocate util.ContainerDevices
+ device *util.DeviceUsage
+ }{
+ allocated: &util.PodDevices{
+ MetaxGPUDevice: util.PodSingleDevice{
+ util.ContainerDevices{
+ {
+ Idx: int(0),
+ Type: MetaxGPUDevice,
+ UUID: "test-0000",
+ Usedcores: int32(1),
+ Usedmem: int32(1000),
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{},
+ toAllocate: util.ContainerDevices{},
+ device: &util.DeviceUsage{
+ Type: MetaxGPUDevice,
+ ID: "test-0000",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "allocated id is different from device id",
+ args: struct {
+ allocated *util.PodDevices
+ request util.ContainerDeviceRequest
+ toAllocate util.ContainerDevices
+ device *util.DeviceUsage
+ }{
+ allocated: &util.PodDevices{
+ MetaxGPUDevice: util.PodSingleDevice{
+ util.ContainerDevices{
+ {
+ Idx: int(0),
+ Type: MetaxGPUDevice,
+ UUID: "test-0000",
+ Usedcores: int32(1),
+ Usedmem: int32(1000),
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{},
+ toAllocate: util.ContainerDevices{},
+ device: &util.DeviceUsage{
+ Type: MetaxGPUDevice,
+ ID: "test-1111",
+ },
+ },
+ want: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := MetaxDevices{}
+ result := dev.customFilterRule(test.args.allocated, test.args.request, test.args.toAllocate, test.args.device)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func Test_ScoreNode(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ node *corev1.Node
+ podDevices util.PodSingleDevice
+ policy string
+ }
+ want float32
+ }{
+ {
+ name: "policy is binpack",
+ args: struct {
+ node *corev1.Node
+ podDevices util.PodSingleDevice
+ policy string
+ }{
+ node: &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "metax-tech.com/gpu.topology.losses": "{\"1\":100,\"2\":200}",
+ },
+ },
+ },
+ podDevices: util.PodSingleDevice{
+ util.ContainerDevices{
+ {
+ Idx: int(0),
+ UUID: "test-0",
+ Type: MetaxGPUDevice,
+ Usedmem: int32(1000),
+ Usedcores: int32(1),
+ },
+ {
+ Idx: int(1),
+ UUID: "test-1",
+ Type: MetaxGPUDevice,
+ Usedmem: int32(1000),
+ Usedcores: int32(1),
+ },
+ },
+ },
+ policy: "binpack",
+ },
+ want: float32(1800),
+ },
+ {
+ name: "policy is spread",
+ args: struct {
+ node *corev1.Node
+ podDevices util.PodSingleDevice
+ policy string
+ }{
+ node: &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "metax-tech.com/gpu.topology.scores": "{\"1\":100,\"2\":200}",
+ },
+ },
+ },
+ podDevices: util.PodSingleDevice{
+ util.ContainerDevices{
+ {
+ Idx: int(0),
+ UUID: "test-0",
+ Type: MetaxGPUDevice,
+ Usedmem: int32(1000),
+ Usedcores: int32(1),
+ },
+ },
+ },
+ policy: "spread",
+ },
+ want: float32(1900),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := MetaxDevices{}
+ result := dev.ScoreNode(test.args.node, test.args.podDevices, []*util.DeviceUsage{}, test.args.policy)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func TestMetaxDevices_Fit(t *testing.T) {
+ config := MetaxConfig{
+ ResourceCountName: "metax-tech.com/gpu",
+ }
+ dev := InitMetaxDevice(config)
+
+ tests := []struct {
+ name string
+ devices []*util.DeviceUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ wantFit bool
+ wantLen int
+ wantDevIDs []string
+ wantReason string
+ }{
+ {
+ name: "fit success",
+ devices: []*util.DeviceUsage{
+ {
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxGPUDevice,
+ Health: true,
+ },
+ {
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxGPUDevice,
+ Health: true,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 64,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MetaxGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-1"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: memory not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MetaxGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientMemory",
+ },
+ {
+ name: "fit fail: core not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1024,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: MetaxGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MetaxGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientCore",
+ },
+ {
+ name: "fit fail: type mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ Type: MetaxGPUDevice,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: "OtherType",
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTypeMismatch",
+ },
+ {
+ name: "fit fail: card overused",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 100,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MetaxGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTimeSlicingExhausted",
+ },
+ {
+ name: "fit success: but core limit can't exceed 100",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 120,
+ Type: MetaxGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: card exclusively",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 100,
+ Type: MetaxGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 ExclusiveDeviceAllocateConflict",
+ },
+ {
+ name: "fit fail: CardComputeUnitsExhausted",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: MetaxGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 0,
+ Type: MetaxGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardComputeUnitsExhausted",
+ },
+ {
+ name: "fit fail: AllocatedCardsInsufficientRequest",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: MetaxGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 20,
+ Type: MetaxGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 AllocatedCardsInsufficientRequest",
+ },
+ {
+ name: "fit success: memory percentage",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: MetaxGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 10,
+ Coresreq: 20,
+ Type: MetaxGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ allocated := &util.PodDevices{}
+ fit, result, reason := dev.Fit(test.devices, test.request, test.annos, &corev1.Pod{}, &util.NodeInfo{}, allocated)
+ if fit != test.wantFit {
+ t.Errorf("Fit: got %v, want %v", fit, test.wantFit)
+ }
+ if test.wantFit {
+ if len(result[MetaxGPUDevice]) != test.wantLen {
+ t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[MetaxGPUDevice]))
+ }
+ for idx, id := range test.wantDevIDs {
+ if id != result[MetaxGPUDevice][idx].UUID {
+ t.Errorf("expected device id: %s, got device id %s", id, result[MetaxGPUDevice][idx].UUID)
+ }
+ }
+ }
+
+ if reason != test.wantReason {
+ t.Errorf("expected reason: %s, got reason: %s", test.wantReason, reason)
+ }
+ })
+ }
+}
+
+func TestMetaxDevices_AddResourceUsage(t *testing.T) {
+ tests := []struct {
+ name string
+ deviceUsage *util.DeviceUsage
+ ctr *util.ContainerDevice
+ wantErr bool
+ wantUsage *util.DeviceUsage
+ }{
+ {
+ name: "test add resource usage",
+ deviceUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 0,
+ Usedcores: 15,
+ Usedmem: 2000,
+ },
+ ctr: &util.ContainerDevice{
+ UUID: "dev-0",
+ Usedcores: 50,
+ Usedmem: 1024,
+ },
+ wantUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 1,
+ Usedcores: 65,
+ Usedmem: 3024,
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &MetaxDevices{}
+ if err := dev.AddResourceUsage(&corev1.Pod{}, tt.deviceUsage, tt.ctr); (err != nil) != tt.wantErr {
+ t.Errorf("AddResourceUsage() error=%v, wantErr %v", err, tt.wantErr)
+ }
+ if !tt.wantErr {
+ if tt.deviceUsage.Usedcores != tt.wantUsage.Usedcores {
+ t.Errorf("expected used cores: %d, got used cores %d", tt.wantUsage.Usedcores, tt.deviceUsage.Usedcores)
+ }
+ if tt.deviceUsage.Usedmem != tt.wantUsage.Usedmem {
+ t.Errorf("expected used mem: %d, got used mem %d", tt.wantUsage.Usedmem, tt.deviceUsage.Usedmem)
+ }
+ if tt.deviceUsage.Used != tt.wantUsage.Used {
+ t.Errorf("expected used: %d, got used %d", tt.wantUsage.Used, tt.deviceUsage.Used)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/device/metax/protocol.go b/pkg/device/metax/protocol.go
new file mode 100644
index 000000000..cf3071a94
--- /dev/null
+++ b/pkg/device/metax/protocol.go
@@ -0,0 +1,136 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metax
+
+import (
+ "fmt"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+const (
+ MetaxSDeviceAnno = "metax-tech.com/node-gpu-devices"
+ MetaxAllocatedSDevices = "metax-tech.com/gpu-devices-allocated"
+ MetaxPredicateTime = "metax-tech.com/predicate-time"
+
+ MetaxUseUUID = "metax-tech.com/use-gpuuuid"
+ MetaxNoUseUUID = "metax-tech.com/nouse-gpuuuid"
+
+ MetaxSGPUQosPolicy = "metax-tech.com/sgpu-qos-policy"
+ MetaxSGPUTopologyAware = "metax-tech.com/sgpu-topology-aware"
+)
+
+const (
+ BestEffort = "best-effort"
+ FixedShare = "fixed-share"
+ BurstShare = "burst-share"
+)
+
+type MetaxSDeviceInfo struct {
+ UUID string `json:"uuid"`
+ BDF string `json:"bdf,omitempty"`
+ Model string `json:"model,omitempty"`
+ TotalDevCount int32 `json:"totalDevCount,omitempty"`
+ TotalCompute int32 `json:"totalCompute,omitempty"`
+ TotalVRam int32 `json:"totalVRam,omitempty"`
+ AvailableDevCount int32 `json:"availableDevCount,omitempty"`
+ AvailableCompute int32 `json:"availableCompute,omitempty"`
+ AvailableVRam int32 `json:"availableVRam,omitempty"`
+ Numa int32 `json:"numa,omitempty"`
+ Healthy bool `json:"healthy,omitempty"`
+ QosPolicy string `json:"qosPolicy,omitempty"`
+ LinkZone int32 `json:"linkZone,omitempty"`
+}
+type NodeMetaxSDeviceInfo []*MetaxSDeviceInfo
+
+type ContainerMetaxSDevice struct {
+ UUID string `json:"uuid"`
+ Compute int32 `json:"compute,omitempty"`
+ VRam int32 `json:"vRam,omitempty"`
+}
+type ContainerMetaxSDevices []ContainerMetaxSDevice
+type PodMetaxSDevice []ContainerMetaxSDevices
+
+func (ni NodeMetaxSDeviceInfo) String() string {
+ str := "\n"
+
+ for _, i := range ni {
+ str += fmt.Sprintf("MetaxSDeviceInfo[%s]: TotalDevCount=%d, TotalCompute=%d, TotalVRam=%d, Numa=%d, Healthy=%t, QosPolicy=%s, LinkZone=%d\n",
+ i.UUID, i.TotalDevCount, i.TotalCompute, i.TotalVRam, i.Numa, i.Healthy, i.QosPolicy, i.LinkZone)
+ }
+
+ return str
+}
+
+func (sdev *PodMetaxSDevice) String() string {
+ str := "\nPodMetaxSDevice:\n"
+
+ for ctrIdx, ctrDevices := range *sdev {
+ str += fmt.Sprintf(" container[%d]:\n", ctrIdx)
+
+ for _, device := range ctrDevices {
+ str += fmt.Sprintf(" SDevice[%s]: Compute=%d, VRam=%d\n",
+ device.UUID, device.Compute, device.VRam)
+ }
+ }
+
+ return str
+}
+
+func convertMetaxSDeviceToHAMIDevice(metaxSDevices []*MetaxSDeviceInfo) []*util.DeviceInfo {
+ hamiDevices := make([]*util.DeviceInfo, len(metaxSDevices))
+
+ for idx, sdevice := range metaxSDevices {
+ hamiDevices[idx] = &util.DeviceInfo{
+ ID: sdevice.UUID,
+ Index: uint(idx),
+ Count: sdevice.TotalDevCount,
+ Devmem: sdevice.TotalVRam,
+ Devcore: sdevice.TotalCompute,
+ Type: MetaxSGPUDevice,
+ Numa: int(sdevice.Numa),
+ Mode: "",
+ MIGTemplate: []util.Geometry{},
+ Health: sdevice.Healthy,
+ DeviceVendor: MetaxSGPUDevice,
+ CustomInfo: map[string]any{
+ "QosPolicy": sdevice.QosPolicy,
+ "Model": sdevice.Model,
+ "LinkZone": sdevice.LinkZone,
+ },
+ }
+ }
+
+ return hamiDevices
+}
+
+func convertHAMIPodDeviceToMetaxPodDevice(hamiPodDevices util.PodSingleDevice) PodMetaxSDevice {
+ metaxDevices := make(PodMetaxSDevice, len(hamiPodDevices))
+
+ for ctrIdx, ctrDevices := range hamiPodDevices {
+ metaxDevices[ctrIdx] = make(ContainerMetaxSDevices, len(ctrDevices))
+ for deviceIdx, device := range ctrDevices {
+ metaxDevices[ctrIdx][deviceIdx] = ContainerMetaxSDevice{
+ UUID: device.UUID,
+ VRam: device.Usedmem,
+ Compute: device.Usedcores,
+ }
+ }
+ }
+
+ return metaxDevices
+}
diff --git a/pkg/device/metax/protocol_test.go b/pkg/device/metax/protocol_test.go
new file mode 100644
index 000000000..81e556b04
--- /dev/null
+++ b/pkg/device/metax/protocol_test.go
@@ -0,0 +1,198 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metax
+
+import (
+ "reflect"
+ "testing"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+func TestConvertMetaxSDeviceToHAMIDevice(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ metax []*MetaxSDeviceInfo
+
+ expected []*util.DeviceInfo
+ }{
+ {
+ name: "base test",
+ metax: []*MetaxSDeviceInfo{
+ {
+ UUID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a9",
+ BDF: "0000:0a:00.0",
+ Model: "native",
+ TotalDevCount: 16,
+ TotalCompute: 100,
+ TotalVRam: 64 * 1024,
+ AvailableDevCount: 6,
+ AvailableCompute: 50,
+ AvailableVRam: 32 * 1024,
+ Numa: 0,
+ Healthy: true,
+ QosPolicy: BestEffort,
+ LinkZone: 1,
+ },
+ {
+ UUID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a8",
+ BDF: "0000:09:00.0",
+ Model: "sgpu",
+ TotalDevCount: 8,
+ TotalCompute: 100,
+ TotalVRam: 32 * 1024,
+ AvailableDevCount: 6,
+ AvailableCompute: 50,
+ AvailableVRam: 32 * 1024,
+ Numa: -1,
+ Healthy: false,
+ QosPolicy: BurstShare,
+ LinkZone: 2,
+ },
+ },
+ expected: []*util.DeviceInfo{
+ {
+ ID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a9",
+ Index: 0,
+ Count: 16,
+ Devmem: 64 * 1024,
+ Devcore: 100,
+ Type: MetaxSGPUDevice,
+ Numa: 0,
+ Mode: "",
+ MIGTemplate: []util.Geometry{},
+ Health: true,
+ DeviceVendor: MetaxSGPUDevice,
+ CustomInfo: map[string]any{
+ "QosPolicy": BestEffort,
+ "Model": "native",
+ "LinkZone": int32(1),
+ },
+ },
+ {
+ ID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a8",
+ Index: 1,
+ Count: 8,
+ Devmem: 32 * 1024,
+ Devcore: 100,
+ Type: MetaxSGPUDevice,
+ Numa: -1,
+ Mode: "",
+ MIGTemplate: []util.Geometry{},
+ Health: false,
+ DeviceVendor: MetaxSGPUDevice,
+ CustomInfo: map[string]any{
+ "QosPolicy": BurstShare,
+ "Model": "sgpu",
+ "LinkZone": int32(2),
+ },
+ },
+ },
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ result := convertMetaxSDeviceToHAMIDevice(ts.metax)
+
+ if !reflect.DeepEqual(ts.expected, result) {
+ t.Errorf("convertMetaxSDeviceToHAMIDevice failed: result %v, expected %v",
+ result, ts.expected)
+ }
+ })
+ }
+}
+
+func TestConvertHAMIPodDeviceToMetaxPodDevice(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ hami util.PodSingleDevice
+
+ expected PodMetaxSDevice
+ }{
+ {
+ name: "base test",
+ hami: util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a0",
+ Type: MetaxGPUDevice,
+ Usedmem: 10,
+ Usedcores: 50,
+ },
+ {
+ Idx: 1,
+ UUID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a1",
+ Type: MetaxGPUDevice,
+ Usedmem: 1024,
+ Usedcores: 30,
+ },
+ },
+ {
+ {
+ Idx: 3,
+ UUID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a3",
+ Type: MetaxGPUDevice,
+ Usedmem: 10 * 1024,
+ Usedcores: 20,
+ },
+ {
+ Idx: 7,
+ UUID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a7",
+ Type: MetaxGPUDevice,
+ Usedmem: 64 * 1024,
+ Usedcores: 80,
+ },
+ },
+ },
+ expected: PodMetaxSDevice{
+ {
+ {
+ UUID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a0",
+ Compute: 50,
+ VRam: 10,
+ },
+ {
+ UUID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a1",
+ Compute: 30,
+ VRam: 1024,
+ },
+ },
+ {
+ {
+ UUID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a3",
+ Compute: 20,
+ VRam: 10 * 1024,
+ },
+ {
+ UUID: "GPU-a16ac188-0592-5c8f-2b6e-8bd8e7a604a7",
+ Compute: 80,
+ VRam: 64 * 1024,
+ },
+ },
+ },
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ result := convertHAMIPodDeviceToMetaxPodDevice(ts.hami)
+
+ if !reflect.DeepEqual(ts.expected, result) {
+ t.Errorf("convertMetaxSDeviceToHAMIDevice failed: result %v, expected %v",
+ result, ts.expected)
+ }
+ })
+ }
+}
diff --git a/pkg/device/metax/qos.go b/pkg/device/metax/qos.go
new file mode 100644
index 000000000..0ec813e5b
--- /dev/null
+++ b/pkg/device/metax/qos.go
@@ -0,0 +1,75 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metax
+
+import (
+ "sync"
+
+ "k8s.io/klog/v2"
+)
+
+type JitteryQosCache struct {
+ sync.Mutex
+ cache map[string]string
+}
+
+func NewJitteryQosCache() *JitteryQosCache {
+ return &JitteryQosCache{
+ cache: map[string]string{},
+ }
+}
+
+func (c *JitteryQosCache) Sync(devices []*MetaxSDeviceInfo) {
+ c.Lock()
+ defer c.Unlock()
+
+ isSync := false
+
+ for _, dev := range devices {
+ expectedQos, ok := c.cache[dev.UUID]
+
+ if ok {
+ if expectedQos == dev.QosPolicy {
+ delete(c.cache, dev.UUID)
+ klog.Infof("%T: device[%s] qos changed to expected [%s], delete data",
+ c, dev.UUID, dev.QosPolicy)
+
+ isSync = true
+ }
+ }
+ }
+
+ if isSync {
+ klog.Infof("%T: sync done, current cache: %v", c, c.cache)
+ }
+}
+
+func (c *JitteryQosCache) Add(uuid string, expectedQos string) {
+ c.Lock()
+ defer c.Unlock()
+
+ c.cache[uuid] = expectedQos
+ klog.Infof("%T: device[%s] add to cache, expected qos [%s]", c, uuid, expectedQos)
+}
+
+func (c *JitteryQosCache) Get(uuid string) (string, bool) {
+ c.Lock()
+ defer c.Unlock()
+
+ qos, ok := c.cache[uuid]
+ return qos, ok
+}
diff --git a/pkg/device/metax/qos_test.go b/pkg/device/metax/qos_test.go
new file mode 100644
index 000000000..6031b890b
--- /dev/null
+++ b/pkg/device/metax/qos_test.go
@@ -0,0 +1,161 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metax
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestJitteryQosCacheSync(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ devices []*MetaxSDeviceInfo
+ currentCache map[string]string
+
+ expectedCache map[string]string
+ }{
+ {
+ name: "no sync",
+ devices: []*MetaxSDeviceInfo{
+ {
+ UUID: "GPU-123",
+ QosPolicy: BestEffort,
+ },
+ {
+ UUID: "GPU-456",
+ QosPolicy: BestEffort,
+ },
+ {
+ UUID: "GPU-789",
+ QosPolicy: BestEffort,
+ },
+ },
+ currentCache: map[string]string{
+ "GPU-123": FixedShare,
+ "GPU-789": BurstShare,
+ },
+
+ expectedCache: map[string]string{
+ "GPU-123": FixedShare,
+ "GPU-789": BurstShare,
+ },
+ },
+ {
+ name: "sync sccuess",
+ devices: []*MetaxSDeviceInfo{
+ {
+ UUID: "GPU-123",
+ QosPolicy: BestEffort,
+ },
+ {
+ UUID: "GPU-456",
+ QosPolicy: BestEffort,
+ },
+ {
+ UUID: "GPU-789",
+ QosPolicy: BestEffort,
+ },
+ },
+ currentCache: map[string]string{
+ "GPU-123": BestEffort,
+ "GPU-789": BurstShare,
+ },
+
+ expectedCache: map[string]string{
+ "GPU-789": BurstShare,
+ },
+ },
+ {
+ name: "sync sccuess",
+ devices: []*MetaxSDeviceInfo{
+ {
+ UUID: "GPU-123",
+ QosPolicy: BestEffort,
+ },
+ {
+ UUID: "GPU-456",
+ QosPolicy: BestEffort,
+ },
+ {
+ UUID: "GPU-789",
+ QosPolicy: BestEffort,
+ },
+ },
+ currentCache: map[string]string{
+ "GPU-123": BestEffort,
+ "GPU-789": BestEffort,
+ },
+
+ expectedCache: map[string]string{},
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ jqCache := JitteryQosCache{
+ cache: ts.currentCache,
+ }
+
+ jqCache.Sync(ts.devices)
+
+ if !reflect.DeepEqual(ts.expectedCache, jqCache.cache) {
+ t.Errorf("JitteryQosCache Sync failed: result %v, expected %v",
+ jqCache.cache, ts.expectedCache)
+ }
+ })
+ }
+}
+
+func TestGet(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ currentCache map[string]string
+ uuid string
+
+ expectedValue string
+ expectedOk bool
+ }{
+ {
+ name: "test get",
+ currentCache: map[string]string{
+ "GPU-123": FixedShare,
+ "GPU-789": BurstShare,
+ },
+ uuid: "GPU-789",
+
+ expectedValue: BurstShare,
+ expectedOk: true,
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ jqCache := JitteryQosCache{
+ cache: ts.currentCache,
+ }
+
+ resValue, resOk := jqCache.Get(ts.uuid)
+
+ if resValue != ts.expectedValue {
+ t.Errorf("Get failed: result %v, expected %v",
+ resValue, ts.expectedValue)
+ }
+
+ if resOk != ts.expectedOk {
+ t.Errorf("Get failed: result %v, expected %v",
+ resOk, ts.expectedOk)
+ }
+ })
+ }
+}
diff --git a/pkg/device/metax/score.go b/pkg/device/metax/score.go
new file mode 100644
index 000000000..3c80ee2ed
--- /dev/null
+++ b/pkg/device/metax/score.go
@@ -0,0 +1,66 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metax
+
+import "fmt"
+
+const DirectLinkScore = 10
+
+type LinkDevice struct {
+ uuid string
+ linkZone int32
+}
+
+func (from *LinkDevice) score(to *LinkDevice) int {
+ if from.uuid == to.uuid {
+ return 0
+ }
+
+ if from.linkZone == 0 || to.linkZone == 0 {
+ return 0
+ }
+
+ if from.linkZone == to.linkZone {
+ return DirectLinkScore
+ } else {
+ return 0
+ }
+}
+
+type LinkDevices []*LinkDevice
+
+func (devs LinkDevices) Score() int {
+ score := 0
+
+ for i := 0; i < len(devs); i++ {
+ for j := i + 1; j < len(devs); j++ {
+ score += devs[i].score(devs[j])
+ }
+ }
+
+ return score
+}
+
+func (devs LinkDevices) String() string {
+ str := "["
+ for _, dev := range devs {
+ str += fmt.Sprintf("%v", *dev)
+ }
+ str += "]"
+
+ return str
+}
diff --git a/pkg/device/metax/sdevice.go b/pkg/device/metax/sdevice.go
new file mode 100644
index 000000000..dde8dbf3f
--- /dev/null
+++ b/pkg/device/metax/sdevice.go
@@ -0,0 +1,650 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metax
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "maps"
+ "slices"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/common"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/nodelock"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/klog/v2"
+)
+
+const (
+ MetaxSGPUCommonWord = "Metax-SGPU"
+ MetaxSGPUDevice = "Metax-SGPU"
+
+ MetaxNodeLock = "hami.io/mutex.lock"
+)
+
+var (
+ MetaxResourceNameVCount string
+ MetaxResourceNameVCore string
+ MetaxResourceNameVMemory string
+ MetaxTopologyAware bool
+)
+
+type MetaxSDevices struct {
+ jqCache *JitteryQosCache
+}
+
+func InitMetaxSDevice(config MetaxConfig) *MetaxSDevices {
+ MetaxResourceNameVCount = config.ResourceVCountName
+ MetaxResourceNameVCore = config.ResourceVCoreName
+ MetaxResourceNameVMemory = config.ResourceVMemoryName
+ MetaxTopologyAware = config.TopologyAware
+
+ util.InRequestDevices[MetaxSGPUDevice] = "hami.io/metax-sgpu-devices-to-allocate"
+ util.SupportDevices[MetaxSGPUDevice] = "hami.io/metax-sgpu-devices-allocated"
+
+ return &MetaxSDevices{
+ jqCache: NewJitteryQosCache(),
+ }
+}
+
+func (sdev *MetaxSDevices) CommonWord() string {
+ return MetaxSGPUCommonWord
+}
+
+func (sdev *MetaxSDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
+ _, ok := ctr.Resources.Limits[corev1.ResourceName(MetaxResourceNameVCount)]
+ if !ok {
+ return false, nil
+ }
+
+ qos, ok := p.Annotations[MetaxSGPUQosPolicy]
+ if !ok {
+ if p.Annotations == nil {
+ p.Annotations = make(map[string]string)
+ }
+
+ p.Annotations[MetaxSGPUQosPolicy] = BestEffort
+ return true, nil
+ }
+
+ if qos == BestEffort ||
+ qos == FixedShare ||
+ qos == BurstShare {
+ return true, nil
+ } else {
+ return true, fmt.Errorf("%s must be set one of [%s, %s, %s]",
+ MetaxSGPUQosPolicy, BestEffort, FixedShare, BurstShare)
+ }
+}
+
+func (sdev *MetaxSDevices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) {
+ metaxSDevices, err := sdev.getMetaxSDevices(n)
+ if err != nil {
+ return []*util.DeviceInfo{}, err
+ }
+
+ sdev.jqCache.Sync(metaxSDevices)
+
+ return convertMetaxSDeviceToHAMIDevice(metaxSDevices), nil
+}
+
+func (sdev *MetaxSDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd util.PodDevices) map[string]string {
+ devlist, ok := pd[MetaxSGPUDevice]
+ if ok && len(devlist) > 0 {
+ deviceStr := util.EncodePodSingleDevice(devlist)
+
+ // hami
+ (*annoinput)[util.InRequestDevices[MetaxSGPUDevice]] = deviceStr
+ (*annoinput)[util.SupportDevices[MetaxSGPUDevice]] = deviceStr
+ klog.Infof("pod add annotation key [%s, %s], values is [%s]",
+ util.InRequestDevices[MetaxSGPUDevice], util.SupportDevices[MetaxSGPUDevice], deviceStr)
+
+ // metax
+ metaxPodDevice := convertHAMIPodDeviceToMetaxPodDevice(devlist)
+ klog.Infof("metaxPodDevice allocated info: %s", metaxPodDevice.String())
+
+ byte, err := json.Marshal(metaxPodDevice)
+ if err != nil {
+ klog.Errorf("metaxPodDevice marshal failed, origin values is [%s]", deviceStr)
+ }
+
+ (*annoinput)[MetaxAllocatedSDevices] = string(byte)
+ (*annoinput)[MetaxPredicateTime] = strconv.FormatInt(time.Now().UnixNano(), 10)
+
+ sdev.addJitteryQos(pod.Annotations[MetaxSGPUQosPolicy], devlist)
+ }
+
+ return *annoinput
+}
+
+func (sdev *MetaxSDevices) LockNode(n *corev1.Node, p *corev1.Pod) error {
+ found := false
+
+ for _, val := range p.Spec.Containers {
+ if (sdev.GenerateResourceRequests(&val).Nums) > 0 {
+ found = true
+ break
+ }
+ }
+
+ if !found {
+ return nil
+ }
+
+ return nodelock.LockNode(n.Name, MetaxNodeLock, p)
+}
+
+func (sdev *MetaxSDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
+ found := false
+
+ for _, val := range p.Spec.Containers {
+ if (sdev.GenerateResourceRequests(&val).Nums) > 0 {
+ found = true
+ break
+ }
+ }
+
+ if !found {
+ return nil
+ }
+
+ return nodelock.ReleaseNodeLock(n.Name, MetaxNodeLock, p, false)
+}
+
+func (sdev *MetaxSDevices) NodeCleanUp(nn string) error {
+ return nil
+}
+
+func (sdev *MetaxSDevices) checkType(annos map[string]string, d util.DeviceUsage, n util.ContainerDeviceRequest) bool {
+ if strings.Compare(n.Type, MetaxSGPUDevice) == 0 {
+ if !d.Health {
+ klog.Infof("device[%s] unhealthy, check failed", d.ID)
+ return false
+ }
+
+ if sdev.checkDeviceQos(annos[MetaxSGPUQosPolicy], d, n) {
+ return true
+ } else {
+ return false
+ }
+ }
+
+ return false
+}
+
+func (sdev *MetaxSDevices) checkUUID(annos map[string]string, d util.DeviceUsage) bool {
+ useUUIDAnno, ok := annos[MetaxUseUUID]
+ if ok {
+ klog.V(5).Infof("check UUID for metax, useUUID[%s], deviceID[%s]", useUUIDAnno, d.ID)
+
+ useUUIDs := strings.Split(useUUIDAnno, ",")
+ if slices.Contains(useUUIDs, d.ID) {
+ klog.V(5).Infof("check UUID pass, the deviceID[%s]", d.ID)
+ return true
+ }
+ return false
+ }
+
+ noUseUUIDAnno, ok := annos[MetaxNoUseUUID]
+ if ok {
+ klog.V(5).Infof("check UUID for metax, nouseUUID[%s], deviceID[%s]", noUseUUIDAnno, d.ID)
+
+ noUseUUIDs := strings.Split(noUseUUIDAnno, ",")
+ if slices.Contains(noUseUUIDs, d.ID) {
+ klog.V(5).Infof("check UUID failed to pass, the deviceID[%s]", d.ID)
+ return false
+ }
+ return true
+ }
+
+ return true
+}
+
+func (sdev *MetaxSDevices) checkIndex(annos map[string]string, d util.DeviceUsage) bool {
+ return true
+}
+
+func (sdev *MetaxSDevices) CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ devices, _ := sdev.getMetaxSDevices(*n)
+
+ return len(devices) > 0, true
+}
+
+func (sdev *MetaxSDevices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest {
+ value, ok := ctr.Resources.Limits[corev1.ResourceName(MetaxResourceNameVCount)]
+ if !ok {
+ return util.ContainerDeviceRequest{}
+ }
+
+ count, ok := value.AsInt64()
+ if !ok {
+ klog.Errorf("container<%s> resource<%s> cannot decode to int64",
+ ctr.Name, MetaxResourceNameVCount)
+ return util.ContainerDeviceRequest{}
+ }
+
+ core := int64(100)
+ coreQuantity, ok := ctr.Resources.Limits[corev1.ResourceName(MetaxResourceNameVCore)]
+ if ok {
+ if v, ok := coreQuantity.AsInt64(); ok {
+ core = v
+ }
+ }
+
+ mem := int64(0)
+ memQuantity, ok := ctr.Resources.Limits[corev1.ResourceName(MetaxResourceNameVMemory)]
+ if ok {
+ if v, ok := memQuantity.AsInt64(); ok {
+ hasUnit := strings.IndexFunc(memQuantity.String(), func(c rune) bool {
+ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+ }) >= 0
+
+ // if user not set unit, default unit is Gi
+ if hasUnit {
+ mem = v / 1024 / 1024
+ } else {
+ mem = v * 1024
+ }
+ }
+ }
+
+ memPercent := 0
+ if mem == 0 {
+ memPercent = 100
+ }
+
+ klog.Infof("container<%s> request",
+ ctr.Name, count, mem, memPercent, core)
+ return util.ContainerDeviceRequest{
+ Nums: int32(count),
+ Type: MetaxSGPUDevice,
+ Memreq: int32(mem),
+ MemPercentagereq: int32(memPercent),
+ Coresreq: int32(core),
+ }
+}
+
+func (sdev *MetaxSDevices) ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32 {
+ if !needScore(podDevices) {
+ return 0
+ }
+
+ // TODO: score should not depend on policy
+ // we have to give it a smaller value because of Spread policy
+ weight := 10000
+ if policy == string(util.NodeSchedulerPolicySpread) {
+ weight = -10000
+ }
+
+ return float32(weight * scoreExclusiveDevices(podDevices, previous))
+}
+
+func (sdev *MetaxSDevices) AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error {
+ n.Used++
+ n.Usedcores += ctr.Usedcores
+ n.Usedmem += ctr.Usedmem
+
+ if value, ok := n.CustomInfo["QosPolicy"]; ok {
+ if qos, ok := value.(string); ok {
+ expectedQos := pod.Annotations[MetaxSGPUQosPolicy]
+ if ctr.Usedcores == 100 {
+ expectedQos = ""
+ }
+
+ n.CustomInfo["QosPolicy"] = expectedQos
+ klog.Infof("device[%s] temp changed qos [%s] to [%s]", n.ID, qos, expectedQos)
+ }
+ }
+
+ return nil
+}
+
+func (mats *MetaxSDevices) Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string) {
+ klog.Infof("pod[%v] container request[%v] devices fit", klog.KObj(pod), request)
+
+ reason := make(map[string]int)
+ candidateDevices := util.ContainerDevices{}
+
+ for i := len(devices) - 1; i >= 0; i-- {
+ dev := devices[i]
+
+ if !mats.checkType(annos, *dev, request) {
+ reason[common.CardTypeMismatch]++
+ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, request.Type)
+ continue
+ }
+
+ if !mats.checkUUID(annos, *dev) {
+ reason[common.CardUUIDMismatch]++
+ klog.V(5).InfoS(common.CardUUIDMismatch, "pod", klog.KObj(pod), "device", dev.ID, "current device info is:", *dev)
+ continue
+ }
+
+ if dev.Count <= dev.Used {
+ reason[common.CardTimeSlicingExhausted]++
+ klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
+ continue
+ }
+
+ memreq := int32(0)
+ if request.Memreq > 0 {
+ memreq = request.Memreq
+ } else {
+ memreq = dev.Totalmem * request.MemPercentagereq / 100
+ }
+
+ if dev.Totalmem-dev.Usedmem < memreq {
+ reason[common.CardInsufficientMemory]++
+ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq)
+ continue
+ }
+
+ if dev.Totalcore-dev.Usedcores < request.Coresreq {
+ reason[common.CardInsufficientCore]++
+ klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", request.Coresreq)
+ continue
+ }
+
+ // Coresreq=100 indicates it want this card exclusively
+ if dev.Totalcore == 100 && request.Coresreq == 100 && dev.Used > 0 {
+ reason[common.ExclusiveDeviceAllocateConflict]++
+ klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
+ continue
+ }
+
+ // You can't allocate core=0 job to an already full GPU
+ if dev.Totalcore != 0 && dev.Usedcores == dev.Totalcore && request.Coresreq == 0 {
+ reason[common.CardComputeUnitsExhausted]++
+ klog.V(5).InfoS(common.CardComputeUnitsExhausted, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+
+ ctrDevice := util.ContainerDevice{
+ Idx: int(dev.Index),
+ UUID: dev.ID,
+ Type: dev.Type,
+ Usedmem: memreq,
+ Usedcores: request.Coresreq,
+ CustomInfo: maps.Clone(dev.CustomInfo),
+ }
+
+ // WorkAround: add Pod annotations into ContainerDevice to pass annotations in `ScoreNode`
+ if ctrDevice.CustomInfo == nil {
+ ctrDevice.CustomInfo = make(map[string]any)
+ }
+ ctrDevice.CustomInfo["Pod.Annotations"] = pod.Annotations
+
+ candidateDevices = append(candidateDevices, ctrDevice)
+ }
+
+ klog.V(5).Infof("pod[%v] candidate devices: %v", klog.KObj(pod), candidateDevices)
+
+ if len(candidateDevices) < int(request.Nums) {
+ if len(candidateDevices) > 0 {
+ reason[common.AllocatedCardsInsufficientRequest] = len(candidateDevices)
+ }
+
+ klog.V(5).Infof("pod[%v] fit devices Insufficient: request[%d], fit[%d]", klog.KObj(pod), request.Nums, len(candidateDevices))
+ return false, map[string]util.ContainerDevices{request.Type: candidateDevices}, common.GenReason(reason, len(devices))
+ }
+
+ bestDevices := candidateDevices[0:request.Nums]
+ if request.Coresreq == 100 {
+ bestDevices = prioritizeExclusiveDevices(candidateDevices, int(request.Nums))
+ }
+
+ klog.Infof("pod[%v] devices fit success, fit devices[%v]", klog.KObj(pod), bestDevices)
+ return true, map[string]util.ContainerDevices{request.Type: bestDevices}, ""
+}
+
+func (sdev *MetaxSDevices) getMetaxSDevices(n corev1.Node) ([]*MetaxSDeviceInfo, error) {
+ anno, ok := n.Annotations[MetaxSDeviceAnno]
+ if !ok {
+ return []*MetaxSDeviceInfo{}, errors.New("annos not found " + MetaxSDeviceAnno)
+ }
+
+ metaxSDevices := []*MetaxSDeviceInfo{}
+ err := json.Unmarshal([]byte(anno), &metaxSDevices)
+ if err != nil {
+ klog.ErrorS(err, "failed to unmarshal metax sdevices", "node", n.Name, "sdevice annotation", anno)
+ return []*MetaxSDeviceInfo{}, err
+ }
+
+ if len(metaxSDevices) == 0 {
+ klog.InfoS("no metax sgpu device found", "node", n.Name, "sdevice annotation", anno)
+ return []*MetaxSDeviceInfo{}, errors.New("no sdevice found on node")
+ }
+
+ klog.V(5).Infof("node[%s] metax sdevice information: %s", n.Name, NodeMetaxSDeviceInfo(metaxSDevices).String())
+ return metaxSDevices, nil
+}
+
+func (sdev *MetaxSDevices) checkDeviceQos(reqQos string, usage util.DeviceUsage, request util.ContainerDeviceRequest) bool {
+ if usage.Used == 0 {
+ klog.Infof("device[%s] not use, it can switch to any qos", usage.ID)
+ return true
+ }
+
+ if request.Coresreq == 100 {
+ klog.Infoln("request exclusive device, no need verify qos")
+ return true
+ }
+
+ devQos := ""
+ if qos, ok := sdev.jqCache.Get(usage.ID); ok {
+ devQos = qos
+ } else {
+ if value, ok := usage.CustomInfo["QosPolicy"]; ok {
+ if qos, ok := value.(string); ok {
+ devQos = qos
+ }
+ }
+ }
+
+ klog.Infof("device[%s]: devQos [%s], reqQos [%s]", usage.ID, devQos, reqQos)
+ if devQos == "" || reqQos == devQos {
+ return true
+ } else {
+ return false
+ }
+}
+
+func (sdev *MetaxSDevices) addJitteryQos(reqQos string, devs util.PodSingleDevice) {
+ for _, ctrdev := range devs {
+ for _, dev := range ctrdev {
+ if value, ok := dev.CustomInfo["QosPolicy"]; ok {
+ if currentQos, ok := value.(string); ok {
+ expectedQos := reqQos
+ if dev.Usedcores == 100 {
+ expectedQos = ""
+ }
+
+ if currentQos != expectedQos {
+ sdev.jqCache.Add(dev.UUID, expectedQos)
+ klog.Infof("device[%s] add to cache, expectedQos[%s] not equal to currentQos[%s]",
+ dev.UUID, expectedQos, currentQos)
+ }
+ }
+ }
+ }
+ }
+}
+
+func prioritizeExclusiveDevices(candidateDevices util.ContainerDevices, require int) util.ContainerDevices {
+ if len(candidateDevices) <= require {
+ return candidateDevices
+ }
+
+ linkDevicesMap := map[int32]util.ContainerDevices{}
+ for _, device := range candidateDevices {
+ linkZone := int32(0)
+ if v, ok := device.CustomInfo["LinkZone"]; ok {
+ if v, ok := v.(int32); ok {
+ linkZone = v
+ }
+ }
+
+ linkDevicesMap[linkZone] = append(linkDevicesMap[linkZone], device)
+ }
+
+ linkDevices := []util.ContainerDevices{}
+ otherDevices := util.ContainerDevices{}
+ for link, devices := range linkDevicesMap {
+ if link <= 0 {
+ otherDevices = append(otherDevices, devices...)
+ } else {
+ linkDevices = append(linkDevices, devices)
+ }
+ }
+
+ sort.Slice(linkDevices, func(i int, j int) bool {
+ return len(linkDevices[i]) < len(linkDevices[j])
+ })
+
+ klog.V(5).Infof("linkDevices: %v, otherDevices: %v", linkDevices, otherDevices)
+
+ // 1. pickup devices within MetaLink
+ for _, devices := range linkDevices {
+ if len(devices) >= require {
+ klog.V(5).Infof("prioritize exclusive devices: best result, within metalink")
+ return devices[0:require]
+ }
+ }
+
+ prioritizeDevices := util.ContainerDevices{}
+
+ // 2. pickup devices cross MetaLink
+ for _, devices := range linkDevices {
+ for _, device := range devices {
+ prioritizeDevices = append(prioritizeDevices, device)
+
+ if len(prioritizeDevices) >= require {
+ klog.V(5).Infof("prioritize exclusive devices: general result, cross metalink")
+ return prioritizeDevices
+ }
+ }
+ }
+
+ // 3. if not satisfied, pick up devices no MetaLink
+ for _, device := range otherDevices {
+ prioritizeDevices = append(prioritizeDevices, device)
+
+ if len(prioritizeDevices) >= require {
+ klog.V(5).Infof("prioritize exclusive devices: bad result, some devices no metalink")
+ return prioritizeDevices
+ }
+ }
+
+ return candidateDevices[0:require]
+}
+
+func needScore(podDevices util.PodSingleDevice) bool {
+ enableTopoAware := MetaxTopologyAware
+
+ for _, ctrDevices := range podDevices {
+ for _, device := range ctrDevices {
+ if device.Usedcores == 100 {
+ if annotations, ok := device.CustomInfo["Pod.Annotations"].(map[string]string); ok {
+ if value, ok := annotations[MetaxSGPUTopologyAware]; ok {
+ if enable, err := strconv.ParseBool(value); err == nil {
+ enableTopoAware = enable
+ }
+ }
+ }
+
+ return enableTopoAware
+ }
+ }
+ }
+
+ return false
+}
+
+func scoreExclusiveDevices(podDevices util.PodSingleDevice, previous []*util.DeviceUsage) int {
+ availableDevices := LinkDevices{}
+ allocatedDevices := LinkDevices{}
+ restDevices := LinkDevices{}
+
+ for _, ctrDevices := range podDevices {
+ for _, device := range ctrDevices {
+ if device.Usedcores == 100 {
+ linkZone := int32(0)
+ if v, ok := device.CustomInfo["LinkZone"]; ok {
+ if v, ok := v.(int32); ok {
+ linkZone = v
+ }
+ }
+
+ allocatedDevices = append(allocatedDevices, &LinkDevice{
+ uuid: device.UUID,
+ linkZone: linkZone,
+ })
+ }
+ }
+ }
+
+ for _, dev := range previous {
+ if dev.Used == 0 {
+ linkZone := int32(0)
+ if v, ok := dev.CustomInfo["LinkZone"]; ok {
+ if v, ok := v.(int32); ok {
+ linkZone = v
+ }
+ }
+
+ availableDevices = append(availableDevices, &LinkDevice{
+ uuid: dev.ID,
+ linkZone: linkZone,
+ })
+
+ find := false
+ for _, allocatedDev := range allocatedDevices {
+ if allocatedDev.uuid == dev.ID {
+ find = true
+ break
+ }
+ }
+
+ if !find {
+ restDevices = append(restDevices, &LinkDevice{
+ uuid: dev.ID,
+ linkZone: linkZone,
+ })
+ }
+ }
+ }
+
+ klog.V(5).Infof("calcScore devices >>> available: %s, allocated: %s, rest: %s",
+ availableDevices, allocatedDevices, restDevices)
+
+ availableScore := availableDevices.Score()
+ allocatedScore := allocatedDevices.Score()
+ restScore := restDevices.Score()
+ lossScore := availableScore - allocatedScore - restScore
+
+ result := 10*allocatedScore - lossScore
+ klog.V(5).Infof("calcScore result[%d] >>> availableScore[%d], allocatedScore[%d], restScore[%d], lossScore[%d]",
+ result, availableScore, allocatedScore, restScore, lossScore)
+
+ return result
+}
diff --git a/pkg/device/metax/sdevice_test.go b/pkg/device/metax/sdevice_test.go
new file mode 100644
index 000000000..51b5dfd01
--- /dev/null
+++ b/pkg/device/metax/sdevice_test.go
@@ -0,0 +1,1840 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metax
+
+import (
+ "flag"
+ "fmt"
+ "reflect"
+ "testing"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestMutateAdmission(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ container *corev1.Container
+ pod *corev1.Pod
+
+ expectedFound bool
+ expectedError string
+ expectedPod *corev1.Pod
+ }{
+ {
+ name: "no sgpu resource",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ },
+ },
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ MetaxSGPUQosPolicy: BestEffort,
+ },
+ },
+ },
+
+ expectedFound: false,
+ expectedError: "",
+ expectedPod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ MetaxSGPUQosPolicy: BestEffort,
+ },
+ },
+ },
+ },
+ {
+ name: "qos policy error",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/sgpu": resource.MustParse("1"),
+ },
+ },
+ },
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ MetaxSGPUQosPolicy: "best-effortx",
+ },
+ },
+ },
+
+ expectedFound: true,
+ expectedError: fmt.Sprintf("%s must be set one of [%s, %s, %s]",
+ MetaxSGPUQosPolicy, BestEffort, FixedShare, BurstShare),
+ expectedPod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ MetaxSGPUQosPolicy: "best-effortx",
+ },
+ },
+ },
+ },
+ {
+ name: "no qos policy",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/sgpu": resource.MustParse("1"),
+ },
+ },
+ },
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{},
+ },
+ },
+
+ expectedFound: true,
+ expectedError: "",
+ expectedPod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ MetaxSGPUQosPolicy: BestEffort,
+ },
+ },
+ },
+ },
+ {
+ name: "pod annotation nil",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/sgpu": resource.MustParse("1"),
+ },
+ },
+ },
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{},
+ },
+
+ expectedFound: true,
+ expectedError: "",
+ expectedPod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ MetaxSGPUQosPolicy: BestEffort,
+ },
+ },
+ },
+ },
+ {
+ name: "qos policy fit",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/sgpu": resource.MustParse("1"),
+ },
+ },
+ },
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ MetaxSGPUQosPolicy: BurstShare,
+ },
+ },
+ },
+
+ expectedFound: true,
+ expectedError: "",
+ expectedPod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ MetaxSGPUQosPolicy: BurstShare,
+ },
+ },
+ },
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ metaxSDevice := &MetaxSDevices{}
+ fs := flag.FlagSet{}
+ ParseConfig(&fs)
+
+ resFound, resErr := metaxSDevice.MutateAdmission(ts.container, ts.pod)
+
+ if resFound != ts.expectedFound {
+ t.Errorf("MutateAdmission failed: resFound %v, expectedFound %v",
+ resFound, ts.expectedFound)
+ }
+
+ resErrString := ""
+ if resErr != nil {
+ resErrString = resErr.Error()
+ }
+
+ if resErrString != ts.expectedError {
+ t.Errorf("MutateAdmission failed: resErr %v, expectedError %v",
+ resErr, ts.expectedError)
+ }
+
+ if !reflect.DeepEqual(ts.expectedPod, ts.pod) {
+ t.Errorf("MutateAdmission failed: result %v, expected %v",
+ ts.pod, ts.expectedPod)
+ }
+ })
+ }
+}
+
+func TestGenerateResourceRequests(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ container *corev1.Container
+
+ expected util.ContainerDeviceRequest
+ }{
+ {
+ name: "one full sgpu test",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/sgpu": resource.MustParse("1"),
+ },
+ },
+ },
+
+ expected: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: MetaxSGPUDevice,
+ Memreq: 0,
+ MemPercentagereq: 100,
+ Coresreq: 100,
+ },
+ },
+ {
+ name: "two full sgpu test",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/sgpu": resource.MustParse("2"),
+ },
+ },
+ },
+
+ expected: util.ContainerDeviceRequest{
+ Nums: 2,
+ Type: MetaxSGPUDevice,
+ Memreq: 0,
+ MemPercentagereq: 100,
+ Coresreq: 100,
+ },
+ },
+ {
+ name: "one sgpu test set vcore",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/sgpu": resource.MustParse("1"),
+ "metax-tech.com/vcore": resource.MustParse("30"),
+ },
+ },
+ },
+
+ expected: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: MetaxSGPUDevice,
+ Memreq: 0,
+ MemPercentagereq: 100,
+ Coresreq: 30,
+ },
+ },
+ {
+ name: "one sgpu test set vmemory",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/sgpu": resource.MustParse("1"),
+ "metax-tech.com/vmemory": resource.MustParse("16"),
+ },
+ },
+ },
+
+ expected: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: MetaxSGPUDevice,
+ Memreq: 16 * 1024,
+ MemPercentagereq: 0,
+ Coresreq: 100,
+ },
+ },
+ {
+ name: "one sgpu test set vcore&vmemory",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/sgpu": resource.MustParse("1"),
+ "metax-tech.com/vcore": resource.MustParse("60"),
+ "metax-tech.com/vmemory": resource.MustParse("16"),
+ },
+ },
+ },
+
+ expected: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: MetaxSGPUDevice,
+ Memreq: 16 * 1024,
+ MemPercentagereq: 0,
+ Coresreq: 60,
+ },
+ },
+ {
+ name: "one sgpu test set vcore&vmemory, mem unit Mi",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/sgpu": resource.MustParse("1"),
+ "metax-tech.com/vcore": resource.MustParse("60"),
+ "metax-tech.com/vmemory": resource.MustParse("1024Mi"),
+ },
+ },
+ },
+
+ expected: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: MetaxSGPUDevice,
+ Memreq: 1024,
+ MemPercentagereq: 0,
+ Coresreq: 60,
+ },
+ },
+ {
+ name: "one sgpu test set vcore&vmemory, mem unit Gi",
+ container: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "metax-tech.com/sgpu": resource.MustParse("1"),
+ "metax-tech.com/vcore": resource.MustParse("60"),
+ "metax-tech.com/vmemory": resource.MustParse("16Gi"),
+ },
+ },
+ },
+
+ expected: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: MetaxSGPUDevice,
+ Memreq: 16 * 1024,
+ MemPercentagereq: 0,
+ Coresreq: 60,
+ },
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ metaxSDevice := &MetaxSDevices{}
+ fs := flag.FlagSet{}
+ ParseConfig(&fs)
+
+ result := metaxSDevice.GenerateResourceRequests(ts.container)
+
+ if !reflect.DeepEqual(ts.expected, result) {
+ t.Errorf("GenerateResourceRequests failed: result %v, expected %v",
+ result, ts.expected)
+ }
+ })
+ }
+}
+
+func TestGetMetaxSDevices(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ node corev1.Node
+
+ expected []*MetaxSDeviceInfo
+ }{
+ {
+ name: "test normal node",
+ node: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "normal",
+ Annotations: map[string]string{
+ MetaxSDeviceAnno: "[{\"uuid\": \"GPU-123\", \"model\": \"sgpu\", \"totalDevCount\": 16, \"totalCompute\": 100, \"bdf\": \"0000:44:00.0\", \"totalVRam\" : 32768, \"numa\": 1, \"healthy\": true, \"qosPolicy\": \"fixed-share\"}]",
+ },
+ },
+ },
+
+ expected: []*MetaxSDeviceInfo{
+ {
+ UUID: "GPU-123",
+ BDF: "0000:44:00.0",
+ Model: "sgpu",
+ TotalDevCount: 16,
+ TotalCompute: 100,
+ TotalVRam: 32768,
+ AvailableDevCount: 0,
+ AvailableCompute: 0,
+ AvailableVRam: 0,
+ Numa: 1,
+ Healthy: true,
+ QosPolicy: FixedShare,
+ },
+ },
+ },
+ {
+ name: "test annotaions nil",
+ node: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "normal",
+ Annotations: map[string]string{},
+ },
+ },
+
+ expected: []*MetaxSDeviceInfo{},
+ },
+ {
+ name: "test Unmarshal fail",
+ node: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "normal",
+ Annotations: map[string]string{
+ MetaxSDeviceAnno: "",
+ },
+ },
+ },
+
+ expected: []*MetaxSDeviceInfo{},
+ },
+ {
+ name: "test devices len 0",
+ node: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "normal",
+ Annotations: map[string]string{
+ MetaxSDeviceAnno: "[]",
+ },
+ },
+ },
+
+ expected: []*MetaxSDeviceInfo{},
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ dev := &MetaxSDevices{}
+ got, _ := dev.getMetaxSDevices(ts.node)
+
+ if !reflect.DeepEqual(got, ts.expected) {
+ t.Errorf("getMetaxSDevices failed: result %v, expected %v",
+ got, ts.expected)
+ }
+ })
+ }
+}
+
+func TestCheckDeviceQos(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ reqQos string
+ usage util.DeviceUsage
+ request util.ContainerDeviceRequest
+
+ expected bool
+ }{
+ {
+ name: "check no use device",
+ reqQos: BestEffort,
+ usage: util.DeviceUsage{
+ ID: "GPU-123",
+ Used: 0,
+ CustomInfo: map[string]any{
+ "QosPolicy": BurstShare,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Coresreq: 50,
+ },
+
+ expected: true,
+ },
+ {
+ name: "check request exclusive",
+ reqQos: BestEffort,
+ usage: util.DeviceUsage{
+ ID: "GPU-123",
+ Used: 2,
+ CustomInfo: map[string]any{
+ "QosPolicy": BurstShare,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Coresreq: 100,
+ },
+
+ expected: true,
+ },
+ {
+ name: "check fail",
+ reqQos: BestEffort,
+ usage: util.DeviceUsage{
+ ID: "GPU-123",
+ Used: 2,
+ CustomInfo: map[string]any{
+ "QosPolicy": BurstShare,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Coresreq: 50,
+ },
+
+ expected: false,
+ },
+ {
+ name: "check pass",
+ reqQos: BestEffort,
+ usage: util.DeviceUsage{
+ ID: "GPU-123",
+ Used: 2,
+ CustomInfo: map[string]any{
+ "QosPolicy": BestEffort,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Coresreq: 50,
+ },
+
+ expected: true,
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ metaxSDevice := &MetaxSDevices{
+ jqCache: NewJitteryQosCache(),
+ }
+
+ res := metaxSDevice.checkDeviceQos(ts.reqQos, ts.usage, ts.request)
+ if res != ts.expected {
+ t.Errorf("checkDeviceQos failed: result %v, expected %v",
+ res, ts.expected)
+ }
+ })
+ }
+}
+
+func TestAddJitteryQos(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ reqQos string
+ devs util.PodSingleDevice
+
+ expectedCache map[string]string
+ }{
+ {
+ name: "request BestEffort",
+ reqQos: BestEffort,
+ devs: util.PodSingleDevice{
+ {
+ {
+ UUID: "GPU-123",
+ Usedcores: 50,
+ CustomInfo: map[string]any{
+ "QosPolicy": BestEffort,
+ },
+ },
+ {
+ UUID: "GPU-456",
+ Usedcores: 50,
+ CustomInfo: map[string]any{
+ "QosPolicy": BurstShare,
+ },
+ },
+ },
+ {
+ {
+ UUID: "GPU-789",
+ Usedcores: 100,
+ CustomInfo: map[string]any{
+ "QosPolicy": BestEffort,
+ },
+ },
+ },
+ },
+
+ expectedCache: map[string]string{
+ "GPU-456": BestEffort,
+ "GPU-789": "",
+ },
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ metaxSDevice := &MetaxSDevices{
+ jqCache: NewJitteryQosCache(),
+ }
+ metaxSDevice.addJitteryQos(ts.reqQos, ts.devs)
+
+ if !reflect.DeepEqual(metaxSDevice.jqCache.cache, ts.expectedCache) {
+ t.Errorf("addJitteryQos failed: result %v, expected %v",
+ metaxSDevice.jqCache.cache, ts.expectedCache)
+ }
+ })
+ }
+}
+
+func TestMetaxSDevices_Fit(t *testing.T) {
+ config := MetaxConfig{
+ ResourceVCountName: "metax-tech.com/sgpu",
+ ResourceVCoreName: "metax-tech.com/vcore",
+ ResourceVMemoryName: "metax-tech.com/vmemory",
+ }
+ dev := InitMetaxSDevice(config)
+
+ tests := []struct {
+ name string
+ devices []*util.DeviceUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ wantFit bool
+ wantLen int
+ wantDevIDs []string
+ wantReason string
+ }{
+ {
+ name: "fit success",
+ devices: []*util.DeviceUsage{
+ {
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ },
+ {
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 64,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MetaxSGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-1"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: memory not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MetaxSGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientMemory",
+ },
+ {
+ name: "fit fail: core not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1024,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MetaxSGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientCore",
+ },
+ {
+ name: "fit fail: type mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ Type: MetaxSGPUDevice,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: "OtherType",
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTypeMismatch",
+ },
+ {
+ name: "fit fail: device unhealthy",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: false,
+ Type: MetaxSGPUDevice,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: MetaxSGPUDevice,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTypeMismatch",
+ },
+ {
+ name: "fit fail: card overused",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 100,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MetaxSGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTimeSlicingExhausted",
+ },
+ {
+ name: "fit success: but core limit can't exceed 100",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 120,
+ Type: MetaxSGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientCore",
+ },
+ {
+ name: "fit fail: card exclusively",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 100,
+ Type: MetaxSGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 ExclusiveDeviceAllocateConflict",
+ },
+ {
+ name: "fit fail: user assign use uuid mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MetaxSGPUDevice,
+ },
+ annos: map[string]string{MetaxUseUUID: "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: user assign no use uuid match",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MetaxSGPUDevice,
+ },
+ annos: map[string]string{MetaxNoUseUUID: "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: CardComputeUnitsExhausted",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 0,
+ Type: MetaxSGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardComputeUnitsExhausted",
+ },
+ {
+ name: "fit fail: AllocatedCardsInsufficientRequest",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 20,
+ Type: MetaxSGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 AllocatedCardsInsufficientRequest",
+ },
+ {
+ name: "fit success: memory percentage",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: MetaxSGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 10,
+ Coresreq: 20,
+ Type: MetaxSGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ allocated := &util.PodDevices{}
+ fit, result, reason := dev.Fit(test.devices, test.request, test.annos, &corev1.Pod{}, &util.NodeInfo{}, allocated)
+ if fit != test.wantFit {
+ t.Errorf("Fit: got %v, want %v", fit, test.wantFit)
+ }
+ if test.wantFit {
+ if len(result[MetaxSGPUDevice]) != test.wantLen {
+ t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[MetaxSGPUDevice]))
+ }
+ for idx, id := range test.wantDevIDs {
+ if id != result[MetaxSGPUDevice][idx].UUID {
+ t.Errorf("expected device id: %s, got device id %s", id, result[MetaxSGPUDevice][idx].UUID)
+ }
+ }
+ }
+
+ if reason != test.wantReason {
+ t.Errorf("expected reason: %s, got reason: %s", test.wantReason, reason)
+ }
+ })
+ }
+}
+
+func TestMetaxSDevices_AddResourceUsage(t *testing.T) {
+ tests := []struct {
+ name string
+ deviceUsage *util.DeviceUsage
+ ctr *util.ContainerDevice
+ wantErr bool
+ wantUsage *util.DeviceUsage
+ }{
+ {
+ name: "test add resource usage",
+ deviceUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 0,
+ Usedcores: 15,
+ Usedmem: 2000,
+ },
+ ctr: &util.ContainerDevice{
+ UUID: "dev-0",
+ Usedcores: 50,
+ Usedmem: 1024,
+ },
+ wantUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 1,
+ Usedcores: 65,
+ Usedmem: 3024,
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &MetaxDevices{}
+ if err := dev.AddResourceUsage(&corev1.Pod{}, tt.deviceUsage, tt.ctr); (err != nil) != tt.wantErr {
+ t.Errorf("AddResourceUsage() error=%v, wantErr %v", err, tt.wantErr)
+ }
+ if !tt.wantErr {
+ if tt.deviceUsage.Usedcores != tt.wantUsage.Usedcores {
+ t.Errorf("expected used cores: %d, got used cores %d", tt.wantUsage.Usedcores, tt.deviceUsage.Usedcores)
+ }
+ if tt.deviceUsage.Usedmem != tt.wantUsage.Usedmem {
+ t.Errorf("expected used mem: %d, got used mem %d", tt.wantUsage.Usedmem, tt.deviceUsage.Usedmem)
+ }
+ if tt.deviceUsage.Used != tt.wantUsage.Used {
+ t.Errorf("expected used: %d, got used %d", tt.wantUsage.Used, tt.deviceUsage.Used)
+ }
+ }
+ })
+ }
+}
+
+func TestPrioritizeExclusiveDevices(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ candidateDevices util.ContainerDevices
+ require int
+
+ expectedDevices util.ContainerDevices
+ }{
+ {
+ name: "require one device",
+ candidateDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-1",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-2",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-5",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+ require: 1,
+
+ expectedDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-5",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+ },
+ {
+ name: "require two device",
+ candidateDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-1",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-2",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-5",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-6",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-7",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+ require: 2,
+
+ expectedDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-1",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-2",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ },
+ },
+ {
+ name: "require four device, best result",
+ candidateDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-1",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-2",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-5",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-6",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-7",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-8",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+ require: 4,
+
+ expectedDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-5",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-6",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-7",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-8",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+ },
+ {
+ name: "require four device, general result",
+ candidateDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-1",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-2",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-5",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-6",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-7",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+ require: 4,
+
+ expectedDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-1",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-2",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-5",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-6",
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+ },
+ {
+ name: "no metalink, require two device",
+ candidateDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-5",
+ CustomInfo: map[string]any{"LinkZone": int32(0)},
+ },
+ {
+ UUID: "GPU-6",
+ CustomInfo: map[string]any{"LinkZone": int32(0)},
+ },
+ {
+ UUID: "GPU-7",
+ CustomInfo: map[string]any{"LinkZone": int32(0)},
+ },
+ },
+ require: 2,
+
+ expectedDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-5",
+ CustomInfo: map[string]any{"LinkZone": int32(0)},
+ },
+ {
+ UUID: "GPU-6",
+ CustomInfo: map[string]any{"LinkZone": int32(0)},
+ },
+ },
+ },
+ {
+ name: "part metalink, require two device, best result",
+ candidateDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-3",
+ CustomInfo: map[string]any{"LinkZone": int32(0)},
+ },
+ {
+ UUID: "GPU-4",
+ CustomInfo: map[string]any{"LinkZone": int32(0)},
+ },
+ {
+ UUID: "GPU-7",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-8",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ },
+ require: 2,
+
+ expectedDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-7",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-8",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ },
+ },
+ {
+ name: "part metalink, require four device, bad result",
+ candidateDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-3",
+ CustomInfo: map[string]any{"LinkZone": int32(0)},
+ },
+ {
+ UUID: "GPU-4",
+ CustomInfo: map[string]any{"LinkZone": int32(0)},
+ },
+ {
+ UUID: "GPU-6",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-7",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-8",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ },
+ require: 4,
+
+ expectedDevices: util.ContainerDevices{
+ {
+ UUID: "GPU-6",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-7",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-8",
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-3",
+ CustomInfo: map[string]any{"LinkZone": int32(0)},
+ },
+ },
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ result := prioritizeExclusiveDevices(ts.candidateDevices, ts.require)
+
+ if !reflect.DeepEqual(result, ts.expectedDevices) {
+ t.Errorf("prioritizeExclusiveDevices failed: result %v, expected %v",
+ result, ts.expectedDevices)
+ }
+ })
+ }
+}
+
+func TestNeedScore(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ podDevices util.PodSingleDevice
+
+ expected bool
+ }{
+ {
+ name: "enable, allocate 100core",
+ podDevices: util.PodSingleDevice{
+ {
+ {
+ Usedcores: 100,
+ CustomInfo: map[string]any{
+ "Pod.Annotations": map[string]string{
+ MetaxSGPUTopologyAware: "true",
+ },
+ },
+ },
+ },
+ },
+
+ expected: true,
+ },
+ {
+ name: "disable, allocate 100core",
+ podDevices: util.PodSingleDevice{
+ {
+ {
+ Usedcores: 100,
+ CustomInfo: map[string]any{
+ "Pod.Annotations": map[string]string{
+ MetaxSGPUTopologyAware: "false",
+ },
+ },
+ },
+ },
+ },
+
+ expected: false,
+ },
+ {
+ name: "enable, allocate 99core",
+ podDevices: util.PodSingleDevice{
+ {
+ {
+ Usedcores: 99,
+ CustomInfo: map[string]any{
+ "Pod.Annotations": map[string]string{
+ MetaxSGPUTopologyAware: "true",
+ },
+ },
+ },
+ },
+ },
+
+ expected: false,
+ },
+ {
+ name: "enable, container[0]: 99core, container[1]: 100core",
+ podDevices: util.PodSingleDevice{
+ {
+ {
+ Usedcores: 99,
+ CustomInfo: map[string]any{
+ "Pod.Annotations": map[string]string{
+ MetaxSGPUTopologyAware: "true",
+ },
+ },
+ },
+ },
+ {
+ {
+ Usedcores: 100,
+ CustomInfo: map[string]any{
+ "Pod.Annotations": map[string]string{
+ MetaxSGPUTopologyAware: "true",
+ },
+ },
+ },
+ },
+ },
+
+ expected: true,
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ result := needScore(ts.podDevices)
+
+ if result != ts.expected {
+ t.Errorf("needScore failed: result %v, expected %v",
+ result, ts.expected)
+ }
+ })
+ }
+}
+
+func TestScoreExclusiveDevices(t *testing.T) {
+ for _, ts := range []struct {
+ name string
+ podDevices util.PodSingleDevice
+ previous []*util.DeviceUsage
+
+ expectedScore int
+ }{
+ {
+ name: "allocate one device, rest zero device",
+ podDevices: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ UUID: "GPU-4",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ },
+ },
+ previous: []*util.DeviceUsage{
+ {
+ ID: "GPU-1",
+ Used: 1,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-2",
+ Used: 1,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-3",
+ Used: 1,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-4",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ },
+
+ expectedScore: 0,
+ },
+ {
+ name: "allocate one device, rest three device",
+ podDevices: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ UUID: "GPU-4",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ },
+ },
+ previous: []*util.DeviceUsage{
+ {
+ ID: "GPU-1",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-2",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-3",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-4",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ },
+
+ expectedScore: -30,
+ },
+ {
+ name: "allocate two device, best result",
+ podDevices: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ UUID: "GPU-3",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-4",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ },
+ },
+ previous: []*util.DeviceUsage{
+ {
+ ID: "GPU-1",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-2",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-3",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-4",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ },
+
+ expectedScore: 60,
+ },
+ {
+ name: "allocate two device, bad result",
+ podDevices: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ UUID: "GPU-4",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-5",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+ },
+ previous: []*util.DeviceUsage{
+ {
+ ID: "GPU-1",
+ Used: 1,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-2",
+ Used: 1,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-3",
+ Used: 1,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-4",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-5",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ ID: "GPU-6",
+ Used: 1,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ ID: "GPU-7",
+ Used: 1,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ ID: "GPU-8",
+ Used: 1,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+
+ expectedScore: 0,
+ },
+ {
+ name: "allocate four device, best result",
+ podDevices: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ UUID: "GPU-1",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-2",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-3",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-4",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ },
+ },
+ previous: []*util.DeviceUsage{
+ {
+ ID: "GPU-1",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-2",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-3",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-4",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-5",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+
+ expectedScore: 600,
+ },
+ {
+ name: "allocate four device, bad result",
+ podDevices: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ UUID: "GPU-3",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-4",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-5",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-6",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+ },
+ previous: []*util.DeviceUsage{
+ {
+ ID: "GPU-2",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-3",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-4",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-5",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ ID: "GPU-6",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+
+ expectedScore: 180,
+ },
+ {
+ name: "allocate eight device",
+ podDevices: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ UUID: "GPU-1",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-2",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-3",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-4",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ UUID: "GPU-5",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-6",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-7",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ UUID: "GPU-8",
+ Usedcores: 100,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+ },
+ previous: []*util.DeviceUsage{
+ {
+ ID: "GPU-1",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-2",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-3",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-4",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(1)},
+ },
+ {
+ ID: "GPU-5",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ ID: "GPU-6",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ ID: "GPU-7",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ {
+ ID: "GPU-8",
+ Used: 0,
+ CustomInfo: map[string]any{"LinkZone": int32(2)},
+ },
+ },
+
+ expectedScore: 1200,
+ },
+ } {
+ t.Run(ts.name, func(t *testing.T) {
+ result := scoreExclusiveDevices(ts.podDevices, ts.previous)
+
+ if result != ts.expectedScore {
+ t.Errorf("scoreExclusiveDevices failed: result %v, expected %v",
+ result, ts.expectedScore)
+ }
+ })
+ }
+}
diff --git a/pkg/device/mthreads/device.go b/pkg/device/mthreads/device.go
new file mode 100644
index 000000000..8a9874184
--- /dev/null
+++ b/pkg/device/mthreads/device.go
@@ -0,0 +1,392 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mthreads
+
+import (
+ "errors"
+ "flag"
+ "fmt"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/common"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ "k8s.io/klog/v2"
+)
+
+type MthreadsDevices struct {
+}
+
+const (
+ MthreadsGPUDevice = "Mthreads"
+ MthreadsGPUCommonWord = "Mthreads"
+ MthreadsDeviceSelection = "mthreads.com/gpu-index"
+ // MthreadsUseUUID is user can use specify Mthreads device for set Mthreads UUID.
+ MthreadsUseUUID = "mthreads.ai/use-gpuuuid"
+ // MthreadsNoUseUUID is user can not use specify Mthreads device for set Mthreads UUID.
+ MthreadsNoUseUUID = "mthreads.ai/nouse-gpuuuid"
+ MthreadsAssignedGPUIndex = "mthreads.com/gpu-index"
+ MthreadsAssignedNode = "mthreads.com/predicate-node"
+ MthreadsPredicateTime = "mthreads.com/predicate-time"
+ coresPerMthreadsGPU = 16
+ memoryPerMthreadsGPU = 96
+)
+
+var (
+ MthreadsResourceCount string
+ MthreadsResourceMemory string
+ MthreadsResourceCores string
+ legalMemoryslices = []int64{2, 4, 8, 16, 32, 64, 96}
+)
+
+type MthreadsConfig struct {
+ ResourceCountName string `yaml:"resourceCountName"`
+ ResourceMemoryName string `yaml:"resourceMemoryName"`
+ ResourceCoreName string `yaml:"resourceCoreName"`
+}
+
+func InitMthreadsDevice(config MthreadsConfig) *MthreadsDevices {
+ MthreadsResourceCount = config.ResourceCountName
+ MthreadsResourceCores = config.ResourceCoreName
+ MthreadsResourceMemory = config.ResourceMemoryName
+ util.InRequestDevices[MthreadsGPUDevice] = "hami.io/mthreads-vgpu-devices-to-allocate"
+ util.SupportDevices[MthreadsGPUDevice] = "hami.io/mthreads-vgpu-devices-allocated"
+ return &MthreadsDevices{}
+}
+
+func (dev *MthreadsDevices) CommonWord() string {
+ return MthreadsGPUCommonWord
+}
+
+func ParseConfig(fs *flag.FlagSet) {
+ fs.StringVar(&MthreadsResourceCount, "mthreads-name", "mthreads.com/vgpu", "mthreads resource count")
+ fs.StringVar(&MthreadsResourceMemory, "mthreads-memory", "mthreads.com/sgpu-memory", "mthreads memory resource")
+ fs.StringVar(&MthreadsResourceCores, "mthreads-cores", "mthreads.com/sgpu-core", "mthreads core resource")
+}
+
+func (dev *MthreadsDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
+ count, ok := ctr.Resources.Limits[corev1.ResourceName(MthreadsResourceCount)]
+ if ok {
+ if count.Value() > 1 {
+ ctr.Resources.Limits[corev1.ResourceName(MthreadsResourceCores)] = *resource.NewQuantity(count.Value()*int64(coresPerMthreadsGPU), resource.DecimalSI)
+ ctr.Resources.Limits[corev1.ResourceName(MthreadsResourceMemory)] = *resource.NewQuantity(count.Value()*int64(memoryPerMthreadsGPU), resource.DecimalSI)
+ p.Annotations["mthreads.com/request-gpu-num"] = fmt.Sprint(count.Value())
+ return ok, nil
+ }
+ mem, memok := ctr.Resources.Limits[corev1.ResourceName(MthreadsResourceMemory)]
+ if !memok {
+ ctr.Resources.Limits[corev1.ResourceName(MthreadsResourceCores)] = *resource.NewQuantity(count.Value()*int64(coresPerMthreadsGPU), resource.DecimalSI)
+ ctr.Resources.Limits[corev1.ResourceName(MthreadsResourceMemory)] = *resource.NewQuantity(count.Value()*int64(memoryPerMthreadsGPU), resource.DecimalSI)
+ } else {
+ memnum, _ := mem.AsInt64()
+ found := slices.Contains(legalMemoryslices, memnum)
+ if !found {
+ return true, errors.New("sGPU memory request value is invalid, valid values are [1, 2, 4, 8, 16, 32, 64, 96]")
+ }
+ }
+ }
+ return ok, nil
+}
+
+func (dev *MthreadsDevices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) {
+ nodedevices := []*util.DeviceInfo{}
+ i := 0
+ cores, ok := n.Status.Capacity.Name(corev1.ResourceName(MthreadsResourceCores), resource.DecimalSI).AsInt64()
+ if !ok || cores == 0 {
+ return []*util.DeviceInfo{}, fmt.Errorf("device not found %s", MthreadsResourceCores)
+ }
+ memoryTotal, _ := n.Status.Capacity.Name(corev1.ResourceName(MthreadsResourceMemory), resource.DecimalSI).AsInt64()
+ for int64(i)*coresPerMthreadsGPU < cores {
+ nodedevices = append(nodedevices, &util.DeviceInfo{
+ Index: uint(i),
+ ID: n.Name + "-mthreads-" + fmt.Sprint(i),
+ Count: 100,
+ Devmem: int32(memoryTotal * 512 * coresPerMthreadsGPU / cores),
+ Devcore: coresPerMthreadsGPU,
+ Type: MthreadsGPUDevice,
+ Numa: 0,
+ Health: true,
+ })
+ i++
+ }
+ return nodedevices, nil
+}
+
+func (dev *MthreadsDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd util.PodDevices) map[string]string {
+ devlist, ok := pd[MthreadsGPUDevice]
+ if ok && len(devlist) > 0 {
+ (*annoinput)[util.SupportDevices[MthreadsGPUDevice]] = util.EncodePodSingleDevice(devlist)
+ for _, dp := range devlist {
+ if len(dp) > 0 {
+ value := ""
+ for _, val := range dp {
+ value = value + fmt.Sprint(val.Idx) + ","
+ }
+ if len(value) > 0 {
+ (*annoinput)[MthreadsAssignedGPUIndex] = strings.TrimRight(value, ",")
+ //(*annoinput)[MthreadsAssignedNode]=
+ tmp := strconv.FormatInt(time.Now().UnixNano(), 10)
+ (*annoinput)[MthreadsPredicateTime] = tmp
+ (*annoinput)[MthreadsAssignedNode] = (*annoinput)[util.AssignedNodeAnnotations]
+ }
+ }
+ }
+ }
+ klog.Infoln("annoinput", (*annoinput))
+ return *annoinput
+}
+
+func (dev *MthreadsDevices) LockNode(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *MthreadsDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
+ return nil
+}
+
+func (dev *MthreadsDevices) NodeCleanUp(nn string) error {
+ return nil
+}
+
+func (dev *MthreadsDevices) checkType(annos map[string]string, d util.DeviceUsage, n util.ContainerDeviceRequest) (bool, bool, bool) {
+ if strings.Compare(n.Type, MthreadsGPUDevice) == 0 {
+ return true, true, false
+ }
+ return false, false, false
+}
+
+func (dev *MthreadsDevices) checkUUID(annos map[string]string, d util.DeviceUsage) bool {
+ userUUID, ok := annos[MthreadsUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for Mthreads user uuid [%s], device id is %s", userUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ userUUIDs := strings.Split(userUUID, ",")
+ return slices.Contains(userUUIDs, d.ID)
+ }
+
+ noUserUUID, ok := annos[MthreadsNoUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for Iluvatar not user uuid [%s], device id is %s", noUserUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ noUserUUIDs := strings.Split(noUserUUID, ",")
+ return !slices.Contains(noUserUUIDs, d.ID)
+ }
+ return true
+}
+
+func (dev *MthreadsDevices) checkIndex(annos map[string]string, d util.DeviceUsage) bool {
+ return true
+}
+
+func (dev *MthreadsDevices) CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ return true, true
+}
+
+func (dev *MthreadsDevices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest {
+ klog.Info("Start to count mthreads devices for container ", ctr.Name)
+ mthreadsResourceCount := corev1.ResourceName(MthreadsResourceCount)
+ mthreadsResourceMem := corev1.ResourceName(MthreadsResourceMemory)
+ mthreadsResourceCores := corev1.ResourceName(MthreadsResourceCores)
+ v, ok := ctr.Resources.Limits[mthreadsResourceCount]
+ if !ok {
+ v, ok = ctr.Resources.Requests[mthreadsResourceCount]
+ }
+ if ok {
+ if n, ok := v.AsInt64(); ok {
+ klog.InfoS("Detected mthreads device request",
+ "container", ctr.Name,
+ "deviceCount", n)
+ memnum := 0
+ mem, ok := ctr.Resources.Limits[mthreadsResourceMem]
+ if !ok {
+ mem, ok = ctr.Resources.Requests[mthreadsResourceMem]
+ }
+ if ok {
+ memnums, ok := mem.AsInt64()
+ if ok {
+ memnum = int(memnums) * 512
+ klog.InfoS("Memory allocation calculated",
+ "container", ctr.Name,
+ "requestedMem", memnums,
+ "allocatedMem", memnum)
+ }
+ }
+ corenum := int32(0)
+ core, ok := ctr.Resources.Limits[mthreadsResourceCores]
+ if !ok {
+ core, ok = ctr.Resources.Requests[mthreadsResourceCores]
+ }
+ if ok {
+ corenums, ok := core.AsInt64()
+ if ok {
+ corenum = int32(corenums)
+ }
+ }
+
+ mempnum := 0
+ if memnum == 0 {
+ mempnum = 100
+ }
+
+ return util.ContainerDeviceRequest{
+ Nums: int32(n),
+ Type: MthreadsGPUDevice,
+ Memreq: int32(memnum) / int32(n),
+ MemPercentagereq: int32(mempnum),
+ Coresreq: corenum / int32(n),
+ }
+ }
+ }
+ return util.ContainerDeviceRequest{}
+}
+
+func (dev *MthreadsDevices) customFilterRule(allocated *util.PodDevices, request util.ContainerDeviceRequest, toAllocate util.ContainerDevices, device *util.DeviceUsage) bool {
+ for _, ctrs := range (*allocated)[device.Type] {
+ for _, ctrdev := range ctrs {
+ if strings.Compare(ctrdev.UUID, device.ID) != 0 {
+ klog.InfoS("Mthreads needs all devices on a device", "used", ctrdev.UUID, "allocating", device.ID)
+ return false
+ }
+ }
+ }
+ return true
+}
+
+func (dev *MthreadsDevices) ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32 {
+ return 0
+}
+
+func (dev *MthreadsDevices) AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error {
+ n.Used++
+ n.Usedcores += ctr.Usedcores
+ n.Usedmem += ctr.Usedmem
+ return nil
+}
+
+func (mth *MthreadsDevices) Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string) {
+ k := request
+ originReq := k.Nums
+ prevnuma := -1
+ klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
+ var tmpDevs map[string]util.ContainerDevices
+ tmpDevs = make(map[string]util.ContainerDevices)
+ reason := make(map[string]int)
+ for i := len(devices) - 1; i >= 0; i-- {
+ dev := devices[i]
+ klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
+
+ klog.V(3).InfoS("Type check", "device", dev.Type, "req", k.Type)
+ if !strings.Contains(dev.Type, k.Type) {
+ reason[common.CardTypeMismatch]++
+ continue
+ }
+
+ _, found, numa := mth.checkType(annos, *dev, k)
+ if !found {
+ reason[common.CardTypeMismatch]++
+ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, k.Type)
+ continue
+ }
+ if numa && prevnuma != dev.Numa {
+ if k.Nums != originReq {
+ reason[common.NumaNotFit] += len(tmpDevs)
+ klog.V(5).InfoS(common.NumaNotFit, "pod", klog.KObj(pod), "device", dev.ID, "k.nums", k.Nums, "numa", numa, "prevnuma", prevnuma, "device numa", dev.Numa)
+ }
+ k.Nums = originReq
+ prevnuma = dev.Numa
+ tmpDevs = make(map[string]util.ContainerDevices)
+ }
+ if !mth.checkUUID(annos, *dev) {
+ reason[common.CardUUIDMismatch]++
+ klog.V(5).InfoS(common.CardUUIDMismatch, "pod", klog.KObj(pod), "device", dev.ID, "current device info is:", *dev)
+ continue
+ }
+
+ memreq := int32(0)
+ if dev.Count <= dev.Used {
+ reason[common.CardTimeSlicingExhausted]++
+ klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
+ continue
+ }
+ if k.Coresreq > 100 {
+ klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Coresreq = 100
+ //return false, tmpDevs
+ }
+ if k.Memreq > 0 {
+ memreq = k.Memreq
+ }
+ if k.MemPercentagereq != 101 && k.Memreq == 0 {
+ //This incurs an issue
+ memreq = dev.Totalmem * k.MemPercentagereq / 100
+ }
+ if dev.Totalmem-dev.Usedmem < memreq {
+ reason[common.CardInsufficientMemory]++
+ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq)
+ continue
+ }
+ if dev.Totalcore-dev.Usedcores < k.Coresreq {
+ reason[common.CardInsufficientCore]++
+ klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", k.Coresreq)
+ continue
+ }
+ // Coresreq=100 indicates it want this card exclusively
+ if dev.Totalcore == 100 && k.Coresreq == 100 && dev.Used > 0 {
+ reason[common.ExclusiveDeviceAllocateConflict]++
+ klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
+ continue
+ }
+ // You can't allocate core=0 job to an already full GPU
+ if dev.Totalcore != 0 && dev.Usedcores == dev.Totalcore && k.Coresreq == 0 {
+ reason[common.CardComputeUnitsExhausted]++
+ klog.V(5).InfoS(common.CardComputeUnitsExhausted, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+
+ if !mth.customFilterRule(allocated, request, tmpDevs[k.Type], dev) {
+ reason[common.CardNotFoundCustomFilterRule]++
+ klog.V(5).InfoS(common.CardNotFoundCustomFilterRule, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+
+ if k.Nums > 0 {
+ klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Nums--
+ tmpDevs[k.Type] = append(tmpDevs[k.Type], util.ContainerDevice{
+ Idx: int(dev.Index),
+ UUID: dev.ID,
+ Type: k.Type,
+ Usedmem: memreq,
+ Usedcores: k.Coresreq,
+ })
+ }
+ if k.Nums == 0 {
+ klog.V(4).InfoS("device allocate success", "pod", klog.KObj(pod), "allocate device", tmpDevs)
+ return true, tmpDevs, ""
+ }
+ }
+ if len(tmpDevs) > 0 {
+ reason[common.AllocatedCardsInsufficientRequest] = len(tmpDevs)
+ klog.V(5).InfoS(common.AllocatedCardsInsufficientRequest, "pod", klog.KObj(pod), "request", originReq, "allocated", len(tmpDevs))
+ }
+ return false, tmpDevs, common.GenReason(reason, len(devices))
+}
diff --git a/pkg/device/mthreads/device_test.go b/pkg/device/mthreads/device_test.go
new file mode 100644
index 000000000..ea8497e53
--- /dev/null
+++ b/pkg/device/mthreads/device_test.go
@@ -0,0 +1,1016 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mthreads
+
+import (
+ "flag"
+ "testing"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "k8s.io/apimachinery/pkg/api/resource"
+)
+
+func Test_MutateAdmission(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }
+ want bool
+ err error
+ }{
+ {
+ name: "set to resources limit",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "mthreads.com/vgpu": *resource.NewQuantity(2, resource.DecimalSI),
+ "mthreads.com/sgpu-memory": *resource.NewQuantity(2, resource.DecimalSI),
+ "mthreads.com/sgpu-core": *resource.NewQuantity(1, resource.DecimalSI),
+ },
+ },
+ },
+ p: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "mthreads.com/request-gpu-num": "test123",
+ },
+ },
+ },
+ },
+ want: true,
+ },
+ {
+ name: "don't set to count limit",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "mthreads.com/sgpu-memory": *resource.NewQuantity(2, resource.DecimalSI),
+ "mthreads.com/sgpu-core": *resource.NewQuantity(1, resource.DecimalSI),
+ },
+ },
+ },
+ p: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "mthreads.com/request-gpu-num": "test123",
+ },
+ },
+ },
+ },
+ want: false,
+ },
+ {
+ name: "don't set to memory limit",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "mthreads.com/vgpu": *resource.NewQuantity(1, resource.DecimalSI),
+ "mthreads.com/sgpu-core": *resource.NewQuantity(1, resource.DecimalSI),
+ },
+ },
+ },
+ p: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "mthreads.com/request-gpu-num": "test123",
+ },
+ },
+ },
+ },
+ want: true,
+ },
+ {
+ name: "count less than one",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "mthreads.com/vgpu": *resource.NewQuantity(1, resource.DecimalSI),
+ "mthreads.com/sgpu-memory": *resource.NewQuantity(2, resource.DecimalSI),
+ "mthreads.com/sgpu-core": *resource.NewQuantity(1, resource.DecimalSI),
+ },
+ },
+ },
+ p: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "mthreads.com/request-gpu-num": "test123",
+ },
+ },
+ },
+ },
+ want: true,
+ },
+ {
+ name: "memory no exist legalMemoryslices",
+ args: struct {
+ ctr *corev1.Container
+ p *corev1.Pod
+ }{
+ ctr: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "mthreads.com/vgpu": *resource.NewQuantity(1, resource.DecimalSI),
+ "mthreads.com/sgpu-memory": *resource.NewQuantity(3, resource.DecimalSI),
+ "mthreads.com/sgpu-core": *resource.NewQuantity(1, resource.DecimalSI),
+ },
+ },
+ },
+ p: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "mthreads.com/request-gpu-num": "test123",
+ },
+ },
+ },
+ },
+ want: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ config := MthreadsConfig{
+ ResourceCountName: "mthreads.com/vgpu",
+ ResourceMemoryName: "mthreads.com/sgpu-memory",
+ ResourceCoreName: "mthreads.com/sgpu-core",
+ }
+ InitMthreadsDevice(config)
+ dev := MthreadsDevices{}
+ result, _ := dev.MutateAdmission(test.args.ctr, test.args.p)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_GetNodeDevices(t *testing.T) {
+ tests := []struct {
+ name string
+ args corev1.Node
+ want []*util.DeviceInfo
+ }{
+ {
+ name: "get node device",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test",
+ },
+ Status: corev1.NodeStatus{
+ Capacity: corev1.ResourceList{
+ "mthreads.com/sgpu-memory": *resource.NewQuantity(1, resource.DecimalSI),
+ "mthreads.com/sgpu-core": *resource.NewQuantity(1, resource.DecimalSI),
+ },
+ },
+ },
+ want: []*util.DeviceInfo{
+ {
+ Index: uint(0),
+ ID: "test-mthreads-0",
+ Count: int32(100),
+ Devmem: int32(8192),
+ Devcore: int32(16),
+ Type: MthreadsGPUDevice,
+ Numa: 0,
+ Health: true,
+ },
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := MthreadsDevices{}
+ fs := flag.FlagSet{}
+ ParseConfig(&fs)
+ result, _ := dev.GetNodeDevices(test.args)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func Test_PatchAnnotations(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annoinput *map[string]string
+ pd util.PodDevices
+ }
+ want map[string]string
+ }{
+ {
+ name: "exist device",
+ args: struct {
+ annoinput *map[string]string
+ pd util.PodDevices
+ }{
+ annoinput: &map[string]string{},
+ pd: util.PodDevices{
+ MthreadsGPUDevice: util.PodSingleDevice{
+ util.ContainerDevices{
+ {
+ Idx: 0,
+ UUID: "test1",
+ Type: MthreadsGPUDevice,
+ Usedmem: int32(1000),
+ Usedcores: int32(1),
+ },
+ },
+ },
+ },
+ },
+ want: map[string]string{
+ util.SupportDevices[MthreadsGPUDevice]: "test1,Mthreads,1000,1:;",
+ "mthreads.com/gpu-index": "0",
+ "mthreads.com/predicate-node": "",
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := MthreadsDevices{}
+ result := dev.PatchAnnotations(&corev1.Pod{}, test.args.annoinput, test.args.pd)
+ assert.Equal(t, result[dev.CommonWord()], test.want[dev.CommonWord()])
+ assert.Equal(t, result["mthreads.com/gpu-index"], test.want["mthreads.com/gpu-index"])
+ assert.Equal(t, result["mthreads.com/predicate-node"], test.want["mthreads.com/predicate-node"])
+ })
+ }
+}
+
+func Test_checkType(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }
+ want1 bool
+ want2 bool
+ want3 bool
+ }{
+ {
+ name: "the same type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: MthreadsGPUDevice,
+ },
+ },
+ want1: true,
+ want2: true,
+ want3: false,
+ },
+ {
+ name: "the different type",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ n util.ContainerDeviceRequest
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ n: util.ContainerDeviceRequest{
+ Type: "test111",
+ },
+ },
+ want1: false,
+ want2: false,
+ want3: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := MthreadsDevices{}
+ result1, result2, result3 := dev.checkType(test.args.annos, test.args.d, test.args.n)
+ assert.Equal(t, result1, test.want1)
+ assert.Equal(t, result2, test.want2)
+ assert.Equal(t, result3, test.want3)
+ })
+ }
+}
+
+func Test_checkUUID(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "no annos",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{},
+ d: util.DeviceUsage{},
+ },
+ want: true,
+ },
+ {
+ name: "use id the same as device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ MthreadsUseUUID: "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test1",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "use id the different from device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ MthreadsUseUUID: "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test2",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "no use id the same as device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ MthreadsNoUseUUID: "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test1",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "no use id the different from device id",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ MthreadsNoUseUUID: "test1",
+ },
+ d: util.DeviceUsage{
+ ID: "test2",
+ },
+ },
+ want: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := MthreadsDevices{}
+ result := dev.checkUUID(test.args.annos, test.args.d)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_GenerateResourceRequests(t *testing.T) {
+ tests := []struct {
+ name string
+ args *corev1.Container
+ want util.ContainerDeviceRequest
+ }{
+ {
+ name: "all resources set to limit and request",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "mthreads.com/vgpu": resource.MustParse("1"),
+ "mthreads.com/sgpu-memory": resource.MustParse("1000"),
+ "mthreads.com/sgpu-core": resource.MustParse("1"),
+ },
+ Requests: corev1.ResourceList{
+ "mthreads.com/vgpu": resource.MustParse("1"),
+ "mthreads.com/sgpu-memory": resource.MustParse("1000"),
+ "mthreads.com/sgpu-core": resource.MustParse("1"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: MthreadsGPUDevice,
+ Memreq: int32(512000),
+ MemPercentagereq: int32(0),
+ Coresreq: int32(1),
+ },
+ },
+ {
+ name: "all resources don't set to limit,count don't set to request",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ Requests: corev1.ResourceList{
+ "mthreads.com/sgpu-memory": resource.MustParse("1000"),
+ "mthreads.com/sgpu-core": resource.MustParse("1"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{},
+ },
+ {
+ name: "all resources don't set to limit,cores and memory don't set to request",
+ args: &corev1.Container{
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ Requests: corev1.ResourceList{
+ "mthreads.com/vgpu": resource.MustParse("1"),
+ },
+ },
+ },
+ want: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: MthreadsGPUDevice,
+ Memreq: int32(0),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(0),
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ config := MthreadsConfig{
+ ResourceCountName: "mthreads.com/vgpu",
+ ResourceMemoryName: "mthreads.com/sgpu-memory",
+ ResourceCoreName: "mthreads.com/sgpu-core",
+ }
+ InitMthreadsDevice(config)
+ dev := MthreadsDevices{}
+ result := dev.GenerateResourceRequests(test.args)
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func Test_CustomFilterRule(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ allocated *util.PodDevices
+ request util.ContainerDeviceRequest
+ toAllocate util.ContainerDevices
+ device *util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "allocate device",
+ args: struct {
+ allocated *util.PodDevices
+ request util.ContainerDeviceRequest
+ toAllocate util.ContainerDevices
+ device *util.DeviceUsage
+ }{
+ allocated: &util.PodDevices{
+ MthreadsGPUDevice: util.PodSingleDevice{
+ util.ContainerDevices{
+ {
+ UUID: "test123",
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{},
+ toAllocate: util.ContainerDevices{},
+ device: &util.DeviceUsage{
+ ID: "test123",
+ Type: MthreadsGPUDevice,
+ },
+ },
+ want: true,
+ },
+ {
+ name: "don't allocate device",
+ args: struct {
+ allocated *util.PodDevices
+ request util.ContainerDeviceRequest
+ toAllocate util.ContainerDevices
+ device *util.DeviceUsage
+ }{
+ allocated: &util.PodDevices{
+ MthreadsGPUDevice: util.PodSingleDevice{
+ util.ContainerDevices{
+ {
+ UUID: "test456",
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{},
+ toAllocate: util.ContainerDevices{},
+ device: &util.DeviceUsage{
+ ID: "test123",
+ Type: MthreadsGPUDevice,
+ },
+ },
+ want: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dev := MthreadsDevices{}
+ result := dev.customFilterRule(test.args.allocated, test.args.request, test.args.toAllocate, test.args.device)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func TestDevices_Fit(t *testing.T) {
+ config := MthreadsConfig{
+ ResourceCountName: "mthreads.com/vgpu",
+ ResourceMemoryName: "mthreads.com/sgpu-memory",
+ ResourceCoreName: "mthreads.com/sgpu-core",
+ }
+ dev := InitMthreadsDevice(config)
+
+ tests := []struct {
+ name string
+ devices []*util.DeviceUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ wantFit bool
+ wantLen int
+ wantDevIDs []string
+ wantReason string
+ }{
+ {
+ name: "fit success",
+ devices: []*util.DeviceUsage{
+ {
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ },
+ {
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 64,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MthreadsGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-1"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: memory not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MthreadsGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientMemory",
+ },
+ {
+ name: "fit fail: core not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1024,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MthreadsGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientCore",
+ },
+ {
+ name: "fit fail: type mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ Type: MthreadsGPUDevice,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: "OtherType",
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTypeMismatch",
+ },
+ {
+ name: "fit fail: user assign use uuid mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MthreadsGPUDevice,
+ },
+ annos: map[string]string{MthreadsUseUUID: "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: user assign no use uuid match",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MthreadsGPUDevice,
+ },
+ annos: map[string]string{MthreadsNoUseUUID: "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: card overused",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 100,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: MthreadsGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTimeSlicingExhausted",
+ },
+ {
+ name: "fit success: but core limit can't exceed 100",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 120,
+ Type: MthreadsGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: card exclusively",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 100,
+ Type: MthreadsGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 ExclusiveDeviceAllocateConflict",
+ },
+ {
+ name: "fit fail: CardComputeUnitsExhausted",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 0,
+ Type: MthreadsGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardComputeUnitsExhausted",
+ },
+ {
+ name: "fit fail: AllocatedCardsInsufficientRequest",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 20,
+ Type: MthreadsGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 AllocatedCardsInsufficientRequest",
+ },
+ {
+ name: "fit success: memory percentage",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: MthreadsGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 10,
+ Coresreq: 20,
+ Type: MthreadsGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ allocated := &util.PodDevices{}
+ fit, result, reason := dev.Fit(test.devices, test.request, test.annos, &corev1.Pod{}, &util.NodeInfo{}, allocated)
+ if fit != test.wantFit {
+ t.Errorf("Fit: got %v, want %v", fit, test.wantFit)
+ }
+ if test.wantFit {
+ if len(result[MthreadsGPUDevice]) != test.wantLen {
+ t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[MthreadsGPUDevice]))
+ }
+ for idx, id := range test.wantDevIDs {
+ if id != result[MthreadsGPUDevice][idx].UUID {
+ t.Errorf("expected device id: %s, got device id %s", id, result[MthreadsGPUDevice][idx].UUID)
+ }
+ }
+ }
+
+ if reason != test.wantReason {
+ t.Errorf("expected reason: %s, got reason: %s", test.wantReason, reason)
+ }
+ })
+ }
+}
+
+func TestDevices_AddResourceUsage(t *testing.T) {
+ tests := []struct {
+ name string
+ deviceUsage *util.DeviceUsage
+ ctr *util.ContainerDevice
+ wantErr bool
+ wantUsage *util.DeviceUsage
+ }{
+ {
+ name: "test add resource usage",
+ deviceUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 0,
+ Usedcores: 15,
+ Usedmem: 2000,
+ },
+ ctr: &util.ContainerDevice{
+ UUID: "dev-0",
+ Usedcores: 50,
+ Usedmem: 1024,
+ },
+ wantUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 1,
+ Usedcores: 65,
+ Usedmem: 3024,
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &MthreadsDevices{}
+ if err := dev.AddResourceUsage(&corev1.Pod{}, tt.deviceUsage, tt.ctr); (err != nil) != tt.wantErr {
+ t.Errorf("AddResourceUsage() error=%v, wantErr %v", err, tt.wantErr)
+ }
+ if !tt.wantErr {
+ if tt.deviceUsage.Usedcores != tt.wantUsage.Usedcores {
+ t.Errorf("expected used cores: %d, got used cores %d", tt.wantUsage.Usedcores, tt.deviceUsage.Usedcores)
+ }
+ if tt.deviceUsage.Usedmem != tt.wantUsage.Usedmem {
+ t.Errorf("expected used mem: %d, got used mem %d", tt.wantUsage.Usedmem, tt.deviceUsage.Usedmem)
+ }
+ if tt.deviceUsage.Used != tt.wantUsage.Used {
+ t.Errorf("expected used: %d, got used %d", tt.wantUsage.Used, tt.deviceUsage.Used)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/device/nvidia/calculate_score.go b/pkg/device/nvidia/calculate_score.go
new file mode 100644
index 000000000..54a874e19
--- /dev/null
+++ b/pkg/device/nvidia/calculate_score.go
@@ -0,0 +1,274 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvidia
+
+import (
+ "errors"
+ "fmt"
+
+ "github.com/NVIDIA/go-nvlib/pkg/nvlib/device"
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+)
+
+// Device represents a GPU device as reported by NVML, including all of its
+// Point-to-Point link information.
+type Device struct {
+ nvlibDevice
+ Index int
+ Links map[int][]P2PLink
+}
+
+// DeviceList stores an ordered list of devices.
+type DeviceList []*Device
+
+// Filter filters out the selected devices from the list.
+// Note that the specified uuids must exist in the list of devices.
+func (d DeviceList) Filter(uuids []string) (DeviceList, error) {
+ var filtered DeviceList
+ for _, uuid := range uuids {
+ for _, device := range d {
+ if device.UUID == uuid {
+ filtered = append(filtered, device)
+ break
+ }
+ }
+ if len(filtered) == 0 || filtered[len(filtered)-1].UUID != uuid {
+ return nil, fmt.Errorf("no device with uuid: %v", uuid)
+ }
+ }
+
+ return filtered, nil
+}
+
+// P2PLink represents a Point-to-Point link between two GPU devices. The link
+// is between the Device struct this struct is embedded in and the GPU Device
+// contained in the P2PLink struct itself.
+type P2PLink struct {
+ GPU *Device
+ Type P2PLinkType
+}
+
+type nvlibDevice struct {
+ device.Device
+ // The previous binding implementation used to cache specific device properties.
+ // These should be considered deprecated and the functions associated with device.Device
+ // should be used instead.
+ UUID string
+ PCI struct {
+ BusID string
+ }
+ CPUAffinity *uint
+}
+
+// deviceListBuilder stores the options required to build a list of linked devices.
+type deviceListBuilder struct {
+ nvmllib nvml.Interface
+ devicelib device.Interface
+}
+
+// NewDevices creates a list of Devices from all available nvml.Devices using the specified options.
+func NewDevices() (DeviceList, error) {
+ o := &deviceListBuilder{}
+ o.nvmllib = nvml.New()
+ o.devicelib = device.New(o.nvmllib)
+ return o.build()
+}
+
+// build uses the configured options to build a DeviceList.
+func (o *deviceListBuilder) build() (DeviceList, error) {
+ if err := o.nvmllib.Init(); !errors.Is(err, nvml.SUCCESS) {
+ return nil, fmt.Errorf("error calling nvml.Init: %v", err)
+ }
+ defer func() {
+ _ = o.nvmllib.Shutdown()
+ }()
+
+ nvmlDevices, err := o.devicelib.GetDevices()
+ if err != nil {
+ return nil, fmt.Errorf("failed to get devices: %v", err)
+ }
+
+ var devices DeviceList
+ for i, d := range nvmlDevices {
+ device, err := newDevice(i, d)
+ if err != nil {
+ return nil, fmt.Errorf("failed to construct linked device: %v", err)
+ }
+ devices = append(devices, device)
+ }
+
+ for i, d1 := range nvmlDevices {
+ for j, d2 := range nvmlDevices {
+ if i != j {
+ p2plink, err := GetP2PLink(d1, d2)
+ if err != nil {
+ return nil, fmt.Errorf("error getting P2PLink for devices (%v, %v): %v", i, j, err)
+ }
+ if p2plink != P2PLinkUnknown {
+ devices[i].Links[j] = append(devices[i].Links[j], P2PLink{devices[j], p2plink})
+ }
+
+ nvlink, err := GetNVLink(d1, d2)
+ if err != nil {
+ return nil, fmt.Errorf("error getting NVLink for devices (%v, %v): %v", i, j, err)
+ }
+ if nvlink != P2PLinkUnknown {
+ devices[i].Links[j] = append(devices[i].Links[j], P2PLink{devices[j], nvlink})
+ }
+ }
+ }
+ }
+
+ return devices, nil
+}
+
+// newDevice constructs a Device for the specified index and nvml Device.
+func newDevice(i int, d device.Device) (*Device, error) {
+ uuid, ret := d.GetUUID()
+ if !errors.Is(ret, nvml.SUCCESS) {
+ return nil, fmt.Errorf("failed to get device uuid: %v", ret)
+ }
+ pciInfo, ret := d.GetPciInfo()
+ if !errors.Is(ret, nvml.SUCCESS) {
+ return nil, fmt.Errorf("failed to get device pci info: %v", ret)
+ }
+
+ device := Device{
+ nvlibDevice: nvlibDevice{
+ Device: d,
+ UUID: uuid,
+ PCI: struct{ BusID string }{BusID: PciInfo(pciInfo).BusID()},
+ CPUAffinity: PciInfo(pciInfo).CPUAffinity(),
+ },
+ Index: i,
+ Links: make(map[int][]P2PLink),
+ }
+
+ return &device, nil
+}
+
+type ListDeviceScore []DeviceScore
+
+type DeviceScore struct {
+ UUID string `json:"uuid"`
+ // Score is record and other gpu communications score,
+ // the value bigger communications bandwidth the higher.
+ Score map[string]int `json:"score"`
+}
+
+func CalculateGPUScore(available []string) (ListDeviceScore, error) {
+ linkedDevices, err := NewDevices()
+ if err != nil {
+ return nil, err
+ }
+ requiredDevices, err := linkedDevices.Filter(available)
+ if err != nil {
+ return nil, err
+ }
+ score := calculateGPUScore(requiredDevices)
+ return score, nil
+}
+
+func calculateGPUScore(devices []*Device) ListDeviceScore {
+ ds := make(ListDeviceScore, len(devices))
+ for indexI, gpuI := range devices {
+ score := make(map[string]int)
+ for indexJ, gpuJ := range devices {
+ if indexI == indexJ {
+ continue
+ }
+ score[gpuJ.UUID] = calculateGPUPairScore(gpuI, gpuJ)
+ }
+ ds[indexI] = DeviceScore{
+ Score: score,
+ UUID: gpuI.UUID,
+ }
+ }
+ return ds
+}
+
+func calculateGPUPairScore(gpu0 *Device, gpu1 *Device) int {
+ if gpu0 == nil || gpu1 == nil {
+ return 0
+ }
+
+ if gpu0 == gpu1 {
+ return 0
+ }
+
+ if len(gpu0.Links[gpu1.Index]) != len(gpu1.Links[gpu0.Index]) {
+ err := fmt.Errorf("internal error in bestEffort GPU allocator: all P2PLinks between 2 GPUs should be bidirectional")
+ panic(err)
+ }
+
+ score := 0
+
+ for _, link := range gpu0.Links[gpu1.Index] {
+ switch link.Type {
+ case P2PLinkCrossCPU:
+ score += 10
+ case P2PLinkSameCPU:
+ score += 20
+ case P2PLinkHostBridge:
+ score += 30
+ case P2PLinkMultiSwitch:
+ score += 40
+ case P2PLinkSingleSwitch:
+ score += 50
+ case P2PLinkSameBoard:
+ score += 60
+ case SingleNVLINKLink:
+ score += 100
+ case TwoNVLINKLinks:
+ score += 200
+ case ThreeNVLINKLinks:
+ score += 300
+ case FourNVLINKLinks:
+ score += 400
+ case FiveNVLINKLinks:
+ score += 500
+ case SixNVLINKLinks:
+ score += 600
+ case SevenNVLINKLinks:
+ score += 700
+ case EightNVLINKLinks:
+ score += 800
+ case NineNVLINKLinks:
+ score += 900
+ case TenNVLINKLinks:
+ score += 1000
+ case ElevenNVLINKLinks:
+ score += 1100
+ case TwelveNVLINKLinks:
+ score += 1200
+ case ThirteenNVLINKLinks:
+ score += 1300
+ case FourteenNVLINKLinks:
+ score += 1400
+ case FifteenNVLINKLinks:
+ score += 1500
+ case SixteenNVLINKLinks:
+ score += 1600
+ case SeventeenNVLINKLinks:
+ score += 1700
+ case EighteenNVLINKLinks:
+ score += 1800
+ }
+ }
+
+ return score
+}
diff --git a/pkg/device/nvidia/calculate_score_test.go b/pkg/device/nvidia/calculate_score_test.go
new file mode 100644
index 000000000..115b6f4d1
--- /dev/null
+++ b/pkg/device/nvidia/calculate_score_test.go
@@ -0,0 +1,132 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nvidia
+
+import (
+ "testing"
+
+ "gotest.tools/v3/assert"
+)
+
+func Test_CalculateGPUScore(t *testing.T) {
+ score, err := CalculateGPUScore([]string{"GPU-ebe7c3f7-303d-558d-435e-99a160631fe4"})
+ t.Log(score)
+ t.Log(err)
+}
+
+// Test_calculateGPUScore.
+// gpu_0 gpu_1 gpu_2 gpu_3.
+//
+// gpu_0 N-A NV1 NV1 NV2.
+// gpu_1 NV1 N-A NV2 NV1.
+// gpu_2 NV1 NV2 N-A NV2.
+// gpu_3 NV2 NV1 NV2 N-A.
+func Test_calculateGPUScore(t *testing.T) {
+ tests := []struct {
+ name string
+ args []*Device
+ want ListDeviceScore
+ }{
+ {
+ name: "nvlink test",
+ args: []*Device{
+ {
+ Index: 0,
+ nvlibDevice: nvlibDevice{
+ UUID: "gpu0",
+ },
+ Links: map[int][]P2PLink{
+ 1: {{Type: SingleNVLINKLink}},
+ 2: {{Type: SingleNVLINKLink}},
+ 3: {{Type: TwoNVLINKLinks}},
+ },
+ },
+ {
+ Index: 1,
+ nvlibDevice: nvlibDevice{
+ UUID: "gpu1",
+ },
+ Links: map[int][]P2PLink{
+ 0: {{Type: SingleNVLINKLink}},
+ 2: {{Type: TwoNVLINKLinks}},
+ 3: {{Type: SingleNVLINKLink}},
+ },
+ },
+ {
+ Index: 2,
+ nvlibDevice: nvlibDevice{
+ UUID: "gpu2",
+ },
+ Links: map[int][]P2PLink{
+ 0: {{Type: SingleNVLINKLink}},
+ 1: {{Type: TwoNVLINKLinks}},
+ 3: {{Type: TwoNVLINKLinks}},
+ },
+ },
+ {
+ Index: 3,
+ nvlibDevice: nvlibDevice{
+ UUID: "gpu3",
+ },
+ Links: map[int][]P2PLink{
+ 0: {{Type: TwoNVLINKLinks}},
+ 1: {{Type: SingleNVLINKLink}},
+ 2: {{Type: TwoNVLINKLinks}},
+ },
+ },
+ },
+ want: ListDeviceScore{
+ {
+ UUID: "gpu0",
+ Score: map[string]int{"gpu1": 100, "gpu2": 100, "gpu3": 200},
+ },
+ {
+ UUID: "gpu1",
+ Score: map[string]int{"gpu0": 100, "gpu2": 200, "gpu3": 100},
+ },
+ {
+ UUID: "gpu2",
+ Score: map[string]int{"gpu0": 100, "gpu1": 200, "gpu3": 200},
+ },
+ {
+ UUID: "gpu3",
+ Score: map[string]int{"gpu0": 200, "gpu1": 100, "gpu2": 200},
+ },
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ scoreList := make(ListDeviceScore, len(test.args))
+ for i, gpuI := range test.args {
+ score := make(map[string]int)
+ for j, gpuJ := range test.args {
+ if i == j {
+ continue
+ }
+ score[gpuJ.UUID] = calculateGPUPairScore(gpuI, gpuJ)
+ }
+ scoreList[i] = DeviceScore{
+ UUID: gpuI.UUID,
+ Score: score,
+ }
+ }
+ t.Log(scoreList)
+ assert.DeepEqual(t, scoreList, test.want)
+ })
+ }
+}
diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go
new file mode 100644
index 000000000..5690df7a1
--- /dev/null
+++ b/pkg/device/nvidia/device.go
@@ -0,0 +1,847 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nvidia
+
+import (
+ "errors"
+ "flag"
+ "fmt"
+ "slices"
+ "strconv"
+ "strings"
+
+ spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/device/common"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/config"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/nodelock"
+)
+
+const (
+ HandshakeAnnos = "hami.io/node-handshake"
+ RegisterAnnos = "hami.io/node-nvidia-register"
+ RegisterGPUPairScore = "hami.io/node-nvidia-score"
+ NvidiaGPUDevice = "NVIDIA"
+ NvidiaGPUCommonWord = "GPU"
+ GPUInUse = "nvidia.com/use-gputype"
+ GPUNoUse = "nvidia.com/nouse-gputype"
+ NumaBind = "nvidia.com/numa-bind"
+ NodeLockNvidia = "hami.io/mutex.lock"
+ // GPUUseUUID is user can use specify GPU device for set GPU UUID.
+ GPUUseUUID = "nvidia.com/use-gpuuuid"
+ // GPUNoUseUUID is user can not use specify GPU device for set GPU UUID.
+ GPUNoUseUUID = "nvidia.com/nouse-gpuuuid"
+ // GPUUseIndexes is user can use specify GPU device for set GPU indexes.
+ GPUUseIndexes = "nvidia.com/use-gpuindexes"
+ // GPUNoUseIndexes is user can not use specify GPU device for set GPU indexes.
+ GPUNoUseIndexes = "nvidia.com/nouse-gpuindexes"
+ AllocateMode = "nvidia.com/vgpu-mode"
+
+ MigMode = "mig"
+ HamiCoreMode = "hami-core"
+ MpsMode = "mps"
+)
+
+var (
+ NodeName string
+ RuntimeSocketFlag string
+ DisableCoreLimit *bool
+
+ // DevicePluginFilterDevice need device-plugin filter this device, don't register this device.
+ DevicePluginFilterDevice *FilterDevice
+)
+
+type MigPartedSpec struct {
+ Version string `json:"version" yaml:"version"`
+ MigConfigs map[string]MigConfigSpecSlice `json:"mig-configs,omitempty" yaml:"mig-configs,omitempty"`
+}
+
+// MigConfigSpec defines the spec to declare the desired MIG configuration for a set of GPUs.
+type MigConfigSpec struct {
+ DeviceFilter any `json:"device-filter,omitempty" yaml:"device-filter,flow,omitempty"`
+ Devices []int32 `json:"devices" yaml:"devices,flow"`
+ MigEnabled bool `json:"mig-enabled" yaml:"mig-enabled"`
+ MigDevices map[string]int32 `json:"mig-devices" yaml:"mig-devices"`
+}
+
+// MigConfigSpecSlice represents a slice of 'MigConfigSpec'.
+type MigConfigSpecSlice []MigConfigSpec
+
+// GPUCoreUtilizationPolicy is set nvidia gpu core isolation policy.
+type GPUCoreUtilizationPolicy string
+
+const (
+ DefaultCorePolicy GPUCoreUtilizationPolicy = "default"
+ ForceCorePolicy GPUCoreUtilizationPolicy = "force"
+ DisableCorePolicy GPUCoreUtilizationPolicy = "disable"
+)
+
+type LibCudaLogLevel string
+
+const (
+ Error LibCudaLogLevel = "0"
+ Warnings LibCudaLogLevel = "1"
+ Infos LibCudaLogLevel = "3"
+ Debugs LibCudaLogLevel = "4"
+)
+
+type NvidiaConfig struct {
+ // These configs are shared and can be overritten by Nodeconfig.
+ NodeDefaultConfig `yaml:",inline"`
+ ResourceCountName string `yaml:"resourceCountName"`
+ ResourceMemoryName string `yaml:"resourceMemoryName"`
+ ResourceCoreName string `yaml:"resourceCoreName"`
+ ResourceMemoryPercentageName string `yaml:"resourceMemoryPercentageName"`
+ ResourcePriority string `yaml:"resourcePriorityName"`
+ OverwriteEnv bool `yaml:"overwriteEnv"`
+ DefaultMemory int32 `yaml:"defaultMemory"`
+ DefaultCores int32 `yaml:"defaultCores"`
+ DefaultGPUNum int32 `yaml:"defaultGPUNum"`
+ // TODO Whether these should be removed
+ DisableCoreLimit bool `yaml:"disableCoreLimit"`
+ MigGeometriesList []util.AllowedMigGeometries `yaml:"knownMigGeometries"`
+ // GPUCorePolicy through webhook automatic injected to container env
+ GPUCorePolicy GPUCoreUtilizationPolicy `yaml:"gpuCorePolicy"`
+ // RuntimeClassName is the name of the runtime class to be added to pod.spec.runtimeClassName
+ RuntimeClassName string `yaml:"runtimeClassName"`
+}
+
+// These configs can be sepecified for each node by using Nodeconfig.
+type NodeDefaultConfig struct {
+ DeviceSplitCount *uint `yaml:"deviceSplitCount" json:"devicesplitcount"`
+ DeviceMemoryScaling *float64 `yaml:"deviceMemoryScaling" json:"devicememoryscaling"`
+ DeviceCoreScaling *float64 `yaml:"deviceCoreScaling" json:"devicecorescaling"`
+ // LogLevel is LIBCUDA_LOG_LEVEL value
+ LogLevel *LibCudaLogLevel `yaml:"libCudaLogLevel" json:"libcudaloglevel"`
+}
+
+type FilterDevice struct {
+ // UUID is the device ID.
+ UUID []string `json:"uuid"`
+ // Index is the device index.
+ Index []uint `json:"index"`
+}
+
+type DevicePluginConfigs struct {
+ Nodeconfig []struct {
+ // These configs is shared and will overrite those in NvidiaConfig.
+ NodeDefaultConfig `json:",inline"`
+ Name string `json:"name"`
+ OperatingMode string `json:"operatingmode"`
+ Migstrategy string `json:"migstrategy"`
+ FilterDevice *FilterDevice `json:"filterdevices"`
+ } `json:"nodeconfig"`
+}
+
+type DeviceConfig struct {
+ *spec.Config
+
+ ResourceName *string
+ DebugMode *bool
+}
+
+type NvidiaGPUDevices struct {
+ config NvidiaConfig
+}
+
+func InitNvidiaDevice(nvconfig NvidiaConfig) *NvidiaGPUDevices {
+ klog.InfoS("initializing nvidia device", "resourceName", nvconfig.ResourceCountName, "resourceMem", nvconfig.ResourceMemoryName, "DefaultGPUNum", nvconfig.DefaultGPUNum)
+ util.InRequestDevices[NvidiaGPUDevice] = "hami.io/vgpu-devices-to-allocate"
+ util.SupportDevices[NvidiaGPUDevice] = "hami.io/vgpu-devices-allocated"
+ util.HandshakeAnnos[NvidiaGPUDevice] = HandshakeAnnos
+ return &NvidiaGPUDevices{
+ config: nvconfig,
+ }
+}
+
+func (dev *NvidiaGPUDevices) CommonWord() string {
+ return NvidiaGPUCommonWord
+}
+
+func ParseConfig(fs *flag.FlagSet) {
+}
+
+func FilterDeviceToRegister(uuid, indexStr string) bool {
+ if DevicePluginFilterDevice == nil || (len(DevicePluginFilterDevice.UUID) == 0 && len(DevicePluginFilterDevice.Index) == 0) {
+ return false
+ }
+ uuidMap, indexMap := make(map[string]struct{}), make(map[uint]struct{})
+ for _, u := range DevicePluginFilterDevice.UUID {
+ uuidMap[u] = struct{}{}
+ }
+ for _, index := range DevicePluginFilterDevice.Index {
+ indexMap[index] = struct{}{}
+ }
+ if uuid != "" {
+ if _, ok := uuidMap[uuid]; ok {
+ return true
+ }
+ }
+ if indexStr != "" {
+ index, err := strconv.Atoi(indexStr)
+ if err != nil {
+ klog.Errorf("Error converting index to int: %v", err)
+ return false
+ }
+ if _, ok := indexMap[uint(index)]; ok {
+ return true
+ }
+ }
+ return false
+}
+
+func (dev *NvidiaGPUDevices) NodeCleanUp(nn string) error {
+ return util.MarkAnnotationsToDelete(HandshakeAnnos, nn)
+}
+
+func (dev *NvidiaGPUDevices) CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ return util.CheckHealth(devType, n)
+}
+
+func (dev *NvidiaGPUDevices) LockNode(n *corev1.Node, p *corev1.Pod) error {
+ found := false
+ for _, val := range p.Spec.Containers {
+ if (dev.GenerateResourceRequests(&val).Nums) > 0 {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil
+ }
+ return nodelock.LockNode(n.Name, NodeLockNvidia, p)
+}
+
+func (dev *NvidiaGPUDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error {
+ found := false
+ for _, val := range p.Spec.Containers {
+ if (dev.GenerateResourceRequests(&val).Nums) > 0 {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil
+ }
+ return nodelock.ReleaseNodeLock(n.Name, NodeLockNvidia, p, false)
+}
+
+func (dev *NvidiaGPUDevices) GetNodeDevices(n corev1.Node) ([]*util.DeviceInfo, error) {
+ devEncoded, ok := n.Annotations[RegisterAnnos]
+ if !ok {
+ return []*util.DeviceInfo{}, errors.New("annos not found " + RegisterAnnos)
+ }
+ nodedevices, err := util.DecodeNodeDevices(devEncoded)
+ if err != nil {
+ klog.ErrorS(err, "failed to decode node devices", "node", n.Name, "device annotation", devEncoded)
+ return []*util.DeviceInfo{}, err
+ }
+ if len(nodedevices) == 0 {
+ klog.InfoS("no nvidia gpu device found", "node", n.Name, "device annotation", devEncoded)
+ return []*util.DeviceInfo{}, errors.New("no gpu found on node")
+ }
+ for _, val := range nodedevices {
+ if val.Mode == MigMode {
+ val.MIGTemplate = make([]util.Geometry, 0)
+ for _, migTemplates := range dev.config.MigGeometriesList {
+ found := false
+ for _, migDevices := range migTemplates.Models {
+ if strings.Contains(val.Type, migDevices) {
+ found = true
+ break
+ }
+ }
+ if found {
+ val.MIGTemplate = append(val.MIGTemplate, migTemplates.Geometries...)
+ break
+ }
+ }
+ }
+ }
+
+ pairScores, ok := n.Annotations[RegisterGPUPairScore]
+ if !ok {
+ klog.Warning("no topology score found", "node", n.Name)
+ } else {
+ devicePairScores, err := util.DecodePairScores(pairScores)
+ if err != nil {
+ klog.ErrorS(err, "failed to decode pair scores", "node", n.Name, "pair scores", pairScores)
+ return []*util.DeviceInfo{}, err
+ }
+ if devicePairScores != nil {
+ // fit pair score to device info
+ for _, deviceInfo := range nodedevices {
+ uuid := deviceInfo.ID
+
+ for _, devicePairScore := range *devicePairScores {
+ if devicePairScore.ID == uuid {
+ deviceInfo.DevicePairScore = devicePairScore
+ break
+ }
+ }
+ }
+ }
+ }
+ devDecoded := util.EncodeNodeDevices(nodedevices)
+ klog.V(5).InfoS("nodes device information", "node", n.Name, "nodedevices", devDecoded)
+ return nodedevices, nil
+}
+
+func (dev *NvidiaGPUDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
+ /*gpu related */
+ priority, ok := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourcePriority)]
+ if ok {
+ ctr.Env = append(ctr.Env, corev1.EnvVar{
+ Name: util.TaskPriority,
+ Value: fmt.Sprint(priority.Value()),
+ })
+ }
+
+ if dev.config.GPUCorePolicy != "" &&
+ dev.config.GPUCorePolicy != DefaultCorePolicy {
+ ctr.Env = append(ctr.Env, corev1.EnvVar{
+ Name: util.CoreLimitSwitch,
+ Value: string(dev.config.GPUCorePolicy),
+ })
+ }
+
+ hasResource := dev.mutateContainerResource(ctr)
+
+ if hasResource {
+ // Set runtime class name if it is not set by user and the runtime class name is configured
+ if p.Spec.RuntimeClassName == nil && dev.config.RuntimeClassName != "" {
+ p.Spec.RuntimeClassName = &dev.config.RuntimeClassName
+ }
+ }
+
+ if !hasResource && dev.config.OverwriteEnv {
+ ctr.Env = append(ctr.Env, corev1.EnvVar{
+ Name: "NVIDIA_VISIBLE_DEVICES",
+ Value: "none",
+ })
+ }
+ return hasResource, nil
+}
+
+func (dev *NvidiaGPUDevices) mutateContainerResource(ctr *corev1.Container) bool {
+ _, resourceNameOK := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceCountName)]
+ if resourceNameOK {
+ return true
+ }
+
+ _, resourceCoresOK := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceCoreName)]
+ _, resourceMemOK := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceMemoryName)]
+ _, resourceMemPercentageOK := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceMemoryPercentageName)]
+
+ if resourceCoresOK || resourceMemOK || resourceMemPercentageOK {
+ if dev.config.DefaultGPUNum > 0 {
+ ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceCountName)] = *resource.NewQuantity(int64(dev.config.DefaultGPUNum), resource.BinarySI)
+ return true
+ }
+ }
+ return false
+}
+
+func checkGPUtype(annos map[string]string, cardtype string) bool {
+ cardtype = strings.ToUpper(cardtype)
+ if inuse, ok := annos[GPUInUse]; ok {
+ useTypes := strings.Split(inuse, ",")
+ if !slices.ContainsFunc(useTypes, func(useType string) bool {
+ return strings.Contains(cardtype, strings.ToUpper(useType))
+ }) {
+ return false
+ }
+ }
+ if unuse, ok := annos[GPUNoUse]; ok {
+ unuseTypes := strings.Split(unuse, ",")
+ if slices.ContainsFunc(unuseTypes, func(unuseType string) bool {
+ return strings.Contains(cardtype, strings.ToUpper(unuseType))
+ }) {
+ return false
+ }
+ }
+ return true
+}
+
+func assertNuma(annos map[string]string) bool {
+ numabind, ok := annos[NumaBind]
+ if ok {
+ enforce, err := strconv.ParseBool(numabind)
+ if err == nil && enforce {
+ return true
+ }
+ }
+ return false
+}
+
+func (dev *NvidiaGPUDevices) checkType(annos map[string]string, d util.DeviceUsage, n util.ContainerDeviceRequest) (bool, bool) {
+ typeCheck := checkGPUtype(annos, d.Type)
+ mode, ok := annos[AllocateMode]
+ if ok && !strings.Contains(mode, d.Mode) {
+ typeCheck = false
+ }
+ if strings.Compare(n.Type, NvidiaGPUDevice) == 0 {
+ return typeCheck, assertNuma(annos)
+ }
+ return false, false
+}
+
+func (dev *NvidiaGPUDevices) checkUUID(annos map[string]string, d util.DeviceUsage) bool {
+ userUUID, ok := annos[GPUUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for nvidia user uuid [%s], device id is %s", userUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ userUUIDs := strings.Split(userUUID, ",")
+ return slices.Contains(userUUIDs, d.ID)
+ }
+
+ noUserUUID, ok := annos[GPUNoUseUUID]
+ if ok {
+ klog.V(5).Infof("check uuid for nvidia not user uuid [%s], device id is %s", noUserUUID, d.ID)
+ // use , symbol to connect multiple uuid
+ noUserUUIDs := strings.Split(noUserUUID, ",")
+ return !slices.Contains(noUserUUIDs, d.ID)
+ }
+
+ return true
+}
+
+func (dev *NvidiaGPUDevices) checkIndex(annos map[string]string, d util.DeviceUsage) bool {
+ userIndexes, ok := annos[GPUUseIndexes]
+ if ok {
+ klog.V(5).Infof("check index for nvidia user indexes [%s], device index is %d", userIndexes, d.Index)
+ return util.ContainsTargetNonNegativeInt(userIndexes, ",", int(d.Index))
+ }
+
+ noUserIndexes, ok := annos[GPUNoUseIndexes]
+ if ok {
+ klog.V(5).Infof("check index for nvidia not user indexes [%s], device index is %d", noUserIndexes, d.Index)
+ return !util.ContainsTargetNonNegativeInt(noUserIndexes, ",", int(d.Index))
+ }
+
+ return true
+}
+
+func (dev *NvidiaGPUDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd util.PodDevices) map[string]string {
+ devlist, ok := pd[NvidiaGPUDevice]
+ if ok && len(devlist) > 0 {
+ deviceStr := util.EncodePodSingleDevice(devlist)
+ (*annoinput)[util.InRequestDevices[NvidiaGPUDevice]] = deviceStr
+ (*annoinput)[util.SupportDevices[NvidiaGPUDevice]] = deviceStr
+ klog.V(5).Infof("pod add notation key [%s], values is [%s]", util.InRequestDevices[NvidiaGPUDevice], deviceStr)
+ klog.V(5).Infof("pod add notation key [%s], values is [%s]", util.SupportDevices[NvidiaGPUDevice], deviceStr)
+ }
+ return *annoinput
+}
+
+func (dev *NvidiaGPUDevices) GenerateResourceRequests(ctr *corev1.Container) util.ContainerDeviceRequest {
+ resourceName := corev1.ResourceName(dev.config.ResourceCountName)
+ resourceMem := corev1.ResourceName(dev.config.ResourceMemoryName)
+ resourceMemPercentage := corev1.ResourceName(dev.config.ResourceMemoryPercentageName)
+ resourceCores := corev1.ResourceName(dev.config.ResourceCoreName)
+ v, ok := ctr.Resources.Limits[resourceName]
+ if !ok {
+ v, ok = ctr.Resources.Requests[resourceName]
+ }
+ if ok {
+ if n, ok := v.AsInt64(); ok {
+ memnum := 0
+ mem, ok := ctr.Resources.Limits[resourceMem]
+ if !ok {
+ mem, ok = ctr.Resources.Requests[resourceMem]
+ }
+ if ok {
+ memnums, ok := mem.AsInt64()
+ if ok {
+ memnum = int(memnums)
+ }
+ }
+ mempnum := int32(101)
+ mem, ok = ctr.Resources.Limits[resourceMemPercentage]
+ if !ok {
+ mem, ok = ctr.Resources.Requests[resourceMemPercentage]
+ }
+ if ok {
+ mempnums, ok := mem.AsInt64()
+ if ok {
+ mempnum = int32(mempnums)
+ }
+ }
+ if mempnum == 101 && memnum == 0 {
+ if dev.config.DefaultMemory != 0 {
+ memnum = int(dev.config.DefaultMemory)
+ } else {
+ mempnum = 100
+ }
+ }
+ corenum := dev.config.DefaultCores
+ core, ok := ctr.Resources.Limits[resourceCores]
+ if !ok {
+ core, ok = ctr.Resources.Requests[resourceCores]
+ }
+ if ok {
+ corenums, ok := core.AsInt64()
+ if ok {
+ corenum = int32(corenums)
+ }
+ }
+ return util.ContainerDeviceRequest{
+ Nums: int32(n),
+ Type: NvidiaGPUDevice,
+ Memreq: int32(memnum),
+ MemPercentagereq: int32(mempnum),
+ Coresreq: int32(corenum),
+ }
+ }
+ }
+ return util.ContainerDeviceRequest{}
+}
+
+func (dev *NvidiaGPUDevices) CustomFilterRule(allocated *util.PodDevices, request util.ContainerDeviceRequest, toAllocate util.ContainerDevices, device *util.DeviceUsage) bool {
+ //memreq := request.Memreq
+ deviceUsageSnapshot := device.MigUsage
+ deviceUsageCurrent := util.MigInUse{
+ UsageList: make(util.MIGS, 0),
+ }
+ deviceUsageCurrent.UsageList = append(deviceUsageCurrent.UsageList, deviceUsageSnapshot.UsageList...)
+ if device.Mode == MigMode {
+ if len(deviceUsageCurrent.UsageList) == 0 {
+ tmpfound := false
+ for tidx, templates := range device.MigTemplate {
+ if templates[0].Memory < request.Memreq {
+ continue
+ } else {
+ util.PlatternMIG(&deviceUsageCurrent, device.MigTemplate, tidx)
+ tmpfound = true
+ break
+ }
+ }
+ if !tmpfound {
+ klog.Infoln("MIG entry no template fit", deviceUsageCurrent.UsageList, "request=", request)
+ }
+ }
+ for _, val := range toAllocate {
+ found := false
+ for idx := range deviceUsageCurrent.UsageList {
+ if !deviceUsageCurrent.UsageList[idx].InUse && deviceUsageCurrent.UsageList[idx].Memory >= val.Usedmem {
+ deviceUsageCurrent.UsageList[idx].InUse = true
+ found = true
+ break
+ }
+ }
+ if !found {
+ klog.Infoln("MIG entry not found", deviceUsageCurrent.UsageList)
+ return false
+ }
+ }
+ for idx := range deviceUsageCurrent.UsageList {
+ if !deviceUsageCurrent.UsageList[idx].InUse && deviceUsageCurrent.UsageList[idx].Memory >= request.Memreq {
+ deviceUsageCurrent.UsageList[idx].InUse = true
+ klog.Infoln("MIG entry device usage true=", deviceUsageCurrent.UsageList, "request", request, "toAllocate", toAllocate)
+ return true
+ }
+ }
+ klog.Infoln("MIG entry device usage false=", deviceUsageCurrent.UsageList)
+ return false
+ }
+ return true
+}
+
+func (dev *NvidiaGPUDevices) ScoreNode(node *corev1.Node, podDevices util.PodSingleDevice, previous []*util.DeviceUsage, policy string) float32 {
+ return 0
+}
+
+func (dev *NvidiaGPUDevices) migNeedsReset(n *util.DeviceUsage) bool {
+ if len(n.MigUsage.UsageList) == 0 {
+ return true
+ }
+ for _, val := range n.MigUsage.UsageList {
+ if val.InUse {
+ return false
+ }
+ }
+ n.MigUsage.UsageList = make(util.MIGS, 0)
+ return true
+}
+
+func (dev *NvidiaGPUDevices) AddResourceUsage(pod *corev1.Pod, n *util.DeviceUsage, ctr *util.ContainerDevice) error {
+ n.Used++
+ if n.Mode == MigMode {
+ if dev.migNeedsReset(n) {
+ for tidx, templates := range n.MigTemplate {
+ if templates[0].Memory < ctr.Usedmem {
+ continue
+ } else {
+ util.PlatternMIG(&n.MigUsage, n.MigTemplate, tidx)
+ ctr.Usedmem = n.MigUsage.UsageList[0].Memory
+ if !strings.Contains(ctr.UUID, "[") {
+ ctr.UUID = ctr.UUID + "[" + fmt.Sprint(tidx) + "-0]"
+ }
+ n.MigUsage.Index = int32(tidx)
+ n.MigUsage.UsageList[0].InUse = true
+ break
+ }
+ }
+ } else {
+ found := false
+ for idx, val := range n.MigUsage.UsageList {
+ if !val.InUse && val.Memory >= ctr.Usedmem {
+ n.MigUsage.UsageList[idx].InUse = true
+ ctr.Usedmem = n.MigUsage.UsageList[idx].Memory
+ if !strings.Contains(ctr.UUID, "[") {
+ ctr.UUID = ctr.UUID + "[" + fmt.Sprint(n.MigUsage.Index) + "-" + fmt.Sprint(idx) + "]"
+ }
+ found = true
+ break
+ }
+ }
+ if !found {
+ return errors.New("mig template allocate resource fail")
+ }
+ }
+ }
+ n.Usedcores += ctr.Usedcores
+ n.Usedmem += ctr.Usedmem
+ return nil
+}
+
+func (nv *NvidiaGPUDevices) Fit(devices []*util.DeviceUsage, request util.ContainerDeviceRequest, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, allocated *util.PodDevices) (bool, map[string]util.ContainerDevices, string) {
+ k := request
+ originReq := k.Nums
+ prevnuma := -1
+ klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
+ var tmpDevs map[string]util.ContainerDevices
+ tmpDevs = make(map[string]util.ContainerDevices)
+ reason := make(map[string]int)
+ needTopology := util.GetGPUSchedulerPolicyByPod(config.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyTopology.String()
+ for i := len(devices) - 1; i >= 0; i-- {
+ dev := devices[i]
+ klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
+
+ found, numa := nv.checkType(annos, *dev, k)
+ if !found {
+ reason[common.CardTypeMismatch]++
+ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, k.Type)
+ continue
+ }
+ if numa && prevnuma != dev.Numa {
+ if k.Nums != originReq {
+ reason[common.NumaNotFit] += len(tmpDevs)
+ klog.V(5).InfoS(common.NumaNotFit, "pod", klog.KObj(pod), "device", dev.ID, "k.nums", k.Nums, "numa", numa, "prevnuma", prevnuma, "device numa", dev.Numa)
+ }
+ k.Nums = originReq
+ prevnuma = dev.Numa
+ tmpDevs = make(map[string]util.ContainerDevices)
+ }
+ if !nv.checkUUID(annos, *dev) {
+ reason[common.CardUUIDMismatch]++
+ klog.V(5).InfoS(common.CardUUIDMismatch, "pod", klog.KObj(pod), "device", dev.ID, "current device info is:", *dev)
+ continue
+ }
+ if !nv.checkIndex(annos, *dev) {
+ reason[common.CardIndexMismatch]++
+ klog.V(5).InfoS(common.CardIndexMismatch, "pod", klog.KObj(pod), "device", dev.ID, "current device info is:", *dev)
+ continue
+ }
+
+ memreq := int32(0)
+ if dev.Count <= dev.Used {
+ reason[common.CardTimeSlicingExhausted]++
+ klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
+ continue
+ }
+ if k.Coresreq > 100 {
+ klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
+ k.Coresreq = 100
+ //return false, tmpDevs
+ }
+ if k.Memreq > 0 {
+ memreq = k.Memreq
+ }
+ if k.MemPercentagereq != 101 && k.Memreq == 0 {
+ //This incurs an issue
+ memreq = dev.Totalmem * k.MemPercentagereq / 100
+ }
+ if dev.Totalmem-dev.Usedmem < memreq {
+ reason[common.CardInsufficientMemory]++
+ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq)
+ continue
+ }
+ if dev.Totalcore-dev.Usedcores < k.Coresreq {
+ reason[common.CardInsufficientCore]++
+ klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", k.Coresreq)
+ continue
+ }
+ // Coresreq=100 indicates it want this card exclusively
+ if dev.Totalcore == 100 && k.Coresreq == 100 && dev.Used > 0 {
+ reason[common.ExclusiveDeviceAllocateConflict]++
+ klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
+ continue
+ }
+ // You can't allocate core=0 job to an already full GPU
+ if dev.Totalcore != 0 && dev.Usedcores == dev.Totalcore && k.Coresreq == 0 {
+ reason[common.CardComputeUnitsExhausted]++
+ klog.V(5).InfoS(common.CardComputeUnitsExhausted, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+ if !nv.CustomFilterRule(allocated, request, tmpDevs[k.Type], dev) {
+ reason[common.CardNotFoundCustomFilterRule]++
+ klog.V(5).InfoS(common.CardNotFoundCustomFilterRule, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
+ continue
+ }
+
+ if k.Nums > 0 {
+ klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
+ if !needTopology {
+ k.Nums--
+ }
+ tmpDevs[k.Type] = append(tmpDevs[k.Type], util.ContainerDevice{
+ Idx: int(dev.Index),
+ UUID: dev.ID,
+ Type: k.Type,
+ Usedmem: memreq,
+ Usedcores: k.Coresreq,
+ })
+ }
+ if k.Nums == 0 && !needTopology {
+ klog.V(4).InfoS("device allocate success", "pod", klog.KObj(pod), "allocate device", tmpDevs)
+ return true, tmpDevs, ""
+ }
+ if dev.Mode == "mig" {
+ i++
+ }
+ }
+ if needTopology {
+ if len(tmpDevs[k.Type]) == int(originReq) {
+ klog.V(5).InfoS("device allocate success", "pod", klog.KObj(pod), "allocate device", tmpDevs)
+ return true, tmpDevs, ""
+ }
+ if len(tmpDevs[k.Type]) > int(originReq) {
+ if originReq == 1 {
+ // If requesting a device, select the card with the worst connection to other cards (lowest total score).
+ lowestDevices := computeWorstSignleCard(nodeInfo, request, tmpDevs)
+ tmpDevs[k.Type] = lowestDevices
+ klog.V(5).InfoS("device allocate success", "pod", klog.KObj(pod), "worst device", lowestDevices)
+ } else {
+ // If requesting multiple devices, select the best combination of cards.
+ combinations := generateCombinations(request, tmpDevs)
+ combination := computeBestCombination(nodeInfo, combinations)
+ tmpDevs[k.Type] = combination
+ klog.V(5).InfoS("device allocate success", "pod", klog.KObj(pod), "best device combination", tmpDevs)
+ }
+ return true, tmpDevs, ""
+ }
+ }
+ if len(tmpDevs) > 0 {
+ reason[common.AllocatedCardsInsufficientRequest] = len(tmpDevs)
+ klog.V(5).InfoS(common.AllocatedCardsInsufficientRequest, "pod", klog.KObj(pod), "request", originReq, "allocated", len(tmpDevs))
+ }
+ return false, tmpDevs, common.GenReason(reason, len(devices))
+}
+
+func generateCombinations(request util.ContainerDeviceRequest, tmpDevs map[string]util.ContainerDevices) []util.ContainerDevices {
+ k := request
+ num := int(k.Nums)
+ devices := tmpDevs[k.Type]
+ // This code mainly performs permutations and combinations to generate all non-repetitive subsets.
+ // For example, if the request is 3 GPUs, and the available GPUs are ["GPU0", "GPU1", "GPU2", "GPU3"],
+ // it will generate all combinations of 3 GPUs from the available GPUs.
+ // Result：[["GPU0", "GPU1", "GPU2"],["GPU0", "GPU1", "GPU3"],["GPU0", "GPU2", "GPU3"],["GPU1", "GPU2", "GPU3"]]
+ var result []util.ContainerDevices
+ var helper func(util.ContainerDevices, int, int, util.ContainerDevices)
+
+ helper = func(arr util.ContainerDevices, start, k int, current util.ContainerDevices) {
+ if k == 0 {
+ temp := make(util.ContainerDevices, len(current))
+ copy(temp, current)
+ result = append(result, temp)
+ return
+ }
+
+ for i := start; i <= len(arr)-k; i++ {
+ current = append(current, arr[i])
+ helper(arr, i+1, k-1, current)
+ current = current[:len(current)-1]
+ }
+ }
+
+ helper(devices, 0, num, util.ContainerDevices{})
+ return result
+}
+
+func getDevicePairScoreMap(nodeInfo *util.NodeInfo) map[string]*util.DevicePairScore {
+ deviceScoreMap := make(map[string]*util.DevicePairScore)
+
+ for _, dev := range nodeInfo.Devices {
+ deviceScoreMap[dev.ID] = &dev.DevicePairScore
+ }
+
+ return deviceScoreMap
+}
+
+func computeWorstSignleCard(nodeInfo *util.NodeInfo, request util.ContainerDeviceRequest, tmpDevs map[string]util.ContainerDevices) util.ContainerDevices {
+ worstScore := -1
+ worstDevices := util.ContainerDevices{}
+ deviceScoreMap := getDevicePairScoreMap(nodeInfo)
+ // Iterate through all devices to find the one with the lowest score
+ devices := tmpDevs[request.Type]
+
+ for _, dev1 := range devices {
+ totalScore := 0
+ scoreMapDev1 := deviceScoreMap[dev1.UUID]
+ for _, dev2 := range devices {
+ if dev1.UUID == dev2.UUID {
+ continue
+ }
+ totalScore += scoreMapDev1.Scores[dev2.UUID]
+ }
+ if totalScore < worstScore || worstScore == -1 {
+ worstScore = totalScore
+ worstDevices = util.ContainerDevices{dev1}
+ }
+ }
+ return worstDevices
+}
+
+func computeBestCombination(nodeInfo *util.NodeInfo, combinations []util.ContainerDevices) util.ContainerDevices {
+ bestScore := 0
+ bestCombination := util.ContainerDevices{}
+ deviceScoreMap := getDevicePairScoreMap(nodeInfo)
+ // Iterate through all combinations to find the one with the highest score
+ for _, partition := range combinations {
+ totalScore := 0
+
+ for i := 0; i < len(partition)-1; i++ {
+ dev1 := partition[i]
+ scoreMapDev1 := deviceScoreMap[dev1.UUID]
+ for z := i + 1; z < len(partition); z++ {
+ dev2 := partition[z]
+ totalScore += scoreMapDev1.Scores[dev2.UUID]
+ }
+ }
+
+ if totalScore > bestScore {
+ bestScore = totalScore
+ bestCombination = partition
+ }
+ }
+ return bestCombination
+}
diff --git a/pkg/device/nvidia/device_test.go b/pkg/device/nvidia/device_test.go
new file mode 100644
index 000000000..0ba80efbb
--- /dev/null
+++ b/pkg/device/nvidia/device_test.go
@@ -0,0 +1,1167 @@
+/*/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nvidia
+
+import (
+ "errors"
+ "testing"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+func Test_DefaultResourceNum(t *testing.T) {
+ v := *resource.NewQuantity(1, resource.BinarySI)
+ vv, ok := v.AsInt64()
+ assert.Equal(t, ok, true)
+ assert.Equal(t, vv, int64(1))
+}
+
+func Test_MutateAdmission(t *testing.T) {
+ tests := []struct {
+ name string
+ args *corev1.Container
+ want bool
+ }{
+ {
+ name: "having ResourceName set to resource limits.",
+ args: &corev1.Container{
+ Name: "test",
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "nvidia.com/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ want: true,
+ },
+ {
+ name: "don't having ResourceName, but having ResourceCores set to resource limits",
+ args: &corev1.Container{
+ Name: "test",
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "nvidia.com/gpucores": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ want: true,
+ },
+ {
+ name: "don't having ResourceName, but having ResourceMem set to resource limits",
+ args: &corev1.Container{
+ Name: "test",
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "nvidia.com/gpumem": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ want: true,
+ },
+ {
+ name: "don't having ResourceName, but having ResourceMemPercentage set to resource limits",
+ args: &corev1.Container{
+ Name: "test",
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "nvidia.com/gpumem-percentage": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ want: true,
+ },
+ {
+ name: "don't having math resources.",
+ args: &corev1.Container{
+ Name: "test",
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ },
+ },
+ want: false,
+ },
+ }
+
+ gpuDevices := &NvidiaGPUDevices{
+ config: NvidiaConfig{
+ ResourceCountName: "nvidia.com/gpu",
+ ResourceMemoryName: "nvidia.com/gpumem",
+ ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
+ ResourceCoreName: "nvidia.com/gpucores",
+ DefaultGPUNum: int32(1),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ got, _ := gpuDevices.MutateAdmission(test.args, &corev1.Pod{})
+ if test.want != got {
+ t.Fatalf("exec MutateAdmission method expect return is %+v, but got is %+v", test.want, got)
+ }
+ })
+ }
+}
+
+func Test_checkUUID(t *testing.T) {
+ gpuDevices := &NvidiaGPUDevices{
+ config: NvidiaConfig{
+ ResourceCountName: "nvidia.com/gpu",
+ ResourceMemoryName: "nvidia.com/gpumem",
+ ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
+ ResourceCoreName: "nvidia.com/gpucores",
+ DefaultGPUNum: int32(1),
+ },
+ }
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "don't set GPUUseUUID and GPUNoUseUUID annotation",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: make(map[string]string),
+ d: util.DeviceUsage{},
+ },
+ want: true,
+ },
+ {
+ name: "use set GPUUseUUID don't set GPUNoUseUUID annotation,device match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ GPUUseUUID: "abc,123",
+ },
+ d: util.DeviceUsage{
+ ID: "abc",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "use set GPUUseUUID don't set GPUNoUseUUID annotation,device don't match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ GPUUseUUID: "abc,123",
+ },
+ d: util.DeviceUsage{
+ ID: "1abc",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "use don't set GPUUseUUID set GPUNoUseUUID annotation,device match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ GPUNoUseUUID: "abc,123",
+ },
+ d: util.DeviceUsage{
+ ID: "abc",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "use don't set GPUUseUUID set GPUNoUseUUID annotation,device don't match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ GPUNoUseUUID: "abc,123",
+ },
+ d: util.DeviceUsage{
+ ID: "1abc",
+ },
+ },
+ want: true,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ got := gpuDevices.checkUUID(test.args.annos, test.args.d)
+ assert.Equal(t, test.want, got)
+ })
+ }
+}
+
+func Test_checkIndex(t *testing.T) {
+ gpuDevices := &NvidiaGPUDevices{
+ config: NvidiaConfig{
+ ResourceCountName: "nvidia.com/gpu",
+ ResourceMemoryName: "nvidia.com/gpumem",
+ ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
+ ResourceCoreName: "nvidia.com/gpucores",
+ DefaultGPUNum: int32(1),
+ },
+ }
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "don't set GPUUseIndex and GPUNoUseIndex annotation",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: make(map[string]string),
+ d: util.DeviceUsage{},
+ },
+ want: true,
+ },
+ {
+ name: "use set GPUUseIndex don't set GPUNoUseIndex annotation,device match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ GPUUseIndexes: "0,1",
+ },
+ d: util.DeviceUsage{
+ ID: "abc",
+ Index: 0,
+ },
+ },
+ want: true,
+ },
+ {
+ name: "use set GPUUseIndex don't set GPUNoUseIndex annotation,device don't match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ GPUUseIndexes: "1",
+ },
+ d: util.DeviceUsage{
+ ID: "abc",
+ Index: 0,
+ },
+ },
+ want: false,
+ },
+ {
+ name: "use don't set GPUUseIndex set GPUNoUseIndex annotation,device match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ GPUNoUseIndexes: "0",
+ },
+ d: util.DeviceUsage{
+ ID: "abc",
+ Index: 0,
+ },
+ },
+ want: false,
+ },
+ {
+ name: "use don't set GPUUseIndex set GPUNoUseIndex annotation,device don't match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ GPUNoUseIndexes: "0",
+ },
+ d: util.DeviceUsage{
+ ID: "abc",
+ Index: 1,
+ },
+ },
+ want: true,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ got := gpuDevices.checkIndex(test.args.annos, test.args.d)
+ assert.Equal(t, test.want, got)
+ })
+ }
+}
+
+func Test_checkType(t *testing.T) {
+ gpuDevices := &NvidiaGPUDevices{
+ config: NvidiaConfig{
+ ResourceCountName: "nvidia.com/gpu",
+ ResourceMemoryName: "nvidia.com/gpumem",
+ ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
+ ResourceCoreName: "nvidia.com/gpucores",
+ DefaultGPUNum: int32(1),
+ },
+ }
+ tests := []struct {
+ name string
+ args struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }
+ want bool
+ }{
+ {
+ name: "use set GPUInUse don't set GPUNoUse annotation,device match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ GPUInUse: "A10",
+ },
+ d: util.DeviceUsage{
+ Type: "NVIDIA A100",
+ },
+ },
+ want: true,
+ },
+ {
+ name: "use set GPUInUse set GPUNoUse annotation,device don't match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ GPUInUse: "A10",
+ GPUNoUse: "A100",
+ },
+ d: util.DeviceUsage{
+ Type: "NVIDIA A100",
+ },
+ },
+ want: false,
+ },
+ {
+ name: "use set GPUInUse set GPUNoUse annotation,device match",
+ args: struct {
+ annos map[string]string
+ d util.DeviceUsage
+ }{
+ annos: map[string]string{
+ GPUInUse: "A10",
+ GPUNoUse: "A100",
+ },
+ d: util.DeviceUsage{
+ Type: "NVIDIA A10",
+ },
+ },
+ want: true,
+ },
+ }
+ req := util.ContainerDeviceRequest{
+ Type: NvidiaGPUDevice,
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ got, _ := gpuDevices.checkType(test.args.annos, test.args.d, req)
+ assert.Equal(t, test.want, got)
+ })
+ }
+}
+
+func Test_FilterDeviceToRegister(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ uuid string
+ idx string
+ *FilterDevice
+ }
+ want bool
+ }{
+ {
+ name: "filter is nil",
+ args: struct {
+ uuid string
+ idx string
+ *FilterDevice
+ }{
+ uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76",
+ idx: "0",
+ FilterDevice: nil,
+ },
+ want: false,
+ },
+ {
+ name: "uuid is empty",
+ args: struct {
+ uuid string
+ idx string
+ *FilterDevice
+ }{
+ uuid: "",
+ idx: "0",
+ FilterDevice: &FilterDevice{
+ UUID: []string{"GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76"},
+ },
+ },
+ want: false,
+ },
+ {
+ name: "uuid is not in filter",
+ args: struct {
+ uuid string
+ idx string
+ *FilterDevice
+ }{
+ uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76",
+ idx: "0",
+ FilterDevice: &FilterDevice{
+ UUID: []string{"GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b77"},
+ },
+ },
+ want: false,
+ },
+ {
+ name: "uuid is in filter",
+ args: struct {
+ uuid string
+ idx string
+ *FilterDevice
+ }{
+ uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76",
+ idx: "0",
+ FilterDevice: &FilterDevice{
+ UUID: []string{"GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76"},
+ },
+ },
+ want: true,
+ },
+ {
+ name: "idx is empty",
+ args: struct {
+ uuid string
+ idx string
+ *FilterDevice
+ }{
+ uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76",
+ idx: "",
+ FilterDevice: &FilterDevice{
+ Index: []uint{0},
+ },
+ },
+ want: false,
+ },
+ {
+ name: "idx is not in filter",
+ args: struct {
+ uuid string
+ idx string
+ *FilterDevice
+ }{
+ uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76",
+ idx: "0",
+ FilterDevice: &FilterDevice{
+ Index: []uint{1},
+ },
+ },
+ want: false,
+ },
+ {
+ name: "idx is in filter",
+ args: struct {
+ uuid string
+ idx string
+ *FilterDevice
+ }{
+ uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76",
+ idx: "0",
+ FilterDevice: &FilterDevice{
+ Index: []uint{0},
+ },
+ },
+ want: true,
+ },
+ {
+ name: "idx is invalid",
+ args: struct {
+ uuid string
+ idx string
+ *FilterDevice
+ }{
+ uuid: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76",
+ idx: "a",
+ FilterDevice: &FilterDevice{
+ Index: []uint{0},
+ },
+ },
+ want: false,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ DevicePluginFilterDevice = test.args.FilterDevice
+ got := FilterDeviceToRegister(test.args.uuid, test.args.idx)
+ assert.DeepEqual(t, test.want, got)
+ })
+ }
+}
+
+func Test_InitNvidiaDevice(t *testing.T) {
+ tests := []struct {
+ name string
+ args NvidiaConfig
+ want *NvidiaGPUDevices
+ }{
+ {
+ name: "test with vaild configuration",
+ args: NvidiaConfig{
+ ResourceCountName: "nvidia.com/gpu",
+ ResourceMemoryName: "nvidia.com/gpumem",
+ DefaultGPUNum: int32(1),
+ },
+ want: &NvidiaGPUDevices{
+ config: NvidiaConfig{
+ ResourceCountName: "nvidia.com/gpu",
+ ResourceMemoryName: "nvidia.com/gpumem",
+ DefaultGPUNum: int32(1),
+ },
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ devices := InitNvidiaDevice(test.args)
+ if devices == nil {
+ t.Fatalf("Expected devices to be initialized")
+ }
+ assert.DeepEqual(t, test.want.config, devices.config)
+ assert.Equal(t, "hami.io/vgpu-devices-to-allocate", util.InRequestDevices[NvidiaGPUDevice], "Expected InRequestDevices to be set")
+ assert.Equal(t, "hami.io/vgpu-devices-allocated", util.SupportDevices[NvidiaGPUDevice], "Expected SupportDevices to be set")
+ assert.Equal(t, HandshakeAnnos, util.HandshakeAnnos[NvidiaGPUDevice], "Expected HandshakeAnnos to be set")
+ })
+ }
+}
+
+func Test_PatchAnnotations(t *testing.T) {
+ InitNvidiaDevice(NvidiaConfig{})
+
+ tests := []struct {
+ name string
+ args struct {
+ annoinput map[string]string
+ pd util.PodDevices
+ }
+ want map[string]string
+ }{
+ {
+ name: "exist device",
+ args: struct {
+ annoinput map[string]string
+ pd util.PodDevices
+ }{
+ annoinput: map[string]string{},
+ pd: util.PodDevices{
+ NvidiaGPUDevice: util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ Idx: 0,
+ UUID: "nvidia-device-0",
+ Type: "NVIDIA",
+ Usedmem: 2000,
+ Usedcores: 1,
+ },
+ },
+ },
+ },
+ },
+ want: map[string]string{
+ util.InRequestDevices[NvidiaGPUDevice]: "nvidia-device-0,NVIDIA,2000,1:;",
+ util.SupportDevices[NvidiaGPUDevice]: "nvidia-device-0,NVIDIA,2000,1:;",
+ },
+ },
+ {
+ name: "no device",
+ args: struct {
+ annoinput map[string]string
+ pd util.PodDevices
+ }{
+ annoinput: map[string]string{},
+ pd: util.PodDevices{},
+ },
+ want: map[string]string{},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ gpuDevices := &NvidiaGPUDevices{}
+ result := gpuDevices.PatchAnnotations(&corev1.Pod{}, &test.args.annoinput, test.args.pd)
+
+ assert.Equal(t, len(test.want), len(result), "Expected length of result to match want")
+ for k, v := range test.want {
+ assert.Equal(t, v, result[k], "pod add annotation key [%s], values is [%s]", k, result[k])
+ }
+ })
+ }
+
+}
+
+func Test_GetNodeDevices(t *testing.T) {
+ tests := []struct {
+ name string
+ args corev1.Node
+ want []*util.DeviceInfo
+ err error
+ }{
+ {
+ name: "exist gpu devices",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-01",
+ Annotations: map[string]string{
+ RegisterAnnos: "GPU-0,5,8192,100,NVIDIA-Tesla P4,0,true:",
+ },
+ },
+ },
+ want: []*util.DeviceInfo{
+ {
+ ID: "GPU-0",
+ Count: 5,
+ Devmem: 8192,
+ Devcore: 100,
+ Type: "NVIDIA-Tesla P4",
+ Numa: 0,
+ Health: true,
+ },
+ },
+ err: nil,
+ },
+ {
+ name: "no gpu devices",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-02",
+ Annotations: map[string]string{
+ RegisterAnnos: "",
+ },
+ },
+ },
+ want: []*util.DeviceInfo{},
+ err: errors.New("failed to decode node devices"),
+ },
+ {
+ name: "no annotation",
+ args: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-03",
+ Annotations: map[string]string{},
+ },
+ },
+ want: []*util.DeviceInfo{},
+ err: errors.New("annos not found " + RegisterAnnos),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ gpuDevices := &NvidiaGPUDevices{}
+ result, err := gpuDevices.GetNodeDevices(test.args)
+ if (err != nil) != (test.err != nil) {
+ t.Errorf("GetNodeDevices error = %v, want %v", err, test.err)
+ }
+ if len(result) != len(test.want) {
+ t.Errorf("GetNodeDevices got %d devices, want %d", len(result), len(test.want))
+ return
+ }
+ if err == nil && len(result) != 0 {
+ for k, v := range test.want {
+ assert.Equal(t, v.Index, result[k].Index)
+ assert.Equal(t, v.ID, result[k].ID)
+ assert.Equal(t, v.Devcore, result[k].Devcore)
+ assert.Equal(t, v.Health, result[k].Health)
+ assert.Equal(t, v.Numa, result[k].Numa)
+ assert.Equal(t, v.Type, result[k].Type)
+ assert.Equal(t, v.Count, result[k].Count)
+ }
+ }
+ })
+ }
+}
+
+func TestDevices_Fit(t *testing.T) {
+ config := NvidiaConfig{
+ ResourceCountName: "nvidia.com/gpu",
+ ResourceMemoryName: "nvidia.com/gpumem",
+ ResourceCoreName: "nvidia.com/gpumem",
+ ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
+ }
+ dev := InitNvidiaDevice(config)
+
+ tests := []struct {
+ name string
+ devices []*util.DeviceUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ wantFit bool
+ wantLen int
+ wantDevIDs []string
+ wantReason string
+ }{
+ {
+ name: "fit success",
+ devices: []*util.DeviceUsage{
+ {
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ },
+ {
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 64,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: NvidiaGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-1"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: memory not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: NvidiaGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientMemory",
+ },
+ {
+ name: "fit fail: core not enough",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1024,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: NvidiaGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardInsufficientCore",
+ },
+ {
+ name: "fit fail: type mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 128,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Health: true,
+ Type: NvidiaGPUDevice,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: "OtherType",
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTypeMismatch",
+ },
+ {
+ name: "fit fail: user assign use uuid mismatch",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-1",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: NvidiaGPUDevice,
+ },
+ annos: map[string]string{GPUUseUUID: "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: user assign no use uuid match",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: NvidiaGPUDevice,
+ },
+ annos: map[string]string{GPUNoUseUUID: "dev-0"},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardUuidMismatch",
+ },
+ {
+ name: "fit fail: card overused",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 100,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 50,
+ Type: NvidiaGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardTimeSlicingExhausted",
+ },
+ {
+ name: "fit success: but core limit can't exceed 100",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 0,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 120,
+ Type: NvidiaGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ {
+ name: "fit fail: card exclusively",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 100,
+ Type: NvidiaGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 ExclusiveDeviceAllocateConflict",
+ },
+ {
+ name: "fit fail: CardComputeUnitsExhausted",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 0,
+ Type: NvidiaGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 CardComputeUnitsExhausted",
+ },
+ {
+ name: "fit fail: AllocatedCardsInsufficientRequest",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 2,
+ Memreq: 512,
+ MemPercentagereq: 0,
+ Coresreq: 20,
+ Type: NvidiaGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: false,
+ wantLen: 0,
+ wantDevIDs: []string{},
+ wantReason: "1/1 AllocatedCardsInsufficientRequest",
+ },
+ {
+ name: "fit success: memory percentage",
+ devices: []*util.DeviceUsage{{
+ ID: "dev-0",
+ Index: 0,
+ Used: 20,
+ Count: 100,
+ Usedmem: 0,
+ Totalmem: 1280,
+ Totalcore: 100,
+ Usedcores: 10,
+ Numa: 0,
+ Type: NvidiaGPUDevice,
+ Health: true,
+ }},
+ request: util.ContainerDeviceRequest{
+ Nums: 1,
+ Memreq: 0,
+ MemPercentagereq: 10,
+ Coresreq: 20,
+ Type: NvidiaGPUDevice,
+ },
+ annos: map[string]string{},
+ wantFit: true,
+ wantLen: 1,
+ wantDevIDs: []string{"dev-0"},
+ wantReason: "",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ allocated := &util.PodDevices{}
+ fit, result, reason := dev.Fit(test.devices, test.request, test.annos, &corev1.Pod{}, &util.NodeInfo{}, allocated)
+ if fit != test.wantFit {
+ t.Errorf("Fit: got %v, want %v", fit, test.wantFit)
+ }
+ if test.wantFit {
+ if len(result[NvidiaGPUDevice]) != test.wantLen {
+ t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[NvidiaGPUDevice]))
+ }
+ for idx, id := range test.wantDevIDs {
+ if id != result[NvidiaGPUDevice][idx].UUID {
+ t.Errorf("expected device id: %s, got device id %s", id, result[NvidiaGPUDevice][idx].UUID)
+ }
+ }
+ }
+
+ if reason != test.wantReason {
+ t.Errorf("expected reason: %s, got reason: %s", test.wantReason, reason)
+ }
+ })
+ }
+}
+
+func TestDevices_AddResourceUsage(t *testing.T) {
+ tests := []struct {
+ name string
+ deviceUsage *util.DeviceUsage
+ ctr *util.ContainerDevice
+ wantErr bool
+ wantUsage *util.DeviceUsage
+ }{
+ {
+ name: "test add resource usage",
+ deviceUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 0,
+ Usedcores: 15,
+ Usedmem: 2000,
+ },
+ ctr: &util.ContainerDevice{
+ UUID: "dev-0",
+ Usedcores: 50,
+ Usedmem: 1024,
+ },
+ wantUsage: &util.DeviceUsage{
+ ID: "dev-0",
+ Used: 1,
+ Usedcores: 65,
+ Usedmem: 3024,
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dev := &NvidiaGPUDevices{}
+ if err := dev.AddResourceUsage(&corev1.Pod{}, tt.deviceUsage, tt.ctr); (err != nil) != tt.wantErr {
+ t.Errorf("AddResourceUsage() error=%v, wantErr %v", err, tt.wantErr)
+ }
+ if !tt.wantErr {
+ if tt.deviceUsage.Usedcores != tt.wantUsage.Usedcores {
+ t.Errorf("expected used cores: %d, got used cores %d", tt.wantUsage.Usedcores, tt.deviceUsage.Usedcores)
+ }
+ if tt.deviceUsage.Usedmem != tt.wantUsage.Usedmem {
+ t.Errorf("expected used mem: %d, got used mem %d", tt.wantUsage.Usedmem, tt.deviceUsage.Usedmem)
+ }
+ if tt.deviceUsage.Used != tt.wantUsage.Used {
+ t.Errorf("expected used: %d, got used %d", tt.wantUsage.Used, tt.deviceUsage.Used)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/device/nvidia/links.go b/pkg/device/nvidia/links.go
new file mode 100644
index 000000000..e7a8917ed
--- /dev/null
+++ b/pkg/device/nvidia/links.go
@@ -0,0 +1,287 @@
+/*
+*
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*
+*/
+
+package nvidia
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "os"
+ "strconv"
+ "strings"
+
+ "github.com/NVIDIA/go-nvlib/pkg/nvlib/device"
+ "github.com/NVIDIA/go-nvml/pkg/nvml"
+
+ safecast "github.com/ccoveille/go-safecast"
+)
+
+// P2PLinkType defines the link information between two devices.
+type P2PLinkType uint
+
+// The following constants define the nature of a link between two devices.
+// These include peer-2-peer and NVLink information.
+const (
+ P2PLinkUnknown P2PLinkType = iota
+ P2PLinkCrossCPU
+ P2PLinkSameCPU
+ P2PLinkHostBridge
+ P2PLinkMultiSwitch
+ P2PLinkSingleSwitch
+ P2PLinkSameBoard
+ SingleNVLINKLink
+ TwoNVLINKLinks
+ ThreeNVLINKLinks
+ FourNVLINKLinks
+ FiveNVLINKLinks
+ SixNVLINKLinks
+ SevenNVLINKLinks
+ EightNVLINKLinks
+ NineNVLINKLinks
+ TenNVLINKLinks
+ ElevenNVLINKLinks
+ TwelveNVLINKLinks
+ ThirteenNVLINKLinks
+ FourteenNVLINKLinks
+ FifteenNVLINKLinks
+ SixteenNVLINKLinks
+ SeventeenNVLINKLinks
+ EighteenNVLINKLinks
+)
+
+// String returns the string representation of the P2PLink type.
+func (l P2PLinkType) String() string {
+ switch l {
+ case P2PLinkCrossCPU:
+ return "P2PLinkCrossCPU"
+ case P2PLinkSameCPU:
+ return "P2PLinkSameCPU"
+ case P2PLinkHostBridge:
+ return "P2PLinkHostBridge"
+ case P2PLinkMultiSwitch:
+ return "P2PLinkMultiSwitch"
+ case P2PLinkSingleSwitch:
+ return "P2PLinkSingleSwitch"
+ case P2PLinkSameBoard:
+ return "P2PLinkSameBoard"
+ case SingleNVLINKLink:
+ return "SingleNVLINKLink"
+ case TwoNVLINKLinks:
+ return "TwoNVLINKLinks"
+ case ThreeNVLINKLinks:
+ return "ThreeNVLINKLinks"
+ case FourNVLINKLinks:
+ return "FourNVLINKLinks"
+ case FiveNVLINKLinks:
+ return "FiveNVLINKLinks"
+ case SixNVLINKLinks:
+ return "SixNVLINKLinks"
+ case SevenNVLINKLinks:
+ return "SevenNVLINKLinks"
+ case EightNVLINKLinks:
+ return "EightNVLINKLinks"
+ case NineNVLINKLinks:
+ return "NineNVLINKLinks"
+ case TenNVLINKLinks:
+ return "TenNVLINKLinks"
+ case ElevenNVLINKLinks:
+ return "ElevenNVLINKLinks"
+ case TwelveNVLINKLinks:
+ return "TwelveNVLINKLinks"
+ case ThirteenNVLINKLinks:
+ return "ThirteenNVLINKLinks"
+ case FourteenNVLINKLinks:
+ return "FourteenNVLINKLinks"
+ case FifteenNVLINKLinks:
+ return "FifteenNVLINKLinks"
+ case SixteenNVLINKLinks:
+ return "SixteenNVLINKLinks"
+ case SeventeenNVLINKLinks:
+ return "SeventeenNVLINKLinks"
+ case EighteenNVLINKLinks:
+ return "EighteenNVLINKLinks"
+ default:
+ return fmt.Sprintf("UNKNOWN (%v)", uint(l))
+ }
+}
+
+// GetP2PLink gets the peer-to-peer connectivity between two devices.
+func GetP2PLink(dev1 device.Device, dev2 device.Device) (P2PLinkType, error) {
+ level, ret := dev1.GetTopologyCommonAncestor(dev2)
+ if !errors.Is(ret, nvml.SUCCESS) {
+ return P2PLinkUnknown, fmt.Errorf("failed to get commmon anscestor: %v", ret)
+ }
+
+ switch level {
+ case nvml.TOPOLOGY_INTERNAL:
+ return P2PLinkSameBoard, nil
+ case nvml.TOPOLOGY_SINGLE:
+ return P2PLinkSingleSwitch, nil
+ case nvml.TOPOLOGY_MULTIPLE:
+ return P2PLinkMultiSwitch, nil
+ case nvml.TOPOLOGY_HOSTBRIDGE:
+ return P2PLinkHostBridge, nil
+ case nvml.TOPOLOGY_NODE: // NVML_TOPOLOGY_CPU was renamed NVML_TOPOLOGY_NODE
+ return P2PLinkSameCPU, nil
+ case nvml.TOPOLOGY_SYSTEM:
+ return P2PLinkCrossCPU, nil
+
+ }
+
+ return P2PLinkUnknown, fmt.Errorf("unknown topology level: %v", level)
+}
+
+// GetNVLink gets the number of NVLinks between the specified devices.
+func GetNVLink(dev1 device.Device, dev2 device.Device) (P2PLinkType, error) {
+ pciInfos, err := getAllNvLinkRemotePciInfo(dev1)
+ if err != nil {
+ return P2PLinkUnknown, fmt.Errorf("failed to get nvlink remote pci info: %v", err)
+ }
+
+ dev2PciInfo, ret := dev2.GetPciInfo()
+ if !errors.Is(ret, nvml.SUCCESS) {
+ return P2PLinkUnknown, fmt.Errorf("failed to get pci info: %v", ret)
+ }
+ dev2BusID := PciInfo(dev2PciInfo).BusID()
+
+ nvlink := P2PLinkUnknown
+ for _, pciInfo := range pciInfos {
+ if pciInfo.BusID() != dev2BusID {
+ continue
+ }
+ switch nvlink {
+ case P2PLinkUnknown:
+ nvlink = SingleNVLINKLink
+ case SingleNVLINKLink:
+ nvlink = TwoNVLINKLinks
+ case TwoNVLINKLinks:
+ nvlink = ThreeNVLINKLinks
+ case ThreeNVLINKLinks:
+ nvlink = FourNVLINKLinks
+ case FourNVLINKLinks:
+ nvlink = FiveNVLINKLinks
+ case FiveNVLINKLinks:
+ nvlink = SixNVLINKLinks
+ case SixNVLINKLinks:
+ nvlink = SevenNVLINKLinks
+ case SevenNVLINKLinks:
+ nvlink = EightNVLINKLinks
+ case EightNVLINKLinks:
+ nvlink = NineNVLINKLinks
+ case NineNVLINKLinks:
+ nvlink = TenNVLINKLinks
+ case TenNVLINKLinks:
+ nvlink = ElevenNVLINKLinks
+ case ElevenNVLINKLinks:
+ nvlink = TwelveNVLINKLinks
+ case TwelveNVLINKLinks:
+ nvlink = ThirteenNVLINKLinks
+ case ThirteenNVLINKLinks:
+ nvlink = FourteenNVLINKLinks
+ case FourteenNVLINKLinks:
+ nvlink = FifteenNVLINKLinks
+ case FifteenNVLINKLinks:
+ nvlink = SixteenNVLINKLinks
+ case SixteenNVLINKLinks:
+ nvlink = SeventeenNVLINKLinks
+ case SeventeenNVLINKLinks:
+ nvlink = EighteenNVLINKLinks
+ }
+ }
+ // TODO(klueska): Handle NVSwitch semantics
+
+ return nvlink, nil
+}
+
+// getAllNvLinkRemotePciInfo returns the PCI info for all devices attached to the specified device by an NVLink.
+func getAllNvLinkRemotePciInfo(dev device.Device) ([]PciInfo, error) {
+ var pciInfos []PciInfo
+ for i := 0; i < nvml.NVLINK_MAX_LINKS; i++ {
+ state, ret := dev.GetNvLinkState(i)
+ if errors.Is(ret, nvml.ERROR_NOT_SUPPORTED) || errors.Is(ret, nvml.ERROR_INVALID_ARGUMENT) {
+ continue
+ }
+ if !errors.Is(ret, nvml.SUCCESS) {
+ return nil, fmt.Errorf("failed to get nvlink state: %v", ret)
+ }
+ if state != nvml.FEATURE_ENABLED {
+ continue
+ }
+ pciInfo, ret := dev.GetNvLinkRemotePciInfo(i)
+ if errors.Is(ret, nvml.ERROR_NOT_SUPPORTED) || errors.Is(ret, nvml.ERROR_INVALID_ARGUMENT) {
+ continue
+ }
+ if !errors.Is(ret, nvml.SUCCESS) {
+ return nil, fmt.Errorf("failed to get remote pci info: %v", ret)
+ }
+ pciInfos = append(pciInfos, PciInfo(pciInfo))
+ }
+
+ return pciInfos, nil
+}
+
+// PciInfo is a type alias to nvml.PciInfo to allow for functions to be defined on the type.
+type PciInfo nvml.PciInfo
+
+// BusID provides a utility function that returns the string representation of the bus ID.
+// Note that the []int8 slice member is named BusId.
+func (p PciInfo) BusID() string {
+ var bytes []byte
+ for _, b := range p.BusId {
+ if byte(b) == '\x00' {
+ break
+ }
+ bytes = append(bytes, byte(b))
+ }
+ id := strings.ToLower(string(bytes))
+
+ if id != "0000" {
+ id = strings.TrimPrefix(id, "0000")
+ }
+ return id
+}
+
+// CPUAffinity returns the CPU affinity associated with a specified PCI device.
+// If NUMA information is not available, this returns nil.
+func (p PciInfo) CPUAffinity() *uint {
+ node := p.NumaNode()
+ if node < 0 {
+ return nil
+ }
+ affinity, err := safecast.ToUint(node)
+ if err != nil {
+ return nil
+ }
+ return &affinity
+}
+
+// NumaNode returns the numa node associates with a PCI device.
+// If numa is unsupported, -1 is returned.
+func (p PciInfo) NumaNode() int64 {
+ // Read the numa_node file associated with the PCI Device Info
+ b, err := os.ReadFile(fmt.Sprintf("/sys/bus/pci/devices/%s/numa_node", p.BusID()))
+ if err != nil {
+ return -1
+ }
+ node, err := strconv.ParseInt(string(bytes.TrimSpace(b)), 10, 64)
+ if err != nil {
+ return -1
+ }
+ return node
+}
diff --git a/pkg/k8sutil/client.go b/pkg/k8sutil/client.go
deleted file mode 100644
index 86715acfc..000000000
--- a/pkg/k8sutil/client.go
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package k8sutil
-
-import (
- "k8s.io/client-go/kubernetes"
- "k8s.io/client-go/rest"
- "k8s.io/client-go/tools/clientcmd"
- "os"
- "path/filepath"
-)
-
-// NewClient connects to an API server
-func NewClient() (kubernetes.Interface, error) {
- kubeConfig := os.Getenv("KUBECONFIG")
- if kubeConfig == "" {
- kubeConfig = filepath.Join(os.Getenv("HOME"), ".kube", "config")
- }
- config, err := rest.InClusterConfig()
- if err != nil {
- config, err = clientcmd.BuildConfigFromFlags("", kubeConfig)
- if err != nil {
- return nil, err
- }
- }
- client, err := kubernetes.NewForConfig(config)
- return client, err
-}
diff --git a/pkg/k8sutil/pod.go b/pkg/k8sutil/pod.go
index 90bee783a..af7750e7c 100644
--- a/pkg/k8sutil/pod.go
+++ b/pkg/k8sutil/pod.go
@@ -1,45 +1,63 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package k8sutil
import (
- corev1 "k8s.io/api/core/v1"
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/klog/v2"
)
-func ResourceNums(pod *corev1.Pod, resourceName corev1.ResourceName) (counts []int) {
- counts = make([]int, len(pod.Spec.Containers))
- for i := 0; i < len(pod.Spec.Containers); i++ {
- v, ok := pod.Spec.Containers[i].Resources.Limits[resourceName]
- if !ok {
- v, ok = pod.Spec.Containers[i].Resources.Requests[resourceName]
- }
- if ok {
- if n, ok := v.AsInt64(); ok {
- counts[i] = int(n)
- }
- }
- }
- return counts
+func Resourcereqs(pod *corev1.Pod) (counts util.PodDeviceRequests) {
+ counts = make(util.PodDeviceRequests, len(pod.Spec.Containers))
+ klog.V(4).InfoS("Processing resource requirements",
+ "pod", klog.KObj(pod),
+ "containerCount", len(pod.Spec.Containers))
+ //Count Nvidia GPU
+ cnt := int32(0)
+ for i := range pod.Spec.Containers {
+ devices := device.GetDevices()
+ counts[i] = make(util.ContainerDeviceRequests)
+ klog.V(5).InfoS("Processing container resources",
+ "pod", klog.KObj(pod),
+ "containerIndex", i,
+ "containerName", pod.Spec.Containers[i].Name)
+ for idx, val := range devices {
+ request := val.GenerateResourceRequests(&pod.Spec.Containers[i])
+ if request.Nums > 0 {
+ cnt += request.Nums
+ counts[i][idx] = request
+ }
+ }
+ }
+ if cnt == 0 {
+ klog.V(4).InfoS("No device requests found", "pod", klog.KObj(pod))
+ } else {
+ klog.V(4).InfoS("Resource requirements collected", "pod", klog.KObj(pod), "requests", counts)
+ }
+ return counts
}
func IsPodInTerminatedState(pod *corev1.Pod) bool {
- return pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodSucceeded
+ return pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodSucceeded
}
func AllContainersCreated(pod *corev1.Pod) bool {
- return len(pod.Status.ContainerStatuses) >= len(pod.Spec.Containers)
+ return len(pod.Status.ContainerStatuses) >= len(pod.Spec.Containers)
}
diff --git a/pkg/k8sutil/pod_test.go b/pkg/k8sutil/pod_test.go
new file mode 100644
index 000000000..5d64f1808
--- /dev/null
+++ b/pkg/k8sutil/pod_test.go
@@ -0,0 +1,421 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8sutil
+
+import (
+ "testing"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ "k8s.io/klog/v2"
+)
+
+func Test_Resourcereqs(t *testing.T) {
+ config := &device.Config{
+ NvidiaConfig: nvidia.NvidiaConfig{
+ ResourceCountName: "hami.io/gpu",
+ ResourceMemoryName: "hami.io/gpumem",
+ ResourceMemoryPercentageName: "hami.io/gpumem-percentage",
+ ResourceCoreName: "hami.io/gpucores",
+ DefaultMemory: 0,
+ DefaultCores: 0,
+ DefaultGPUNum: 1,
+ },
+ }
+
+ if err := device.InitDevicesWithConfig(config); err != nil {
+ klog.Fatalf("Failed to initialize devices with config: %v", err)
+ }
+
+ tests := []struct {
+ name string
+ args *corev1.Pod
+ want util.PodDeviceRequests
+ }{
+ {
+ name: "don't resource",
+ args: &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "cpu": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ want: []util.ContainerDeviceRequests{{}},
+ },
+ {
+ name: "one container use gpu",
+ args: &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ want: []util.ContainerDeviceRequests{
+ {
+ nvidia.NvidiaGPUDevice: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ MemPercentagereq: 101,
+ Coresreq: 30,
+ },
+ },
+ },
+ },
+ {
+ name: "two container only one container use gpu",
+ args: &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "cpu": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ want: []util.ContainerDeviceRequests{
+ {
+ nvidia.NvidiaGPUDevice: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ MemPercentagereq: 101,
+ Coresreq: 30,
+ },
+ },
+ {},
+ },
+ },
+ {
+ name: "three containers gpu container first",
+ args: &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "cpu": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "memory": *resource.NewQuantity(2000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ want: []util.ContainerDeviceRequests{
+ {
+ nvidia.NvidiaGPUDevice: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ MemPercentagereq: 101,
+ Coresreq: 30,
+ },
+ },
+ {},
+ {},
+ },
+ },
+ {
+ name: "three containers gpu container in the middle",
+ args: &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "cpu": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "memory": *resource.NewQuantity(2000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ want: []util.ContainerDeviceRequests{
+ {},
+ {
+ nvidia.NvidiaGPUDevice: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ MemPercentagereq: 101,
+ Coresreq: 30,
+ },
+ },
+ {},
+ },
+ },
+ {
+ name: "three containers gpu container last",
+ args: &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "cpu": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "memory": *resource.NewQuantity(2000, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ want: []util.ContainerDeviceRequests{
+ {},
+ {},
+ {
+ nvidia.NvidiaGPUDevice: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ MemPercentagereq: 101,
+ Coresreq: 30,
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ got := Resourcereqs(test.args)
+ assert.DeepEqual(t, test.want, got)
+ })
+ }
+}
+func Test_IsPodInTerminatedState(t *testing.T) {
+ tests := []struct {
+ name string
+ args *corev1.Pod
+ want bool
+ }{
+ {
+ name: "pod in failed state",
+ args: &corev1.Pod{
+ Status: corev1.PodStatus{
+ Phase: corev1.PodFailed,
+ },
+ },
+ want: true,
+ },
+ {
+ name: "pod in succeeded state",
+ args: &corev1.Pod{
+ Status: corev1.PodStatus{
+ Phase: corev1.PodSucceeded,
+ },
+ },
+ want: true,
+ },
+ {
+ name: "pod in running state",
+ args: &corev1.Pod{
+ Status: corev1.PodStatus{
+ Phase: corev1.PodRunning,
+ },
+ },
+ want: false,
+ },
+ {
+ name: "pod in pending state",
+ args: &corev1.Pod{
+ Status: corev1.PodStatus{
+ Phase: corev1.PodPending,
+ },
+ },
+ want: false,
+ },
+ {
+ name: "pod in unknown state",
+ args: &corev1.Pod{
+ Status: corev1.PodStatus{
+ Phase: corev1.PodUnknown,
+ },
+ },
+ want: false,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ got := IsPodInTerminatedState(test.args)
+ assert.Equal(t, test.want, got)
+ })
+ }
+}
+func Test_AllContainersCreated(t *testing.T) {
+ tests := []struct {
+ name string
+ args *corev1.Pod
+ want bool
+ }{
+ {
+ name: "all containers created",
+ args: &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {},
+ {},
+ },
+ },
+ Status: corev1.PodStatus{
+ ContainerStatuses: []corev1.ContainerStatus{
+ {},
+ {},
+ },
+ },
+ },
+ want: true,
+ },
+ {
+ name: "not all containers created",
+ args: &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {},
+ {},
+ },
+ },
+ Status: corev1.PodStatus{
+ ContainerStatuses: []corev1.ContainerStatus{
+ {},
+ },
+ },
+ },
+ want: false,
+ },
+ {
+ name: "no containers created",
+ args: &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {},
+ },
+ },
+ Status: corev1.PodStatus{
+ ContainerStatuses: []corev1.ContainerStatus{},
+ },
+ },
+ want: false,
+ },
+ {
+ name: "more container statuses than containers",
+ args: &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {},
+ },
+ },
+ Status: corev1.PodStatus{
+ ContainerStatuses: []corev1.ContainerStatus{
+ {},
+ {},
+ },
+ },
+ },
+ want: true,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ got := AllContainersCreated(test.args)
+ assert.Equal(t, test.want, got)
+ })
+ }
+}
diff --git a/pkg/monitor/nvidia/cudevshr.go b/pkg/monitor/nvidia/cudevshr.go
new file mode 100644
index 000000000..8b6ce5aa6
--- /dev/null
+++ b/pkg/monitor/nvidia/cudevshr.go
@@ -0,0 +1,336 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nvidia
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "syscall"
+ "time"
+ "unsafe"
+
+ v0 "github.com/Project-HAMi/HAMi/pkg/monitor/nvidia/v0"
+ v1 "github.com/Project-HAMi/HAMi/pkg/monitor/nvidia/v1"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/client-go/informers"
+ "k8s.io/client-go/kubernetes"
+ corelisters "k8s.io/client-go/listers/core/v1"
+ "k8s.io/client-go/tools/cache"
+ "k8s.io/client-go/tools/clientcmd"
+ "k8s.io/klog/v2"
+)
+
+const SharedRegionMagicFlag = 19920718
+
+type headerT struct {
+ initializedFlag int32
+ majorVersion int32
+ minorVersion int32
+}
+
+type UsageInfo interface {
+ DeviceMax() int
+ DeviceNum() int
+ DeviceMemoryContextSize(idx int) uint64
+ DeviceMemoryModuleSize(idx int) uint64
+ DeviceMemoryBufferSize(idx int) uint64
+ DeviceMemoryOffset(idx int) uint64
+ DeviceMemoryTotal(idx int) uint64
+ DeviceSmUtil(idx int) uint64
+ SetDeviceSmLimit(l uint64)
+ IsValidUUID(idx int) bool
+ DeviceUUID(idx int) string
+ DeviceMemoryLimit(idx int) uint64
+ SetDeviceMemoryLimit(l uint64)
+ LastKernelTime() int64
+ //UsedMemory(idx int) (uint64, error)
+ GetPriority() int
+ GetRecentKernel() int32
+ SetRecentKernel(v int32)
+ GetUtilizationSwitch() int32
+ SetUtilizationSwitch(v int32)
+}
+
+type ContainerUsage struct {
+ PodUID string
+ ContainerName string
+ data []byte
+ Info UsageInfo
+}
+
+type ContainerLister struct {
+ containerPath string
+ containers map[string]*ContainerUsage
+ mutex sync.Mutex
+ clientset *kubernetes.Clientset
+ nodeName string
+
+ // Fields for the informer-based pod cache mechanism
+ informerFactory informers.SharedInformerFactory
+ podInformer cache.SharedIndexInformer
+ podLister corelisters.PodLister
+ podListerSynced cache.InformerSynced
+ stopCh chan struct{}
+}
+
+var resyncInterval time.Duration = 5 * time.Minute
+
+func init() {
+ if os.Getenv("HAMI_RESYNC_INTERVAL") != "" {
+ // If HAMI_RESYNC_INTERVAL is set, parse it
+ if interval, err := time.ParseDuration(os.Getenv("HAMI_RESYNC_INTERVAL")); err == nil {
+ resyncInterval = interval
+ } else {
+ klog.Warningf("Invalid HAMI_RESYNC_INTERVAL value: %s, using default %v", os.Getenv("HAMI_RESYNC_INTERVAL"), resyncInterval)
+ }
+ }
+}
+
+func NewContainerLister() (*ContainerLister, error) {
+ hookPath, ok := os.LookupEnv("HOOK_PATH")
+ if !ok {
+ return nil, fmt.Errorf("HOOK_PATH not set")
+ }
+ config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG"))
+ if err != nil {
+ klog.Errorf("Failed to build kubeconfig: %v", err)
+ return nil, err
+ }
+ clientset, err := kubernetes.NewForConfig(config)
+ if err != nil {
+ klog.Errorf("Failed to build clientset: %v", err)
+ return nil, err
+ }
+
+ nodeName := os.Getenv(util.NodeNameEnvName)
+ if nodeName == "" {
+ return nil, fmt.Errorf("env %s not set", util.NodeNameEnvName)
+ }
+
+ lister := &ContainerLister{
+ containerPath: filepath.Join(hookPath, "containers"),
+ containers: make(map[string]*ContainerUsage),
+ clientset: clientset,
+ nodeName: nodeName,
+ stopCh: make(chan struct{}),
+ }
+
+ // Initialize the informer
+ if err := lister.initInformerWithConfig(resyncInterval); err != nil {
+ return nil, err
+ }
+
+ return lister, nil
+}
+
+func (l *ContainerLister) Lock() {
+ l.mutex.Lock()
+}
+
+func (l *ContainerLister) UnLock() {
+ l.mutex.Unlock()
+}
+
+func (l *ContainerLister) ListContainers() map[string]*ContainerUsage {
+ return l.containers
+}
+
+func (l *ContainerLister) Clientset() *kubernetes.Clientset {
+ return l.clientset
+}
+
+func (l *ContainerLister) Update() error {
+
+ l.mutex.Lock()
+ defer l.mutex.Unlock()
+
+ entries, err := os.ReadDir(l.containerPath)
+ if err != nil {
+ return err
+ }
+
+ pods, err := l.podLister.List(labels.Everything())
+ if err != nil {
+ return fmt.Errorf("failed to list pods: %v", err)
+ }
+
+ podUIDs := make(map[string]bool, len(pods))
+ for _, pod := range pods {
+ podUIDs[string(pod.UID)] = true
+ }
+
+ for _, entry := range entries {
+ if !entry.IsDir() {
+ continue
+ }
+ dirName := filepath.Join(l.containerPath, entry.Name())
+ podUID := strings.Split(entry.Name(), "_")[0]
+ if !podUIDs[podUID] {
+ dirInfo, err := os.Stat(dirName)
+ if err == nil && dirInfo.ModTime().Add(resyncInterval).After(time.Now()) {
+ continue
+ }
+ klog.Infof("Removing dirname %s in monitorpath", dirName)
+ if c, ok := l.containers[entry.Name()]; ok {
+ syscall.Munmap(c.data)
+ delete(l.containers, entry.Name())
+ }
+ _ = os.RemoveAll(dirName)
+ continue
+ }
+ if _, ok := l.containers[entry.Name()]; ok {
+ continue
+ }
+ usage, err := loadCache(dirName)
+ if err != nil {
+ klog.Errorf("Failed to load cache: %s, error: %v", dirName, err)
+ continue
+ }
+ if usage == nil {
+ // no cuInit in container
+ continue
+ }
+ usage.PodUID = podUID
+ usage.ContainerName = strings.Split(entry.Name(), "_")[1]
+ l.containers[entry.Name()] = usage
+ klog.Infof("Adding ctr dirname %s in monitorpath", dirName)
+ }
+ return nil
+}
+
+func loadCache(fpath string) (*ContainerUsage, error) {
+ klog.Infof("Checking path %s", fpath)
+ files, err := os.ReadDir(fpath)
+ if err != nil {
+ return nil, err
+ }
+ if len(files) > 2 {
+ return nil, errors.New("cache num not matched")
+ }
+ if len(files) == 0 {
+ return nil, nil
+ }
+ cacheFile := ""
+ for _, val := range files {
+ if strings.Contains(val.Name(), "libvgpu.so") {
+ continue
+ }
+ if !strings.Contains(val.Name(), ".cache") {
+ continue
+ }
+ cacheFile = filepath.Join(fpath, val.Name())
+ break
+ }
+ if cacheFile == "" {
+ klog.Infof("No cache file in %s", fpath)
+ return nil, nil
+ }
+ info, err := os.Stat(cacheFile)
+ if err != nil {
+ klog.Errorf("Failed to stat cache file: %s, error: %v", cacheFile, err)
+ return nil, err
+ }
+ if info.Size() < int64(unsafe.Sizeof(headerT{})) {
+ return nil, fmt.Errorf("cache file size %d too small", info.Size())
+ }
+ f, err := os.OpenFile(cacheFile, os.O_RDWR, 0666)
+ if err != nil {
+ klog.Errorf("Failed to open cache file: %s, error: %v", cacheFile, err)
+ return nil, err
+ }
+ defer func(f *os.File) {
+ _ = f.Close()
+ }(f)
+ usage := &ContainerUsage{}
+ usage.data, err = syscall.Mmap(int(f.Fd()), 0, int(info.Size()), syscall.PROT_WRITE|syscall.PROT_READ, syscall.MAP_SHARED)
+ if err != nil {
+ klog.Errorf("Failed to mmap cache file: %s, error: %v", cacheFile, err)
+ return nil, err
+ }
+ head := (*headerT)(unsafe.Pointer(&usage.data[0]))
+ if head.initializedFlag != SharedRegionMagicFlag {
+ _ = syscall.Munmap(usage.data)
+ return nil, fmt.Errorf("cache file magic flag not matched")
+ }
+ if info.Size() == 1197897 {
+ klog.Infoln("casting......v0")
+ usage.Info = v0.CastSpec(usage.data)
+ } else if head.majorVersion == 1 {
+ klog.Infoln("casting......v1")
+ usage.Info = v1.CastSpec(usage.data)
+ } else {
+ _ = syscall.Munmap(usage.data)
+ return nil, fmt.Errorf("unknown cache file size %d version %d.%d", info.Size(), head.majorVersion, head.minorVersion)
+ }
+ return usage, nil
+}
+
+func (l *ContainerLister) initInformerWithConfig(resyncInterval time.Duration) error {
+ // Create informer factory with a longer resync period to reduce API calls
+ l.informerFactory = informers.NewSharedInformerFactoryWithOptions(
+ l.clientset,
+ resyncInterval,
+ informers.WithTweakListOptions(func(options *metav1.ListOptions) {
+ options.FieldSelector = fmt.Sprintf("spec.nodeName=%s", l.nodeName)
+ }),
+ )
+
+ podInformer := l.informerFactory.Core().V1().Pods()
+ l.podInformer = podInformer.Informer()
+ l.podLister = podInformer.Lister()
+ l.podListerSynced = l.podInformer.HasSynced
+
+ l.podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+ DeleteFunc: l.onPodDelete,
+ })
+
+ // Start the informer
+ l.informerFactory.Start(l.stopCh)
+
+ // Wait for cache sync
+ if !cache.WaitForCacheSync(l.stopCh, l.podListerSynced) {
+ return fmt.Errorf("failed to sync pod informer cache")
+ }
+
+ klog.Info("Pod informer started successfully")
+ return nil
+}
+
+func (l *ContainerLister) onPodDelete(obj interface{}) {
+ pod, ok := obj.(*corev1.Pod)
+ if !ok {
+ tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+ if !ok {
+ klog.Errorf("couldn't get object from tombstone %+v", obj)
+ return
+ }
+ pod, ok = tombstone.Obj.(*corev1.Pod)
+ if !ok {
+ klog.Errorf("tombstone contained object that is not a Pod: %+v", obj)
+ return
+ }
+ }
+ klog.V(5).Infof("Pod removed: %s/%s", pod.Namespace, pod.Name)
+}
diff --git a/pkg/monitor/nvidia/v0/spec.go b/pkg/monitor/nvidia/v0/spec.go
new file mode 100644
index 000000000..9163e7627
--- /dev/null
+++ b/pkg/monitor/nvidia/v0/spec.go
@@ -0,0 +1,192 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v0
+
+import "unsafe"
+
+const maxDevices = 16
+
+type deviceMemory struct {
+ contextSize uint64
+ moduleSize uint64
+ bufferSize uint64
+ offset uint64
+ total uint64
+}
+
+type deviceUtilization struct {
+ decUtil uint64
+ encUtil uint64
+ smUtil uint64
+}
+
+type shrregProcSlotT struct {
+ pid int32
+ hostpid int32
+ used [16]deviceMemory
+ monitorused [16]uint64
+ deviceUtil [16]deviceUtilization
+ status int32
+}
+
+type uuid struct {
+ uuid [96]byte
+}
+
+type semT struct {
+ sem [32]byte
+}
+
+type sharedRegionT struct {
+ initializedFlag int32
+ smInitFlag int32
+ ownerPid uint32
+ sem semT
+ num uint64
+ uuids [16]uuid
+
+ limit [16]uint64
+ smLimit [16]uint64
+ procs [1024]shrregProcSlotT
+
+ procnum int32
+ utilizationSwitch int32
+ recentKernel int32
+ priority int32
+}
+
+type Spec struct {
+ sr *sharedRegionT
+}
+
+func (s Spec) DeviceMax() int {
+ return maxDevices
+}
+
+func (s Spec) DeviceNum() int {
+ return int(s.sr.num)
+}
+
+func (s Spec) DeviceMemoryContextSize(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.used[idx].contextSize
+ }
+ return v
+}
+
+func (s Spec) DeviceMemoryModuleSize(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.used[idx].moduleSize
+ }
+ return v
+}
+
+func (s Spec) DeviceMemoryBufferSize(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.used[idx].bufferSize
+ }
+ return v
+}
+
+func (s Spec) DeviceMemoryOffset(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.used[idx].offset
+ }
+ return v
+}
+
+func (s Spec) DeviceMemoryTotal(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.used[idx].total
+ }
+ return v
+}
+
+func (s Spec) DeviceSmUtil(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.deviceUtil[idx].smUtil
+ }
+ return v
+}
+
+func (s Spec) SetDeviceSmLimit(l uint64) {
+ idx := uint64(0)
+ for idx < s.sr.num {
+ s.sr.smLimit[idx] = l
+ idx += 1
+ }
+}
+
+func (s Spec) IsValidUUID(idx int) bool {
+ return s.sr.uuids[idx].uuid[0] != 0
+}
+
+func (s Spec) DeviceUUID(idx int) string {
+ return string(s.sr.uuids[idx].uuid[:])
+}
+
+func (s Spec) DeviceMemoryLimit(idx int) uint64 {
+ return s.sr.limit[idx]
+}
+
+func (s Spec) SetDeviceMemoryLimit(l uint64) {
+ idx := uint64(0)
+ for idx < s.sr.num {
+ s.sr.limit[idx] = l
+ idx += 1
+ }
+}
+
+func (s Spec) LastKernelTime() int64 {
+ return 0
+}
+
+func CastSpec(data []byte) Spec {
+ return Spec{
+ sr: (*sharedRegionT)(unsafe.Pointer(&data[0])),
+ }
+}
+
+// func (s *SharedRegionT) UsedMemory(idx int) (uint64, error) {
+// return 0, nil
+// }
+
+func (s Spec) GetPriority() int {
+ return int(s.sr.priority)
+}
+
+func (s Spec) GetRecentKernel() int32 {
+ return s.sr.recentKernel
+}
+
+func (s Spec) SetRecentKernel(v int32) {
+ s.sr.recentKernel = v
+}
+
+func (s Spec) GetUtilizationSwitch() int32 {
+ return s.sr.utilizationSwitch
+}
+
+func (s Spec) SetUtilizationSwitch(v int32) {
+ s.sr.utilizationSwitch = v
+}
diff --git a/pkg/monitor/nvidia/v0/spec_test.go b/pkg/monitor/nvidia/v0/spec_test.go
new file mode 100644
index 000000000..5234a8fe7
--- /dev/null
+++ b/pkg/monitor/nvidia/v0/spec_test.go
@@ -0,0 +1,569 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v0
+
+import (
+ "reflect"
+ "testing"
+)
+
+type specTest struct {
+ name string
+ spec *Spec
+ input int
+ expected any
+}
+
+func TestSpec_DeviceMax(t *testing.T) {
+ tests := []specTest{
+ {name: "max devices is 8", spec: &Spec{sr: &sharedRegionT{num: 8}}, expected: 16},
+ {name: "max devices is 16", spec: &Spec{sr: &sharedRegionT{num: 16}}, expected: 16},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.DeviceMax()
+ if actual != tt.expected {
+ t.Errorf("DeviceMax() = %d, want %d", actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_DeviceNum(t *testing.T) {
+ tests := []specTest{
+ {name: "device num is 4", spec: &Spec{sr: &sharedRegionT{num: 4}}, expected: 4},
+ {name: "device num is 8", spec: &Spec{sr: &sharedRegionT{num: 8}}, expected: 8},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.DeviceNum()
+ if actual != tt.expected {
+ t.Errorf("DeviceNum() = %d, want %d", actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_DeviceMemoryContextSize(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "device memory context size for index 1",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {used: [16]deviceMemory{{contextSize: 100}, {contextSize: 200}}},
+ {used: [16]deviceMemory{{contextSize: 300}, {contextSize: 400}}},
+ },
+ }},
+ input: 1,
+ expected: uint64(600),
+ },
+ {
+ name: "device memory context size for index 0",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {used: [16]deviceMemory{{contextSize: 100}, {contextSize: 200}}},
+ {used: [16]deviceMemory{{contextSize: 300}, {contextSize: 400}}},
+ },
+ }},
+ input: 0,
+ expected: uint64(400),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.DeviceMemoryContextSize(tt.input)
+ if actual != tt.expected {
+ t.Errorf("DeviceMemoryContextSize(%d) = %d, want %d", tt.input, actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_DeviceMemoryModuleSize(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "device memory module size for index 1",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {used: [16]deviceMemory{{moduleSize: 100}, {moduleSize: 200}}},
+ {used: [16]deviceMemory{{moduleSize: 300}, {moduleSize: 400}}},
+ },
+ }},
+ input: 1,
+ expected: uint64(600),
+ },
+ {
+ name: "device memory module size for index 0",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {used: [16]deviceMemory{{moduleSize: 100}, {moduleSize: 200}}},
+ {used: [16]deviceMemory{{moduleSize: 300}, {moduleSize: 400}}},
+ },
+ }},
+ input: 0,
+ expected: uint64(400),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.DeviceMemoryModuleSize(tt.input)
+ if actual != tt.expected {
+ t.Errorf("DeviceMemoryModuleSize(%d) = %d, want %d", tt.input, actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_DeviceMemoryBufferSize(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "device memory buffer size for index 1",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {used: [16]deviceMemory{{bufferSize: 100}, {bufferSize: 200}}},
+ {used: [16]deviceMemory{{bufferSize: 300}, {bufferSize: 400}}},
+ },
+ }},
+ input: 1,
+ expected: uint64(600),
+ },
+ {
+ name: "device memory buffer size for index 0",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {used: [16]deviceMemory{{bufferSize: 100}, {bufferSize: 200}}},
+ {used: [16]deviceMemory{{bufferSize: 300}, {bufferSize: 400}}},
+ },
+ }},
+ input: 0,
+ expected: uint64(400),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.DeviceMemoryBufferSize(tt.input)
+ if actual != tt.expected {
+ t.Errorf("DeviceMemoryBufferSize(%d) = %d, want %d", tt.input, actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_DeviceMemoryOffset(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "device memory offset for index 1",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {used: [16]deviceMemory{{offset: 100}, {offset: 200}}},
+ {used: [16]deviceMemory{{offset: 300}, {offset: 400}}},
+ },
+ }},
+ input: 1,
+ expected: uint64(600),
+ },
+ {
+ name: "device memory offset for index 0",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {used: [16]deviceMemory{{offset: 100}, {offset: 200}}},
+ {used: [16]deviceMemory{{offset: 300}, {offset: 400}}},
+ },
+ }},
+ input: 0,
+ expected: uint64(400),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.DeviceMemoryOffset(tt.input)
+ if actual != tt.expected {
+ t.Errorf("DeviceMemoryOffset(%d) = %d, want %d", tt.input, actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_DeviceMemoryTotal(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "device memory total for index 1",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {used: [16]deviceMemory{{total: 100}, {total: 200}}},
+ {used: [16]deviceMemory{{total: 300}, {total: 400}}},
+ },
+ }},
+ input: 1,
+ expected: uint64(600),
+ },
+ {
+ name: "device memory total for index 0",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {used: [16]deviceMemory{{total: 100}, {total: 200}}},
+ {used: [16]deviceMemory{{total: 300}, {total: 400}}},
+ },
+ }},
+ input: 0,
+ expected: uint64(400),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.DeviceMemoryTotal(tt.input)
+ if actual != tt.expected {
+ t.Errorf("DeviceMemoryTotal(%d) = %d, want %d", tt.input, actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_DeviceSmUtil(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "device sm util for index 1",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {deviceUtil: [16]deviceUtilization{{smUtil: 100}, {smUtil: 200}}},
+ {deviceUtil: [16]deviceUtilization{{smUtil: 300}, {smUtil: 400}}},
+ },
+ }},
+ input: 1,
+ expected: uint64(600),
+ },
+ {
+ name: "device sm util for index 0",
+ spec: &Spec{sr: &sharedRegionT{
+ num: 2,
+ procs: [1024]shrregProcSlotT{
+ {deviceUtil: [16]deviceUtilization{{smUtil: 100}, {smUtil: 200}}},
+ {deviceUtil: [16]deviceUtilization{{smUtil: 300}, {smUtil: 400}}},
+ },
+ }},
+ input: 0,
+ expected: uint64(400),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.DeviceSmUtil(tt.input)
+ if actual != tt.expected {
+ t.Errorf("DeviceSmUtil(%d) = %d, want %d", tt.input, actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestDeviceMemoryLimit(t *testing.T) {
+ testCases := []struct {
+ name string
+ idx int
+ expectedLimit uint64
+ }{
+ {"Test index 0", 0, 1024},
+ {"Test index 1", 1, 2048},
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+
+ mockSR := &sharedRegionT{
+ limit: [16]uint64{1024, 2048, 3072, 4096},
+ }
+
+ s := Spec{
+ sr: mockSR,
+ }
+
+ limit := s.DeviceMemoryLimit(tc.idx)
+
+ if limit != tc.expectedLimit {
+ t.Errorf("DeviceMemoryLimit(%d) = %d; want %d", tc.idx, limit, tc.expectedLimit)
+ }
+ })
+ }
+}
+
+func TestSpec_SetDeviceSmLimit(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "set device sm limit to 1000",
+ spec: &Spec{sr: &sharedRegionT{num: 2}},
+ input: 1000,
+ expected: [16]uint64{1000, 1000},
+ },
+ {
+ name: "set device sm limit to 2000",
+ spec: &Spec{sr: &sharedRegionT{num: 3}},
+ input: 2000,
+ expected: [16]uint64{2000, 2000, 2000},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tt.spec.SetDeviceSmLimit(uint64(tt.input))
+ actual := tt.spec.sr.smLimit
+ expected, ok := tt.expected.([16]uint64)
+ if !ok {
+ t.Errorf("TestSpec_SetDeviceSmLimit: type assertion failed for expected value")
+ }
+ for i := range int(tt.spec.sr.num) {
+ if actual[i] != expected[i] {
+ t.Errorf("SetDeviceSmLimit(%d) failed: actual[%d] = %d, want %d", tt.input, i, actual[i], expected[i])
+ }
+ }
+ })
+ }
+}
+
+func TestSpec_IsValidUUID(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "valid UUID",
+ spec: &Spec{sr: &sharedRegionT{uuids: [16]uuid{{uuid: [96]byte{1}}}}},
+ input: 0,
+ expected: true,
+ },
+ {
+ name: "invalid UUID",
+ spec: &Spec{sr: &sharedRegionT{uuids: [16]uuid{{uuid: [96]byte{0}}}}},
+ input: 0,
+ expected: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.IsValidUUID(tt.input)
+ if actual != tt.expected {
+ t.Errorf("IsValidUUID(%d) = %v, want %v", tt.input, actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_DeviceUUID(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "device UUID for index 0",
+ spec: &Spec{sr: &sharedRegionT{uuids: [16]uuid{{uuid: [96]byte{'a', 'b', 'c', 'd'}}}}},
+ input: 0,
+ expected: "abcd",
+ },
+ {
+ name: "device UUID for index 1",
+ spec: &Spec{sr: &sharedRegionT{uuids: [16]uuid{{uuid: [96]byte{'e', 'f', 'g', 'h'}}, {uuid: [96]byte{'i', 'j', 'k', 'l'}}}}},
+ input: 1,
+ expected: "ijkl",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.DeviceUUID(tt.input)
+ if actual[:4] != tt.expected {
+ t.Errorf("DeviceUUID(%d) = %s, want %s", tt.input, actual[:4], tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_GetPriority(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "get priority",
+ spec: &Spec{
+ sr: &sharedRegionT{priority: 5},
+ },
+ expected: 5,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.GetPriority()
+ if actual != tt.expected {
+ t.Errorf("GetPriority() = %d, want %d", actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_GetRecentKernel(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "get recent kernel",
+ spec: &Spec{
+ sr: &sharedRegionT{recentKernel: 12345},
+ },
+ expected: int32(12345),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.GetRecentKernel()
+ if actual != tt.expected {
+ t.Errorf("GetRecentKernel() = %d, want %d", actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_SetRecentKernel(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "set recent kernel",
+ spec: &Spec{
+ sr: &sharedRegionT{},
+ },
+ input: 67890,
+ expected: int32(67890),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tt.spec.SetRecentKernel(int32(tt.input))
+ actual := tt.spec.GetRecentKernel()
+ if actual != tt.expected {
+ t.Errorf("SetRecentKernel(%d) failed: actual = %d, want %d", tt.input, actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_GetUtilizationSwitch(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "get utilization switch",
+ spec: &Spec{
+ sr: &sharedRegionT{utilizationSwitch: 1},
+ },
+ expected: int32(1),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.GetUtilizationSwitch()
+ if actual != tt.expected {
+ t.Errorf("GetUtilizationSwitch() = %d, want %d", actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_SetUtilizationSwitch(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "set utilization switch",
+ spec: &Spec{
+ sr: &sharedRegionT{},
+ },
+ input: 2,
+ expected: int32(2),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tt.spec.SetUtilizationSwitch(int32(tt.input))
+ actual := tt.spec.GetUtilizationSwitch()
+ if actual != tt.expected {
+ t.Errorf("SetUtilizationSwitch(%d) failed: actual = %d, want %d", tt.input, actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_SetDeviceMemoryLimit(t *testing.T) {
+ tests := []struct {
+ name string
+ spec *Spec
+ input uint64
+ expected []uint64
+ }{
+ {
+ name: "set device memory limit",
+ spec: &Spec{
+ sr: &sharedRegionT{
+ num: 3,
+ limit: [16]uint64{
+ 100, 200, 300,
+ },
+ },
+ },
+ input: 500,
+ expected: []uint64{
+ 500, 500, 500,
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tt.spec.SetDeviceMemoryLimit(tt.input)
+ actual := tt.spec.sr.limit[:tt.spec.sr.num]
+ if !reflect.DeepEqual(actual, tt.expected) {
+ t.Errorf("SetDeviceMemoryLimit(%d) failed: actual = %v, want %v", tt.input, actual, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSpec_LastKernelTime(t *testing.T) {
+ tests := []specTest{
+ {
+ name: "last kernel time",
+ spec: &Spec{
+ sr: &sharedRegionT{},
+ },
+ expected: int64(0),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ actual := tt.spec.LastKernelTime()
+ if actual != tt.expected {
+ t.Errorf("LastKernelTime() = %d, want %d", actual, tt.expected)
+ }
+ })
+ }
+}
diff --git a/pkg/monitor/nvidia/v1/spec.go b/pkg/monitor/nvidia/v1/spec.go
new file mode 100644
index 000000000..5bfcd9363
--- /dev/null
+++ b/pkg/monitor/nvidia/v1/spec.go
@@ -0,0 +1,199 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import "unsafe"
+
+const maxDevices = 16
+
+type deviceMemory struct {
+ contextSize uint64
+ moduleSize uint64
+ bufferSize uint64
+ offset uint64
+ total uint64
+ unused [3]uint64
+}
+
+type deviceUtilization struct {
+ decUtil uint64
+ encUtil uint64
+ smUtil uint64
+ unused [3]uint64
+}
+
+type shrregProcSlotT struct {
+ pid int32
+ hostpid int32
+ used [16]deviceMemory
+ monitorused [16]uint64
+ deviceUtil [16]deviceUtilization
+ status int32
+ unused [3]uint64
+}
+
+type uuid struct {
+ uuid [96]byte
+}
+
+type semT struct {
+ sem [32]byte
+}
+
+type sharedRegionT struct {
+ initializedFlag int32
+ majorVersion int32
+ minorVersion int32
+ smInitFlag int32
+ ownerPid uint32
+ sem semT
+ num uint64
+ uuids [16]uuid
+
+ limit [16]uint64
+ smLimit [16]uint64
+ procs [1024]shrregProcSlotT
+
+ procnum int32
+ utilizationSwitch int32
+ recentKernel int32
+ priority int32
+ lastKernelTime int64
+ unused [4]uint64
+}
+
+type Spec struct {
+ sr *sharedRegionT
+}
+
+func (s Spec) DeviceMax() int {
+ return maxDevices
+}
+
+func (s Spec) DeviceNum() int {
+ return int(s.sr.num)
+}
+
+func (s Spec) DeviceMemoryContextSize(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.used[idx].contextSize
+ }
+ return v
+}
+
+func (s Spec) DeviceMemoryModuleSize(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.used[idx].moduleSize
+ }
+ return v
+}
+
+func (s Spec) DeviceMemoryBufferSize(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.used[idx].bufferSize
+ }
+ return v
+}
+
+func (s Spec) DeviceMemoryOffset(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.used[idx].offset
+ }
+ return v
+}
+
+func (s Spec) DeviceMemoryTotal(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.used[idx].total
+ }
+ return v
+}
+
+func (s Spec) DeviceSmUtil(idx int) uint64 {
+ v := uint64(0)
+ for _, p := range s.sr.procs {
+ v += p.deviceUtil[idx].smUtil
+ }
+ return v
+}
+
+func (s Spec) SetDeviceSmLimit(l uint64) {
+ idx := uint64(0)
+ for idx < s.sr.num {
+ s.sr.smLimit[idx] = l
+ idx += 1
+ }
+}
+
+func (s Spec) IsValidUUID(idx int) bool {
+ return s.sr.uuids[idx].uuid[0] != 0
+}
+
+func (s Spec) DeviceUUID(idx int) string {
+ return string(s.sr.uuids[idx].uuid[:])
+}
+
+func (s Spec) DeviceMemoryLimit(idx int) uint64 {
+ return s.sr.limit[idx]
+}
+
+func (s Spec) SetDeviceMemoryLimit(l uint64) {
+ idx := uint64(0)
+ for idx < s.sr.num {
+ s.sr.limit[idx] = l
+ idx += 1
+ }
+}
+
+func (s Spec) LastKernelTime() int64 {
+ return s.sr.lastKernelTime
+}
+
+func CastSpec(data []byte) Spec {
+ return Spec{
+ sr: (*sharedRegionT)(unsafe.Pointer(&data[0])),
+ }
+}
+
+// func (s *SharedRegionT) UsedMemory(idx int) (uint64, error) {
+// return 0, nil
+// }
+
+func (s Spec) GetPriority() int {
+ return int(s.sr.priority)
+}
+
+func (s Spec) GetRecentKernel() int32 {
+ return s.sr.recentKernel
+}
+
+func (s Spec) SetRecentKernel(v int32) {
+ s.sr.recentKernel = v
+}
+
+func (s Spec) GetUtilizationSwitch() int32 {
+ return s.sr.utilizationSwitch
+}
+
+func (s Spec) SetUtilizationSwitch(v int32) {
+ s.sr.utilizationSwitch = v
+}
diff --git a/pkg/monitor/nvidia/v1/spec_test.go b/pkg/monitor/nvidia/v1/spec_test.go
new file mode 100644
index 000000000..548f403c1
--- /dev/null
+++ b/pkg/monitor/nvidia/v1/spec_test.go
@@ -0,0 +1,1073 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+ "testing"
+
+ "gotest.tools/v3/assert"
+)
+
+func Test_DeviceMax(t *testing.T) {
+ tests := []struct {
+ name string
+ args *Spec
+ want int
+ }{
+ {
+ name: "device max is 8",
+ args: &Spec{
+ sr: &sharedRegionT{
+ num: 8,
+ },
+ },
+ want: maxDevices,
+ },
+ {
+ name: "device max is 16",
+ args: &Spec{
+ sr: &sharedRegionT{
+ num: 16,
+ },
+ },
+ want: maxDevices,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := Spec{}
+ result := s.DeviceMax()
+ if result != test.want {
+ t.Errorf("DeviceMax is %d, want is %d", result, test.want)
+ }
+ })
+ }
+}
+
+func Test_DeviceNum(t *testing.T) {
+ tests := []struct {
+ name string
+ args *Spec
+ want int
+ }{
+ {
+ name: "device num is 2",
+ args: &Spec{
+ sr: &sharedRegionT{
+ num: 2,
+ },
+ },
+ want: int(2),
+ },
+ {
+ name: "device num is 4",
+ args: &Spec{
+ sr: &sharedRegionT{
+ num: 4,
+ },
+ },
+ want: int(4),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := Spec{
+ sr: &sharedRegionT{
+ num: test.args.sr.num,
+ },
+ }
+ result := s.DeviceNum()
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_DeviceMemoryContextSize(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ idx int
+ spec *Spec
+ }
+ want uint64
+ }{
+ {
+ name: "device memory context size for idx 0",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(0),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ used: [16]deviceMemory{
+ {
+ contextSize: 100,
+ },
+ {
+ contextSize: 200,
+ },
+ },
+ },
+ {
+ used: [16]deviceMemory{
+ {
+ contextSize: 100,
+ },
+ {
+ contextSize: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(200),
+ },
+ {
+ name: "device memory context size for idx 1",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(1),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ used: [16]deviceMemory{
+ {
+ contextSize: 100,
+ },
+ {
+ contextSize: 200,
+ },
+ },
+ },
+ {
+ used: [16]deviceMemory{
+ {
+ contextSize: 100,
+ },
+ {
+ contextSize: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(400),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ result := s.DeviceMemoryContextSize(test.args.idx)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_DeviceMemoryModuleSize(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ idx int
+ spec *Spec
+ }
+ want uint64
+ }{
+ {
+ name: "device memory module size for idx 0",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(0),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ used: [16]deviceMemory{
+ {
+ moduleSize: 100,
+ },
+ {
+ moduleSize: 200,
+ },
+ },
+ },
+ {
+ used: [16]deviceMemory{
+ {
+ moduleSize: 100,
+ },
+ {
+ moduleSize: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(200),
+ },
+ {
+ name: "device memory module size for idx 1",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(1),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ used: [16]deviceMemory{
+ {
+ moduleSize: 100,
+ },
+ {
+ moduleSize: 200,
+ },
+ },
+ },
+ {
+ used: [16]deviceMemory{
+ {
+ moduleSize: 100,
+ },
+ {
+ moduleSize: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(400),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ result := s.DeviceMemoryModuleSize(test.args.idx)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_DeviceMemoryBufferSize(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ idx int
+ spec *Spec
+ }
+ want uint64
+ }{
+ {
+ name: "device memory buffer size for idx 0",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(0),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ used: [16]deviceMemory{
+ {
+ bufferSize: 100,
+ },
+ {
+ bufferSize: 200,
+ },
+ },
+ },
+ {
+ used: [16]deviceMemory{
+ {
+ bufferSize: 100,
+ },
+ {
+ bufferSize: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(200),
+ },
+ {
+ name: "device memory module size for idx 1",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(1),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ used: [16]deviceMemory{
+ {
+ bufferSize: 100,
+ },
+ {
+ bufferSize: 200,
+ },
+ },
+ },
+ {
+ used: [16]deviceMemory{
+ {
+ bufferSize: 100,
+ },
+ {
+ bufferSize: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(400),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ result := s.DeviceMemoryBufferSize(test.args.idx)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_DeviceMemoryOffset(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ idx int
+ spec *Spec
+ }
+ want uint64
+ }{
+ {
+ name: "device memory offset for idx 0",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(0),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ used: [16]deviceMemory{
+ {
+ offset: 100,
+ },
+ {
+ offset: 200,
+ },
+ },
+ },
+ {
+ used: [16]deviceMemory{
+ {
+ offset: 100,
+ },
+ {
+ offset: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(200),
+ },
+ {
+ name: "device memory offset for idx 1",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(1),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ used: [16]deviceMemory{
+ {
+ offset: 100,
+ },
+ {
+ offset: 200,
+ },
+ },
+ },
+ {
+ used: [16]deviceMemory{
+ {
+ offset: 100,
+ },
+ {
+ offset: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(400),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ result := s.DeviceMemoryOffset(test.args.idx)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_DeviceMemoryTotal(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ idx int
+ spec *Spec
+ }
+ want uint64
+ }{
+ {
+ name: "device memory total for idx 0",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(0),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ used: [16]deviceMemory{
+ {
+ total: 100,
+ },
+ {
+ total: 200,
+ },
+ },
+ },
+ {
+ used: [16]deviceMemory{
+ {
+ total: 100,
+ },
+ {
+ total: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(200),
+ },
+ {
+ name: "device memory total for idx 1",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(1),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ used: [16]deviceMemory{
+ {
+ total: 100,
+ },
+ {
+ total: 200,
+ },
+ },
+ },
+ {
+ used: [16]deviceMemory{
+ {
+ total: 100,
+ },
+ {
+ total: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(400),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ result := s.DeviceMemoryTotal(test.args.idx)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_DeviceSmUtil(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ idx int
+ spec *Spec
+ }
+ want uint64
+ }{
+ {
+ name: "device sm util for idx 0",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(0),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ deviceUtil: [16]deviceUtilization{
+ {
+ smUtil: 100,
+ },
+ {
+ smUtil: 200,
+ },
+ },
+ },
+ {
+ deviceUtil: [16]deviceUtilization{
+ {
+ smUtil: 100,
+ },
+ {
+ smUtil: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(200),
+ },
+ {
+ name: "device sm util for idx 1",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: int(1),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ procs: [1024]shrregProcSlotT{
+ {
+ deviceUtil: [16]deviceUtilization{
+ {
+ smUtil: 100,
+ },
+ {
+ smUtil: 200,
+ },
+ },
+ },
+ {
+ deviceUtil: [16]deviceUtilization{
+ {
+ smUtil: 100,
+ },
+ {
+ smUtil: 200,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ want: uint64(400),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ result := s.DeviceSmUtil(test.args.idx)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_SetDeviceSmLimit(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ l uint64
+ spec *Spec
+ }
+ want [16]uint64
+ }{
+ {
+ name: "set device sm limit to 300",
+ args: struct {
+ l uint64
+ spec *Spec
+ }{
+ l: uint64(300),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ num: 2,
+ smLimit: [16]uint64{},
+ },
+ },
+ },
+ want: [16]uint64{300, 300},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ s.SetDeviceSmLimit(test.args.l)
+ result := test.args.spec.sr.smLimit
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func Test_IsValidUUID(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ idx int
+ spec *Spec
+ }
+ want bool
+ }{
+ {
+ name: "set vaild uuid",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: 0,
+ spec: &Spec{
+ sr: &sharedRegionT{
+ uuids: [16]uuid{
+ {
+ uuid: [96]byte{
+ 1,
+ },
+ },
+ },
+ },
+ },
+ },
+ want: true,
+ },
+ {
+ name: "set invaild uuid",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: 0,
+ spec: &Spec{
+ sr: &sharedRegionT{
+ uuids: [16]uuid{
+ {
+ uuid: [96]byte{
+ 0,
+ },
+ },
+ },
+ },
+ },
+ },
+ want: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ result := s.IsValidUUID(test.args.idx)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_DeviceUUID(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ idx int
+ spec *Spec
+ }
+ want string
+ }{
+ {
+ name: "device uuid for idx 0",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: 0,
+ spec: &Spec{
+ sr: &sharedRegionT{
+ uuids: [16]uuid{
+ {
+ uuid: [96]byte{
+ 'a', '1', 'b', '2',
+ },
+ },
+ },
+ },
+ },
+ },
+ want: "a1b2",
+ },
+ {
+ name: "device uuid for idx 1",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: 1,
+ spec: &Spec{
+ sr: &sharedRegionT{
+ uuids: [16]uuid{
+ {
+ uuid: [96]byte{
+ 'a', '1', 'b', '2',
+ },
+ },
+ {
+ uuid: [96]byte{
+ 'c', '3', 'd', '4',
+ },
+ },
+ },
+ },
+ },
+ },
+ want: "c3d4",
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ result := s.DeviceUUID(test.args.idx)
+ assert.Equal(t, result[:4], test.want)
+ })
+ }
+}
+
+func Test_DeviceMemoryLimit(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ idx int
+ spec *Spec
+ }
+ want uint64
+ }{
+ {
+ name: "device memory limit for idx 0",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: 0,
+ spec: &Spec{
+ sr: &sharedRegionT{
+ limit: [16]uint64{
+ 100,
+ },
+ },
+ },
+ },
+ want: uint64(100),
+ },
+ {
+ name: "device memory limit for idx 1",
+ args: struct {
+ idx int
+ spec *Spec
+ }{
+ idx: 1,
+ spec: &Spec{
+ sr: &sharedRegionT{
+ limit: [16]uint64{
+ 100, 200,
+ },
+ },
+ },
+ },
+ want: uint64(200),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ result := s.DeviceMemoryLimit(test.args.idx)
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_SetDeviceMemoryLimit(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ l uint64
+ spec *Spec
+ }
+ want [16]uint64
+ }{
+ {
+ name: "set device memory limit to 1024",
+ args: struct {
+ l uint64
+ spec *Spec
+ }{
+ l: uint64(1024),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ num: 1,
+ },
+ },
+ },
+ want: [16]uint64{1024},
+ },
+ {
+ name: "set device memory limit to 2048",
+ args: struct {
+ l uint64
+ spec *Spec
+ }{
+ l: uint64(2048),
+ spec: &Spec{
+ sr: &sharedRegionT{
+ num: 2,
+ },
+ },
+ },
+ want: [16]uint64{2048, 2048},
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ s.SetDeviceMemoryLimit(test.args.l)
+ result := test.args.spec.sr.limit
+ assert.DeepEqual(t, result, test.want)
+ })
+ }
+}
+
+func Test_LastKernelTime(t *testing.T) {
+ tests := []struct {
+ name string
+ args *Spec
+ want int64
+ }{
+ {
+ name: "last kernel time",
+ args: &Spec{
+ sr: &sharedRegionT{
+ lastKernelTime: int64(1234),
+ },
+ },
+ want: int64(1234),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args
+ result := s.LastKernelTime()
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_GetPriority(t *testing.T) {
+ tests := []struct {
+ name string
+ args Spec
+ want int
+ }{
+ {
+ name: "get priority",
+ args: Spec{
+ sr: &sharedRegionT{
+ priority: int32(1),
+ },
+ },
+ want: int(1),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args
+ result := s.GetPriority()
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_GetRecentKernel(t *testing.T) {
+ tests := []struct {
+ name string
+ args Spec
+ want int32
+ }{
+ {
+ name: "get recent kernel",
+ args: Spec{
+ sr: &sharedRegionT{
+ recentKernel: int32(1234),
+ },
+ },
+ want: int32(1234),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args
+ result := s.GetRecentKernel()
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_SetRecentKernel(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ v int32
+ spec Spec
+ }
+ want int32
+ }{
+ {
+ name: "get recent kernel",
+ args: struct {
+ v int32
+ spec Spec
+ }{
+ v: int32(1111),
+ spec: Spec{
+ sr: &sharedRegionT{},
+ },
+ },
+ want: int32(1111),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := test.args.spec
+ s.SetRecentKernel(test.args.v)
+ result := test.args.spec.sr.recentKernel
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_GetUtilizationSwitch(t *testing.T) {
+ tests := []struct {
+ name string
+ args Spec
+ want int32
+ }{
+ {
+ name: "get utilzation switch",
+ args: Spec{
+ sr: &sharedRegionT{
+ utilizationSwitch: int32(1234),
+ },
+ },
+ want: int32(1234),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result := test.args.GetUtilizationSwitch()
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
+
+func Test_SetUtilizationSwitch(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ v int32
+ spec Spec
+ }
+ want int32
+ }{
+ {
+ name: "set utilzation switch",
+ args: struct {
+ v int32
+ spec Spec
+ }{
+ v: int32(3333),
+ spec: Spec{
+ sr: &sharedRegionT{},
+ },
+ },
+ want: int32(3333),
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ test.args.spec.SetUtilizationSwitch(test.args.v)
+ result := test.args.spec.sr.utilizationSwitch
+ assert.Equal(t, result, test.want)
+ })
+ }
+}
diff --git a/pkg/oci/runtime.go b/pkg/oci/runtime.go
new file mode 100644
index 000000000..89df5aa1d
--- /dev/null
+++ b/pkg/oci/runtime.go
@@ -0,0 +1,23 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+// Runtime is an interface for a runtime shim. The Exec method accepts a list
+// of command line arguments, and returns an error / nil.
+type Runtime interface {
+ Exec([]string) error
+}
diff --git a/pkg/oci/runtime_exec.go b/pkg/oci/runtime_exec.go
new file mode 100644
index 000000000..74086c51c
--- /dev/null
+++ b/pkg/oci/runtime_exec.go
@@ -0,0 +1,79 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+ "fmt"
+ "os"
+ "syscall"
+
+ log "github.com/sirupsen/logrus"
+)
+
+// SyscallExecRuntime wraps the path that a binary and defines the semantics for how to exec into it.
+// This can be used to wrap an OCI-compliant low-level runtime binary, allowing it to be used through the
+// Runtime internface.
+type SyscallExecRuntime struct {
+ logger *log.Logger
+ path string
+ // exec is used for testing. This defaults to syscall.Exec
+ exec func(argv0 string, argv []string, envv []string) error
+}
+
+var _ Runtime = (*SyscallExecRuntime)(nil)
+
+// NewSyscallExecRuntime creates a SyscallExecRuntime for the specified path with the standard logger.
+func NewSyscallExecRuntime(path string) (Runtime, error) {
+ return NewSyscallExecRuntimeWithLogger(log.StandardLogger(), path)
+}
+
+// NewSyscallExecRuntimeWithLogger creates a SyscallExecRuntime for the specified logger and path.
+func NewSyscallExecRuntimeWithLogger(logger *log.Logger, path string) (Runtime, error) {
+ info, err := os.Stat(path)
+ if err != nil {
+ return nil, fmt.Errorf("invalid path '%v': %v", path, err)
+ }
+ if info.IsDir() || info.Mode()&0111 == 0 {
+ return nil, fmt.Errorf("specified path '%v' is not an executable file", path)
+ }
+
+ shim := SyscallExecRuntime{
+ logger: logger,
+ path: path,
+ exec: syscall.Exec,
+ }
+
+ return &shim, nil
+}
+
+// Exec exces into the binary at the path from the SyscallExecRuntime struct, passing it the supplied arguments
+// after ensuring that the first argument is the path of the target binary.
+func (s SyscallExecRuntime) Exec(args []string) error {
+ runtimeArgs := []string{s.path}
+ if len(args) > 1 {
+ runtimeArgs = append(runtimeArgs, args[1:]...)
+ }
+
+ err := s.exec(s.path, runtimeArgs, os.Environ())
+ if err != nil {
+ return fmt.Errorf("could not exec '%v': %v", s.path, err)
+ }
+
+ // syscall.Exec is not expected to return. This is an error state regardless of whether
+ // err is nil or not.
+ return fmt.Errorf("unexpected return from exec '%v'", s.path)
+}
diff --git a/pkg/oci/runtime_exec_test.go b/pkg/oci/runtime_exec_test.go
new file mode 100644
index 000000000..83ac64a2e
--- /dev/null
+++ b/pkg/oci/runtime_exec_test.go
@@ -0,0 +1,100 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+package oci
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ testlog "github.com/sirupsen/logrus/hooks/test"
+ "github.com/stretchr/testify/require"
+)
+
+func TestSyscallExecConstructor(t *testing.T) {
+ r, err := NewSyscallExecRuntime("////an/invalid/path")
+ require.Error(t, err)
+ require.Nil(t, r)
+
+ r, err = NewSyscallExecRuntime("/tmp")
+ require.Error(t, err)
+ require.Nil(t, r)
+
+ r, err = NewSyscallExecRuntime("/dev/null")
+ require.Error(t, err)
+ require.Nil(t, r)
+
+ r, err = NewSyscallExecRuntime("/bin/sh")
+ require.NoError(t, err)
+
+ f, ok := r.(*SyscallExecRuntime)
+ require.True(t, ok)
+
+ require.Equal(t, "/bin/sh", f.path)
+}
+
+func TestSyscallExecForwardsArgs(t *testing.T) {
+ logger, _ := testlog.NewNullLogger()
+ f := SyscallExecRuntime{
+ logger: logger,
+ path: "runtime",
+ }
+
+ testCases := []struct {
+ returnError error
+ args []string
+ errorPrefix string
+ }{
+ {
+ returnError: nil,
+ errorPrefix: "unexpected return from exec",
+ },
+ {
+ returnError: fmt.Errorf("error from exec"),
+ errorPrefix: "could not exec",
+ },
+ {
+ returnError: nil,
+ args: []string{"otherargv0"},
+ errorPrefix: "unexpected return from exec",
+ },
+ {
+ returnError: nil,
+ args: []string{"otherargv0", "arg1", "arg2", "arg3"},
+ errorPrefix: "unexpected return from exec",
+ },
+ }
+
+ for i, tc := range testCases {
+ execMock := WithMockExec(f, tc.returnError)
+
+ err := execMock.Exec(tc.args)
+
+ require.Errorf(t, err, "%d: %v", i, tc)
+ require.Truef(t, strings.HasPrefix(err.Error(), tc.errorPrefix), "%d: %v", i, tc)
+ if tc.returnError != nil {
+ require.Truef(t, strings.HasSuffix(err.Error(), tc.returnError.Error()), "%d: %v", i, tc)
+ }
+
+ require.Equalf(t, f.path, execMock.argv0, "%d: %v", i, tc)
+ require.Equalf(t, f.path, execMock.argv[0], "%d: %v", i, tc)
+
+ require.LessOrEqualf(t, len(tc.args), len(execMock.argv), "%d: %v", i, tc)
+ if len(tc.args) > 1 {
+ require.Equalf(t, tc.args[1:], execMock.argv[1:], "%d: %v", i, tc)
+ }
+ }
+}
diff --git a/pkg/oci/runtime_mock.go b/pkg/oci/runtime_mock.go
new file mode 100644
index 000000000..fc48efd87
--- /dev/null
+++ b/pkg/oci/runtime_mock.go
@@ -0,0 +1,49 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+// MockExecRuntime wraps a SyscallExecRuntime, intercepting the exec call for testing.
+type MockExecRuntime struct {
+ SyscallExecRuntime
+ execMock
+}
+
+// WithMockExec wraps a specified SyscallExecRuntime with a mocked exec function for testing.
+func WithMockExec(e SyscallExecRuntime, execResult error) *MockExecRuntime {
+ m := MockExecRuntime{
+ SyscallExecRuntime: e,
+ execMock: execMock{result: execResult},
+ }
+ // overrdie the exec function to the mocked exec function.
+ m.SyscallExecRuntime.exec = m.execMock.exec
+ return &m
+}
+
+type execMock struct {
+ argv0 string
+ argv []string
+ envv []string
+ result error
+}
+
+func (m *execMock) exec(argv0 string, argv []string, envv []string) error {
+ m.argv0 = argv0
+ m.argv = argv
+ m.envv = envv
+
+ return m.result
+}
diff --git a/pkg/oci/spec.go b/pkg/oci/spec.go
new file mode 100644
index 000000000..e163ee1b7
--- /dev/null
+++ b/pkg/oci/spec.go
@@ -0,0 +1,102 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+
+ oci "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// SpecModifier is a function that accepts a pointer to an OCI Srec and returns an
+// error. The intention is that the function would modify the spec in-place.
+type SpecModifier func(*oci.Spec) error
+
+// Spec defines the operations to be performed on an OCI specification.
+type Spec interface {
+ Load() error
+ Flush() error
+ Modify(SpecModifier) error
+}
+
+type fileSpec struct {
+ *oci.Spec
+ path string
+}
+
+var _ Spec = (*fileSpec)(nil)
+
+// NewSpecFromFile creates an object that encapsulates a file-backed OCI spec.
+// This can be used to read from the file, modify the spec, and write to the
+// same file.
+func NewSpecFromFile(filepath string) Spec {
+ oci := fileSpec{
+ path: filepath,
+ }
+
+ return &oci
+}
+
+// Load reads the contents of an OCI spec from file to be referenced internally.
+// The file is opened "read-only".
+func (s *fileSpec) Load() error {
+ specFile, err := os.Open(s.path)
+ if err != nil {
+ return fmt.Errorf("error opening OCI specification file: %v", err)
+ }
+ defer specFile.Close()
+
+ decoder := json.NewDecoder(specFile)
+
+ var spec oci.Spec
+ err = decoder.Decode(&spec)
+ if err != nil {
+ return fmt.Errorf("error reading OCI specification from file: %v", err)
+ }
+
+ s.Spec = &spec
+ return nil
+}
+
+// Modify applies the specified SpecModifier to the stored OCI specification.
+func (s *fileSpec) Modify(f SpecModifier) error {
+ if s.Spec == nil {
+ return fmt.Errorf("no spec loaded for modification")
+ }
+ return f(s.Spec)
+}
+
+// Flush writes the stored OCI specification to the filepath specified by the path member.
+// The file is truncated upon opening, overwriting any existing contents.
+func (s fileSpec) Flush() error {
+ specFile, err := os.Create(s.path)
+ if err != nil {
+ return fmt.Errorf("error opening OCI specification file: %v", err)
+ }
+ defer specFile.Close()
+
+ encoder := json.NewEncoder(specFile)
+
+ err = encoder.Encode(s.Spec)
+ if err != nil {
+ return fmt.Errorf("error writing OCI specification to file: %v", err)
+ }
+
+ return nil
+}
diff --git a/pkg/oci/spec_mock.go b/pkg/oci/spec_mock.go
new file mode 100644
index 000000000..544d184c7
--- /dev/null
+++ b/pkg/oci/spec_mock.go
@@ -0,0 +1,70 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+ oci "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// MockSpec provides a simple mock for an OCI spec to be used in testing.
+// It also implements the SpecModifier interface.
+type MockSpec struct {
+ *oci.Spec
+ MockLoad mockFunc
+ MockFlush mockFunc
+ MockModify mockFunc
+}
+
+var _ Spec = (*MockSpec)(nil)
+
+// NewMockSpec constructs a MockSpec to be used in testing as a Spec.
+func NewMockSpec(spec *oci.Spec, flushResult error, modifyResult error) *MockSpec {
+ s := MockSpec{
+ Spec: spec,
+ MockFlush: mockFunc{result: flushResult},
+ MockModify: mockFunc{result: modifyResult},
+ }
+
+ return &s
+}
+
+// Load invokes the mocked Load function to return the predefined error / result.
+func (s *MockSpec) Load() error {
+ return s.MockLoad.call()
+}
+
+// Flush invokes the mocked Load function to return the predefined error / result.
+func (s *MockSpec) Flush() error {
+ return s.MockFlush.call()
+}
+
+// Modify applies the specified SpecModifier to the spec and invokes the
+// mocked modify function to return the predefined error / result.
+func (s *MockSpec) Modify(f SpecModifier) error {
+ f(s.Spec)
+ return s.MockModify.call()
+}
+
+type mockFunc struct {
+ Callcount int
+ result error
+}
+
+func (m *mockFunc) call() error {
+ m.Callcount++
+ return m.result
+}
diff --git a/pkg/scheduler/config/config.go b/pkg/scheduler/config/config.go
index f706d1470..20545e8f3 100644
--- a/pkg/scheduler/config/config.go
+++ b/pkg/scheduler/config/config.go
@@ -1,23 +1,50 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package config
+import (
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
var (
- HttpBind string
- GrpcBind string
- SchedulerName string
+ QPS float32
+ Burst int
+ Timeout int
+ HTTPBind string
+ SchedulerName string
+ MetricsBindAddress string
+
+ DefaultMem int32
+ DefaultCores int32
+ DefaultResourceNum int32
+
+ // NodeSchedulerPolicy is config this scheduler node to use `binpack` or `spread`. default value is binpack.
+ NodeSchedulerPolicy = util.NodeSchedulerPolicyBinpack.String()
+ // GPUSchedulerPolicy is config this scheduler GPU to use `binpack` or `spread`. default value is spread.
+ GPUSchedulerPolicy = util.GPUSchedulerPolicySpread.String()
+
+ // NodeLabelSelector is scheduler filter node by node label.
+ NodeLabelSelector map[string]string
+
+ // NodeLockTimeout is the timeout for node locks.
+ NodeLockTimeout time.Duration
+
+ // If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it, default value is true.
+ ForceOverwriteDefaultScheduler bool
)
diff --git a/pkg/scheduler/event.go b/pkg/scheduler/event.go
new file mode 100644
index 000000000..1e100e4eb
--- /dev/null
+++ b/pkg/scheduler/event.go
@@ -0,0 +1,77 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scheduler
+
+import (
+ "fmt"
+
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/config"
+
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+ v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+ "k8s.io/client-go/tools/record"
+)
+
+// Define events for ResourceBinding, ResourceFilter objects and their associated resources.
+const (
+ // EventReasonFilteringFailed indicates that filtering failed.
+ EventReasonFilteringFailed = "FilteringFailed"
+ // EventReasonFilteringSucceed indicates that filtering succeed.
+ EventReasonFilteringSucceed = "FilteringSucceed"
+
+ // EventReasonBindingFailed indicates that binding failed.
+ EventReasonBindingFailed = "BindingFailed"
+ // EventReasonBindingSucceed indicates that binding succeed.
+ EventReasonBindingSucceed = "BindingSucceed"
+)
+
+func (s *Scheduler) addAllEventHandlers() {
+
+ eventBroadcaster := record.NewBroadcaster()
+ eventBroadcaster.StartStructuredLogging(0)
+ eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: s.kubeClient.CoreV1().Events(metav1.NamespaceAll)})
+ schema := runtime.NewScheme()
+
+ _ = clientgoscheme.AddToScheme(schema)
+ s.eventRecorder = eventBroadcaster.NewRecorder(schema, corev1.EventSource{Component: config.SchedulerName})
+}
+
+func (s *Scheduler) recordScheduleBindingResultEvent(pod *corev1.Pod, eventReason string, nodeResult []string, schedulerErr error) {
+ if pod == nil {
+ return
+ }
+ if schedulerErr == nil {
+ successMsg := fmt.Sprintf("Successfully binding node %v to %v/%v", nodeResult, pod.Namespace, pod.Name)
+ s.eventRecorder.Event(pod, corev1.EventTypeNormal, eventReason, successMsg)
+ } else {
+ s.eventRecorder.Event(pod, corev1.EventTypeWarning, eventReason, schedulerErr.Error())
+ }
+}
+
+func (s *Scheduler) recordScheduleFilterResultEvent(pod *corev1.Pod, eventReason string, successMsg string, schedulerErr error) {
+ if pod == nil {
+ return
+ }
+ if schedulerErr == nil {
+ s.eventRecorder.Event(pod, corev1.EventTypeNormal, eventReason, successMsg)
+ } else {
+ s.eventRecorder.Event(pod, corev1.EventTypeWarning, eventReason, schedulerErr.Error())
+ }
+}
diff --git a/pkg/scheduler/event_test.go b/pkg/scheduler/event_test.go
new file mode 100644
index 000000000..679a652f8
--- /dev/null
+++ b/pkg/scheduler/event_test.go
@@ -0,0 +1,206 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scheduler
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/client-go/kubernetes/fake"
+ v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+ "k8s.io/client-go/tools/record"
+)
+
+func TestRecordScheduleBindingResultEvent(t *testing.T) {
+ tests := []struct {
+ name string
+ pod *corev1.Pod
+ eventReason string
+ nodeResult []string
+ schedulerErr error
+ wantEventType string
+ }{
+ {
+ name: "no pod",
+ pod: nil,
+ },
+ {
+ name: "schedule success",
+ pod: &corev1.Pod{
+ TypeMeta: metav1.TypeMeta{
+ Kind: "Pod",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "default",
+ },
+ },
+ nodeResult: []string{"node-1"},
+ schedulerErr: nil,
+ wantEventType: corev1.EventTypeNormal,
+ },
+ {
+ name: "schedule failed",
+ pod: &corev1.Pod{
+ TypeMeta: metav1.TypeMeta{
+ Kind: "Pod",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "default",
+ },
+ },
+ eventReason: "FailedBinding",
+ schedulerErr: fmt.Errorf("schedule failed"),
+ wantEventType: corev1.EventTypeWarning,
+ },
+ }
+ for _, test := range tests {
+ fakeClient := fake.NewSimpleClientset()
+ eventBroadcaster := record.NewBroadcaster()
+ eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: fakeClient.CoreV1().Events(metav1.NamespaceAll)})
+
+ s := &Scheduler{
+ kubeClient: fakeClient,
+ eventRecorder: eventBroadcaster.NewRecorder(runtime.NewScheme(), corev1.EventSource{Component: "test-fake-scheduler"}),
+ }
+
+ t.Run(test.name, func(t *testing.T) {
+ s.recordScheduleBindingResultEvent(test.pod, test.eventReason, test.nodeResult, test.schedulerErr)
+
+ var events *corev1.EventList
+ var err error
+
+ if test.pod != nil {
+ for range 5 {
+ events, err = fakeClient.CoreV1().Events(test.pod.Namespace).List(context.Background(), metav1.ListOptions{})
+ if err != nil {
+ if len(events.Items) > 0 {
+ break
+ }
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ if err != nil {
+ t.Errorf("get event list err: %v", err)
+ }
+ event := events.Items[0]
+ if test.schedulerErr != nil {
+ assert.Equal(t, event.Type, corev1.EventTypeWarning)
+ } else {
+ assert.Equal(t, event.Type, corev1.EventTypeNormal)
+ }
+ } else {
+ events, _ = fakeClient.CoreV1().Events(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{})
+ assert.Equal(t, len(events.Items), 0)
+ }
+ })
+ }
+}
+
+func TestRecordScheduleFilterResultEvent(t *testing.T) {
+ tests := []struct {
+ name string
+ pod *corev1.Pod
+ eventReason string
+ successMsg string
+ schedulerErr error
+ wantEventType string
+ }{
+ {
+ name: "no pod",
+ pod: nil,
+ },
+ {
+ name: "schedule success",
+ pod: &corev1.Pod{
+ TypeMeta: metav1.TypeMeta{
+ Kind: "Pod",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "default",
+ },
+ },
+ successMsg: "find fit node(node-1)",
+ schedulerErr: nil,
+ wantEventType: corev1.EventTypeNormal,
+ },
+ {
+ name: "schedule failed",
+ pod: &corev1.Pod{
+ TypeMeta: metav1.TypeMeta{
+ Kind: "Pod",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "default",
+ },
+ },
+ eventReason: "FailedBinding",
+ schedulerErr: fmt.Errorf("schedule failed"),
+ wantEventType: corev1.EventTypeWarning,
+ },
+ }
+ for _, test := range tests {
+ fakeClient := fake.NewSimpleClientset()
+ eventBroadcaster := record.NewBroadcaster()
+ eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: fakeClient.CoreV1().Events(metav1.NamespaceAll)})
+
+ s := &Scheduler{
+ kubeClient: fakeClient,
+ eventRecorder: eventBroadcaster.NewRecorder(runtime.NewScheme(), corev1.EventSource{Component: "test-fake-scheduler"}),
+ }
+
+ t.Run(test.name, func(t *testing.T) {
+ s.recordScheduleFilterResultEvent(test.pod, test.eventReason, test.successMsg, test.schedulerErr)
+
+ var events *corev1.EventList
+ var err error
+
+ if test.pod != nil {
+ for range 5 {
+ events, err = fakeClient.CoreV1().Events(test.pod.Namespace).List(context.Background(), metav1.ListOptions{})
+ if err != nil {
+ if len(events.Items) > 0 {
+ break
+ }
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ if err != nil {
+ t.Errorf("get event list err: %v", err)
+ }
+ event := events.Items[0]
+ if test.schedulerErr != nil {
+ assert.Equal(t, event.Type, corev1.EventTypeWarning)
+ } else {
+ assert.Equal(t, event.Type, corev1.EventTypeNormal)
+ }
+ } else {
+ events, _ = fakeClient.CoreV1().Events(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{})
+ assert.Equal(t, len(events.Items), 0)
+ }
+ })
+ }
+}
diff --git a/pkg/scheduler/nodes.go b/pkg/scheduler/nodes.go
index 8b1241888..a268100b5 100644
--- a/pkg/scheduler/nodes.go
+++ b/pkg/scheduler/nodes.go
@@ -1,76 +1,116 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package scheduler
import (
- "fmt"
- "sync"
+ "fmt"
+ "strings"
+ "sync"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/policy"
+ "github.com/Project-HAMi/HAMi/pkg/util"
)
-type DeviceInfo struct {
- ID string
- Count int32
- Health bool
+type NodeUsage struct {
+ Node *corev1.Node
+ Devices policy.DeviceUsageList
}
-type NodeInfo struct {
- ID string
- Devices []DeviceInfo
+type nodeManager struct {
+ nodes map[string]*util.NodeInfo
+ mutex sync.RWMutex
}
-type DeviceUsage struct {
- id string
- used int32
- count int32
- health bool
+func newNodeManager() *nodeManager {
+ return &nodeManager{
+ nodes: make(map[string]*util.NodeInfo),
+ }
}
-type DeviceUsageList []*DeviceUsage
-
-type NodeUsage struct {
- devices DeviceUsageList
+func (m *nodeManager) addNode(nodeID string, nodeInfo *util.NodeInfo) {
+ if nodeInfo == nil || len(nodeInfo.Devices) == 0 {
+ return
+ }
+ m.mutex.Lock()
+ defer m.mutex.Unlock()
+ _, ok := m.nodes[nodeID]
+ if ok {
+ if len(nodeInfo.Devices) > 0 {
+ tmp := make([]util.DeviceInfo, 0, len(nodeInfo.Devices))
+ devices := device.GetDevices()
+ deviceType := ""
+ for _, val := range devices {
+ if strings.Contains(nodeInfo.Devices[0].Type, val.CommonWord()) {
+ deviceType = val.CommonWord()
+ }
+ }
+ for _, val := range m.nodes[nodeID].Devices {
+ if !strings.Contains(val.Type, deviceType) {
+ tmp = append(tmp, val)
+ }
+ }
+ m.nodes[nodeID].Devices = tmp
+ m.nodes[nodeID].Devices = append(m.nodes[nodeID].Devices, nodeInfo.Devices...)
+ }
+ m.nodes[nodeID].Node = nodeInfo.Node
+ } else {
+ m.nodes[nodeID] = nodeInfo
+ }
}
-type nodeManager struct {
- nodes map[string]NodeInfo
- mutex sync.Mutex
-}
+func (m *nodeManager) rmNodeDevices(nodeID string, deviceVendor string) {
+ m.mutex.Lock()
+ defer m.mutex.Unlock()
-func (m *nodeManager) init() {
- m.nodes = make(map[string]NodeInfo)
-}
+ nodeInfo := m.nodes[nodeID]
+ if nodeInfo == nil {
+ return
+ }
+
+ devices := make([]util.DeviceInfo, 0)
+ for _, val := range nodeInfo.Devices {
+ if val.DeviceVendor != deviceVendor {
+ devices = append(devices, val)
+ }
+ }
-func (m *nodeManager) addNode(nodeID string, nodeInfo NodeInfo) {
- m.mutex.Lock()
- defer m.mutex.Unlock()
- m.nodes[nodeID] = nodeInfo
+ if len(devices) == 0 {
+ delete(m.nodes, nodeID)
+ } else {
+ nodeInfo.Devices = devices
+ }
+ klog.InfoS("Removing device from node", "nodeName", nodeID, "deviceVendor", deviceVendor, "remainingDevices", devices)
}
-func (m *nodeManager) delNode(nodeID string) {
- m.mutex.Lock()
- defer m.mutex.Unlock()
- delete(m.nodes, nodeID)
+func (m *nodeManager) GetNode(nodeID string) (*util.NodeInfo, error) {
+ m.mutex.RLock()
+ defer m.mutex.RUnlock()
+ if n, ok := m.nodes[nodeID]; ok {
+ return n, nil
+ }
+ return &util.NodeInfo{}, fmt.Errorf("node %v not found", nodeID)
}
-func (m *nodeManager) GetNode(nodeID string) (NodeInfo, error) {
- m.mutex.Lock()
- defer m.mutex.Unlock()
- if n, ok := m.nodes[nodeID]; ok {
- return n, nil
- }
- return NodeInfo{}, fmt.Errorf("node %v not found", nodeID)
+func (m *nodeManager) ListNodes() (map[string]*util.NodeInfo, error) {
+ m.mutex.RLock()
+ defer m.mutex.RUnlock()
+ return m.nodes, nil
}
diff --git a/pkg/scheduler/nodes_test.go b/pkg/scheduler/nodes_test.go
new file mode 100644
index 000000000..a2489c868
--- /dev/null
+++ b/pkg/scheduler/nodes_test.go
@@ -0,0 +1,324 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scheduler
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+)
+
+func Test_addNode_ListNodes(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ nodeID string
+ nodeInfo util.NodeInfo
+ }
+ want map[string]*util.NodeInfo
+ err error
+ }{
+ {
+ name: "node info is empty",
+ args: struct {
+ nodeID string
+ nodeInfo util.NodeInfo
+ }{
+ nodeID: "node-01",
+ nodeInfo: util.NodeInfo{},
+ },
+ },
+ {
+ name: "test vaild info",
+ args: struct {
+ nodeID string
+ nodeInfo util.NodeInfo
+ }{
+ nodeID: "node-01",
+ nodeInfo: util.NodeInfo{
+ ID: "node-01",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "node-01",
+ },
+ },
+ },
+ },
+ want: map[string]*util.NodeInfo{
+ "node-01": {
+ ID: "test123",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "node-01",
+ },
+ },
+ },
+ },
+ err: nil,
+ },
+ {
+ name: "test the different node id",
+ args: struct {
+ nodeID string
+ nodeInfo util.NodeInfo
+ }{
+ nodeID: "node-02",
+ nodeInfo: util.NodeInfo{
+ ID: "node-02",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "node-02",
+ Count: int32(1),
+ Devcore: int32(1),
+ Devmem: int32(2000),
+ },
+ },
+ },
+ },
+ want: map[string]*util.NodeInfo{
+ "node-01": {
+ ID: "test123",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "GPU-0",
+ Count: int32(1),
+ Devcore: int32(1),
+ Devmem: int32(2000),
+ },
+ },
+ },
+ "node-02": {
+ ID: "node-02",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "node-02",
+ Count: int32(1),
+ Devcore: int32(1),
+ Devmem: int32(2000),
+ },
+ },
+ },
+ },
+ err: nil,
+ },
+ }
+ device.InitDefaultDevices()
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ m := nodeManager{
+ nodes: map[string]*util.NodeInfo{
+ "node-01": {
+ ID: "test123",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "GPU-0",
+ Count: int32(1),
+ Devcore: int32(1),
+ Devmem: int32(2000),
+ },
+ },
+ },
+ },
+ }
+ m.addNode(test.args.nodeID, &test.args.nodeInfo)
+ if len(test.want) != 0 {
+ result, err := m.ListNodes()
+ if err == nil {
+ assert.DeepEqual(t, test.want, result)
+ }
+ }
+ })
+ }
+}
+
+func Test_GetNode(t *testing.T) {
+ tests := []struct {
+ name string
+ args string
+ want *util.NodeInfo
+ err error
+ }{
+ {
+ name: "node not found",
+ args: "node-1111",
+ want: &util.NodeInfo{},
+ err: fmt.Errorf("node %v not found", "node-111"),
+ },
+ {
+ name: "test vaild info",
+ args: "node-04",
+ want: &util.NodeInfo{
+ ID: "node-04",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "GPU-0",
+ Count: int32(1),
+ Devcore: int32(1),
+ Devmem: int32(2000),
+ },
+ },
+ },
+ err: nil,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ m := nodeManager{
+ nodes: map[string]*util.NodeInfo{
+ "node-04": {
+ ID: "node-04",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "GPU-0",
+ Count: int32(1),
+ Devcore: int32(1),
+ Devmem: int32(2000),
+ },
+ },
+ },
+ },
+ }
+ result, err := m.GetNode(test.args)
+ if err != nil {
+ assert.DeepEqual(t, test.want, result)
+ }
+ })
+ }
+}
+
+func Test_rmNodeDevices(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ nodeID string
+ deviceVendor string
+ }
+ }{
+ {
+ name: "no device",
+ args: struct {
+ nodeID string
+ deviceVendor string
+ }{
+ nodeID: "node-06",
+ },
+ },
+ {
+ name: "exist device info",
+ args: struct {
+ nodeID string
+ deviceVendor string
+ }{
+ nodeID: "node-05",
+ deviceVendor: "NVIDIA",
+ },
+ },
+ {
+ name: "the different devicevendor",
+ args: struct {
+ nodeID string
+ deviceVendor string
+ }{
+ nodeID: "node-07",
+ deviceVendor: "NVIDIA",
+ },
+ },
+ {
+ name: "the same of device id no less than one",
+ args: struct {
+ nodeID string
+ deviceVendor string
+ }{
+ nodeID: "node-08",
+ deviceVendor: "NVIDIA",
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ m := nodeManager{
+ nodes: map[string]*util.NodeInfo{
+ "node-05": {
+ ID: "node-05",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "GPU-0",
+ Count: int32(1),
+ Devcore: int32(1),
+ Devmem: int32(2000),
+ DeviceVendor: "NVIDIA",
+ },
+ },
+ },
+ "node-06": {
+ ID: "node-06",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{},
+ },
+ "node-07": {
+ ID: "node-17",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "GPU-0",
+ Count: int32(1),
+ Devcore: int32(1),
+ Devmem: int32(2000),
+ DeviceVendor: "test",
+ },
+ },
+ },
+ "node-08": {
+ ID: "node-08",
+ Node: &corev1.Node{},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "GPU-0",
+ Count: int32(1),
+ Devcore: int32(1),
+ Devmem: int32(2000),
+ DeviceVendor: "NVIDIA",
+ },
+ {
+ ID: "GPU-0",
+ Count: int32(1),
+ Devcore: int32(1),
+ Devmem: int32(2000),
+ DeviceVendor: "NVIDIA",
+ },
+ },
+ },
+ },
+ }
+ m.rmNodeDevices(test.args.nodeID, test.args.deviceVendor)
+ })
+ }
+}
diff --git a/pkg/scheduler/pod_test.go b/pkg/scheduler/pod_test.go
new file mode 100644
index 000000000..0cade230c
--- /dev/null
+++ b/pkg/scheduler/pod_test.go
@@ -0,0 +1,164 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scheduler
+
+import (
+ "reflect"
+ "sync"
+ "testing"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ "github.com/stretchr/testify/assert"
+ k8stypes "k8s.io/apimachinery/pkg/types"
+)
+
+func TestPodInfo(t *testing.T) {
+ tests := []struct {
+ name string
+ podInfo podInfo
+ expected podInfo
+ }{
+ {
+ name: "Empty podInfo",
+ podInfo: podInfo{},
+ expected: podInfo{},
+ },
+ {
+ name: "Filled podInfo",
+ podInfo: podInfo{
+ Namespace: "default",
+ Name: "my-pod",
+ UID: k8stypes.UID("12345678"),
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "device1": {
+ {
+ {
+ Idx: 1,
+ },
+ },
+ },
+ },
+ CtrIDs: []string{"ctr1", "ctr2"},
+ },
+ expected: podInfo{
+ Namespace: "default",
+ Name: "my-pod",
+ UID: k8stypes.UID("12345678"),
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "device1": {
+ {
+ {
+ Idx: 1,
+ },
+ },
+ },
+ },
+ CtrIDs: []string{"ctr1", "ctr2"},
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if !reflect.DeepEqual(tt.podInfo, tt.expected) {
+ t.Errorf("Expected %v, got %v", tt.expected, tt.podInfo)
+ }
+ })
+ }
+}
+
+func TestPodUseDeviceStat(t *testing.T) {
+ tests := []struct {
+ name string
+ stat PodUseDeviceStat
+ expected PodUseDeviceStat
+ }{
+ {
+ name: "Empty PodUseDeviceStat",
+ stat: PodUseDeviceStat{},
+ expected: PodUseDeviceStat{},
+ },
+ {
+ name: "Filled PodUseDeviceStat",
+ stat: PodUseDeviceStat{
+ TotalPod: 10,
+ UseDevicePod: 5,
+ },
+ expected: PodUseDeviceStat{
+ TotalPod: 10,
+ UseDevicePod: 5,
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if !reflect.DeepEqual(tt.stat, tt.expected) {
+ t.Errorf("Expected %v, got %v", tt.expected, tt.stat)
+ }
+ })
+ }
+}
+func TestGetScheduledPods(t *testing.T) {
+ podManager := &podManager{
+ pods: make(map[k8stypes.UID]*podInfo),
+ mutex: sync.RWMutex{},
+ }
+
+ pod1 := &podInfo{
+ Namespace: "default",
+ Name: "pod1",
+ UID: k8stypes.UID("uid1"),
+ NodeID: "node1",
+ Devices: util.PodDevices{"device1": {{}}},
+ CtrIDs: []string{"ctr1"},
+ }
+ pod2 := &podInfo{
+ Namespace: "default",
+ Name: "pod2",
+ UID: k8stypes.UID("uid2"),
+ NodeID: "node2",
+ Devices: util.PodDevices{"device2": {{}}},
+ CtrIDs: []string{"ctr2"},
+ }
+ podManager.pods[pod1.UID] = pod1
+ podManager.pods[pod2.UID] = pod2
+
+ scheduledPods, err := podManager.GetScheduledPods()
+
+ assert.NoError(t, err, "GetScheduledPods should not return an error")
+ assert.NotNil(t, scheduledPods, "The result should not be nil")
+ assert.Equal(t, 2, len(scheduledPods), "The number of scheduled pods should be 2")
+
+ expectedPods := map[k8stypes.UID]*podInfo{
+ pod1.UID: pod1,
+ pod2.UID: pod2,
+ }
+ for uid, pod := range scheduledPods {
+ expectedPod := expectedPods[uid]
+ assert.NotNil(t, expectedPod, "Pod with UID %s should exist in the expected pods", uid)
+ assert.Equal(t, expectedPod.Namespace, pod.Namespace, "Namespace should match")
+ assert.Equal(t, expectedPod.Name, pod.Name, "Name should match")
+ assert.Equal(t, expectedPod.UID, pod.UID, "UID should match")
+ assert.Equal(t, expectedPod.NodeID, pod.NodeID, "NodeID should match")
+ assert.Equal(t, expectedPod.Devices, pod.Devices, "Devices should match")
+ assert.Equal(t, expectedPod.CtrIDs, pod.CtrIDs, "CtrIDs should match")
+ }
+}
diff --git a/pkg/scheduler/pods.go b/pkg/scheduler/pods.go
index 9211cd77b..894ca67d0 100644
--- a/pkg/scheduler/pods.go
+++ b/pkg/scheduler/pods.go
@@ -1,114 +1,150 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package scheduler
import (
- "4pd.io/k8s-vgpu/pkg/api"
- "4pd.io/k8s-vgpu/pkg/util"
- "fmt"
- corev1 "k8s.io/api/core/v1"
- k8stypes "k8s.io/apimachinery/pkg/types"
- "k8s.io/klog/v2"
- "sync"
+ "sync"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ k8stypes "k8s.io/apimachinery/pkg/types"
+ "k8s.io/klog/v2"
)
type podInfo struct {
- namespace string
- name string
- uid k8stypes.UID
- nodeID string
- devices util.PodDevices
- ctrIDs []string
+ Namespace string
+ Name string
+ UID k8stypes.UID
+ NodeID string
+ Devices util.PodDevices
+ CtrIDs []string
}
-type containerInfo struct {
- podUID k8stypes.UID
- ctrIdx int
+// PodUseDeviceStat counts pod use device info.
+type PodUseDeviceStat struct {
+ TotalPod int // Count of all running pods on the current node
+ UseDevicePod int // Count of running pods that use devices
}
type podManager struct {
- pods map[k8stypes.UID]*podInfo
- containers map[string]containerInfo
- mutex sync.Mutex
+ pods map[k8stypes.UID]*podInfo
+ mutex sync.RWMutex
}
-func (m *podManager) init() {
- m.pods = make(map[k8stypes.UID]*podInfo)
- m.containers = make(map[string]containerInfo)
+func newPodManager() *podManager {
+ pm := &podManager{
+ pods: make(map[k8stypes.UID]*podInfo),
+ }
+ klog.InfoS("Pod manager initialized", "podCount", len(pm.pods))
+ return pm
}
func (m *podManager) addPod(pod *corev1.Pod, nodeID string, devices util.PodDevices) {
- m.mutex.Lock()
- defer m.mutex.Unlock()
- pi, ok := m.pods[pod.UID]
- if !ok {
- pi = &podInfo{name: pod.Name, uid: pod.UID}
- m.pods[pod.UID] = pi
- pi.namespace = pod.Namespace
- pi.name = pod.Name
- pi.uid = pod.UID
- pi.nodeID = nodeID
- pi.devices = devices
- pi.ctrIDs = make([]string, len(pod.Spec.Containers))
- for i := 0; i < len(pod.Spec.Containers); i++ {
- c := &pod.Spec.Containers[i]
- if i >= len(devices) {
- klog.Errorf("len(device) != len(containers)")
- continue
- }
- for _, env := range c.Env {
- if env.Name == api.ContainerUID {
- m.containers[env.Value] = containerInfo{
- podUID: pod.UID,
- ctrIdx: i,
- }
- pi.ctrIDs[i] = env.Value
- break
- }
- }
- if len(pi.ctrIDs[i]) == 0 {
- klog.Errorf("not found container uid in container %v/%v/%v", pod.Namespace, pod.Name, c.Name)
- }
- }
- }
+ m.mutex.Lock()
+ defer m.mutex.Unlock()
+
+ _, exists := m.pods[pod.UID]
+ if !exists {
+ pi := &podInfo{
+ Name: pod.Name,
+ UID: pod.UID,
+ Namespace: pod.Namespace,
+ NodeID: nodeID,
+ Devices: devices,
+ }
+ m.pods[pod.UID] = pi
+ klog.InfoS("Pod added",
+ "pod", klog.KRef(pod.Namespace, pod.Name),
+ "nodeID", nodeID,
+ "devices", devices,
+ )
+ } else {
+ m.pods[pod.UID].Devices = devices
+ klog.InfoS("Pod devices updated",
+ "pod", klog.KRef(pod.Namespace, pod.Name),
+ "devices", devices,
+ )
+ }
}
func (m *podManager) delPod(pod *corev1.Pod) {
- m.mutex.Lock()
- defer m.mutex.Unlock()
- pi, ok := m.pods[pod.UID]
- if ok {
- for _, id := range pi.ctrIDs {
- delete(m.containers, id)
- }
- delete(m.pods, pod.UID)
- }
+ m.mutex.Lock()
+ defer m.mutex.Unlock()
+
+ pi, exists := m.pods[pod.UID]
+ if exists {
+ klog.InfoS("Pod deleted",
+ "pod", klog.KRef(pod.Namespace, pod.Name),
+ "nodeID", pi.NodeID,
+ )
+ delete(m.pods, pod.UID)
+ } else {
+ klog.InfoS("Pod not found for deletion",
+ "pod", klog.KRef(pod.Namespace, pod.Name),
+ )
+ }
}
-func (m *podManager) getContainerByUUID(uuid string) (podInfo, int, error) {
- m.mutex.Lock()
- defer m.mutex.Unlock()
- c, ok := m.containers[uuid]
- if !ok {
- return podInfo{}, 0, fmt.Errorf("not found container %v", uuid)
- }
- pi, ok := m.pods[c.podUID]
- if !ok {
- return podInfo{}, 0, fmt.Errorf("not found pod %v", c.podUID)
- }
- return *pi, c.ctrIdx, nil
+func (m *podManager) ListPodsUID() ([]*corev1.Pod, error) {
+ m.mutex.RLock()
+ defer m.mutex.RUnlock()
+
+ pods := make([]*corev1.Pod, 0, len(m.pods))
+ for uid := range m.pods {
+ pods = append(pods, &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ UID: uid,
+ },
+ })
+ }
+ klog.InfoS("Listed pod UIDs",
+ "podCount", len(pods),
+ )
+ return pods, nil
+}
+
+func (m *podManager) ListPodsInfo() []*podInfo {
+ m.mutex.RLock()
+ defer m.mutex.RUnlock()
+
+ pods := make([]*podInfo, 0, len(m.pods))
+ for _, pod := range m.pods {
+ pods = append(pods, pod)
+ klog.V(5).InfoS("Pod info",
+ "pod", klog.KRef(pod.Namespace, pod.Name),
+ "nodeID", pod.NodeID,
+ "devices", pod.Devices,
+ )
+ }
+ klog.V(5).InfoS("Listed pod infos",
+ "podCount", len(pods),
+ )
+ return pods
+}
+
+func (m *podManager) GetScheduledPods() (map[k8stypes.UID]*podInfo, error) {
+ m.mutex.RLock()
+ defer m.mutex.RUnlock()
+
+ podCount := len(m.pods)
+ klog.InfoS("Retrieved scheduled pods",
+ "podCount", podCount,
+ )
+ return m.pods, nil
}
diff --git a/pkg/scheduler/policy/constant.go b/pkg/scheduler/policy/constant.go
new file mode 100644
index 000000000..451a0b2b8
--- /dev/null
+++ b/pkg/scheduler/policy/constant.go
@@ -0,0 +1,45 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package policy
+
+type SchedulerPolicyName string
+
+const (
+ // NodeSchedulerPolicyBinpack is node use binpack scheduler policy.
+ NodeSchedulerPolicyBinpack SchedulerPolicyName = "binpack"
+ // NodeSchedulerPolicySpread is node use spread scheduler policy.
+ NodeSchedulerPolicySpread SchedulerPolicyName = "spread"
+ // GPUSchedulerPolicyBinpack is GPU use binpack scheduler.
+ GPUSchedulerPolicyBinpack SchedulerPolicyName = "binpack"
+ // GPUSchedulerPolicySpread is GPU use spread scheduler.
+ GPUSchedulerPolicySpread SchedulerPolicyName = "spread"
+)
+
+func (s SchedulerPolicyName) String() string {
+ return string(s)
+}
+
+const (
+ // NodeSchedulerPolicyAnnotationKey is user set Pod annotation to change this default node policy.
+ NodeSchedulerPolicyAnnotationKey = "hami.io/node-scheduler-policy"
+ // GPUSchedulerPolicyAnnotationKey is user set Pod annotation to change this default GPU policy.
+ GPUSchedulerPolicyAnnotationKey = "hami.io/gpu-scheduler-policy"
+)
+
+const (
+ Weight int = 10
+)
diff --git a/pkg/scheduler/policy/gpu_policy.go b/pkg/scheduler/policy/gpu_policy.go
new file mode 100644
index 000000000..b7a1746b0
--- /dev/null
+++ b/pkg/scheduler/policy/gpu_policy.go
@@ -0,0 +1,77 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package policy
+
+import (
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ "k8s.io/klog/v2"
+)
+
+type DeviceListsScore struct {
+ Device *util.DeviceUsage
+ // Score recode every device user/allocate score
+ Score float32
+}
+
+type DeviceUsageList struct {
+ DeviceLists []*DeviceListsScore
+ Policy string
+}
+
+func (l DeviceUsageList) Len() int {
+ return len(l.DeviceLists)
+}
+
+func (l DeviceUsageList) Swap(i, j int) {
+ l.DeviceLists[i], l.DeviceLists[j] = l.DeviceLists[j], l.DeviceLists[i]
+}
+
+func (l DeviceUsageList) Less(i, j int) bool {
+ if l.Policy == util.GPUSchedulerPolicyBinpack.String() {
+ if l.DeviceLists[i].Device.Numa == l.DeviceLists[j].Device.Numa {
+ return l.DeviceLists[i].Score < l.DeviceLists[j].Score
+ }
+ return l.DeviceLists[i].Device.Numa > l.DeviceLists[j].Device.Numa
+ }
+ // default policy is spread
+ if l.DeviceLists[i].Device.Numa == l.DeviceLists[j].Device.Numa {
+ return l.DeviceLists[i].Score > l.DeviceLists[j].Score
+ }
+ return l.DeviceLists[i].Device.Numa < l.DeviceLists[j].Device.Numa
+}
+
+func (ds *DeviceListsScore) ComputeScore(requests util.ContainerDeviceRequests) {
+ request, core, mem := int32(0), int32(0), int32(0)
+ // Here we are required to use the same type device
+ for _, container := range requests {
+ request += container.Nums
+ core += container.Coresreq
+ if container.MemPercentagereq != 0 && container.MemPercentagereq != 101 {
+ mem += ds.Device.Totalmem * (container.MemPercentagereq / 100.0)
+ continue
+ }
+ mem += container.Memreq
+ }
+ klog.V(2).Infof("device %s user %d, userCore %d, userMem %d,", ds.Device.ID, ds.Device.Used, ds.Device.Usedcores, ds.Device.Usedmem)
+
+ usedScore := float32(request+ds.Device.Used) / float32(ds.Device.Count)
+ coreScore := float32(core+ds.Device.Usedcores) / float32(ds.Device.Totalcore)
+ memScore := float32(mem+ds.Device.Usedmem) / float32(ds.Device.Totalmem)
+ ds.Score = float32(Weight) * (usedScore + coreScore + memScore)
+ klog.V(2).Infof("device %s computer score is %f", ds.Device.ID, ds.Score)
+}
diff --git a/pkg/scheduler/policy/gpu_policy_test.go b/pkg/scheduler/policy/gpu_policy_test.go
new file mode 100644
index 000000000..041132f07
--- /dev/null
+++ b/pkg/scheduler/policy/gpu_policy_test.go
@@ -0,0 +1,322 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package policy
+
+import (
+ "testing"
+
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+func TestDeviceUsageListLen(t *testing.T) {
+ tests := []struct {
+ name string
+ list DeviceUsageList
+ expected int
+ }{
+ {
+ name: "empty list",
+ list: DeviceUsageList{DeviceLists: []*DeviceListsScore{}},
+ expected: 0,
+ },
+ {
+ name: "list with items",
+ list: DeviceUsageList{
+ DeviceLists: []*DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "device1",
+ Count: 1,
+ Health: true,
+ },
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "device2",
+ Count: 2,
+ Health: false,
+ },
+ },
+ },
+ },
+ expected: 2,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := tt.list.Len(); got != tt.expected {
+ t.Errorf("DeviceUsageList.Len() = %v, want %v", got, tt.expected)
+ }
+ })
+ }
+}
+
+func TestSwap(t *testing.T) {
+ device1 := &util.DeviceUsage{
+ ID: "device1",
+ Index: 1,
+ Used: 50,
+ Count: 100,
+ Usedmem: 512,
+ Totalmem: 1024,
+ Totalcore: 8,
+ Usedcores: 4,
+ Numa: 0,
+ Type: "GPU",
+ Health: true,
+ }
+
+ device2 := &util.DeviceUsage{
+ ID: "device2",
+ Index: 2,
+ Used: 75,
+ Count: 150,
+ Usedmem: 768,
+ Totalmem: 2048,
+ Totalcore: 16,
+ Usedcores: 8,
+ Numa: 1,
+ Type: "CPU",
+ Health: false,
+ }
+
+ dul := &DeviceUsageList{
+ DeviceLists: []*DeviceListsScore{
+ {Device: device1, Score: 0.5},
+ {Device: device2, Score: 0.75},
+ },
+ Policy: "some_policy",
+ }
+
+ dul.Swap(0, 1)
+
+ expectedResult := []*DeviceListsScore{
+ {Device: device2, Score: 0.75},
+ {Device: device1, Score: 0.5},
+ }
+
+ for i, dls := range dul.DeviceLists {
+ if dls.Device.ID != expectedResult[i].Device.ID || dls.Score != expectedResult[i].Score {
+ t.Errorf("TestSwap failed: expected %v, got %v", expectedResult, dul.DeviceLists)
+ break
+ }
+ }
+}
+
+func TestDeviceUsageList_Less(t *testing.T) {
+ tests := []struct {
+ name string
+ policy string
+ deviceLists []*DeviceListsScore
+ expectedLess bool
+ }{
+ {
+ name: "Binpack policy with same Numa",
+ policy: "binpack",
+ deviceLists: []*DeviceListsScore{
+ {Device: &util.DeviceUsage{Numa: 0, Used: 10}, Score: 10},
+ {Device: &util.DeviceUsage{Numa: 0, Used: 20}, Score: 20},
+ },
+ expectedLess: true,
+ },
+ {
+ name: "Binpack policy with different Numa true",
+ policy: "binpack",
+ deviceLists: []*DeviceListsScore{
+ {Device: &util.DeviceUsage{Numa: 1, Used: 10}, Score: 10},
+ {Device: &util.DeviceUsage{Numa: 0, Used: 20}, Score: 20},
+ },
+ expectedLess: true,
+ },
+ {
+ name: "Binpack policy with different Numa false",
+ policy: "binpack",
+ deviceLists: []*DeviceListsScore{
+ {Device: &util.DeviceUsage{Numa: 0, Used: 10}, Score: 10},
+ {Device: &util.DeviceUsage{Numa: 1, Used: 20}, Score: 20},
+ },
+ expectedLess: false,
+ },
+ {
+ name: "Spread policy with same Numa",
+ policy: "spread",
+ deviceLists: []*DeviceListsScore{
+ {Device: &util.DeviceUsage{Numa: 0, Used: 10}, Score: 10},
+ {Device: &util.DeviceUsage{Numa: 0, Used: 20}, Score: 20},
+ },
+ expectedLess: false,
+ },
+ {
+ name: "Spread policy with different Numa false",
+ policy: "spread",
+ deviceLists: []*DeviceListsScore{
+ {Device: &util.DeviceUsage{Numa: 1, Used: 10}, Score: 10},
+ {Device: &util.DeviceUsage{Numa: 0, Used: 20}, Score: 20},
+ },
+ expectedLess: false,
+ },
+ {
+ name: "Spread policy with different Numa true",
+ policy: "spread",
+ deviceLists: []*DeviceListsScore{
+ {Device: &util.DeviceUsage{Numa: 0, Used: 10}, Score: 10},
+ {Device: &util.DeviceUsage{Numa: 1, Used: 20}, Score: 20},
+ },
+ expectedLess: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ l := DeviceUsageList{
+ Policy: tt.policy,
+ DeviceLists: tt.deviceLists,
+ }
+ i, j := 0, 1
+ result := l.Less(i, j)
+ if result != tt.expectedLess {
+ t.Errorf("Expected %v, got %v", tt.expectedLess, result)
+ }
+ })
+ }
+}
+
+func TestComputeScore(t *testing.T) {
+ tests := []struct {
+ name string
+ device *util.DeviceUsage
+ requests util.ContainerDeviceRequests
+ expectedScore float32
+ }{
+ {
+ name: "ContainerDeviceRequests has no data",
+ device: &util.DeviceUsage{
+ ID: "test-device",
+ Totalcore: 4,
+ Totalmem: 8192,
+ Count: 10,
+ Used: 2,
+ Usedcores: 1,
+ Usedmem: 2048,
+ },
+ requests: make(util.ContainerDeviceRequests),
+ expectedScore: 7,
+ },
+ {
+ name: "ContainerDeviceRequests has data",
+ device: &util.DeviceUsage{
+ ID: "test-device",
+ Totalcore: 4,
+ Totalmem: 8192,
+ Count: 10,
+ Used: 2,
+ Usedcores: 1,
+ Usedmem: 2048,
+ },
+ requests: util.ContainerDeviceRequests{
+ "container1": {
+ Nums: 1,
+ Type: "type1",
+ Memreq: 1024,
+ MemPercentagereq: 0,
+ Coresreq: 1,
+ },
+ "container2": {
+ Nums: 2,
+ Type: "type2",
+ Memreq: 0,
+ MemPercentagereq: 50,
+ Coresreq: 2,
+ },
+ },
+ expectedScore: 18.75,
+ },
+ {
+ name: "ContainerDeviceRequests has data",
+ device: &util.DeviceUsage{
+ ID: "test-device",
+ Totalcore: 4,
+ Totalmem: 8192,
+ Count: 10,
+ Used: 2,
+ Usedcores: 1,
+ Usedmem: 2048,
+ },
+ requests: util.ContainerDeviceRequests{
+ "container1": {
+ Nums: 1,
+ Type: "type1",
+ Memreq: 1024,
+ MemPercentagereq: 30,
+ Coresreq: 1,
+ },
+ "container2": {
+ Nums: 2,
+ Type: "type2",
+ Memreq: 200,
+ MemPercentagereq: 101,
+ Coresreq: 3,
+ },
+ },
+ expectedScore: 20.24414,
+ },
+ {
+ name: "ContainerDeviceRequests has data",
+ device: &util.DeviceUsage{
+ ID: "test-device",
+ Totalcore: 4,
+ Totalmem: 8192,
+ Count: 10,
+ Used: 2,
+ Usedcores: 1,
+ Usedmem: 2048,
+ },
+ requests: util.ContainerDeviceRequests{
+ "container1": {
+ Nums: 1,
+ Type: "type1",
+ Memreq: 1024,
+ MemPercentagereq: 30,
+ Coresreq: 1,
+ },
+ "container2": {
+ Nums: 2,
+ Type: "type2",
+ Memreq: 200,
+ MemPercentagereq: 30,
+ Coresreq: 3,
+ },
+ },
+ expectedScore: 20,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ ds := &DeviceListsScore{
+ Device: tt.device,
+ }
+
+ ds.ComputeScore(tt.requests)
+
+ if ds.Score != tt.expectedScore {
+ t.Errorf("ComputeScore() = %v, want %v", ds.Score, tt.expectedScore)
+ }
+ })
+ }
+}
diff --git a/pkg/scheduler/policy/node_policy.go b/pkg/scheduler/policy/node_policy.go
new file mode 100644
index 000000000..84bd64efd
--- /dev/null
+++ b/pkg/scheduler/policy/node_policy.go
@@ -0,0 +1,95 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package policy
+
+import (
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/klog/v2"
+)
+
+type NodeScore struct {
+ NodeID string
+ Node *corev1.Node
+ Devices util.PodDevices
+ // Score recode every node all device user/allocate score
+ Score float32
+}
+
+type NodeScoreList struct {
+ NodeList []*NodeScore
+ Policy string
+}
+
+func (l NodeScoreList) Len() int {
+ return len(l.NodeList)
+}
+
+func (l NodeScoreList) Swap(i, j int) {
+ l.NodeList[i], l.NodeList[j] = l.NodeList[j], l.NodeList[i]
+}
+
+func (l NodeScoreList) Less(i, j int) bool {
+ if l.Policy == util.NodeSchedulerPolicySpread.String() {
+ return l.NodeList[i].Score > l.NodeList[j].Score
+ }
+ // default policy is Binpack
+ return l.NodeList[i].Score < l.NodeList[j].Score
+}
+
+func (ns *NodeScore) OverrideScore(previous []*util.DeviceUsage, policy string) {
+ // current user having request resource
+ devScore := float32(0)
+ for idx, val := range ns.Devices {
+ devScore += device.GetDevices()[idx].ScoreNode(ns.Node, val, previous, policy)
+ }
+ ns.Score += devScore
+ klog.V(2).Infof("node %s default score is %f, computer override score is %f", ns.NodeID, ns.Score-devScore, ns.Score)
+}
+
+func (ns *NodeScore) SnapshotDevice(devices DeviceUsageList) []*util.DeviceUsage {
+ snapshot := []*util.DeviceUsage{}
+ for _, val := range devices.DeviceLists {
+ tmp := *val.Device
+ snapshot = append(snapshot, &tmp)
+ }
+ return snapshot
+}
+
+func (ns *NodeScore) ComputeDefaultScore(devices DeviceUsageList) {
+ used, usedCore, usedMem := int32(0), int32(0), int32(0)
+ for _, device := range devices.DeviceLists {
+ used += device.Device.Used
+ usedCore += device.Device.Usedcores
+ usedMem += device.Device.Usedmem
+ }
+ klog.V(2).Infof("node %s used %d, usedCore %d, usedMem %d,", ns.NodeID, used, usedCore, usedMem)
+
+ total, totalCore, totalMem := int32(0), int32(0), int32(0)
+ for _, deviceLists := range devices.DeviceLists {
+ total += deviceLists.Device.Count
+ totalCore += deviceLists.Device.Totalcore
+ totalMem += deviceLists.Device.Totalmem
+ }
+ useScore := float32(used) / float32(total)
+ coreScore := float32(usedCore) / float32(totalCore)
+ memScore := float32(usedMem) / float32(totalMem)
+ ns.Score = float32(Weight) * (useScore + coreScore + memScore)
+ klog.V(2).Infof("node %s computer default score is %f", ns.NodeID, ns.Score)
+}
diff --git a/pkg/scheduler/policy/node_policy_test.go b/pkg/scheduler/policy/node_policy_test.go
new file mode 100644
index 000000000..5eb4c1c64
--- /dev/null
+++ b/pkg/scheduler/policy/node_policy_test.go
@@ -0,0 +1,387 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package policy
+
+import (
+ "testing"
+
+ "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestNodeScoreListLen(t *testing.T) {
+ tests := []struct {
+ name string
+ list NodeScoreList
+ expected int
+ }{
+ {
+ name: "empty list",
+ list: NodeScoreList{NodeList: []*NodeScore{}, Policy: "default"},
+ expected: 0,
+ },
+ {
+ name: "list with elements",
+ list: NodeScoreList{
+ NodeList: []*NodeScore{
+ {
+ NodeID: "node1",
+ Node: &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{Name: "node1"},
+ Spec: corev1.NodeSpec{},
+ Status: corev1.NodeStatus{},
+ },
+ Devices: util.PodDevices{},
+ Score: 85.5,
+ },
+ {
+ NodeID: "node2",
+ Node: &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{Name: "node2"},
+ Spec: corev1.NodeSpec{},
+ Status: corev1.NodeStatus{},
+ },
+ Devices: util.PodDevices{},
+ Score: 90.0,
+ },
+ {
+ NodeID: "node3",
+ Node: &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{Name: "node3"},
+ Spec: corev1.NodeSpec{},
+ Status: corev1.NodeStatus{},
+ },
+ Devices: util.PodDevices{},
+ Score: 78.0,
+ },
+ },
+ Policy: "custom",
+ },
+ expected: 3,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := tt.list.Len(); got != tt.expected {
+ t.Errorf("NodeScoreList.Len() = %v, want %v", got, tt.expected)
+ }
+ })
+ }
+}
+
+func TestNodeSwap(t *testing.T) {
+ node1 := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{Name: "node1"},
+ }
+ node2 := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{Name: "node2"},
+ }
+
+ nodes := []*NodeScore{
+ {NodeID: "1", Node: node1, Score: 1.0},
+ {NodeID: "2", Node: node2, Score: 2.0},
+ }
+ list := NodeScoreList{NodeList: nodes}
+
+ i, j := 0, 1
+
+ originalI := list.NodeList[i]
+ originalJ := list.NodeList[j]
+
+ list.Swap(i, j)
+
+ if list.NodeList[i] != originalJ || list.NodeList[j] != originalI {
+ t.Errorf("Swap failed: expected (%v, %v), got (%v, %v)", originalJ, originalI, list.NodeList[i], list.NodeList[j])
+ }
+}
+
+func TestLess(t *testing.T) {
+ tests := []struct {
+ name string
+ nodeScoreList NodeScoreList
+ i int
+ j int
+ expected bool
+ }{
+ {
+ name: "Spread strategy, i score higher",
+ nodeScoreList: NodeScoreList{
+ NodeList: []*NodeScore{
+ {NodeID: "node1", Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}}, Score: 20.0},
+ {NodeID: "node2", Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2"}}, Score: 10.0},
+ },
+ Policy: "spread",
+ },
+ i: 0,
+ j: 1,
+ expected: true,
+ },
+ {
+ name: "Spread strategy,j score higher",
+ nodeScoreList: NodeScoreList{
+ NodeList: []*NodeScore{
+ {NodeID: "node1", Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}}, Score: 10.0},
+ {NodeID: "node2", Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2"}}, Score: 20.0},
+ },
+ Policy: "spread",
+ },
+ i: 0,
+ j: 1,
+ expected: false,
+ },
+ {
+ name: "Default strategy (Binpack), i score lower",
+ nodeScoreList: NodeScoreList{
+ NodeList: []*NodeScore{
+ {NodeID: "node1", Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}}, Score: 10.0},
+ {NodeID: "node2", Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2"}}, Score: 20.0},
+ },
+ Policy: "binpack",
+ },
+ i: 0,
+ j: 1,
+ expected: true,
+ },
+ {
+ name: "Default strategy (Binpack), j score lower",
+ nodeScoreList: NodeScoreList{
+ NodeList: []*NodeScore{
+ {NodeID: "node1", Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}}, Score: 20.0},
+ {NodeID: "node2", Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2"}}, Score: 10.0},
+ },
+ Policy: "binpack",
+ },
+ i: 0,
+ j: 1,
+ expected: false,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result := test.nodeScoreList.Less(test.i, test.j)
+ assert.Equal(t, test.expected, result)
+ })
+ }
+}
+
+// setup initializes the devices with a given configuration.
+func setup(t *testing.T, config *device.Config) {
+ if err := device.InitDevicesWithConfig(config); err != nil {
+ klog.Fatalf("Failed to initialize devices with config: %v", err)
+ }
+}
+
+// TestOverrideScore tests the OverrideScore method for different scenarios.
+func TestOverrideScore(t *testing.T) {
+ config := &device.Config{
+ NvidiaConfig: nvidia.NvidiaConfig{
+ ResourceCountName: "hami.io/gpu",
+ ResourceMemoryName: "hami.io/gpumem",
+ ResourceMemoryPercentageName: "hami.io/gpumem-percentage",
+ ResourceCoreName: "hami.io/gpucores",
+ DefaultMemory: 0,
+ DefaultCores: 0,
+ DefaultGPUNum: 1,
+ },
+ }
+ setup(t, config)
+
+ tests := []struct {
+ name string
+ nodeScore *NodeScore
+ devices []*util.DeviceUsage
+ policy string
+ wantScore float32
+ }{
+ {
+ name: "Device score greater than zero",
+ nodeScore: &NodeScore{
+ Node: &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node1",
+ Annotations: map[string]string{
+ "metax-tech.com/gpu.topology.losses": "{\"1\":123,\"2\":321}",
+ },
+ },
+ },
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "DCU": util.PodSingleDevice{
+ util.ContainerDevices{
+ {Idx: 1, UUID: "uuid1", Type: "gpu", Usedmem: 1024, Usedcores: 2},
+ {Idx: 2, UUID: "uuid2", Type: "gpu", Usedmem: 2048, Usedcores: 4},
+ },
+ },
+ "Metax-GPU": util.PodSingleDevice{
+ util.ContainerDevices{
+ {Idx: 1, UUID: "uuid1", Type: "gpu", Usedmem: 1024, Usedcores: 2},
+ {Idx: 2, UUID: "uuid2", Type: "gpu", Usedmem: 2048, Usedcores: 4},
+ },
+ },
+ },
+ Score: 0,
+ },
+ devices: []*util.DeviceUsage{
+ {
+ Count: 4,
+ Totalcore: 8,
+ Totalmem: 4096,
+ Type: "gpu",
+ Used: 0,
+ Usedcores: 0,
+ Usedmem: 0,
+ },
+ },
+ policy: "binpack",
+ wantScore: 1679,
+ },
+ {
+ name: "Device score equal to zero",
+ nodeScore: &NodeScore{
+ Node: &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node1",
+ Annotations: map[string]string{
+ "ccc": "cpu:123,gpu:321",
+ },
+ },
+ },
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "DCU": util.PodSingleDevice{
+ util.ContainerDevices{
+ {Idx: 1, UUID: "uuid1", Type: "gpu", Usedmem: 1024, Usedcores: 2},
+ {Idx: 2, UUID: "uuid2", Type: "gpu", Usedmem: 2048, Usedcores: 4},
+ },
+ },
+ "Metax-GPU": util.PodSingleDevice{
+ util.ContainerDevices{
+ {Idx: 1, UUID: "uuid1", Type: "gpu", Usedmem: 1024, Usedcores: 2},
+ {Idx: 2, UUID: "uuid2", Type: "gpu", Usedmem: 2048, Usedcores: 4},
+ },
+ },
+ },
+ Score: 0,
+ },
+ devices: []*util.DeviceUsage{
+ {
+ Count: 4,
+ Totalcore: 8,
+ Totalmem: 4096,
+ Type: "gpu",
+ Used: 0,
+ Usedcores: 0,
+ Usedmem: 0,
+ },
+ },
+ policy: "binpack",
+ wantScore: 0,
+ },
+ // Add more test cases here to cover other scenarios and policies.
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tt.nodeScore.OverrideScore(tt.devices, tt.policy)
+ if gotScore := tt.nodeScore.Score; gotScore != tt.wantScore {
+ t.Errorf("OverrideScore() gotScore = %v, want %v", gotScore, tt.wantScore)
+ }
+ })
+ }
+}
+
+func TestComputeDefaultScore(t *testing.T) {
+ device1 := &util.DeviceUsage{
+ ID: "device1",
+ Index: 1,
+ Used: 50,
+ Count: 100,
+ Usedmem: 100,
+ Totalmem: 100,
+ Totalcore: 100,
+ Usedcores: 100,
+ Numa: 0,
+ Type: "GPU",
+ Health: true,
+ }
+
+ device2 := &util.DeviceUsage{
+ ID: "device2",
+ Index: 2,
+ Used: 75,
+ Count: 150,
+ Usedmem: 200,
+ Totalmem: 200,
+ Totalcore: 200,
+ Usedcores: 200,
+ Numa: 1,
+ Type: "CPU",
+ Health: false,
+ }
+ tests := []struct {
+ name string
+ nodeScore NodeScore
+ devices DeviceUsageList
+ wantScore float32
+ }{
+ {
+ name: "Test with no devices",
+ nodeScore: NodeScore{
+ NodeID: "node1",
+ Score: 0.0,
+ },
+ devices: DeviceUsageList{
+ DeviceLists: []*DeviceListsScore{
+ {Device: device1, Score: 0},
+ {Device: device2, Score: 0},
+ },
+ },
+ wantScore: 25,
+ },
+ {
+ name: "Test with devices",
+ nodeScore: NodeScore{
+ NodeID: "node2",
+ },
+ devices: DeviceUsageList{
+ DeviceLists: []*DeviceListsScore{
+ {Device: device1, Score: 1},
+ {Device: device2, Score: 1},
+ },
+ },
+ wantScore: 25,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tt.nodeScore.ComputeDefaultScore(tt.devices)
+ if tt.nodeScore.Score != tt.wantScore {
+ t.Errorf("NodeScore.ComputeDefaultScore() = %v, want %v", tt.nodeScore.Score, tt.wantScore)
+ }
+ })
+ }
+}
diff --git a/pkg/scheduler/routes/route.go b/pkg/scheduler/routes/route.go
index 74501ef63..c3c7251ed 100644
--- a/pkg/scheduler/routes/route.go
+++ b/pkg/scheduler/routes/route.go
@@ -1,84 +1,147 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package routes
import (
- "4pd.io/k8s-vgpu/pkg/scheduler"
- "bytes"
- "encoding/json"
- "io"
- "net/http"
-
- "github.com/julienschmidt/httprouter"
- "k8s.io/klog/v2"
- extenderv1 "k8s.io/kube-scheduler/extender/v1"
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+
+ "github.com/julienschmidt/httprouter"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/klog/v2"
+ extenderv1 "k8s.io/kube-scheduler/extender/v1"
+
+ "github.com/Project-HAMi/HAMi/pkg/scheduler"
)
func checkBody(w http.ResponseWriter, r *http.Request) {
- if r.Body == nil {
- http.Error(w, "Please send a request body", 400)
- return
- }
+ if r.Body == nil {
+ http.Error(w, "Please send a request body", 400)
+ return
+ }
}
func PredicateRoute(s *scheduler.Scheduler) httprouter.Handle {
- return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
- checkBody(w, r)
-
- var buf bytes.Buffer
- body := io.TeeReader(r.Body, &buf)
-
- var extenderArgs extenderv1.ExtenderArgs
- var extenderFilterResult *extenderv1.ExtenderFilterResult
-
- if err := json.NewDecoder(body).Decode(&extenderArgs); err != nil {
- extenderFilterResult = &extenderv1.ExtenderFilterResult{
- Error: err.Error(),
- }
- } else {
- extenderFilterResult, err = s.Filter(extenderArgs)
- if err != nil {
- klog.Errorf("pod %v filter error, %v", extenderArgs.Pod.Name, err)
- extenderFilterResult = &extenderv1.ExtenderFilterResult{
- Error: err.Error(),
- }
- }
- }
-
- if resultBody, err := json.Marshal(extenderFilterResult); err != nil {
- klog.Errorf("Failed to marshal extenderFilterResult: %+v, %+v",
- err, extenderFilterResult)
- w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(http.StatusInternalServerError)
- w.Write([]byte(err.Error()))
- } else {
- w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(http.StatusOK)
- w.Write(resultBody)
- }
- }
+ klog.Infoln("Initializing Predicate Route")
+ return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ klog.Infoln("Entering Predicate Route handler")
+ checkBody(w, r)
+
+ var buf bytes.Buffer
+ body := io.TeeReader(r.Body, &buf)
+
+ var extenderArgs extenderv1.ExtenderArgs
+ var extenderFilterResult *extenderv1.ExtenderFilterResult
+
+ if err := json.NewDecoder(body).Decode(&extenderArgs); err != nil {
+ klog.ErrorS(err, "Failed to decode extender arguments")
+ extenderFilterResult = &extenderv1.ExtenderFilterResult{
+ Error: err.Error(),
+ }
+ } else {
+ extenderFilterResult, err = s.Filter(extenderArgs)
+ if err != nil {
+ klog.ErrorS(err, "Filter error for pod", "pod", extenderArgs.Pod.Name)
+ extenderFilterResult = &extenderv1.ExtenderFilterResult{
+ Error: err.Error(),
+ }
+ }
+ }
+
+ if resultBody, err := json.Marshal(extenderFilterResult); err != nil {
+ klog.ErrorS(err, "Failed to marshal extender filter result", "result", extenderFilterResult)
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusInternalServerError)
+ w.Write([]byte(err.Error()))
+ } else {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ w.Write(resultBody)
+ }
+ }
+}
+
+func Bind(s *scheduler.Scheduler) httprouter.Handle {
+ return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+ klog.Infoln("Entering Bind handler")
+ var buf bytes.Buffer
+ body := io.TeeReader(r.Body, &buf)
+ var extenderBindingArgs extenderv1.ExtenderBindingArgs
+ var extenderBindingResult *extenderv1.ExtenderBindingResult
+
+ if err := json.NewDecoder(body).Decode(&extenderBindingArgs); err != nil {
+ klog.ErrorS(err, "Failed to decode extender binding arguments")
+ extenderBindingResult = &extenderv1.ExtenderBindingResult{
+ Error: err.Error(),
+ }
+ } else {
+ extenderBindingResult, err = s.Bind(extenderBindingArgs)
+ if err != nil {
+ klog.ErrorS(err, "Bind error for pod", "pod", extenderBindingArgs.PodName)
+ extenderBindingResult = &extenderv1.ExtenderBindingResult{
+ Error: err.Error(),
+ }
+ }
+ }
+
+ if response, err := json.Marshal(extenderBindingResult); err != nil {
+ klog.ErrorS(err, "Failed to marshal binding result", "result", extenderBindingResult)
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusInternalServerError)
+ errMsg := fmt.Sprintf("{'error':'%s'}", err.Error())
+ w.Write([]byte(errMsg))
+ } else {
+ klog.V(5).InfoS("Returning bind response", "result", extenderBindingResult)
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ w.Write(response)
+ }
+ }
+}
+
+func bind(args extenderv1.ExtenderBindingArgs, bindFunc func(string, string, types.UID, string) error) *extenderv1.ExtenderBindingResult {
+ err := bindFunc(args.PodName, args.PodNamespace, args.PodUID, args.Node)
+ errMsg := ""
+ if err != nil {
+ klog.ErrorS(err, "Bind error", "pod", args.PodName, "namespace", args.PodNamespace, "node", args.Node, "uid", args.PodUID)
+ errMsg = err.Error()
+ }
+ return &extenderv1.ExtenderBindingResult{
+ Error: errMsg,
+ }
}
func WebHookRoute() httprouter.Handle {
- h, err := scheduler.NewWebHook()
- if err != nil {
- klog.Fatalf("new web hook error, %v", err)
- }
- return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
- h.ServeHTTP(w, r)
- }
+ h, err := scheduler.NewWebHook()
+ if err != nil {
+ klog.ErrorS(err, "Failed to create new webhook")
+ }
+ return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ klog.Infof("Handling webhook request on %s", r.URL.Path)
+ h.ServeHTTP(w, r)
+ }
+}
+
+func HealthzRoute() httprouter.Handle {
+ return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ klog.Infoln("Health check endpoint hit")
+ w.WriteHeader(http.StatusOK)
+ }
}
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index 455a3dd0e..f61ea04ee 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -1,295 +1,523 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package scheduler
import (
- "context"
- "encoding/json"
- "fmt"
- "sort"
- "strconv"
- "time"
-
- "4pd.io/k8s-vgpu/pkg/api"
- "4pd.io/k8s-vgpu/pkg/k8sutil"
- "4pd.io/k8s-vgpu/pkg/util"
- corev1 "k8s.io/api/core/v1"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- k8stypes "k8s.io/apimachinery/pkg/types"
- "k8s.io/client-go/informers"
- "k8s.io/client-go/kubernetes"
- listerscorev1 "k8s.io/client-go/listers/core/v1"
- "k8s.io/client-go/tools/cache"
- "k8s.io/klog/v2"
- extenderv1 "k8s.io/kube-scheduler/extender/v1"
+ "context"
+ "fmt"
+ "maps"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/client-go/informers"
+ "k8s.io/client-go/kubernetes"
+ listerscorev1 "k8s.io/client-go/listers/core/v1"
+ "k8s.io/client-go/tools/cache"
+ "k8s.io/client-go/tools/record"
+ "k8s.io/klog/v2"
+ extenderv1 "k8s.io/kube-scheduler/extender/v1"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/k8sutil"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/config"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/policy"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
)
type Scheduler struct {
- nodeManager
- podManager
+ *nodeManager
+ *podManager
+
+ stopCh chan struct{}
+ kubeClient kubernetes.Interface
+ podLister listerscorev1.PodLister
+ nodeLister listerscorev1.NodeLister
+ //Node status returned by filter
+ cachedstatus map[string]*NodeUsage
+ nodeNotify chan struct{}
+ //Node Overview
+ overviewstatus map[string]*NodeUsage
- stopCh chan struct{}
- kubeClient kubernetes.Interface
- podLister listerscorev1.PodLister
- nodeLister listerscorev1.NodeLister
+ eventRecorder record.EventRecorder
}
func NewScheduler() *Scheduler {
- s := &Scheduler{
- stopCh: make(chan struct{}),
- }
- s.nodeManager.init()
- s.podManager.init()
- return s
+ klog.InfoS("Initializing HAMi scheduler")
+ s := &Scheduler{
+ stopCh: make(chan struct{}),
+ cachedstatus: make(map[string]*NodeUsage),
+ nodeNotify: make(chan struct{}, 1),
+ }
+ s.nodeManager = newNodeManager()
+ s.podManager = newPodManager()
+ klog.V(2).InfoS("Scheduler initialized successfully")
+ return s
}
-func check(err error) {
- if err != nil {
- klog.Fatal(err)
- }
+func (s *Scheduler) doNodeNotify() {
+ select {
+ case s.nodeNotify <- struct{}{}:
+ default:
+ }
}
-func (s *Scheduler) onAddPod(obj interface{}) {
- pod, ok := obj.(*corev1.Pod)
- if !ok {
- klog.Errorf("unknown add object type")
- return
- }
- nodeID, ok := pod.Annotations[util.AssignedNodeAnnotations]
- if !ok {
- return
- }
- ids, ok := pod.Annotations[util.AssignedIDsAnnotations]
- if !ok {
- return
- }
- if k8sutil.IsPodInTerminatedState(pod) {
- s.delPod(pod)
- return
- }
- podDev := util.DecodePodDevices(ids)
- s.addPod(pod, nodeID, podDev)
+func (s *Scheduler) onAddPod(obj any) {
+ pod, ok := obj.(*corev1.Pod)
+ if !ok {
+ klog.ErrorS(fmt.Errorf("invalid pod object"), "Failed to process pod addition")
+ return
+ }
+ klog.V(5).InfoS("Pod added", "pod", pod.Name, "namespace", pod.Namespace)
+ nodeID, ok := pod.Annotations[util.AssignedNodeAnnotations]
+ if !ok {
+ return
+ }
+ if k8sutil.IsPodInTerminatedState(pod) {
+ s.delPod(pod)
+ return
+ }
+ podDev, _ := util.DecodePodDevices(util.SupportDevices, pod.Annotations)
+ s.addPod(pod, nodeID, podDev)
}
-func (s *Scheduler) onUpdatePod(_, newObj interface{}) {
- s.onAddPod(newObj)
+func (s *Scheduler) onUpdatePod(_, newObj any) {
+ s.onAddPod(newObj)
}
-func (s *Scheduler) onDelPod(obj interface{}) {
- pod, ok := obj.(*corev1.Pod)
- if !ok {
- klog.Errorf("unknown add object type")
- return
- }
- _, ok = pod.Annotations[util.AssignedNodeAnnotations]
- if !ok {
- return
- }
- s.delPod(pod)
+func (s *Scheduler) onDelPod(obj any) {
+ pod, ok := obj.(*corev1.Pod)
+ if !ok {
+ klog.Errorf("unknown add object type")
+ return
+ }
+ _, ok = pod.Annotations[util.AssignedNodeAnnotations]
+ if !ok {
+ return
+ }
+ s.delPod(pod)
}
func (s *Scheduler) Start() {
- kubeClient, err := k8sutil.NewClient()
- check(err)
- s.kubeClient = kubeClient
- informerFactory := informers.NewSharedInformerFactoryWithOptions(s.kubeClient, time.Hour*1)
- s.podLister = informerFactory.Core().V1().Pods().Lister()
- s.nodeLister = informerFactory.Core().V1().Nodes().Lister()
-
- informer := informerFactory.Core().V1().Pods().Informer()
- informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
- AddFunc: s.onAddPod,
- UpdateFunc: s.onUpdatePod,
- DeleteFunc: s.onDelPod,
- })
-
- informerFactory.Start(s.stopCh)
- informerFactory.WaitForCacheSync(s.stopCh)
+ klog.InfoS("Starting HAMi scheduler components")
+ s.kubeClient = client.GetClient()
+ informerFactory := informers.NewSharedInformerFactoryWithOptions(s.kubeClient, time.Hour*1)
+ s.podLister = informerFactory.Core().V1().Pods().Lister()
+ s.nodeLister = informerFactory.Core().V1().Nodes().Lister()
+
+ informerFactory.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+ AddFunc: s.onAddPod,
+ UpdateFunc: s.onUpdatePod,
+ DeleteFunc: s.onDelPod,
+ })
+ informerFactory.Core().V1().Nodes().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+ AddFunc: func(_ any) { s.doNodeNotify() },
+ UpdateFunc: func(_, _ any) { s.doNodeNotify() },
+ DeleteFunc: func(_ any) { s.doNodeNotify() },
+ })
+ informerFactory.Start(s.stopCh)
+ informerFactory.WaitForCacheSync(s.stopCh)
+ s.addAllEventHandlers()
}
func (s *Scheduler) Stop() {
- close(s.stopCh)
+ close(s.stopCh)
}
-//func (s *Scheduler) assignedNode(pod *corev1.Pod) string {
-// if node, ok := pod.ObjectMeta.Annotations[util.AssignedNodeAnnotations]; ok {
-// return node
-// }
-// return ""
-//}
-func (s *Scheduler) Register(stream api.DeviceService_RegisterServer) error {
- var nodeID string
- for {
- req, err := stream.Recv()
- if err != nil {
- s.delNode(nodeID)
- klog.Infof("node %v leave, %v", nodeID, err)
- _ = stream.SendAndClose(&api.RegisterReply{})
- return err
- }
- klog.V(3).Infof("device register %v", req.String())
- nodeID = req.GetNode()
- nodeInfo := NodeInfo{}
- nodeInfo.ID = nodeID
- nodeInfo.Devices = make([]DeviceInfo, len(req.Devices))
- for i := 0; i < len(req.Devices); i++ {
- nodeInfo.Devices[i] = DeviceInfo{
- ID: req.Devices[i].GetId(),
- Count: req.Devices[i].GetCount(),
- Health: req.Devices[i].GetHealth(),
- }
- }
- s.addNode(nodeID, nodeInfo)
- klog.Infof("node %v come", nodeID)
- }
+func (s *Scheduler) RegisterFromNodeAnnotations() {
+ klog.InfoS("Entering RegisterFromNodeAnnotations")
+ defer klog.InfoS("Exiting RegisterFromNodeAnnotations")
+
+ labelSelector := labels.Set(config.NodeLabelSelector).AsSelector()
+ klog.InfoS("Using label selector for list nodes", "selector", labelSelector.String())
+
+ ticker := time.NewTicker(time.Second * 15)
+ defer ticker.Stop()
+ printedLog := map[string]bool{}
+ for {
+ select {
+ case <-s.nodeNotify:
+ klog.V(5).InfoS("Received node notification")
+ case <-ticker.C:
+ klog.InfoS("Ticker triggered")
+ case <-s.stopCh:
+ klog.InfoS("Received stop signal, exiting RegisterFromNodeAnnotations")
+ return
+ }
+ rawNodes, err := s.nodeLister.List(labelSelector)
+ if err != nil {
+ klog.ErrorS(err, "Failed to list nodes with selector", "selector", labelSelector.String())
+ continue
+ }
+ klog.V(5).InfoS("Listed nodes", "nodeCount", len(rawNodes))
+ var nodeNames []string
+ for _, val := range rawNodes {
+ nodeNames = append(nodeNames, val.Name)
+ klog.V(5).InfoS("Processing node", "nodeName", val.Name)
+
+ for devhandsk, devInstance := range device.GetDevices() {
+ klog.V(5).InfoS("Checking device health", "nodeName", val.Name, "deviceVendor", devhandsk)
+
+ nodedevices, err := devInstance.GetNodeDevices(*val)
+ if err != nil {
+ klog.V(5).InfoS("Failed to get node devices", "nodeName", val.Name, "deviceVendor", devhandsk)
+ continue
+ }
+
+ health, needUpdate := devInstance.CheckHealth(devhandsk, val)
+ klog.V(5).InfoS("Device health check result", "nodeName", val.Name, "deviceVendor", devhandsk, "health", health, "needUpdate", needUpdate)
+
+ if !health {
+ klog.Warning("Device is unhealthy, cleaning up node", "nodeName", val.Name, "deviceVendor", devhandsk)
+ err := devInstance.NodeCleanUp(val.Name)
+ if err != nil {
+ klog.ErrorS(err, "Node cleanup failed", "nodeName", val.Name, "deviceVendor", devhandsk)
+ }
+
+ s.rmNodeDevices(val.Name, devhandsk)
+ continue
+ }
+ if !needUpdate {
+ klog.V(5).InfoS("No update needed for device", "nodeName", val.Name, "deviceVendor", devhandsk)
+ continue
+ }
+ _, ok := util.HandshakeAnnos[devhandsk]
+ if ok {
+ tmppat := make(map[string]string)
+ tmppat[util.HandshakeAnnos[devhandsk]] = "Requesting_" + time.Now().Format(time.DateTime)
+ klog.InfoS("New timestamp for annotation", "nodeName", val.Name, "annotationKey", util.HandshakeAnnos[devhandsk], "annotationValue", tmppat[util.HandshakeAnnos[devhandsk]])
+ n, err := util.GetNode(val.Name)
+ if err != nil {
+ klog.ErrorS(err, "Failed to get node", "nodeName", val.Name)
+ continue
+ }
+ klog.V(5).InfoS("Patching node annotations", "nodeName", val.Name, "annotations", tmppat)
+ if err := util.PatchNodeAnnotations(n, tmppat); err != nil {
+ klog.ErrorS(err, "Failed to patch node annotations", "nodeName", val.Name)
+ }
+ }
+ nodeInfo := &util.NodeInfo{}
+ nodeInfo.ID = val.Name
+ nodeInfo.Node = val
+ klog.V(5).InfoS("Fetching node devices", "nodeName", val.Name, "deviceVendor", devhandsk)
+ nodeInfo.Devices = make([]util.DeviceInfo, 0)
+ for _, deviceinfo := range nodedevices {
+ nodeInfo.Devices = append(nodeInfo.Devices, *deviceinfo)
+ }
+ s.addNode(val.Name, nodeInfo)
+ if s.nodes[val.Name] != nil && len(nodeInfo.Devices) > 0 {
+ if printedLog[val.Name] {
+ klog.V(5).InfoS("Node device updated", "nodeName", val.Name, "deviceVendor", devhandsk, "nodeInfo", nodeInfo, "totalDevices", s.nodes[val.Name].Devices)
+ } else {
+ klog.InfoS("Node device added", "nodeName", val.Name, "deviceVendor", devhandsk, "nodeInfo", nodeInfo, "totalDevices", s.nodes[val.Name].Devices)
+ printedLog[val.Name] = true
+ }
+ }
+ }
+ }
+ _, _, err = s.getNodesUsage(&nodeNames, nil)
+ if err != nil {
+ klog.ErrorS(err, "Failed to get node usage", "nodeNames", nodeNames)
+ }
+ }
}
-func (s *Scheduler) GetContainer(_ context.Context, req *api.GetContainerRequest) (*api.GetContainerReply, error) {
- pi, ctrIdx, err := s.getContainerByUUID(req.Uuid)
- if err != nil {
- return nil, err
- }
- if ctrIdx >= len(pi.devices) {
- return nil, fmt.Errorf("container index error")
- }
- pod, err := s.podLister.Pods(pi.namespace).Get(pi.name)
- if err != nil {
- return nil, err
- }
- if pod == nil || ctrIdx >= len(pi.devices) {
- return nil, fmt.Errorf("container not found")
- }
- rep := api.GetContainerReply{
- DevList: pi.devices[ctrIdx],
- PodUID: string(pod.UID),
- CtrName: pod.Spec.Containers[ctrIdx].Name,
- PodNamespace: pod.Namespace,
- PodName: pod.Name,
- }
- return &rep, nil
+// InspectAllNodesUsage is used by metrics monitor.
+func (s *Scheduler) InspectAllNodesUsage() *map[string]*NodeUsage {
+ return &s.overviewstatus
}
-func (s *Scheduler) getNodesUsage(nodes *[]string) (*map[string]*NodeUsage, map[string]string, error) {
- nodeMap := make(map[string]*NodeUsage)
- failedNodes := make(map[string]string)
- for _, nodeID := range *nodes {
- node, err := s.GetNode(nodeID)
- if err != nil {
- klog.Errorf("get node %v device error, %v", nodeID, err)
- failedNodes[nodeID] = fmt.Sprintf("node unregisterd")
- continue
- }
-
- nodeInfo := &NodeUsage{}
- for _, d := range node.Devices {
- nodeInfo.devices = append(nodeInfo.devices, &DeviceUsage{
- id: d.ID,
- used: 0,
- count: d.Count,
- health: d.Health,
- })
- }
- nodeMap[nodeID] = nodeInfo
- }
- for _, p := range s.pods {
- node, ok := nodeMap[p.nodeID]
- if !ok {
- continue
- }
- for _, ds := range p.devices {
- for _, deviceID := range ds {
- for _, d := range node.devices {
- if d.id == deviceID {
- d.used++
- }
- }
- }
- }
- klog.V(5).Infof("usage: pod %v assigned %v %v", p.name, p.nodeID, p.devices)
- }
- return &nodeMap, failedNodes, nil
+// returns all nodes and its device memory usage, and we filter it with nodeSelector, taints, nodeAffinity
+// unschedulerable and nodeName.
+func (s *Scheduler) getNodesUsage(nodes *[]string, task *corev1.Pod) (*map[string]*NodeUsage, map[string]string, error) {
+ overallnodeMap := make(map[string]*NodeUsage)
+ cachenodeMap := make(map[string]*NodeUsage)
+ failedNodes := make(map[string]string)
+ allNodes, err := s.ListNodes()
+ if err != nil {
+ return &overallnodeMap, failedNodes, err
+ }
+
+ for _, node := range allNodes {
+ nodeInfo := &NodeUsage{}
+ userGPUPolicy := util.GetGPUSchedulerPolicyByPod(config.GPUSchedulerPolicy, task)
+ nodeInfo.Node = node.Node
+ nodeInfo.Devices = policy.DeviceUsageList{
+ Policy: userGPUPolicy,
+ DeviceLists: make([]*policy.DeviceListsScore, 0),
+ }
+ for _, d := range node.Devices {
+ nodeInfo.Devices.DeviceLists = append(nodeInfo.Devices.DeviceLists, &policy.DeviceListsScore{
+ Score: 0,
+ Device: &util.DeviceUsage{
+ ID: d.ID,
+ Index: d.Index,
+ Used: 0,
+ Count: d.Count,
+ Usedmem: 0,
+ Totalmem: d.Devmem,
+ Totalcore: d.Devcore,
+ Usedcores: 0,
+ MigUsage: util.MigInUse{
+ Index: 0,
+ UsageList: make(util.MIGS, 0),
+ },
+ MigTemplate: d.MIGTemplate,
+ Mode: d.Mode,
+ Type: d.Type,
+ Numa: d.Numa,
+ Health: d.Health,
+ CustomInfo: maps.Clone(d.CustomInfo),
+ },
+ })
+ }
+ overallnodeMap[node.ID] = nodeInfo
+ }
+
+ podsInfo := s.ListPodsInfo()
+ for _, p := range podsInfo {
+ node, ok := overallnodeMap[p.NodeID]
+ if !ok {
+ klog.V(5).InfoS("pod allocated unknown node resources",
+ "pod", klog.KRef(p.Namespace, p.Name), "nodeID", p.NodeID)
+ continue
+ }
+ for _, podsingleds := range p.Devices {
+ for _, ctrdevs := range podsingleds {
+ for _, udevice := range ctrdevs {
+ for _, d := range node.Devices.DeviceLists {
+ deviceID := udevice.UUID
+ if strings.Contains(deviceID, "[") {
+ deviceID = strings.Split(deviceID, "[")[0]
+ }
+ if d.Device.ID == deviceID {
+ d.Device.Used++
+ d.Device.Usedmem += udevice.Usedmem
+ d.Device.Usedcores += udevice.Usedcores
+ if strings.Contains(udevice.UUID, "[") {
+ if strings.Compare(d.Device.Mode, "hami-core") == 0 {
+ klog.Errorf("found a mig task running on a hami-core GPU\n")
+ d.Device.Health = false
+ continue
+ }
+ tmpIdx, Instance, _ := util.ExtractMigTemplatesFromUUID(udevice.UUID)
+ if len(d.Device.MigUsage.UsageList) == 0 {
+ util.PlatternMIG(&d.Device.MigUsage, d.Device.MigTemplate, tmpIdx)
+ }
+ d.Device.MigUsage.UsageList[Instance].InUse = true
+ klog.V(5).Infoln("add mig usage", d.Device.MigUsage, "template=", d.Device.MigTemplate, "uuid=", d.Device.ID)
+ }
+ }
+ }
+ }
+ }
+ }
+ klog.V(5).Infof("usage: pod %v assigned %v %v", p.Name, p.NodeID, p.Devices)
+ }
+ s.overviewstatus = overallnodeMap
+ for _, nodeID := range *nodes {
+ node, err := s.GetNode(nodeID)
+ if err != nil {
+ // The identified node does not have a gpu device, so the log here has no practical meaning,increase log priority.
+ klog.V(5).InfoS("node unregistered", "node", nodeID, "error", err)
+ failedNodes[nodeID] = "node unregistered"
+ continue
+ }
+ cachenodeMap[node.ID] = overallnodeMap[node.ID]
+ }
+ s.cachedstatus = cachenodeMap
+ return &cachenodeMap, failedNodes, nil
+}
+
+func (s *Scheduler) getPodUsage() (map[string]PodUseDeviceStat, error) {
+ podUsageStat := make(map[string]PodUseDeviceStat)
+ pods, err := s.podLister.List(labels.NewSelector())
+ if err != nil {
+ return nil, err
+ }
+ for _, pod := range pods {
+ if pod.Status.Phase != corev1.PodSucceeded {
+ continue
+ }
+ podUseDeviceNum := 0
+ if v, ok := pod.Annotations[util.DeviceBindPhase]; ok && v == util.DeviceBindSuccess {
+ podUseDeviceNum = 1
+ }
+ nodeName := pod.Spec.NodeName
+ if _, ok := podUsageStat[nodeName]; !ok {
+ podUsageStat[nodeName] = PodUseDeviceStat{
+ TotalPod: 1,
+ UseDevicePod: podUseDeviceNum,
+ }
+ } else {
+ exist := podUsageStat[nodeName]
+ podUsageStat[nodeName] = PodUseDeviceStat{
+ TotalPod: exist.TotalPod + 1,
+ UseDevicePod: exist.UseDevicePod + podUseDeviceNum,
+ }
+ }
+ }
+ return podUsageStat, nil
+}
+
+func (s *Scheduler) Bind(args extenderv1.ExtenderBindingArgs) (*extenderv1.ExtenderBindingResult, error) {
+ klog.InfoS("Attempting to bind pod to node", "pod", args.PodName, "namespace", args.PodNamespace, "node", args.Node)
+ var res *extenderv1.ExtenderBindingResult
+
+ binding := &corev1.Binding{
+ ObjectMeta: metav1.ObjectMeta{Name: args.PodName, UID: args.PodUID},
+ Target: corev1.ObjectReference{Kind: "Node", Name: args.Node},
+ }
+ current, err := s.kubeClient.CoreV1().Pods(args.PodNamespace).Get(context.Background(), args.PodName, metav1.GetOptions{})
+ if err != nil {
+ klog.ErrorS(err, "Failed to get pod", "pod", args.PodName, "namespace", args.PodNamespace)
+ return &extenderv1.ExtenderBindingResult{Error: err.Error()}, err
+ }
+ klog.InfoS("Trying to get the target node for pod", "pod", args.PodName, "namespace", args.PodNamespace, "node", args.Node)
+ node, err := s.kubeClient.CoreV1().Nodes().Get(context.Background(), args.Node, metav1.GetOptions{})
+ if err != nil {
+ klog.ErrorS(err, "Failed to get node", "node", args.Node)
+ s.recordScheduleBindingResultEvent(current, EventReasonBindingFailed, []string{}, fmt.Errorf("failed to get node %s", args.Node))
+ res = &extenderv1.ExtenderBindingResult{Error: err.Error()}
+ return res, nil
+ }
+
+ tmppatch := map[string]string{
+ util.DeviceBindPhase: "allocating",
+ util.BindTimeAnnotations: strconv.FormatInt(time.Now().Unix(), 10),
+ }
+
+ for _, val := range device.GetDevices() {
+ err = val.LockNode(node, current)
+ if err != nil {
+ klog.ErrorS(err, "Failed to lock node", "node", args.Node, "device", val)
+ goto ReleaseNodeLocks
+ }
+ }
+
+ err = util.PatchPodAnnotations(current, tmppatch)
+ if err != nil {
+ klog.ErrorS(err, "Failed to patch pod annotations", "pod", klog.KObj(current))
+ goto ReleaseNodeLocks
+ }
+
+ err = s.kubeClient.CoreV1().Pods(args.PodNamespace).Bind(context.Background(), binding, metav1.CreateOptions{})
+ if err != nil {
+ klog.ErrorS(err, "Failed to bind pod", "pod", args.PodName, "namespace", args.PodNamespace, "node", args.Node)
+ goto ReleaseNodeLocks
+ }
+
+ s.recordScheduleBindingResultEvent(current, EventReasonBindingSucceed, []string{args.Node}, nil)
+ klog.InfoS("Successfully bound pod to node", "pod", args.PodName, "namespace", args.PodNamespace, "node", args.Node)
+ return &extenderv1.ExtenderBindingResult{Error: ""}, nil
+
+ReleaseNodeLocks:
+ klog.InfoS("Release node locks", "node", args.Node)
+ for _, val := range device.GetDevices() {
+ val.ReleaseNodeLock(node, current)
+ }
+ s.recordScheduleBindingResultEvent(current, EventReasonBindingFailed, []string{}, err)
+ return &extenderv1.ExtenderBindingResult{Error: err.Error()}, nil
}
func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFilterResult, error) {
- klog.Infof("schedule pod %v/%v[%v]", args.Pod.Namespace, args.Pod.Name, args.Pod.UID)
- nums := k8sutil.ResourceNums(args.Pod, corev1.ResourceName(util.ResourceName))
- total := 0
- for _, n := range nums {
- total += n
- }
- if total == 0 {
- klog.V(1).Infof("pod %v not find resource %v", args.Pod.Name, util.ResourceName)
- return &extenderv1.ExtenderFilterResult{
- NodeNames: args.NodeNames,
- FailedNodes: nil,
- Error: "",
- }, nil
- }
- s.delPod(args.Pod)
- nodeUsage, failedNodes, err := s.getNodesUsage(args.NodeNames)
- if err != nil {
- return nil, err
- }
- nodeScores, err := calcScore(nodeUsage, &failedNodes, nums)
- if err != nil {
- return nil, err
- }
- if len(*nodeScores) == 0 {
- return &extenderv1.ExtenderFilterResult{
- FailedNodes: failedNodes,
- }, nil
- }
- sort.Sort(nodeScores)
- m := (*nodeScores)[len(*nodeScores)-1]
- klog.Infof("schedule %v/%v to %v %v", args.Pod.Namespace, args.Pod.Name, m.nodeID, m.devices)
- annotations := make(map[string]string)
- annotations[util.AssignedNodeAnnotations] = m.nodeID
- annotations[util.AssignedTimeAnnotations] = strconv.FormatInt(time.Now().Unix(), 10)
- annotations[util.AssignedIDsAnnotations] = util.EncodePodDevices(m.devices)
- s.addPod(args.Pod, m.nodeID, m.devices)
- err = s.patchPodAnnotations(args.Pod, annotations)
- if err != nil {
- s.delPod(args.Pod)
- return nil, err
- }
- res := extenderv1.ExtenderFilterResult{NodeNames: &[]string{m.nodeID}}
- return &res, nil
+ klog.InfoS("Starting schedule filter process", "pod", args.Pod.Name, "uuid", args.Pod.UID, "namespace", args.Pod.Namespace)
+ resourceReqs := k8sutil.Resourcereqs(args.Pod)
+ resourceReqTotal := 0
+ for _, n := range resourceReqs {
+ for _, k := range n {
+ resourceReqTotal += int(k.Nums)
+ }
+ }
+ if resourceReqTotal == 0 {
+ klog.V(1).InfoS("Pod does not request any resources",
+ "pod", args.Pod.Name)
+ s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", fmt.Errorf("does not request any resource"))
+ return &extenderv1.ExtenderFilterResult{
+ NodeNames: args.NodeNames,
+ FailedNodes: nil,
+ Error: "",
+ }, nil
+ }
+ annos := args.Pod.Annotations
+ s.delPod(args.Pod)
+ nodeUsage, failedNodes, err := s.getNodesUsage(args.NodeNames, args.Pod)
+ if err != nil {
+ s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", err)
+ return nil, err
+ }
+ if len(failedNodes) != 0 {
+ klog.V(5).InfoS("Nodes failed during usage retrieval",
+ "nodes", failedNodes)
+ }
+ nodeScores, err := s.calcScore(nodeUsage, resourceReqs, annos, args.Pod, failedNodes)
+ if err != nil {
+ err := fmt.Errorf("calcScore failed %v for pod %v", err, args.Pod.Name)
+ s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", err)
+ return nil, err
+ }
+ if len((*nodeScores).NodeList) == 0 {
+ klog.V(4).InfoS("No available nodes meet the required scores",
+ "pod", args.Pod.Name)
+ s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", fmt.Errorf("no available node, %d nodes do not meet", len(*args.NodeNames)))
+ return &extenderv1.ExtenderFilterResult{
+ FailedNodes: failedNodes,
+ }, nil
+ }
+ klog.V(4).Infoln("nodeScores_len=", len((*nodeScores).NodeList))
+ sort.Sort(nodeScores)
+ m := (*nodeScores).NodeList[len((*nodeScores).NodeList)-1]
+ klog.InfoS("Scheduling pod to node",
+ "podNamespace", args.Pod.Namespace,
+ "podName", args.Pod.Name,
+ "nodeID", m.NodeID,
+ "devices", m.Devices)
+ annotations := make(map[string]string)
+ annotations[util.AssignedNodeAnnotations] = m.NodeID
+ annotations[util.AssignedTimeAnnotations] = strconv.FormatInt(time.Now().Unix(), 10)
+
+ for _, val := range device.GetDevices() {
+ val.PatchAnnotations(args.Pod, &annotations, m.Devices)
+ }
+
+ s.addPod(args.Pod, m.NodeID, m.Devices)
+ err = util.PatchPodAnnotations(args.Pod, annotations)
+ if err != nil {
+ s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", err)
+ s.delPod(args.Pod)
+ return nil, err
+ }
+ successMsg := genSuccessMsg(len(*args.NodeNames), m.NodeID, nodeScores.NodeList)
+ s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringSucceed, successMsg, nil)
+ res := extenderv1.ExtenderFilterResult{NodeNames: &[]string{m.NodeID}}
+ return &res, nil
}
-func (s *Scheduler) patchPodAnnotations(pod *corev1.Pod, annotations map[string]string) error {
- type patchMetadata struct {
- Annotations map[string]string `json:"annotations,omitempty"`
- }
- type patchPod struct {
- Metadata patchMetadata `json:"metadata"`
- //Spec patchSpec `json:"spec,omitempty"`
- }
-
- p := patchPod{}
- p.Metadata.Annotations = annotations
-
- bytes, err := json.Marshal(p)
- if err != nil {
- return err
- }
- _, err = s.kubeClient.CoreV1().Pods(pod.Namespace).
- Patch(context.Background(), pod.Name, k8stypes.StrategicMergePatchType, bytes, metav1.PatchOptions{})
- if err != nil {
- klog.Infof("patch pod %v failed, %v", pod.Name, err)
- }
- return err
+func genSuccessMsg(totalNodes int, target string, nodes []*policy.NodeScore) string {
+ successMsg := "find fit node(%s), %d nodes not fit, %d nodes fit(%s)"
+ var scores []string
+ for _, no := range nodes {
+ scores = append(scores, fmt.Sprintf("%s:%.2f", no.NodeID, no.Score))
+ }
+ score := strings.Join(scores, ",")
+ return fmt.Sprintf(successMsg, target, totalNodes-len(nodes), len(nodes), score)
}
diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go
new file mode 100644
index 000000000..3cfe7f824
--- /dev/null
+++ b/pkg/scheduler/scheduler_test.go
@@ -0,0 +1,836 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scheduler
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/informers"
+ "k8s.io/client-go/kubernetes/fake"
+ "k8s.io/client-go/tools/cache"
+ "k8s.io/klog/v2"
+ extenderv1 "k8s.io/kube-scheduler/extender/v1"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/policy"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+)
+
+func Test_getNodesUsage(t *testing.T) {
+ nodeMage := newNodeManager()
+ nodeMage.addNode("node1", &util.NodeInfo{
+ ID: "node1",
+ Devices: []util.DeviceInfo{
+ {
+ ID: "GPU0",
+ Index: 0,
+ Count: 10,
+ Devmem: 1024,
+ Devcore: 100,
+ Numa: 1,
+ Mode: "hami",
+ Health: true,
+ },
+ {
+ ID: "GPU1",
+ Index: 1,
+ Count: 10,
+ Devmem: 1024,
+ Devcore: 100,
+ Numa: 1,
+ Mode: "hami",
+ Health: true,
+ },
+ },
+ })
+ podDevces := util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ []util.ContainerDevice{
+ {
+ Idx: 0,
+ UUID: "GPU0",
+ Usedmem: 100,
+ Usedcores: 10,
+ },
+ },
+ },
+ }
+ podMap := newPodManager()
+ podMap.addPod(&corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ UID: "1111",
+ Name: "test1",
+ Namespace: "default",
+ },
+ }, "node1", podDevces)
+ podMap.addPod(&corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ UID: "2222",
+ Name: "test2",
+ Namespace: "default",
+ },
+ }, "node1", podDevces)
+ s := Scheduler{
+ nodeManager: nodeMage,
+ podManager: podMap,
+ }
+ nodes := make([]string, 0)
+ nodes = append(nodes, "node1")
+ cachenodeMap, _, err := s.getNodesUsage(&nodes, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, len(*cachenodeMap), 1)
+ v, ok := (*cachenodeMap)["node1"]
+ assert.Equal(t, ok, true)
+ assert.Equal(t, len(v.Devices.DeviceLists), 2)
+ assert.Equal(t, v.Devices.DeviceLists[0].Device.Used, int32(2))
+ assert.Equal(t, v.Devices.DeviceLists[0].Device.Usedmem, int32(200))
+ assert.Equal(t, v.Devices.DeviceLists[0].Device.Usedcores, int32(20))
+}
+
+// test case matrix
+/**
+| pod name | node name| pod status | annotations | result |
+|--------------|----------|-------------|---------------------------- |-------------------------------------|
+| test-pod-1 | node11 | Succeeded | hami.io/bind-phase:success | node11:{TotalPod:1,UseDevicePod:1} |
+| test-pod-2 | node12 | Running | none | node12:{TotalPod:0;UseDevicePod:0} |
+| test-pod-3 | node13 | Succeeded | none | node13:{TotalPod:1;UseDevicePod:0} |
+test case matrix.
+*/
+
+func Test_getPodUsage(t *testing.T) {
+ s := NewScheduler()
+ client.KubeClient = fake.NewSimpleClientset()
+ s.kubeClient = client.KubeClient
+ informerFactory := informers.NewSharedInformerFactoryWithOptions(client.KubeClient, time.Hour*1)
+ s.podLister = informerFactory.Core().V1().Pods().Lister()
+ informerFactory.Start(s.stopCh)
+ informerFactory.WaitForCacheSync(s.stopCh)
+
+ tests := []struct {
+ name string
+ pods []*corev1.Pod
+ want map[string]PodUseDeviceStat
+ wantErr error
+ }{
+ {
+ name: "One pod running",
+ pods: []*corev1.Pod{
+ {
+ Status: corev1.PodStatus{Phase: corev1.PodRunning},
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod-2",
+ Namespace: "default",
+ UID: "uuid12",
+ },
+ Spec: corev1.PodSpec{
+ NodeName: "node12",
+ },
+ },
+ },
+ want: map[string]PodUseDeviceStat{
+ "node12": {
+ TotalPod: 0, // Running pod does not count
+ UseDevicePod: 0,
+ },
+ },
+ },
+ {
+ name: "one pod succeeded,no annotation",
+ pods: []*corev1.Pod{
+ {
+ Status: corev1.PodStatus{Phase: corev1.PodSucceeded},
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod-3",
+ Namespace: "default",
+ UID: "uuid14",
+ },
+ Spec: corev1.PodSpec{
+ NodeName: "node13",
+ },
+ },
+ },
+ want: map[string]PodUseDeviceStat{
+ "node13": {
+ TotalPod: 1,
+ UseDevicePod: 0, // No annotation
+ },
+ },
+ },
+ {
+ name: "All pods succeeded with device bind success",
+ pods: []*corev1.Pod{
+ {
+ Status: corev1.PodStatus{Phase: corev1.PodSucceeded},
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod-1",
+ Namespace: "default",
+ UID: "uuid11",
+ Annotations: map[string]string{util.DeviceBindPhase: util.DeviceBindSuccess},
+ },
+ Spec: corev1.PodSpec{
+ NodeName: "node11",
+ },
+ },
+ },
+ want: map[string]PodUseDeviceStat{
+ "node11": {
+ TotalPod: 1,
+ UseDevicePod: 1,
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ for _, pod := range test.pods {
+ client.KubeClient.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{})
+ s.addPod(pod, pod.Spec.NodeName, util.PodDevices{})
+ }
+
+ result, err := s.getPodUsage()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ assert.Equal(t, test.want[test.pods[0].Namespace], result[test.pods[0].Namespace])
+ })
+ }
+}
+
+// test case matrix
+/**
+| node policy| gpu policy| node num | per node device | pod use device | device use info | result |
+|------------|------------|----------|-----------------|----------------|---------------------------|--------------|
+| binpack | binpack | 2 | 2 | 1 ｜device1: 25%,device4: 75% ｜ node2-device4|
+| binpack | spread | 2 | 2 | 1 ｜device1: 25%,device4: 75% ｜ node2-device3|
+| spread | binpack | 2 | 2 | 1 ｜device1: 25%,device4: 75% ｜ node1-device1|
+| spread | spread | 2 | 2 | 1 ｜device1: 25%,device4: 75% ｜ node1-device2|
+test case matrix.
+*/
+func Test_Filter(t *testing.T) {
+ s := NewScheduler()
+ client.KubeClient = fake.NewSimpleClientset()
+ s.kubeClient = client.KubeClient
+ informerFactory := informers.NewSharedInformerFactoryWithOptions(client.KubeClient, time.Hour*1)
+ s.podLister = informerFactory.Core().V1().Pods().Lister()
+ informer := informerFactory.Core().V1().Pods().Informer()
+ informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+ AddFunc: s.onAddPod,
+ UpdateFunc: s.onUpdatePod,
+ DeleteFunc: s.onDelPod,
+ })
+ informerFactory.Start(s.stopCh)
+ informerFactory.WaitForCacheSync(s.stopCh)
+ s.addAllEventHandlers()
+ config := &device.Config{
+ NvidiaConfig: nvidia.NvidiaConfig{
+ ResourceCountName: "hami.io/gpu",
+ ResourceMemoryName: "hami.io/gpumem",
+ ResourceMemoryPercentageName: "hami.io/gpumem-percentage",
+ ResourceCoreName: "hami.io/gpucores",
+ DefaultMemory: 0,
+ DefaultCores: 0,
+ DefaultGPUNum: 1,
+ },
+ }
+
+ if err := device.InitDevicesWithConfig(config); err != nil {
+ klog.Fatalf("Failed to initialize devices with config: %v", err)
+ }
+ pod1 := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pod1",
+ UID: "uuid1",
+ Annotations: map[string]string{
+ util.DeviceBindPhase: util.DeviceBindSuccess,
+ },
+ },
+ Spec: corev1.PodSpec{
+ NodeName: "node1",
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(25, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(2000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ Status: corev1.PodStatus{
+ Phase: corev1.PodSucceeded,
+ },
+ }
+ pod2 := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pod2",
+ UID: "uuid2",
+ Annotations: map[string]string{
+ util.DeviceBindPhase: util.DeviceBindSuccess,
+ },
+ },
+ Spec: corev1.PodSpec{
+ NodeName: "node2",
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(75, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(6000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ Status: corev1.PodStatus{
+ Phase: corev1.PodSucceeded,
+ },
+ }
+ pod3 := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pod3",
+ UID: "uuid3",
+ },
+ Spec: corev1.PodSpec{
+ NodeName: "node2",
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{},
+ },
+ },
+ },
+ },
+ Status: corev1.PodStatus{
+ Phase: corev1.PodSucceeded,
+ },
+ }
+ client.KubeClient.CoreV1().Pods(pod1.Namespace).Create(context.Background(), pod1, metav1.CreateOptions{})
+ client.KubeClient.CoreV1().Pods(pod2.Namespace).Create(context.Background(), pod2, metav1.CreateOptions{})
+ client.KubeClient.CoreV1().Pods(pod3.Namespace).Create(context.Background(), pod3, metav1.CreateOptions{})
+
+ initNode := func() {
+ nodes, _ := s.ListNodes()
+ for index := range nodes {
+ s.rmNodeDevices(index, nvidia.NvidiaGPUDevice)
+ }
+ pods, _ := s.ListPodsUID()
+ for index := range pods {
+ s.delPod(pods[index])
+ }
+
+ s.addNode("node1", &util.NodeInfo{
+ ID: "node1",
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "device1",
+ Index: 0,
+ Count: 10,
+ Devmem: 8000,
+ Devcore: 100,
+ Numa: 0,
+ Mode: "hami",
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ DeviceVendor: nvidia.NvidiaGPUDevice,
+ },
+ {
+ ID: "device2",
+ Index: 1,
+ Count: 10,
+ Devmem: 8000,
+ Devcore: 100,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ DeviceVendor: nvidia.NvidiaGPUDevice,
+ },
+ },
+ })
+ s.addNode("node2", &util.NodeInfo{
+ ID: "node2",
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2"}},
+ Devices: []util.DeviceInfo{
+ {
+ ID: "device3",
+ Index: 0,
+ Count: 10,
+ Devmem: 8000,
+ Devcore: 100,
+ Numa: 0,
+ Mode: "hami",
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ {
+ ID: "device4",
+ Index: 1,
+ Count: 10,
+ Devmem: 8000,
+ Devcore: 100,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ },
+ })
+ s.addPod(pod1, "node1", util.PodDevices{
+ nvidia.NvidiaGPUDevice: util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "device1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedmem: 2000,
+ Usedcores: 25,
+ },
+ },
+ },
+ })
+ s.addPod(pod2, "node2", util.PodDevices{
+ nvidia.NvidiaGPUDevice: util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "device4",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedmem: 6000,
+ Usedcores: 75,
+ },
+ },
+ },
+ })
+ }
+
+ tests := []struct {
+ name string
+ args extenderv1.ExtenderArgs
+ want *extenderv1.ExtenderFilterResult
+ wantPodAnnotationDeviceID string
+ wantErr error
+ }{
+ {
+ name: "node use binpack gpu use binpack policy",
+ args: extenderv1.ExtenderArgs{
+ Pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ UID: "test1-uid1",
+ Annotations: map[string]string{
+ policy.GPUSchedulerPolicyAnnotationKey: util.GPUSchedulerPolicyBinpack.String(),
+ policy.NodeSchedulerPolicyAnnotationKey: util.NodeSchedulerPolicyBinpack.String(),
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(20, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ NodeNames: &[]string{"node1", "node2"},
+ },
+ wantErr: nil,
+ want: &extenderv1.ExtenderFilterResult{
+ NodeNames: &[]string{"node2"},
+ },
+ wantPodAnnotationDeviceID: "device4",
+ },
+ {
+ name: "node use binpack gpu use spread policy",
+ args: extenderv1.ExtenderArgs{
+ Pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test2",
+ UID: "test2-uid2",
+ Annotations: map[string]string{
+ policy.GPUSchedulerPolicyAnnotationKey: util.GPUSchedulerPolicySpread.String(),
+ policy.NodeSchedulerPolicyAnnotationKey: util.NodeSchedulerPolicyBinpack.String(),
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(20, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ NodeNames: &[]string{"node1", "node2"},
+ },
+ wantErr: nil,
+ want: &extenderv1.ExtenderFilterResult{
+ NodeNames: &[]string{"node2"},
+ },
+ wantPodAnnotationDeviceID: "device3",
+ },
+ {
+ name: "node use spread gpu use binpack policy",
+ args: extenderv1.ExtenderArgs{
+ Pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test3",
+ UID: "test3-uid3",
+ Annotations: map[string]string{
+ policy.GPUSchedulerPolicyAnnotationKey: util.GPUSchedulerPolicyBinpack.String(),
+ policy.NodeSchedulerPolicyAnnotationKey: util.NodeSchedulerPolicySpread.String(),
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(20, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ NodeNames: &[]string{"node1", "node2"},
+ },
+ wantErr: nil,
+ want: &extenderv1.ExtenderFilterResult{
+ NodeNames: &[]string{"node1"},
+ },
+ wantPodAnnotationDeviceID: "device1",
+ },
+ {
+ name: "node use spread gpu use spread policy",
+ args: extenderv1.ExtenderArgs{
+ Pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test4",
+ UID: "test4-uid4",
+ Annotations: map[string]string{
+ policy.GPUSchedulerPolicyAnnotationKey: util.GPUSchedulerPolicySpread.String(),
+ policy.NodeSchedulerPolicyAnnotationKey: util.NodeSchedulerPolicySpread.String(),
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(20, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ NodeNames: &[]string{"node1", "node2"},
+ },
+ wantErr: nil,
+ want: &extenderv1.ExtenderFilterResult{
+ NodeNames: &[]string{"node1"},
+ },
+ wantPodAnnotationDeviceID: "device2",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ initNode()
+ client.KubeClient.CoreV1().Pods(test.args.Pod.Namespace).Create(context.Background(), test.args.Pod, metav1.CreateOptions{})
+ got, gotErr := s.Filter(test.args)
+ assert.DeepEqual(t, test.wantErr, gotErr)
+ assert.DeepEqual(t, test.want, got)
+ getPod, _ := client.KubeClient.CoreV1().Pods(test.args.Pod.Namespace).Get(context.Background(), test.args.Pod.Name, metav1.GetOptions{})
+ podDevices, _ := util.DecodePodDevices(util.SupportDevices, getPod.Annotations)
+ assert.DeepEqual(t, test.wantPodAnnotationDeviceID, podDevices["NVIDIA"][0][0].UUID)
+ })
+ }
+}
+
+func Test_RegisterFromNodeAnnotations(t *testing.T) {
+ tests := []struct {
+ name string
+ Scheduler *Scheduler
+ want func(node *corev1.Node) bool
+ }{
+ {
+ name: "test node handshake annotations layout",
+ Scheduler: func() *Scheduler {
+ s := NewScheduler()
+ s.stopCh = make(chan struct{})
+ s.nodeNotify = make(chan struct{})
+ client.KubeClient = fake.NewSimpleClientset()
+ s.kubeClient = client.KubeClient
+ informerFactory := informers.NewSharedInformerFactoryWithOptions(client.KubeClient, time.Hour*1)
+ s.nodeLister = informerFactory.Core().V1().Nodes().Lister()
+
+ // Create a node
+ node := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node",
+ Annotations: map[string]string{
+ "hami.io/node-handshake": "Requesting_2025-06-13 09:07:40",
+ "hami.io/node-handshake-dcu": "Requesting_2025-06-13 09:07:40",
+ },
+ },
+ }
+ _, err := client.KubeClient.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{})
+ if err != nil {
+ t.Errorf("failed to create node: %v", err)
+ return nil
+ }
+
+ // Add node to informer cache
+ err = informerFactory.Core().V1().Nodes().Informer().GetIndexer().Add(node)
+ if err != nil {
+ t.Errorf("failed to add node to indexer: %v", err)
+ return nil
+ }
+
+ // Start informer factory to sync cache
+ informerFactory.Start(s.stopCh)
+ informerFactory.WaitForCacheSync(s.stopCh)
+
+ return s
+ }(),
+ want: func(node *corev1.Node) bool {
+ handshakeTimeStr, okHami := node.Annotations["hami.io/node-handshake"]
+ if !okHami {
+ t.Errorf("missing annotation: hami.io/node-handshake")
+ return false
+ }
+ dcuTimeStr, okDcu := node.Annotations["hami.io/node-handshake-dcu"]
+ if !okDcu {
+ t.Errorf("missing annotation: hami.io/node-handshake-dcu")
+ return false
+ }
+ _, errHami := time.Parse(time.DateTime, strings.TrimPrefix(handshakeTimeStr, "Requesting_"))
+ _, errDcu := time.Parse(time.DateTime, strings.TrimPrefix(dcuTimeStr, "Requesting_"))
+ if errHami != nil {
+ t.Errorf("invalid time format in annotation 'hami.io/node-handshake': %v", errHami)
+ return false
+ }
+ if errDcu != nil {
+ t.Errorf("invalid time format in annotation 'hami.io/node-handshake-dcu': %v", errDcu)
+ return false
+ }
+ return true
+ },
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ // Graceful shutdown after 5 seconds
+ time.AfterFunc(5*time.Second, func() {
+ close(test.Scheduler.stopCh)
+ })
+
+ // Notify node annotations
+ go func() {
+ test.Scheduler.nodeNotify <- struct{}{}
+ }()
+
+ // Invoke the method to test
+ test.Scheduler.RegisterFromNodeAnnotations()
+
+ // Get the node to verify annotations
+ node, err := test.Scheduler.kubeClient.CoreV1().Nodes().Get(context.TODO(), "node", metav1.GetOptions{})
+ if err != nil {
+ t.Errorf("failed to get node: %v", err)
+ return
+ }
+
+ // Verify the annotations
+ assert.Equal(t, test.want(node), true)
+ })
+ }
+}
+
+func Test_RegisterFromNodeAnnotations_NIL(t *testing.T) {
+ // Define a helper function to create a scheduler with a node that has nil annotations.
+ createSchedulerWithNilAnnotations := func() *Scheduler {
+ s := NewScheduler()
+ s.stopCh = make(chan struct{})
+ s.nodeNotify = make(chan struct{})
+
+ client.KubeClient = fake.NewSimpleClientset()
+ s.kubeClient = client.KubeClient
+
+ informerFactory := informers.NewSharedInformerFactoryWithOptions(client.KubeClient, time.Hour)
+ s.nodeLister = informerFactory.Core().V1().Nodes().Lister()
+
+ // Create a node without annotations (nil annotations)
+ node := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "node-nil-annotations",
+ },
+ }
+
+ // Create the node and add it to the indexer
+ _, err := s.kubeClient.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{})
+ if err != nil {
+ t.Fatalf("failed to create node: %v", err)
+ }
+ err = informerFactory.Core().V1().Nodes().Informer().GetIndexer().Add(node)
+ if err != nil {
+ t.Fatalf("failed to add node to indexer: %v", err)
+ }
+
+ // Start informer factory to sync cache
+ informerFactory.Start(s.stopCh)
+
+ // Check if cache sync was successful
+ _ = informerFactory.WaitForCacheSync(s.stopCh)
+
+ return s
+ }
+
+ tests := []struct {
+ name string
+ Scheduler *Scheduler
+ want func(*corev1.Node) bool
+ }{
+ {
+ name: "test nil annotations handling",
+ Scheduler: createSchedulerWithNilAnnotations(),
+ want: func(node *corev1.Node) bool {
+ if node == nil {
+ t.Errorf("node is nil")
+ return false
+ }
+
+ // Check if RegisterFromNodeAnnotations handles nil annotations gracefully
+ if node.Annotations == nil {
+ t.Logf("node annotations are nil, checking if handled properly...")
+ return true // Adjust based on expected behavior
+ }
+
+ // If annotations exist, check for specific annotations
+ handshakeTimeStr, okHami := node.Annotations["hami.io/node-handshake"]
+ dcuTimeStr, okDcu := node.Annotations["hami.io/node-handshake-dcu"]
+
+ // Here you can define what should happen when annotations are present but not set
+ if !okHami || !okDcu {
+ t.Logf("expected annotations are missing, checking if handled properly...")
+ return true // Adjust based on expected behavior
+ }
+
+ // Verify time format in annotations if they exist
+ _, errHami := time.Parse(time.DateTime, strings.TrimPrefix(handshakeTimeStr, "Requesting_"))
+ _, errDcu := time.Parse(time.DateTime, strings.TrimPrefix(dcuTimeStr, "Requesting_"))
+
+ if errHami != nil {
+ t.Errorf("invalid time format in annotation 'hami.io/node-handshake': %v", errHami)
+ return false
+ }
+ if errDcu != nil {
+ t.Errorf("invalid time format in annotation 'hami.io/node-handshake-dcu': %v", errDcu)
+ return false
+ }
+
+ return true
+ },
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ defer func() {
+ if r := recover(); r != nil {
+ t.Errorf("Recovered from panic: %v", r)
+ }
+ }()
+
+ // Ensure scheduler starts before running the test
+ time.AfterFunc(5*time.Second, func() {
+ close(test.Scheduler.stopCh)
+ })
+
+ // Notify node annotations after a short delay to ensure scheduler is ready
+ go func() {
+ time.Sleep(100 * time.Millisecond) // Give some time for scheduler to start
+ test.Scheduler.nodeNotify <- struct{}{}
+ }()
+
+ // Invoke the method to test
+ test.Scheduler.RegisterFromNodeAnnotations()
+
+ // Get the node to verify annotations
+ node, err := test.Scheduler.kubeClient.CoreV1().Nodes().Get(context.TODO(), "node-nil-annotations", metav1.GetOptions{})
+ require.NoError(t, err) // Use require to fail fast on error
+ require.NotNil(t, node) // Ensure node is not nil
+
+ // Verify the annotations
+ if !test.want(node) {
+ t.Errorf("annotations validation failed")
+ }
+ })
+ }
+}
diff --git a/pkg/scheduler/score.go b/pkg/scheduler/score.go
index 7d9c63847..0cf8ed1ac 100644
--- a/pkg/scheduler/score.go
+++ b/pkg/scheduler/score.go
@@ -1,94 +1,202 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package scheduler
import (
- "4pd.io/k8s-vgpu/pkg/util"
- "sort"
+ "sort"
+ "strings"
+ "sync"
+
+ corev1 "k8s.io/api/core/v1"
+ utilerrors "k8s.io/apimachinery/pkg/util/errors"
+ "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/config"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/policy"
+ "github.com/Project-HAMi/HAMi/pkg/util"
)
-type NodeScore struct {
- nodeID string
- devices util.PodDevices
- score float32
+func viewStatus(usage NodeUsage) {
+ klog.V(5).Info("devices status")
+ for _, val := range usage.Devices.DeviceLists {
+ klog.V(5).InfoS("device status", "device id", val.Device.ID, "device detail", val)
+ }
}
-type NodeScoreList []*NodeScore
+const (
+ cardTypeMismatch = "CardTypeMismatch"
+ cardUUIDMismatch = "CardUuidMismatch"
+ cardTimeSlicingExhausted = "CardTimeSlicingExhausted"
+ cardComputeUnitsExhausted = "CardComputeUnitsExhausted"
+ cardInsufficientMemory = "CardInsufficientMemory"
+ cardInsufficientCore = "CardInsufficientCore"
+ numaNotFit = "NumaNotFit"
+ exclusiveDeviceAllocateConflict = "ExclusiveDeviceAllocateConflict"
+ cardNotFoundCustomFilterRule = "CardNotFoundCustomFilterRule"
+ nodeInsufficientDevice = "NodeInsufficientDevice"
+ allocatedCardsInsufficientRequest = "AllocatedCardsInsufficientRequest"
+ nodeUnfitPod = "NodeUnfitPod"
+ nodeFitPod = "NodeFitPod"
+)
-func (l DeviceUsageList) Len() int {
- return len(l)
+func getNodeResources(list NodeUsage, t string) []*util.DeviceUsage {
+ l := []*util.DeviceUsage{}
+ for _, val := range list.Devices.DeviceLists {
+ if strings.Contains(val.Device.Type, t) {
+ l = append(l, val.Device)
+ }
+ }
+ return l
}
-func (l DeviceUsageList) Swap(i, j int) {
- l[i], l[j] = l[j], l[i]
+func fitInDevices(node *NodeUsage, requests util.ContainerDeviceRequests, annos map[string]string, pod *corev1.Pod, nodeInfo *util.NodeInfo, devinput *util.PodDevices) (bool, string) {
+ //devmap := make(map[string]util.ContainerDevices)
+ devs := util.ContainerDevices{}
+ total, totalCore, totalMem := int32(0), int32(0), int32(0)
+ free, freeCore, freeMem := int32(0), int32(0), int32(0)
+ sums := 0
+ // computer all device score for one node
+ for index := range node.Devices.DeviceLists {
+ node.Devices.DeviceLists[index].ComputeScore(requests)
+ }
+ //This loop is for requests for different devices
+ for _, k := range requests {
+ sums += int(k.Nums)
+ if int(k.Nums) > len(node.Devices.DeviceLists) {
+ klog.V(5).InfoS(nodeInsufficientDevice, "pod", klog.KObj(pod), "request devices nums", k.Nums, "node device nums", len(node.Devices.DeviceLists))
+ return false, nodeInsufficientDevice
+ }
+ sort.Sort(node.Devices)
+ _, ok := device.GetDevices()[k.Type]
+ if !ok {
+ return false, "Device type not found"
+ }
+ fit, tmpDevs, devreason := device.GetDevices()[k.Type].Fit(getNodeResources(*node, k.Type), k, annos, pod, nodeInfo, devinput)
+ reason := "node:" + node.Node.Name + " " + "resaon:" + devreason
+ if fit {
+ for idx, val := range tmpDevs[k.Type] {
+ for nidx, v := range node.Devices.DeviceLists {
+ //bc node.Devices has been sorted, so we should find out the correct device
+ if v.Device.ID != val.UUID {
+ continue
+ }
+ total += v.Device.Count
+ totalCore += v.Device.Totalcore
+ totalMem += v.Device.Totalmem
+ free += v.Device.Count - v.Device.Used
+ freeCore += v.Device.Totalcore - v.Device.Usedcores
+ freeMem += v.Device.Totalmem - v.Device.Usedmem
+ err := device.GetDevices()[k.Type].AddResourceUsage(pod, node.Devices.DeviceLists[nidx].Device, &tmpDevs[k.Type][idx])
+ if err != nil {
+ klog.Errorf("AddResourceUsage failed:%s", err.Error())
+ return false, "AddResourceUsage failed"
+ }
+ klog.Infoln("After AddResourceUsage:", node.Devices.DeviceLists[nidx].Device)
+ }
+ }
+ devs = append(devs, tmpDevs[k.Type]...)
+ } else {
+ return false, reason
+ }
+ (*devinput)[k.Type] = append((*devinput)[k.Type], devs)
+ }
+ return true, ""
}
-func (l DeviceUsageList) Less(i, j int) bool {
- return l[i].count-l[i].used < l[j].count-l[j].used
-}
+func (s *Scheduler) calcScore(nodes *map[string]*NodeUsage, resourceReqs util.PodDeviceRequests, annos map[string]string, task *corev1.Pod, failedNodes map[string]string) (*policy.NodeScoreList, error) {
+ userNodePolicy := config.NodeSchedulerPolicy
+ if annos != nil {
+ if value, ok := annos[policy.NodeSchedulerPolicyAnnotationKey]; ok {
+ userNodePolicy = value
+ }
+ }
+ res := policy.NodeScoreList{
+ Policy: userNodePolicy,
+ NodeList: make([]*policy.NodeScore, 0),
+ }
-func (l NodeScoreList) Len() int {
- return len(l)
-}
+ wg := sync.WaitGroup{}
+ fitNodesMutex := sync.Mutex{}
+ failedNodesMutex := sync.Mutex{}
+ errCh := make(chan error, len(*nodes))
+ for nodeID, node := range *nodes {
+ wg.Add(1)
+ go func(nodeID string, node *NodeUsage) {
+ defer wg.Done()
-func (l NodeScoreList) Swap(i, j int) {
- l[i], l[j] = l[j], l[i]
-}
+ viewStatus(*node)
+ score := policy.NodeScore{NodeID: nodeID, Node: node.Node, Devices: make(util.PodDevices), Score: 0}
+ score.ComputeDefaultScore(node.Devices)
+ snapshot := score.SnapshotDevice(node.Devices)
-func (l NodeScoreList) Less(i, j int) bool {
- return l[i].score < l[j].score
-}
+ nodeInfo, err := s.GetNode(nodeID)
+ if err != nil {
+ klog.ErrorS(err, "Failed to get node", "nodeID", nodeID)
+ errCh <- err
+ return
+ }
+
+ //This loop is for different container request
+ ctrfit := false
+ for ctrid, n := range resourceReqs {
+ sums := 0
+ for _, k := range n {
+ sums += int(k.Nums)
+ }
+
+ if sums == 0 {
+ for idx := range score.Devices {
+ for len(score.Devices[idx]) < ctrid {
+ defaultContainerDevices := util.ContainerDevices{}
+ defaultPodSingleDevice := util.PodSingleDevice{}
+ defaultPodSingleDevice = append(defaultPodSingleDevice, defaultContainerDevices)
+ score.Devices[idx] = append(defaultPodSingleDevice, score.Devices[idx]...)
+ }
+ defaultContainerDevices := util.ContainerDevices{}
+ score.Devices[idx] = append(score.Devices[idx], defaultContainerDevices)
+ }
+ }
+ klog.V(5).InfoS("fitInDevices", "pod", klog.KObj(task), "node", nodeID)
+ fit, reason := fitInDevices(node, n, annos, task, nodeInfo, &score.Devices)
+ ctrfit = fit
+ if !fit {
+ klog.V(4).InfoS(nodeUnfitPod, "pod", klog.KObj(task), "node", nodeID, "reason", reason)
+ failedNodesMutex.Lock()
+ failedNodes[nodeID] = nodeUnfitPod
+ failedNodesMutex.Unlock()
+ break
+ }
+ }
+
+ if ctrfit {
+ fitNodesMutex.Lock()
+ res.NodeList = append(res.NodeList, &score)
+ fitNodesMutex.Unlock()
+ score.OverrideScore(snapshot, userNodePolicy)
+ klog.V(4).InfoS(nodeFitPod, "pod", klog.KObj(task), "node", nodeID, "score", score.Score)
+ }
+ }(nodeID, node)
+ }
+ wg.Wait()
+ close(errCh)
-func calcScore(nodes *map[string]*NodeUsage, errMap *map[string]string, nums []int) (*NodeScoreList, error) {
- res := make(NodeScoreList, 0, len(*nodes))
- for nodeID, node := range *nodes {
- dn := len(node.devices)
- score := NodeScore{nodeID: nodeID, score: 0}
- for _, n := range nums {
- if n == 0 {
- score.devices = append(score.devices, []string{})
- continue
- }
- if n > dn {
- break
- }
- sort.Sort(node.devices)
- if node.devices[dn-n].count <= node.devices[dn-n].used {
- continue
- }
- total := int32(0)
- free := int32(0)
- devs := make([]string, 0, n)
- for i := len(node.devices) - 1; i >= 0; i-- {
- total += node.devices[i].count
- free += node.devices[i].count - node.devices[i].used
- if n > 0 {
- n--
- node.devices[i].used++
- devs = append(devs, node.devices[i].id)
- }
- }
- score.devices = append(score.devices, devs)
- score.score += float32(free) / float32(total)
- score.score += float32(dn - n)
- }
- if len(score.devices) == len(nums) {
- res = append(res, &score)
- }
- }
- return &res, nil
+ var errorsSlice []error
+ for e := range errCh {
+ errorsSlice = append(errorsSlice, e)
+ }
+ return &res, utilerrors.NewAggregate(errorsSlice)
}
diff --git a/pkg/scheduler/score_test.go b/pkg/scheduler/score_test.go
new file mode 100644
index 000000000..a293f183d
--- /dev/null
+++ b/pkg/scheduler/score_test.go
@@ -0,0 +1,3461 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scheduler
+
+import (
+ "strconv"
+ "strings"
+ "testing"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/klog/v2"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/device/hygon"
+ "github.com/Project-HAMi/HAMi/pkg/device/kunlun"
+ "github.com/Project-HAMi/HAMi/pkg/device/metax"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/policy"
+ "github.com/Project-HAMi/HAMi/pkg/util"
+)
+
+func TestMain(m *testing.M) {
+ config := &device.Config{
+ NvidiaConfig: nvidia.NvidiaConfig{
+ ResourceCountName: "hami.io/gpu",
+ ResourceMemoryName: "hami.io/gpumem",
+ ResourceMemoryPercentageName: "hami.io/gpumem-percentage",
+ ResourceCoreName: "hami.io/gpucores",
+ DefaultMemory: 0,
+ DefaultCores: 0,
+ DefaultGPUNum: 1,
+ },
+ }
+
+ if err := device.InitDevicesWithConfig(config); err != nil {
+ klog.Fatalf("Failed to initialize devices with config: %v", err)
+ }
+ m.Run()
+}
+
+// test case matrix
+/**
+| node num | per node device | pod use device | device having use | score |
+|----------|-----------------|----------------|-------------------|-------|
+| 1 node | 1 device | 1 device | no | 5.25 |
+| 1 node | 1 device | 1 device | 50% core, 50% mem | 20.25 |
+| 1 node | 2 device | 1 device | no | 2.625 |
+| 1 node | 2 device | 1 device | 50% core, 50% mem | 10.125 |
+| 1 node | 2 device | 2 device | no | 5.25 |
+| 1 node | 2 device | 2 device | 50% core, 50% mem | 20.25 |
+| 2 node | 1 device | 1 device | no | 5.25 |
+| 2 node | 1 device | 1 device | node1-device1: 50% core, 50% mem, node2-device1: 0% core, 0% mem | node1: 5.25 node2: 5.25 |
+| 2 node | 2 device | 1 device | no | 1,1 |
+| 2 node | 1 device | 1 device | node1-device1: 50% core, 50% mem, node2-device1: 0% core, 0% mem | node1: 20.25 node2: 5.25 |
+test case matrix.
+*/
+func Test_calcScore(t *testing.T) {
+ /*
+ Uncomment this line if you're running this single test.
+ If you're running `make test`, keep this commented out, as there's another test
+ (pkg/k8sutil/pod_test.go) that may cause a DATA RACE when calling device.InitDevices().
+ */
+ //device.InitDevices()
+
+ tests := []struct {
+ name string
+ args struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }
+ wants struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }
+ }{
+ {
+ name: "one node one device one pod one container use one device.",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 0,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "one node one device one pod one container use one device,but this device before having use.",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 5,
+ Count: 10,
+ Usedmem: 4000,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 50,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 15,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "one node two device one pod one container use one device",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid2",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 0,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "one node two device one pod one container use one device,but having use 50%",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 0,
+ Used: 5,
+ Count: 10,
+ Usedmem: 4000,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 50,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 7.5,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "one node two device one pod one container use two device",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 2,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(2, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid2",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 0,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "one node two device one pod one container use two device,but this two device before having use.",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 5,
+ Count: 10,
+ Usedmem: 4000,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 50,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 0,
+ Used: 5,
+ Count: 10,
+ Usedmem: 4000,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 50,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 2,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(2, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid2",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 15,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "two node per node having one device one pod one container use one device",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ "node2": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 0,
+ },
+ {
+ NodeID: "node2",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid2",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 0,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "two node per node having one device one pod one container use one device,one device having use 50%",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 5,
+ Count: 10,
+ Usedmem: 4000,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 50,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ "node2": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 15,
+ },
+ {
+ NodeID: "node2",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid2",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 0,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "one node two device one pod two container use two device",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ nvidia.NvidiaGPUDevice: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ MemPercentagereq: 101,
+ Coresreq: 30,
+ },
+ },
+ {},
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn1",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Name: "gpu-burn2",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{},
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ {},
+ },
+ },
+ Score: 0,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "one node one device one pod with three containers, middle container uses one device.",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {},
+ {
+ nvidia.NvidiaGPUDevice: util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ MemPercentagereq: 101,
+ Coresreq: 30,
+ },
+ },
+ {},
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn1",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{},
+ },
+ {
+ Name: "gpu-burn2",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Name: "gpu-burn3",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{},
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {},
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ {},
+ },
+ },
+ Score: 0,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "one node two device one pod two containers use two device with spread,should not in same device.",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 1,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 8000,
+ Coresreq: 30,
+ },
+ },
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(8000, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Name: "gpu-burn1",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 1,
+ UUID: "uuid2",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 8000,
+ },
+ },
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 0,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "one node one device one pod one container use one device and not enough resource,node should be failed.",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 50, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{},
+ },
+ failedNodes: map[string]string{
+ "node1": nodeUnfitPod,
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "race condition test for failedNodes map with two nodes failing",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 50, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ "node2": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 50, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-race",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{},
+ },
+ failedNodes: map[string]string{
+ "node1": nodeUnfitPod,
+ "node2": nodeUnfitPod,
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "one node one device one pod one container use one device for kunlun.",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 1,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid3",
+ Index: 2,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid4",
+ Index: 3,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid5",
+ Index: 4,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid6",
+ Index: 5,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid7",
+ Index: 6,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid8",
+ Index: 7,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ "node2": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 1,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid3",
+ Index: 2,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid4",
+ Index: 3,
+ Used: 1,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid5",
+ Index: 4,
+ Used: 1,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid6",
+ Index: 5,
+ Used: 1,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid7",
+ Index: 6,
+ Used: 1,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid8",
+ Index: 7,
+ Used: 1,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: kunlun.KunlunGPUDevice,
+ Memreq: 0,
+ Coresreq: 0,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "xpu",
+ Image: "chrstnhntschl/xpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "kunlunxin.com/xpu": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "kunlun": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: kunlun.KunlunGPUDevice,
+ Usedcores: 100,
+ Usedmem: 98304,
+ },
+ },
+ },
+ },
+ Score: 0,
+ },
+ {
+ NodeID: "node2",
+ Devices: util.PodDevices{
+ "kunlun": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: kunlun.KunlunGPUDevice,
+ Usedcores: 100,
+ Usedmem: 98304,
+ },
+ },
+ },
+ },
+ Score: 1006.25,
+ },
+ },
+ },
+ failedNodes: map[string]string{},
+ err: nil,
+ },
+ },
+ {
+ name: "two node eight device one pod two container use one device for kunlun.",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 1,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid3",
+ Index: 2,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid4",
+ Index: 3,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid5",
+ Index: 4,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid6",
+ Index: 5,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid7",
+ Index: 6,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid8",
+ Index: 7,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ "node2": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 1,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid3",
+ Index: 2,
+ Used: 0,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid4",
+ Index: 3,
+ Used: 1,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid5",
+ Index: 4,
+ Used: 1,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid6",
+ Index: 5,
+ Used: 1,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid7",
+ Index: 6,
+ Used: 1,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid8",
+ Index: 7,
+ Used: 1,
+ Count: 1,
+ Usedmem: 0,
+ Totalmem: 98304, // not enough mem
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: kunlun.KunlunGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: kunlun.KunlunGPUDevice,
+ Memreq: 0,
+ Coresreq: 0,
+ },
+ },
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: kunlun.KunlunGPUDevice,
+ Memreq: 0,
+ Coresreq: 0,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "xpu",
+ Image: "chrstnhntschl/xpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "kunlunxin.com/xpu": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Name: "xpu",
+ Image: "chrstnhntschl/xpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "kunlunxin.com/xpu": *resource.NewQuantity(1, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "kunlun": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: kunlun.KunlunGPUDevice,
+ Usedcores: 100,
+ Usedmem: 98304,
+ },
+ },
+ {
+ {
+ Idx: 1,
+ UUID: "uuid2",
+ Type: kunlun.KunlunGPUDevice,
+ Usedcores: 100,
+ Usedmem: 98304,
+ },
+ },
+ },
+ },
+ Score: 1000,
+ },
+ {
+ NodeID: "node2",
+ Devices: util.PodDevices{
+ "kunlun": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: kunlun.KunlunGPUDevice,
+ Usedcores: 100,
+ Usedmem: 98304,
+ },
+ },
+ {
+ {
+ Idx: 1,
+ UUID: "uuid2",
+ Type: kunlun.KunlunGPUDevice,
+ Usedcores: 100,
+ Usedmem: 98304,
+ },
+ },
+ },
+ },
+ Score: 2006.25,
+ },
+ },
+ },
+ failedNodes: map[string]string{},
+ err: nil,
+ },
+ },
+ {
+ name: "two node per node having one device one pod two container use one device",
+ args: struct {
+ nodes *map[string]*NodeUsage
+ nums util.PodDeviceRequests
+ annos map[string]string
+ task *corev1.Pod
+ }{
+ nodes: &map[string]*NodeUsage{
+ "node1": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid1",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ "node2": {
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2"}},
+ Devices: policy.DeviceUsageList{
+ Policy: util.GPUSchedulerPolicySpread.String(),
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "uuid2",
+ Index: 0,
+ Used: 0,
+ Count: 10,
+ Usedmem: 0,
+ Totalmem: 8000,
+ Totalcore: 100,
+ Usedcores: 0,
+ Numa: 0,
+ Type: nvidia.NvidiaGPUDevice,
+ Health: true,
+ },
+ Score: 0,
+ },
+ },
+ },
+ },
+ },
+ nums: util.PodDeviceRequests{
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 30,
+ },
+ },
+ {
+ "hami.io/vgpu-devices-to-allocate": util.ContainerDeviceRequest{
+ Nums: 1,
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: 1000,
+ Coresreq: 40,
+ },
+ },
+ },
+ annos: make(map[string]string),
+ task: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "gpu-burn",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(30, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ {
+ Name: "gpu-burn1",
+ Image: "chrstnhntschl/gpu_burn",
+ Args: []string{"6000"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI),
+ "hami.io/gpucores": *resource.NewQuantity(40, resource.BinarySI),
+ "hami.io/gpumem": *resource.NewQuantity(1000, resource.BinarySI),
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ wants: struct {
+ want *policy.NodeScoreList
+ failedNodes map[string]string
+ err error
+ }{
+ want: &policy.NodeScoreList{
+ Policy: util.NodeSchedulerPolicyBinpack.String(),
+ NodeList: []*policy.NodeScore{
+ {
+ NodeID: "node1",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ {
+ {
+ Idx: 0,
+ UUID: "uuid1",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 40,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 0,
+ },
+ {
+ NodeID: "node2",
+ Devices: util.PodDevices{
+ "NVIDIA": util.PodSingleDevice{
+ {
+ {
+ Idx: 0,
+ UUID: "uuid2",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 30,
+ Usedmem: 1000,
+ },
+ },
+ {
+ {
+ Idx: 0,
+ UUID: "uuid2",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: 40,
+ Usedmem: 1000,
+ },
+ },
+ },
+ },
+ Score: 0,
+ },
+ },
+ },
+ err: nil,
+ },
+ },
+ }
+ s := NewScheduler()
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ for nodeName, nodeUsage := range *(test.args.nodes) {
+ devices := []util.DeviceInfo{}
+ for _, device := range nodeUsage.Devices.DeviceLists {
+ devices = append(devices, util.DeviceInfo{
+ ID: device.Device.ID,
+ })
+ }
+ s.addNode(nodeName, &util.NodeInfo{nodeName, nodeUsage.Node, devices})
+ }
+ failedNodes := map[string]string{}
+ got, gotErr := s.calcScore(test.args.nodes, test.args.nums, test.args.annos, test.args.task, failedNodes)
+ assert.DeepEqual(t, test.wants.err, gotErr)
+ wantMap := make(map[string]*policy.NodeScore)
+ for index, node := range (*(test.wants.want)).NodeList {
+ wantMap[node.NodeID] = (*(test.wants.want)).NodeList[index]
+ }
+ if gotErr == nil && len(got.NodeList) == 0 && len(failedNodes) == 0 {
+ t.Fatal("empty error and empty result")
+ }
+ if len(failedNodes) != 0 {
+ assert.DeepEqual(t, test.wants.failedNodes, failedNodes)
+ return
+ }
+ for i := range got.Len() {
+ gotI := (*(got)).NodeList[i]
+ wantI := wantMap[gotI.NodeID]
+ assert.DeepEqual(t, wantI.NodeID, gotI.NodeID)
+ assert.DeepEqual(t, wantI.Devices, gotI.Devices)
+ assert.DeepEqual(t, wantI.Score, gotI.Score)
+ }
+ })
+ }
+}
+
+func Test_fitInCertainDevice(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }
+ want1 bool
+ want2 map[string]util.ContainerDevices
+ want3 map[string]int
+ }{
+ {
+ name: "allocated device",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-0",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ Totalcore: int32(4),
+ },
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(1),
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{},
+ allocated: &util.PodDevices{},
+ },
+ want1: true,
+ want2: map[string]util.ContainerDevices{
+ "NVIDIA": {
+ {
+ Usedcores: int32(1),
+ Usedmem: int32(1024),
+ Type: nvidia.NvidiaGPUDevice,
+ UUID: "test-0",
+ },
+ },
+ },
+ },
+ {
+ name: "card type don't match",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-0",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ Totalcore: int32(4),
+ },
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: "test",
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(1),
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{},
+ allocated: &util.PodDevices{},
+ },
+ want1: false,
+ want2: map[string]util.ContainerDevices{},
+ want3: map[string]int{cardTypeMismatch: 1},
+ },
+ {
+ name: "device count less than device used",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-0",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(5),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ Totalcore: int32(4),
+ },
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(1),
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{},
+ allocated: &util.PodDevices{},
+ },
+ want1: false,
+ want2: map[string]util.ContainerDevices{},
+ want3: map[string]int{cardTimeSlicingExhausted: 1},
+ },
+ {
+ name: "core limit exceed 100",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-0",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ Totalcore: int32(4),
+ },
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(200),
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{},
+ allocated: &util.PodDevices{},
+ },
+ want1: false,
+ want2: map[string]util.ContainerDevices{},
+ want3: map[string]int{cardInsufficientCore: 1},
+ },
+ {
+ name: "card insufficient remaining memory",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-0",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8000),
+ Usedmem: int32(8000),
+ Usedcores: int32(1),
+ Totalcore: int32(4),
+ },
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(0),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(100),
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{},
+ allocated: &util.PodDevices{},
+ },
+ want1: false,
+ want2: map[string]util.ContainerDevices{},
+ want3: map[string]int{cardInsufficientMemory: 1},
+ },
+ {
+ name: "the container wants exclusive access to an entire card, but the card is already in use",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-0",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(0),
+ Totalcore: int32(100),
+ },
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(100),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(100),
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{},
+ allocated: &util.PodDevices{},
+ },
+ want1: false,
+ want2: map[string]util.ContainerDevices{},
+ want3: map[string]int{exclusiveDeviceAllocateConflict: 1},
+ },
+ {
+ name: "can't allocate core=0 job to an already full GPU",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-0",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ Totalcore: int32(1),
+ },
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(0),
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{},
+ allocated: &util.PodDevices{},
+ },
+ want1: false,
+ want2: map[string]util.ContainerDevices{},
+ want3: map[string]int{cardComputeUnitsExhausted: 1},
+ },
+ {
+ name: "mode is mig",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-0",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ Totalcore: int32(4),
+ Mode: "mig",
+ MigUsage: util.MigInUse{
+ Index: int32(1),
+ UsageList: util.MIGS{
+ {
+ Name: "test6",
+ Memory: int32(2048),
+ InUse: false,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(2),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(1),
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{},
+ allocated: &util.PodDevices{},
+ },
+ want1: false,
+ want2: map[string]util.ContainerDevices{
+ "NVIDIA": {
+ {
+ UUID: "test-0",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: int32(1),
+ Usedmem: int32(1024),
+ },
+ },
+ },
+ want3: map[string]int{cardNotFoundCustomFilterRule: 1, allocatedCardsInsufficientRequest: 1},
+ },
+ {
+ name: "card uuid don't match",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-0",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ Totalcore: int32(4),
+ },
+ },
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(1),
+ },
+ annos: map[string]string{
+ nvidia.GPUUseUUID: "abc",
+ },
+ pod: &corev1.Pod{},
+ allocated: &util.PodDevices{},
+ },
+ want1: false,
+ want2: map[string]util.ContainerDevices{},
+ want3: map[string]int{cardUUIDMismatch: 1},
+ },
+ {
+ name: "numa not fit",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {Device: makeDevice("test-0", 0, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 4)},
+ {Device: makeDevice("test-1", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 4)},
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(2),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(1),
+ },
+ annos: map[string]string{
+ nvidia.GPUInUse: "NVIDIA",
+ nvidia.NumaBind: "true",
+ },
+ pod: &corev1.Pod{},
+ allocated: &util.PodDevices{},
+ },
+ want1: false,
+ want2: map[string]util.ContainerDevices{
+ "NVIDIA": {
+ {
+ UUID: "test-0",
+ Type: nvidia.NvidiaGPUDevice,
+ Usedcores: int32(1),
+ Usedmem: int32(1024),
+ },
+ },
+ },
+ want3: map[string]int{numaNotFit: 1, allocatedCardsInsufficientRequest: 1},
+ },
+ {
+ name: "test device kind of not fit reason",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ // test CardTypeMismatch
+ {Device: makeDevice("a", 0, hygon.HygonDCUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("f", 0, metax.MetaxGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ // test CardUUIDMismatch
+ {Device: makeDevice("b", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("q", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("i", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ // test CardTimeSlicingExhausted
+ {Device: makeDevice("c", 1, nvidia.NvidiaGPUDevice, 4, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("j", 1, nvidia.NvidiaGPUDevice, 4, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("u", 1, nvidia.NvidiaGPUDevice, 4, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("l", 1, nvidia.NvidiaGPUDevice, 4, 4, 8192, 2048, 1, 100)},
+ // test CardInsufficientMemory
+ {Device: makeDevice("d", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 8048, 1, 100)},
+ {Device: makeDevice("m", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 8048, 1, 100)},
+ // test CardInsufficientCore
+ {Device: makeDevice("e", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 90, 100)},
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(2),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(20),
+ },
+ annos: map[string]string{nvidia.GPUUseUUID: "a,f,c,d,e,g,h,j,l,u,m"},
+ pod: &corev1.Pod{},
+ allocated: &util.PodDevices{},
+ },
+ want1: false,
+ want2: map[string]util.ContainerDevices{},
+ want3: map[string]int{cardUUIDMismatch: 3, cardTimeSlicingExhausted: 4,
+ cardInsufficientMemory: 2, cardInsufficientCore: 1},
+ },
+ }
+ for _, test := range tests {
+
+ t.Run(test.name, func(t *testing.T) {
+ gpuDevices := &nvidia.NvidiaGPUDevices{}
+
+ result1, result2, result3 := gpuDevices.Fit(getNodeResources(*test.args.node, nvidia.NvidiaGPUDevice), test.args.request, test.args.annos, test.args.pod, &util.NodeInfo{}, test.args.allocated)
+ assert.DeepEqual(t, result1, test.want1)
+ assert.DeepEqual(t, result2, test.want2)
+ assert.DeepEqual(t, convertReasonToMap(result3), test.want3)
+ })
+ }
+}
+
+func makeDevice(id string, numa int, Type string, used, count, totalmem, usedmem, usedcores, totalcore int) *util.DeviceUsage {
+ return &util.DeviceUsage{
+ ID: id,
+ Numa: numa,
+ Type: Type,
+ Used: int32(used),
+ Count: int32(count),
+ Totalmem: int32(totalmem),
+ Usedmem: int32(usedmem),
+ Usedcores: int32(usedcores),
+ Totalcore: int32(totalcore),
+ }
+}
+
+// convertReasonToMap converts a string in a specific format to a map.
+// The input string should be in the format "cnt/total reason1, cnt/total reason2, ...".
+// This function parses the string and returns a map where the key is the reason and the value is the corresponding count.
+func convertReasonToMap(reason string) map[string]int {
+ var reasonMap map[string]int
+ for r := range strings.SplitSeq(reason, ", ") {
+ parts := strings.SplitN(r, " ", 2)
+ if len(parts) != 2 {
+ continue
+ }
+ countParts := strings.SplitN(parts[0], "/", 2)
+ if len(countParts) != 2 {
+ continue
+ }
+ cnt, err := strconv.Atoi(countParts[0])
+ if err != nil {
+ continue
+ }
+ if reasonMap == nil {
+ reasonMap = make(map[string]int)
+ }
+ reasonMap[parts[1]] = cnt
+ }
+ return reasonMap
+}
+
+func Test_fitInDevices(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ node NodeUsage
+ requests util.ContainerDeviceRequests
+ annos map[string]string
+ pod *corev1.Pod
+ devinput *util.PodDevices
+ }
+ want1 bool
+ want2 string
+ }{
+ {
+ name: "all device score for one node",
+ args: struct {
+ node NodeUsage
+ requests util.ContainerDeviceRequests
+ annos map[string]string
+ pod *corev1.Pod
+ devinput *util.PodDevices
+ }{
+ node: NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-1",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ Totalcore: int32(4),
+ },
+ },
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-2",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ Totalcore: int32(4),
+ },
+ },
+ },
+ },
+ },
+ requests: util.ContainerDeviceRequests{
+ "test-2": {
+ Nums: int32(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(1),
+ },
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{},
+ devinput: &util.PodDevices{},
+ },
+ want1: true,
+ want2: "",
+ },
+ {
+ name: "request devices nums cannot exceed the total number of devices on the node",
+ args: struct {
+ node NodeUsage
+ requests util.ContainerDeviceRequests
+ annos map[string]string
+ pod *corev1.Pod
+ devinput *util.PodDevices
+ }{
+ node: NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-1",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ Totalcore: int32(4),
+ },
+ },
+ },
+ },
+ },
+ requests: util.ContainerDeviceRequests{
+ "test-1": {
+ Nums: int32(2),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(1),
+ },
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{},
+ devinput: &util.PodDevices{},
+ },
+ want1: false,
+ want2: "NodeInsufficientDevice",
+ },
+ {
+ name: "device type the different from request type",
+ args: struct {
+ node NodeUsage
+ requests util.ContainerDeviceRequests
+ annos map[string]string
+ pod *corev1.Pod
+ devinput *util.PodDevices
+ }{
+ node: NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {
+ Device: &util.DeviceUsage{
+ ID: "test-2",
+ Numa: int(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Used: int32(1),
+ Count: int32(4),
+ Totalmem: int32(8192),
+ Usedmem: int32(2048),
+ Usedcores: int32(1),
+ Totalcore: int32(4),
+ },
+ },
+ },
+ },
+ },
+ requests: util.ContainerDeviceRequests{
+ "test-1": {
+ Nums: int32(1),
+ Type: "test",
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(1),
+ },
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{},
+ devinput: &util.PodDevices{},
+ },
+ want1: false,
+ want2: "Device type not found",
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ viewStatus(test.args.node)
+ result1, result2 := fitInDevices(&test.args.node, test.args.requests, test.args.annos, test.args.pod, nil, test.args.devinput)
+ assert.DeepEqual(t, result1, test.want1)
+ assert.DeepEqual(t, result2, test.want2)
+ })
+ }
+}
+
+func Test_Nvidia_GPU_Topology(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ nodeInfo *util.NodeInfo
+ allocated *util.PodDevices
+ }
+ want1 bool
+ want2 map[string]util.ContainerDevices
+ want3 string
+ }{
+ {
+ name: "test nvidia gpu topology-aware",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ nodeInfo *util.NodeInfo
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {Device: makeDevice("a", 0, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("b", 0, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("c", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("d", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("e", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("f", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(3),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(20),
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ util.GPUSchedulerPolicyAnnotationKey: util.GPUSchedulerPolicyTopology.String(),
+ },
+ },
+ },
+ nodeInfo: &util.NodeInfo{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: []util.DeviceInfo{
+ {ID: "a", DevicePairScore: util.DevicePairScore{ID: "a", Scores: map[string]int{"b": 1, "c": 1, "d": 1, "e": 1, "f": 100}}},
+ {ID: "b", DevicePairScore: util.DevicePairScore{ID: "b", Scores: map[string]int{"a": 1, "c": 1, "d": 1, "e": 1, "f": 1}}},
+ {ID: "c", DevicePairScore: util.DevicePairScore{ID: "c", Scores: map[string]int{"a": 1, "b": 1, "d": 1, "e": 1, "f": 100}}},
+ {ID: "d", DevicePairScore: util.DevicePairScore{ID: "d", Scores: map[string]int{"a": 1, "b": 1, "c": 1, "e": 1, "f": 1}}},
+ {ID: "e", DevicePairScore: util.DevicePairScore{ID: "e", Scores: map[string]int{"a": 1, "b": 1, "c": 1, "d": 1, "f": 1}}},
+ {ID: "f", DevicePairScore: util.DevicePairScore{ID: "f", Scores: map[string]int{"a": 100, "b": 1, "c": 100, "d": 1, "e": 1}}},
+ },
+ },
+ allocated: &util.PodDevices{},
+ },
+ want1: true,
+ want2: map[string]util.ContainerDevices{
+ "NVIDIA": {
+ {UUID: "f", Type: "NVIDIA", Usedmem: 1024, Usedcores: 20},
+ {UUID: "c", Type: "NVIDIA", Usedmem: 1024, Usedcores: 20},
+ {UUID: "a", Type: "NVIDIA", Usedmem: 1024, Usedcores: 20},
+ },
+ },
+ want3: "",
+ },
+ {
+ name: "test Single Card Topology ",
+ args: struct {
+ node *NodeUsage
+ request util.ContainerDeviceRequest
+ annos map[string]string
+ pod *corev1.Pod
+ nodeInfo *util.NodeInfo
+ allocated *util.PodDevices
+ }{
+ node: &NodeUsage{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: policy.DeviceUsageList{
+ DeviceLists: []*policy.DeviceListsScore{
+ {Device: makeDevice("a", 0, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("b", 0, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("c", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("d", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("e", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ {Device: makeDevice("f", 1, nvidia.NvidiaGPUDevice, 1, 4, 8192, 2048, 1, 100)},
+ },
+ },
+ },
+ request: util.ContainerDeviceRequest{
+ Nums: int32(1),
+ Type: nvidia.NvidiaGPUDevice,
+ Memreq: int32(1024),
+ MemPercentagereq: int32(100),
+ Coresreq: int32(20),
+ },
+ annos: map[string]string{},
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ util.GPUSchedulerPolicyAnnotationKey: util.GPUSchedulerPolicyTopology.String(),
+ },
+ },
+ },
+ nodeInfo: &util.NodeInfo{
+ Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
+ Devices: []util.DeviceInfo{
+ {ID: "a", DevicePairScore: util.DevicePairScore{ID: "a", Scores: map[string]int{"b": 100, "c": 100, "d": 100, "e": 100, "f": 1}}},
+ {ID: "b", DevicePairScore: util.DevicePairScore{ID: "b", Scores: map[string]int{"a": 100, "c": 100, "d": 100, "e": 100, "f": 1}}},
+ {ID: "c", DevicePairScore: util.DevicePairScore{ID: "c", Scores: map[string]int{"a": 100, "b": 100, "d": 100, "e": 100, "f": 1}}},
+ {ID: "d", DevicePairScore: util.DevicePairScore{ID: "d", Scores: map[string]int{"a": 100, "b": 100, "c": 100, "e": 100, "f": 1}}},
+ {ID: "e", DevicePairScore: util.DevicePairScore{ID: "e", Scores: map[string]int{"a": 100, "b": 100, "c": 100, "d": 100, "f": 1}}},
+ {ID: "f", DevicePairScore: util.DevicePairScore{ID: "f", Scores: map[string]int{"a": 1, "b": 1, "c": 1, "d": 1, "e": 1}}},
+ },
+ },
+ allocated: &util.PodDevices{},
+ },
+ want1: true,
+ want2: map[string]util.ContainerDevices{
+ "NVIDIA": {
+ {UUID: "f", Type: "NVIDIA", Usedmem: 1024, Usedcores: 20},
+ },
+ },
+ want3: "",
+ },
+ }
+ for _, test := range tests {
+
+ t.Run(test.name, func(t *testing.T) {
+ gpuDevices := &nvidia.NvidiaGPUDevices{}
+
+ result1, result2, result3 := gpuDevices.Fit(getNodeResources(*test.args.node, nvidia.NvidiaGPUDevice), test.args.request, test.args.annos, test.args.pod, test.args.nodeInfo, test.args.allocated)
+ assert.DeepEqual(t, result1, test.want1)
+ assert.DeepEqual(t, result2, test.want2)
+ assert.DeepEqual(t, result3, test.want3)
+ })
+ }
+}
diff --git a/pkg/scheduler/webhook.go b/pkg/scheduler/webhook.go
index e32d7fcca..5fc866017 100644
--- a/pkg/scheduler/webhook.go
+++ b/pkg/scheduler/webhook.go
@@ -1,91 +1,105 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package scheduler
import (
- "context"
- "encoding/json"
- "fmt"
- "net/http"
+ "context"
+ "encoding/json"
+ "net/http"
- "4pd.io/k8s-vgpu/pkg/api"
- "4pd.io/k8s-vgpu/pkg/k8sutil"
- "4pd.io/k8s-vgpu/pkg/scheduler/config"
- "4pd.io/k8s-vgpu/pkg/util"
- corev1 "k8s.io/api/core/v1"
- "k8s.io/apimachinery/pkg/runtime"
- clientgoscheme "k8s.io/client-go/kubernetes/scheme"
- "k8s.io/klog/v2"
- "k8s.io/klog/v2/klogr"
- "sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+ "k8s.io/klog/v2"
+ logf "sigs.k8s.io/controller-runtime/pkg/log"
+ "sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/config"
)
+const template = "Processing admission hook for pod %v/%v, UID: %v"
+
type webhook struct {
- decoder *admission.Decoder
+ decoder admission.Decoder
}
func NewWebHook() (*admission.Webhook, error) {
- schema := runtime.NewScheme()
- if err := clientgoscheme.AddToScheme(schema); err != nil {
- return nil, err
- }
- decoder, err := admission.NewDecoder(schema)
- if err != nil {
- return nil, err
- }
- wh := &admission.Webhook{Handler: &webhook{decoder: decoder}}
- _ = wh.InjectLogger(klogr.New())
- return wh, nil
+ logf.SetLogger(klog.NewKlogr())
+ schema := runtime.NewScheme()
+ if err := clientgoscheme.AddToScheme(schema); err != nil {
+ return nil, err
+ }
+ decoder := admission.NewDecoder(schema)
+ wh := &admission.Webhook{Handler: &webhook{decoder: decoder}}
+ return wh, nil
}
func (h *webhook) Handle(_ context.Context, req admission.Request) admission.Response {
- pod := &corev1.Pod{}
- err := h.decoder.Decode(req, pod)
- if err != nil {
- return admission.Errored(http.StatusBadRequest, err)
- }
- if len(pod.Spec.Containers) == 0 {
- return admission.Denied("pod has no containers")
- }
- klog.V(1).Infof("hook %v pod %v/%v", req.UID, req.Namespace, req.Name)
- nums := k8sutil.ResourceNums(pod, corev1.ResourceName(util.ResourceName))
- total := 0
- // use request uid
- uid := req.UID
- for i := 0; i < len(nums); i++ {
- if nums[i] == 0 {
- continue
- }
- total += nums[i]
- c := &pod.Spec.Containers[i]
- c.Env = append(c.Env, corev1.EnvVar{
- Name: api.ContainerUID,
- Value: fmt.Sprintf("%v/%v", uid, c.Name),
- })
- }
- if total == 0 {
- return admission.Allowed(fmt.Sprintf("no resource %v", util.ResourceName))
- }
- if len(config.SchedulerName) > 0 {
- pod.Spec.SchedulerName = config.SchedulerName
- }
- marshaledPod, err := json.Marshal(pod)
- if err != nil {
- return admission.Errored(http.StatusInternalServerError, err)
- }
- return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod)
+ pod := &corev1.Pod{}
+ err := h.decoder.Decode(req, pod)
+ if err != nil {
+ klog.Errorf("Failed to decode request: %v", err)
+ return admission.Errored(http.StatusBadRequest, err)
+ }
+ if len(pod.Spec.Containers) == 0 {
+ klog.Warningf(template+" - Denying admission as pod has no containers", pod.Namespace, pod.Name, pod.UID)
+ return admission.Denied("pod has no containers")
+ }
+ if pod.Spec.SchedulerName != "" &&
+ pod.Spec.SchedulerName != corev1.DefaultSchedulerName || !config.ForceOverwriteDefaultScheduler &&
+ (len(config.SchedulerName) == 0 || pod.Spec.SchedulerName != config.SchedulerName) {
+ klog.Infof(template+" - Pod already has different scheduler assigned", req.Namespace, req.Name, req.UID)
+ return admission.Allowed("pod already has different scheduler assigned")
+ }
+ klog.Infof(template, pod.Namespace, pod.Name, pod.UID)
+ hasResource := false
+ for idx, ctr := range pod.Spec.Containers {
+ c := &pod.Spec.Containers[idx]
+ if ctr.SecurityContext != nil {
+ if ctr.SecurityContext.Privileged != nil && *ctr.SecurityContext.Privileged {
+ klog.Warningf(template+" - Denying admission as container %s is privileged", pod.Namespace, pod.Name, pod.UID, c.Name)
+ continue
+ }
+ }
+ for _, val := range device.GetDevices() {
+ found, err := val.MutateAdmission(c, pod)
+ if err != nil {
+ klog.Errorf("validating pod failed:%s", err.Error())
+ return admission.Errored(http.StatusInternalServerError, err)
+ }
+ hasResource = hasResource || found
+ }
+ }
+
+ if !hasResource {
+ klog.Infof(template+" - Allowing admission for pod: no resource found", pod.Namespace, pod.Name, pod.UID)
+ //return admission.Allowed("no resource found")
+ } else if len(config.SchedulerName) > 0 {
+ pod.Spec.SchedulerName = config.SchedulerName
+ if pod.Spec.NodeName != "" {
+ klog.Infof(template+" - Pod already has node assigned", pod.Namespace, pod.Name, pod.UID)
+ return admission.Denied("pod has node assigned")
+ }
+ }
+ marshaledPod, err := json.Marshal(pod)
+ if err != nil {
+ klog.Errorf(template+" - Failed to marshal pod, error: %v", pod.Namespace, pod.Name, pod.UID, err)
+ return admission.Errored(http.StatusInternalServerError, err)
+ }
+ return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod)
}
diff --git a/pkg/scheduler/webhook_test.go b/pkg/scheduler/webhook_test.go
new file mode 100644
index 000000000..5f0daf0dd
--- /dev/null
+++ b/pkg/scheduler/webhook_test.go
@@ -0,0 +1,243 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scheduler
+
+import (
+ "context"
+ "testing"
+
+ admissionv1 "k8s.io/api/admission/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/serializer"
+ "k8s.io/klog/v2"
+ "sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+ "github.com/Project-HAMi/HAMi/pkg/device"
+ "github.com/Project-HAMi/HAMi/pkg/device/nvidia"
+ "github.com/Project-HAMi/HAMi/pkg/scheduler/config"
+)
+
+func TestHandle(t *testing.T) {
+ // create a Pod object
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "default",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "container1",
+ SecurityContext: &corev1.SecurityContext{
+ Privileged: nil,
+ },
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "nvidia.com/gpu": resource.MustParse("1"),
+ },
+ },
+ },
+ },
+ },
+ }
+
+ // encode the Pod object
+ scheme := runtime.NewScheme()
+ corev1.AddToScheme(scheme)
+ codec := serializer.NewCodecFactory(scheme).LegacyCodec(corev1.SchemeGroupVersion)
+ podBytes, err := runtime.Encode(codec, pod)
+ if err != nil {
+ t.Fatalf("Error encoding pod: %v", err)
+ }
+
+ // create an AdmissionRequest object
+ req := admission.Request{
+ AdmissionRequest: admissionv1.AdmissionRequest{
+ UID: "test-uid",
+ Namespace: "default",
+ Name: "test-pod",
+ Object: runtime.RawExtension{
+ Raw: podBytes,
+ },
+ },
+ }
+
+ // create a WebHook object
+ wh, err := NewWebHook()
+ if err != nil {
+ t.Fatalf("Error creating WebHook: %v", err)
+ }
+
+ // call the Handle method
+ resp := wh.Handle(context.Background(), req)
+ if !resp.Allowed {
+ t.Errorf("Expected allowed response, but got: %v", resp)
+ }
+
+}
+
+func TestPodHasNodeName(t *testing.T) {
+ config.SchedulerName = "hami-scheduler"
+ config.ForceOverwriteDefaultScheduler = true
+ config := &device.Config{
+ NvidiaConfig: nvidia.NvidiaConfig{
+ ResourceCountName: "hami.io/gpu",
+ ResourceMemoryName: "hami.io/gpumem",
+ ResourceMemoryPercentageName: "hami.io/gpumem-percentage",
+ ResourceCoreName: "hami.io/gpucores",
+ DefaultMemory: 0,
+ DefaultCores: 0,
+ DefaultGPUNum: 1,
+ },
+ }
+
+ if err := device.InitDevicesWithConfig(config); err != nil {
+ klog.Fatalf("Failed to initialize devices with config: %v", err)
+ }
+ // create a Pod object
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "default",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "container1",
+ SecurityContext: &corev1.SecurityContext{
+ Privileged: nil,
+ },
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": resource.MustParse("1"),
+ },
+ },
+ },
+ },
+ NodeName: "test-node",
+ },
+ }
+
+ // encode the Pod object
+ scheme := runtime.NewScheme()
+ corev1.AddToScheme(scheme)
+ codec := serializer.NewCodecFactory(scheme).LegacyCodec(corev1.SchemeGroupVersion)
+ podBytes, err := runtime.Encode(codec, pod)
+ if err != nil {
+ t.Fatalf("Error encoding pod: %v", err)
+ }
+
+ // create an AdmissionRequest object
+ req := admission.Request{
+ AdmissionRequest: admissionv1.AdmissionRequest{
+ UID: "test-uid",
+ Namespace: "default",
+ Name: "test-pod",
+ Object: runtime.RawExtension{
+ Raw: podBytes,
+ },
+ },
+ }
+
+ // create a WebHook object
+ wh, err := NewWebHook()
+ if err != nil {
+ t.Fatalf("Error creating WebHook: %v", err)
+ }
+
+ // call the Handle method
+ resp := wh.Handle(context.Background(), req)
+ if resp.Allowed {
+ t.Errorf("Expected denied response, but got: %v", resp)
+ }
+
+}
+
+func TestPodHasDifferentScheduler(t *testing.T) {
+ config.SchedulerName = "hami-scheduler"
+
+ config := &device.Config{
+ NvidiaConfig: nvidia.NvidiaConfig{
+ ResourceCountName: "hami.io/gpu",
+ ResourceMemoryName: "hami.io/gpumem",
+ ResourceMemoryPercentageName: "hami.io/gpumem-percentage",
+ ResourceCoreName: "hami.io/gpucores",
+ DefaultMemory: 0,
+ DefaultCores: 0,
+ DefaultGPUNum: 1,
+ },
+ }
+
+ if err := device.InitDevicesWithConfig(config); err != nil {
+ klog.Fatalf("Failed to initialize devices with config: %v", err)
+ }
+
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "default",
+ },
+ Spec: corev1.PodSpec{
+ SchedulerName: "different-scheduler",
+ Containers: []corev1.Container{
+ {
+ Name: "container1",
+ SecurityContext: &corev1.SecurityContext{
+ Privileged: nil,
+ },
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "hami.io/gpu": resource.MustParse("1"),
+ },
+ },
+ },
+ },
+ },
+ }
+
+ scheme := runtime.NewScheme()
+ corev1.AddToScheme(scheme)
+ codec := serializer.NewCodecFactory(scheme).LegacyCodec(corev1.SchemeGroupVersion)
+ podBytes, err := runtime.Encode(codec, pod)
+ if err != nil {
+ t.Fatalf("Error encoding pod: %v", err)
+ }
+
+ req := admission.Request{
+ AdmissionRequest: admissionv1.AdmissionRequest{
+ UID: "test-uid",
+ Namespace: "default",
+ Name: "test-pod",
+ Object: runtime.RawExtension{
+ Raw: podBytes,
+ },
+ },
+ }
+ wh, err := NewWebHook()
+ if err != nil {
+ t.Fatalf("Error creating WebHook: %v", err)
+ }
+
+ resp := wh.Handle(context.Background(), req)
+
+ if !resp.Allowed {
+ t.Errorf("Expected allowed response for pod with different scheduler, but got: %v", resp)
+ }
+}
diff --git a/pkg/util/client/client.go b/pkg/util/client/client.go
new file mode 100644
index 000000000..759e50d32
--- /dev/null
+++ b/pkg/util/client/client.go
@@ -0,0 +1,101 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "sync"
+
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/rest"
+ "k8s.io/client-go/tools/clientcmd"
+ "k8s.io/klog/v2"
+)
+
+type Client struct {
+ // Embedded kubernetes.Interface to avoid name conflicts.
+ kubernetes.Interface
+ config *rest.Config
+}
+
+var (
+ KubeClient kubernetes.Interface
+ once sync.Once
+)
+
+func init() {
+ KubeClient = nil
+}
+
+// GetClient returns the global Kubernetes client.
+func GetClient() kubernetes.Interface {
+ return KubeClient
+}
+
+// NewClient creates a new Kubernetes client with the given options.
+func NewClient(opts ...Option) (*Client, error) {
+ restConfig, err := loadKubeConfig()
+ if err != nil {
+ return nil, fmt.Errorf("failed to load kubeconfig: %w", err)
+ }
+
+ // Apply WithDefaults option first to set default values.
+ WithDefaults()(restConfig)
+
+ // Then apply user-provided options that will override defaults if specified.
+ for _, opt := range opts {
+ opt(restConfig)
+ }
+
+ clientset, err := kubernetes.NewForConfig(restConfig)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create kubernetes client: %w", err)
+ }
+
+ return &Client{
+ Interface: clientset,
+ config: restConfig,
+ }, nil
+}
+
+// InitGlobalClient initializes the global Kubernetes client with the given options.
+func InitGlobalClient(opts ...Option) {
+ once.Do(func() {
+ client, err := NewClient(opts...)
+ if err != nil {
+ klog.Fatalf("Failed to initialize global client: %v", err)
+ }
+ KubeClient = client.Interface
+ })
+}
+
+// loadKubeConfig loads Kubernetes configuration from the environment or in-cluster.
+func loadKubeConfig() (*rest.Config, error) {
+ kubeConfigPath := os.Getenv("KUBECONFIG")
+ if kubeConfigPath == "" {
+ kubeConfigPath = filepath.Join(os.Getenv("HOME"), ".kube", "config")
+ }
+
+ config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
+ if err != nil {
+ klog.Infof("BuildConfigFromFlags failed for file %s: %v. Using in-cluster config.", kubeConfigPath, err)
+ return rest.InClusterConfig()
+ }
+ return config, nil
+}
diff --git a/pkg/util/client/client_test.go b/pkg/util/client/client_test.go
new file mode 100644
index 000000000..8995fc27c
--- /dev/null
+++ b/pkg/util/client/client_test.go
@@ -0,0 +1,257 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ "gotest.tools/v3/assert"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/rest"
+ "k8s.io/client-go/tools/clientcmd"
+)
+
+// Mock functions for testing.
+var (
+ buildConfigFromFlags = clientcmd.BuildConfigFromFlags
+ inClusterConfig = rest.InClusterConfig
+)
+
+// TestGetClient tests the GetClient function.
+func TestGetClient(t *testing.T) {
+ InitGlobalClient()
+ tests := []struct {
+ name string
+ kubeConfig string
+ buildConfig *rest.Config
+ buildConfigErr error
+ inCluster *rest.Config
+ inClusterErr error
+ expectError bool
+ }{
+ {
+ name: "Success from kubeconfig",
+ kubeConfig: filepath.Join("testdata", "kubeconfig.yaml"),
+ buildConfig: &rest.Config{Host: "https://example.com"},
+ buildConfigErr: nil,
+ inCluster: nil,
+ inClusterErr: nil,
+ expectError: false,
+ },
+ {
+ name: "Fallback to in-cluster config",
+ kubeConfig: filepath.Join("testdata", "invalid_kubeconfig.yaml"),
+ buildConfig: nil,
+ buildConfigErr: errors.New("kubeconfig error"),
+ inCluster: &rest.Config{Host: "https://in-cluster.example.com"},
+ inClusterErr: nil,
+ expectError: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // Mock the clientcmd.BuildConfigFromFlags function.
+ oldBuildConfigFromFlags := buildConfigFromFlags
+ buildConfigFromFlags = func(masterUrl, kubeconfigPath string) (*rest.Config, error) {
+ return tt.buildConfig, tt.buildConfigErr
+ }
+ defer func() { buildConfigFromFlags = oldBuildConfigFromFlags }()
+
+ // Mock the rest.InClusterConfig function.
+ oldInClusterConfig := inClusterConfig
+ inClusterConfig = func() (*rest.Config, error) {
+ return tt.inCluster, tt.inClusterErr
+ }
+ defer func() { inClusterConfig = oldInClusterConfig }()
+
+ // Set the KUBECONFIG environment variable.
+ oldKubeConfig := os.Getenv("KUBECONFIG")
+ os.Setenv("KUBECONFIG", tt.kubeConfig)
+ defer os.Setenv("KUBECONFIG", oldKubeConfig)
+
+ // Call GetClient and check the result.
+ client := GetClient()
+ if tt.expectError {
+ if client != nil {
+ t.Errorf("Expected error, but got a valid client")
+ }
+ } else {
+ if client == nil {
+ t.Errorf("Expected a valid client, but got nil")
+ }
+ }
+ })
+ }
+}
+
+// TestClientWithOptions tests client initialization with options.
+func TestClientWithOptions(t *testing.T) {
+ KubeClient = nil
+ once = sync.Once{}
+
+ timeout := 1
+ client, _ := NewClient(WithTimeout(timeout))
+
+ assert.Equal(t, client.config.Timeout, time.Duration(timeout)*time.Second)
+ assert.Equal(t, client.config.QPS, DefaultQPS)
+ assert.Equal(t, client.config.Burst, DefaultBurst)
+
+ KubeClient = nil
+ once = sync.Once{}
+
+ qps := float32(50.0)
+ client, _ = NewClient(WithQPS(qps))
+
+ assert.Equal(t, client.config.Timeout, time.Duration(DefaultTimeout)*time.Second)
+ assert.Equal(t, client.config.QPS, qps)
+ assert.Equal(t, client.config.Burst, DefaultBurst)
+
+ KubeClient = nil
+ once = sync.Once{}
+ burst := 100
+ client, _ = NewClient(WithBurst(burst))
+
+ assert.Equal(t, client.config.Timeout, time.Duration(DefaultTimeout)*time.Second)
+ assert.Equal(t, client.config.QPS, DefaultQPS)
+ assert.Equal(t, client.config.Burst, burst)
+
+ KubeClient = nil
+ once = sync.Once{}
+ timeout = 2
+ qps = 0.5
+ burst = 100
+ client, _ = NewClient(WithTimeout(timeout), WithQPS(qps), WithBurst(burst))
+
+ assert.Equal(t, client.config.Timeout, time.Duration(timeout)*time.Second)
+ assert.Equal(t, client.config.QPS, qps)
+ assert.Equal(t, client.config.Burst, burst)
+}
+
+// TestClientRealNodePerformance tests the performance with a real Kubernetes cluster if available.
+func TestClientRealNodePerformance(t *testing.T) {
+
+ skipRealClusterTest := true
+ // Skip this test by default as it requires a real Kubernetes cluster.
+ if skipRealClusterTest == true {
+ t.Skip("Skipping real cluster test. Set TEST_WITH_REAL_CLUSTER=true to run this test.")
+ }
+
+ tests := []struct {
+ name string
+ qps float32
+ burst int
+ updates int
+ timeout int
+ }{
+ {
+ name: "Real Cluster - Low QPS and Burst",
+ qps: 1,
+ burst: 1,
+ updates: 10,
+ timeout: 1,
+ },
+ {
+ name: "Real Cluster - Standard Timeout",
+ qps: 5,
+ burst: 10,
+ updates: 10,
+ timeout: 5,
+ },
+ {
+ name: "Real Cluster - High Timeout",
+ qps: 10,
+ burst: 20,
+ updates: 15,
+ timeout: 10,
+ },
+ {
+ name: "Real Cluster - Very Short Timeout",
+ qps: 5,
+ burst: 5,
+ updates: 5,
+ timeout: 1,
+ },
+ }
+
+ labelKey := "test-performance-label"
+ var nodeName string
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ client, err := NewClient(WithQPS(tt.qps), WithBurst(tt.burst), WithTimeout(tt.timeout))
+ if err != nil {
+ t.Fatalf("Failed to create client: %v", err)
+ }
+
+ if nodeName == "" {
+ nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+ if err != nil {
+ t.Fatalf("Failed to list nodes: %v", err)
+ }
+ if len(nodes.Items) == 0 {
+ t.Fatal("No nodes found in the cluster")
+ }
+ nodeName = nodes.Items[0].Name
+ t.Logf("Using node %s for testing", nodeName)
+ }
+ start := time.Now()
+ for i := 0; i < tt.updates; i++ {
+ labelValue := fmt.Sprintf("perf-test-value-%d", i)
+ node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("Failed to get node: %v", err)
+ }
+ if node.Labels == nil {
+ node.Labels = make(map[string]string)
+ }
+ node.Labels[labelKey] = labelValue
+ _, err = client.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{})
+ if err != nil {
+ t.Fatalf("Failed to update node: %v", err)
+ }
+ }
+
+ elapsed := time.Since(start)
+
+ node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("Failed to get node during cleanup: %v", err)
+ }
+ delete(node.Labels, labelKey)
+ _, err = client.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{})
+ if err != nil {
+ t.Fatalf("Failed to cleanup test label: %v", err)
+ }
+
+ opsPerSecond := float64(tt.updates) / elapsed.Seconds()
+
+ t.Logf("Real cluster performance test results for %s:", tt.name)
+ t.Logf(" - QPS: %.1f, Burst: %d", tt.qps, tt.burst)
+ t.Logf(" - Updates performed: %d", tt.updates)
+ t.Logf(" - Total time: %v", elapsed)
+ t.Logf(" - Operations per second: %.2f", opsPerSecond)
+ })
+ }
+}
diff --git a/pkg/util/client/options.go b/pkg/util/client/options.go
new file mode 100644
index 000000000..20680860e
--- /dev/null
+++ b/pkg/util/client/options.go
@@ -0,0 +1,69 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+ "time"
+
+ "k8s.io/client-go/rest"
+)
+
+// Option defines a function type for client configuration options.
+type Option func(*rest.Config)
+
+// Now we use the default values of kubernetes client, unless HAMi has specific requirements.
+const (
+ DefaultQPS float32 = rest.DefaultQPS
+ DefaultBurst int = rest.DefaultBurst
+ DefaultTimeout int = 0 // seconds, 0 means no timeout, follow the default behavior of kubernetes client.
+)
+
+// WithQPS sets the QPS for the client.
+func WithQPS(qps float32) Option {
+ return func(c *rest.Config) {
+ c.QPS = qps
+ }
+}
+
+// WithBurst sets the burst for the client.
+func WithBurst(burst int) Option {
+ return func(c *rest.Config) {
+ c.Burst = burst
+ }
+}
+
+// WithTimeout sets the timeout for the client.
+func WithTimeout(timeout int) Option {
+ return func(c *rest.Config) {
+ c.Timeout = time.Duration(timeout) * time.Second
+ }
+}
+
+// WithDefaults sets default values for the client configuration.
+func WithDefaults() Option {
+ return func(c *rest.Config) {
+ if c.QPS == 0 {
+ c.QPS = DefaultQPS
+ }
+ if c.Burst == 0 {
+ c.Burst = DefaultBurst
+ }
+ if c.Timeout == 0 {
+ c.Timeout = time.Duration(DefaultTimeout) * time.Second
+ }
+ }
+}
diff --git a/pkg/util/client/testdata/invalid_kubeconfig.yaml b/pkg/util/client/testdata/invalid_kubeconfig.yaml
new file mode 100644
index 000000000..482e41e1e
--- /dev/null
+++ b/pkg/util/client/testdata/invalid_kubeconfig.yaml
@@ -0,0 +1,20 @@
+# testdata/invalid_kubeconfig.yaml
+apiVersion: v1
+kind: Config
+clusters:
+ - cluster:
+ # Missing server field or invalid URL
+ # server: http://invalid-url/
+ name: broken-cluster
+ insecure-skip-tls-verify: true
+users:
+ - name: broken-user
+ user:
+ # Invalid or missing token
+ token: not-a-valid-token
+contexts:
+ - context:
+ cluster: non-existent-cluster
+ user: non-existent-user
+ name: broken-context
+# Missing current-context
\ No newline at end of file
diff --git a/pkg/util/client/testdata/kubeconfig.yaml b/pkg/util/client/testdata/kubeconfig.yaml
new file mode 100644
index 000000000..f7f141618
--- /dev/null
+++ b/pkg/util/client/testdata/kubeconfig.yaml
@@ -0,0 +1,18 @@
+# testdata/kubeconfig.yaml
+apiVersion: v1
+kind: Config
+clusters:
+ - cluster:
+ server: https://example.com
+ insecure-skip-tls-verify: true
+ name: example-cluster
+users:
+ - name: example-user
+ user:
+ token: my-token-value
+contexts:
+ - context:
+ cluster: example-cluster
+ user: example-user
+ name: example-context
+current-context: example-context
\ No newline at end of file
diff --git a/pkg/util/flag/flags.go b/pkg/util/flag/flags.go
new file mode 100644
index 000000000..de8d2e12e
--- /dev/null
+++ b/pkg/util/flag/flags.go
@@ -0,0 +1,40 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flag
+
+import (
+ "github.com/spf13/pflag"
+ "github.com/urfave/cli/v2"
+ "k8s.io/klog/v2"
+)
+
+func PrintPFlags(flags *pflag.FlagSet) {
+ flags.VisitAll(func(flag *pflag.Flag) {
+ klog.Infof("FLAG: --%s=%q", flag.Name, flag.Value)
+ })
+}
+
+func PrintCliFlags(c *cli.Context) {
+ for _, flag := range c.App.Flags {
+ names := flag.Names()
+ for _, name := range names {
+ value := c.Generic(name)
+ klog.Infof("FLAG: --%s=%q\n", name, value)
+ }
+
+ }
+}
diff --git a/pkg/util/flag/flags_test.go b/pkg/util/flag/flags_test.go
new file mode 100644
index 000000000..a0f3eefe3
--- /dev/null
+++ b/pkg/util/flag/flags_test.go
@@ -0,0 +1,104 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flag
+
+import (
+ "bytes"
+ "flag"
+ "strings"
+ "testing"
+
+ "github.com/spf13/pflag"
+ "github.com/urfave/cli/v2"
+ "k8s.io/klog/v2"
+)
+
+func TestPrintPFlags(t *testing.T) {
+ var buf bytes.Buffer
+ klog.SetOutput(&buf)
+ klog.LogToStderr(false)
+ defer klog.LogToStderr(true)
+ tests := []struct {
+ name string
+ flags func() *pflag.FlagSet
+ expected string
+ }{
+ {
+ name: "Test with name flags",
+ flags: func() *pflag.FlagSet {
+ fs := pflag.NewFlagSet("test", pflag.ContinueOnError)
+ fs.String("name", "bob", "set name")
+ return fs
+ },
+ expected: `FLAG: --name="bob"`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ buf.Reset()
+ PrintPFlags(tt.flags())
+ if got := buf.String(); !strings.Contains(got, tt.expected) {
+ t.Errorf("PrintPFlags() = %q, want %q", got, tt.expected)
+ }
+ })
+ }
+}
+
+func TestPrintCliFlags(t *testing.T) {
+ var buf bytes.Buffer
+ klog.SetOutput(&buf)
+ klog.LogToStderr(false)
+ defer klog.LogToStderr(true)
+
+ tests := []struct {
+ name string
+ cliCtx func() *cli.Context
+ expected string
+ }{
+ {
+ name: "Test with name flag",
+ cliCtx: func() *cli.Context {
+ app := &cli.App{
+ Flags: []cli.Flag{
+ &cli.StringFlag{
+ Name: "name",
+ Value: "bob",
+ Usage: "set user name",
+ },
+ },
+ }
+ flagSet := flag.NewFlagSet("test", flag.ContinueOnError)
+ flagSet.String("name", "bob", "")
+ return cli.NewContext(app, flagSet, nil)
+ },
+ expected: `FLAG: --name="bob"
+`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ buf.Reset()
+ PrintCliFlags(tt.cliCtx())
+ got := buf.String()
+ if !strings.Contains(got, tt.expected) {
+ t.Errorf("PrintCliFlags() output = %q, want %q", got, tt.expected)
+ }
+ })
+ }
+}
diff --git a/pkg/util/nodelock/nodelock.go b/pkg/util/nodelock/nodelock.go
new file mode 100644
index 000000000..ff168de0e
--- /dev/null
+++ b/pkg/util/nodelock/nodelock.go
@@ -0,0 +1,216 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodelock
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/util/wait"
+ "k8s.io/client-go/util/retry"
+ "k8s.io/klog/v2"
+)
+
+const (
+ NodeLockKey = "hami.io/mutex.lock"
+ NodeLockSep = ","
+)
+
+var (
+ lock sync.Mutex
+ // NodeLockTimeout is the global timeout for node locks.
+ NodeLockTimeout time.Duration = time.Minute * 5
+
+ DefaultStrategy = wait.Backoff{
+ Steps: 5,
+ Duration: 100 * time.Millisecond,
+ Factor: 1.0,
+ Jitter: 0.1,
+ }
+)
+
+func init() {
+ setupNodeLockTimeout()
+}
+
+// setupNodeLockTimeout configures the node lock timeout from the environment.
+func setupNodeLockTimeout() {
+ nodelock := os.Getenv("HAMI_NODELOCK_EXPIRE")
+ if nodelock != "" {
+ d, err := time.ParseDuration(nodelock)
+ if err != nil {
+ klog.ErrorS(err, "Failed to parse HAMI_NODELOCK_EXPIRE, using default", "duration", NodeLockTimeout)
+ } else {
+ NodeLockTimeout = d
+ klog.InfoS("Node lock expiration time set from environment variable", "duration", d)
+ }
+ }
+}
+
+func SetNodeLock(nodeName string, lockname string, pods *corev1.Pod) error {
+ lock.Lock()
+ defer lock.Unlock()
+ ctx := context.Background()
+ node, err := client.GetClient().CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+ if err != nil {
+ return err
+ }
+ if _, ok := node.Annotations[NodeLockKey]; ok {
+ return fmt.Errorf("node %s is locked", nodeName)
+ }
+ err = retry.OnError(DefaultStrategy, func(err error) bool {
+ // Retry on any error
+ return true
+ }, func() error {
+ node, err = client.GetClient().CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+ if err != nil {
+ klog.ErrorS(err, "Failed to get node when retry to patch", "node", nodeName)
+ return err
+ }
+ patchData := fmt.Sprintf(`{"metadata":{"annotations":{"%s":"%s"},"resourceVersion":"%s"}}`, NodeLockKey, GenerateNodeLockKeyByPod(pods), node.ResourceVersion)
+ _, err = client.GetClient().CoreV1().Nodes().Patch(ctx, nodeName, types.MergePatchType, []byte(patchData), metav1.PatchOptions{})
+ if err != nil {
+ klog.ErrorS(err, "Failed to patch node when retry to patch", "node", nodeName)
+ return err
+ }
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("failed to set node lock (node=%s, retry strategy=%+v): %w", nodeName, DefaultStrategy, err)
+ }
+
+ klog.InfoS("Node lock set", "node", nodeName, "podName", pods.Name)
+ return nil
+}
+
+func ReleaseNodeLock(nodeName string, lockname string, pod *corev1.Pod, skipNodeLockOwnerCheck bool) error {
+ lock.Lock()
+ defer lock.Unlock()
+ ctx := context.Background()
+ node, err := client.GetClient().CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+ if err != nil {
+ return err
+ }
+
+ lockStr, ok := node.Annotations[NodeLockKey]
+ if !ok {
+ return nil
+ }
+ if !skipNodeLockOwnerCheck && !strings.Contains(lockStr, pod.Name) {
+ klog.InfoS("NodeLock is not set by this pod", lockStr, "pod", pod.Name)
+ return nil
+ }
+
+ err = retry.OnError(DefaultStrategy, func(err error) bool {
+ // Retry on any error
+ return true
+ }, func() error {
+ node, err = client.GetClient().CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+ if err != nil {
+ klog.ErrorS(err, "Failed to get node when retry to patch", "node", nodeName)
+ return err
+ }
+ patchData := fmt.Sprintf(`{"metadata":{"annotations":{"%s":null},"resourceVersion":"%s"}}`, NodeLockKey, node.ResourceVersion)
+ _, err = client.GetClient().CoreV1().Nodes().Patch(ctx, nodeName, types.MergePatchType, []byte(patchData), metav1.PatchOptions{})
+ if err != nil {
+ klog.ErrorS(err, "Failed to patch node when retry to patch", "node", nodeName)
+ return err
+ }
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("failed to release node lock (node=%s, retry strategy=%+v): %w", nodeName, DefaultStrategy, err)
+ }
+
+ klog.InfoS("Node lock released", "node", nodeName, "podName", pod.Name)
+ return nil
+}
+
+func LockNode(nodeName string, lockname string, pods *corev1.Pod) error {
+ ctx := context.Background()
+ node, err := client.GetClient().CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+ if err != nil {
+ return err
+ }
+ if _, ok := node.Annotations[NodeLockKey]; !ok {
+ return SetNodeLock(nodeName, lockname, pods)
+ }
+ lockTime, ns, previousPodName, err := ParseNodeLock(node.Annotations[NodeLockKey])
+ if err != nil {
+ return err
+ }
+
+ var skipOwnerCheck = false
+ if time.Since(lockTime) > NodeLockTimeout {
+ klog.InfoS("Node lock expired", "node", nodeName, "lockTime", lockTime, "timeout", NodeLockTimeout)
+ skipOwnerCheck = true
+ } else
+ // Check dangling nodeLock
+ if ns != "" && previousPodName != "" && (ns != pods.Namespace || previousPodName != pods.Name) {
+ if _, err := client.GetClient().CoreV1().Pods(ns).Get(ctx, previousPodName, metav1.GetOptions{}); err != nil {
+ if !apierrors.IsNotFound(err) {
+ klog.ErrorS(err, "Failed to get pod of NodeLock", "podName", previousPodName, "namespace", ns)
+ return err
+ }
+ klog.InfoS("Previous pod of NodeLock not found, releasing lock", "podName", previousPodName, "namespace", ns, "nodeLock", node.Annotations[NodeLockKey])
+ skipOwnerCheck = true
+ }
+ }
+
+ if skipOwnerCheck {
+ err = ReleaseNodeLock(nodeName, lockname, pods, true)
+ if err != nil {
+ klog.ErrorS(err, "Failed to release node lock", "node", nodeName)
+ return err
+ }
+ return SetNodeLock(nodeName, lockname, pods)
+ }
+
+ return fmt.Errorf("node %s has been locked within %v", nodeName, NodeLockTimeout)
+}
+
+func ParseNodeLock(value string) (lockTime time.Time, ns, name string, err error) {
+ if !strings.Contains(value, NodeLockSep) {
+ lockTime, err = time.Parse(time.RFC3339, value)
+ return lockTime, "", "", err
+ }
+ s := strings.Split(value, NodeLockSep)
+ if len(s) != 3 {
+ lockTime, err = time.Parse(time.RFC3339, value)
+ return lockTime, "", "", err
+ }
+ lockTime, err = time.Parse(time.RFC3339, s[0])
+ return lockTime, s[1], s[2], err
+}
+
+func GenerateNodeLockKeyByPod(pods *corev1.Pod) string {
+ if pods == nil {
+ return time.Now().Format(time.RFC3339)
+ }
+ ns, name := pods.Namespace, pods.Name
+ return fmt.Sprintf("%s%s%s%s%s", time.Now().Format(time.RFC3339), NodeLockSep, ns, NodeLockSep, name)
+}
diff --git a/pkg/util/nodelock/nodelock_test.go b/pkg/util/nodelock/nodelock_test.go
new file mode 100644
index 000000000..5a27df9fc
--- /dev/null
+++ b/pkg/util/nodelock/nodelock_test.go
@@ -0,0 +1,335 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodelock
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "time"
+
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes/fake"
+
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+)
+
+func Test_LockNode(t *testing.T) {
+ client.KubeClient = fake.NewSimpleClientset()
+ type args struct {
+ nodeName func() string
+ lockname string
+ pods *corev1.Pod
+ }
+ tests := []struct {
+ name string
+ args args
+ wantErr bool
+ }{
+ {
+ name: "node not found",
+ args: args{
+ nodeName: func() string {
+ return "node"
+ },
+ pods: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "hami",
+ Namespace: "hami-ns",
+ },
+ },
+ },
+ wantErr: true,
+ },
+ {
+ name: "node has been locked",
+ args: args{
+ nodeName: func() string {
+ name := "worker-1"
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Annotations: map[string]string{
+ NodeLockKey: GenerateNodeLockKeyByPod(&corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{Name: "hami", Namespace: "hami-ns"},
+ }),
+ },
+ },
+ }, metav1.CreateOptions{})
+ return name
+ },
+ pods: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "hami",
+ Namespace: "hami-ns",
+ },
+ },
+ },
+ wantErr: true,
+ },
+ {
+ name: "node lock is invalid",
+ args: args{
+ nodeName: func() string {
+ name := "worker-2"
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Annotations: map[string]string{
+ NodeLockKey: "lock",
+ },
+ },
+ }, metav1.CreateOptions{})
+ return name
+ },
+ pods: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "hami",
+ Namespace: "hami-ns",
+ },
+ },
+ },
+ wantErr: true,
+ },
+ {
+ name: "successfully set node lock",
+ args: args{
+ nodeName: func() string {
+ name := "worker-3"
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{Name: name, Annotations: map[string]string{}},
+ }, metav1.CreateOptions{})
+ return name
+ },
+ pods: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "hami",
+ Namespace: "hami-ns",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if err := LockNode(tt.args.nodeName(), tt.args.lockname, tt.args.pods); (err != nil) != tt.wantErr {
+ t.Errorf("LockNode() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ })
+ }
+}
+
+func TestLockNodeWithTimeout(t *testing.T) {
+ client.KubeClient = fake.NewSimpleClientset()
+
+ // Set a custom timeout for testing
+ originalTimeout := NodeLockTimeout
+ NodeLockTimeout = time.Minute * 2
+ defer func() {
+ NodeLockTimeout = originalTimeout
+ }()
+
+ nodeName := "test-node-timeout"
+
+ // Create a node with a fresh lock (should not be expired)
+ freshLockTime := time.Now().Format(time.RFC3339)
+ testNamespace := "test-ns"
+ testPodName := "test-pod"
+ lockValue := freshLockTime + NodeLockSep + testNamespace + NodeLockSep + testPodName
+
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: nodeName,
+ Annotations: map[string]string{
+ NodeLockKey: lockValue,
+ },
+ },
+ }, metav1.CreateOptions{})
+
+ // Pod must exist to avoid dangling node lock
+ client.KubeClient.CoreV1().Pods(testNamespace).Create(context.TODO(), &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: testPodName,
+ Namespace: testNamespace,
+ },
+ }, metav1.CreateOptions{})
+
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "new-pod",
+ Namespace: "new-ns",
+ },
+ }
+
+ // Try to lock the node again - this should trigger line 130
+ err := LockNode(nodeName, "", pod)
+
+ // Verify the error contains the NodeLockTimeout value
+ if err == nil {
+ t.Fatal("Expected error but got nil")
+ }
+
+ expectedError := "has been locked within 2m0s"
+ if !strings.Contains(err.Error(), expectedError) {
+ t.Errorf("Expected error to contain '%s', but got: %v", expectedError, err)
+ }
+}
+
+func TestLockNodeWithDangling(t *testing.T) {
+ client.KubeClient = fake.NewSimpleClientset()
+
+ // Set a custom timeout for testing
+ originalTimeout := NodeLockTimeout
+ NodeLockTimeout = time.Minute * 2
+ defer func() {
+ NodeLockTimeout = originalTimeout
+ }()
+
+ nodeName := "test-node-timeout"
+
+ // Create a node with a fresh lock (should not be expired)
+ freshLockTime := time.Now().Format(time.RFC3339)
+ testNamespace := "test-ns"
+ testPodName := "test-pod"
+ lockValue := freshLockTime + NodeLockSep + testNamespace + NodeLockSep + testPodName
+
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: nodeName,
+ Annotations: map[string]string{
+ NodeLockKey: lockValue,
+ },
+ },
+ }, metav1.CreateOptions{})
+
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "new-pod",
+ Namespace: "new-ns",
+ },
+ }
+
+ // Try to lock the node again - this should pass and release the old dangling lock
+ if err := LockNode(nodeName, "", pod); err != nil {
+ t.Fatal("Expected nil but got error")
+ }
+}
+
+func TestReleaseNodeLock(t *testing.T) {
+ client.KubeClient = fake.NewSimpleClientset()
+ type args struct {
+ nodeName func() string
+ lockname string
+ pod *corev1.Pod
+ timeout bool
+ }
+ tests := []struct {
+ name string
+ args args
+ wantErr bool
+ }{
+ {
+ name: "node not found",
+ args: args{
+ nodeName: func() string {
+ return "node"
+ },
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "hami",
+ Namespace: "hami-ns",
+ },
+ },
+ },
+ wantErr: true,
+ },
+ {
+ name: "node is not lock",
+ args: args{
+ nodeName: func() string {
+ name := "worker-1"
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{Name: name, Annotations: map[string]string{}},
+ }, metav1.CreateOptions{})
+ return name
+ },
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "hami",
+ Namespace: "hami-ns",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "node lock is not set by this pod",
+ args: args{
+ nodeName: func() string {
+ name := "worker-2"
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{Name: name, Annotations: map[string]string{
+ NodeLockKey: GenerateNodeLockKeyByPod(&corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{Name: "pod", Namespace: "namespace"},
+ }),
+ }},
+ }, metav1.CreateOptions{})
+ return name
+ },
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "hami",
+ Namespace: "hami-ns",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "successfully release node lock",
+ args: args{
+ nodeName: func() string {
+ name := "worker-3"
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{Name: name, Annotations: map[string]string{
+ NodeLockKey: GenerateNodeLockKeyByPod(&corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{Name: "hami", Namespace: "hami-ns"},
+ }),
+ }},
+ }, metav1.CreateOptions{})
+ return name
+ },
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "hami",
+ Namespace: "hami-ns",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if err := ReleaseNodeLock(tt.args.nodeName(), tt.args.lockname, tt.args.pod, tt.args.timeout); (err != nil) != tt.wantErr {
+ t.Errorf("ReleaseNodeLock() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ })
+ }
+}
diff --git a/pkg/util/types.go b/pkg/util/types.go
index 941206f57..381a55272 100644
--- a/pkg/util/types.go
+++ b/pkg/util/types.go
@@ -1,45 +1,178 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package util
+import (
+ corev1 "k8s.io/api/core/v1"
+)
+
const (
- //ResourceName = "nvidia.com/gpu"
- //ResourceName = "4pd.io/vgpu"
- AssignedTimeAnnotations = "4pd.io/vgpu-time"
- AssignedIDsAnnotations = "4pd.io/vgpu-ids"
- AssignedNodeAnnotations = "4pd.io/vgpu-node"
-
- //TimeLayout = "ANSIC"
- //DefaultTimeout = time.Second * 60
+ //ResourceName = "nvidia.com/gpu"
+ //ResourceName = "hami.io/vgpu".
+ AssignedTimeAnnotations = "hami.io/vgpu-time"
+ AssignedNodeAnnotations = "hami.io/vgpu-node"
+ BindTimeAnnotations = "hami.io/bind-time"
+ DeviceBindPhase = "hami.io/bind-phase"
+
+ DeviceBindAllocating = "allocating"
+ DeviceBindFailed = "failed"
+ DeviceBindSuccess = "success"
+
+ DeviceLimit = 100
+ //TimeLayout = "ANSIC"
+ //DefaultTimeout = time.Second * 60.
+
+ BestEffort string = "best-effort"
+ Restricted string = "restricted"
+ Guaranteed string = "guaranteed"
+
+ // NodeNameEnvName define env var name for use get node name.
+ NodeNameEnvName = "NODE_NAME"
+ TaskPriority = "CUDA_TASK_PRIORITY"
+ CoreLimitSwitch = "GPU_CORE_UTILIZATION_POLICY"
)
var (
- ResourceName string
- DebugMode bool
+ DebugMode bool
+
+ NodeName string
+ RuntimeSocketFlag string
)
-//type ContainerDevices struct {
-// Devices []string `json:"devices,omitempty"`
-//}
-//
-//type PodDevices struct {
-// Containers []ContainerDevices `json:"containers,omitempty"`
-//}
+type ContainerDevice struct {
+ // TODO current Idx cannot use, because EncodeContainerDevices method not encode this filed.
+ Idx int
+ UUID string
+ Type string
+ Usedmem int32
+ Usedcores int32
+ CustomInfo map[string]any
+}
+
+type ContainerDeviceRequest struct {
+ Nums int32
+ Type string
+ Memreq int32
+ MemPercentagereq int32
+ Coresreq int32
+}
+
+type ContainerDevices []ContainerDevice
+type ContainerDeviceRequests map[string]ContainerDeviceRequest
+
+// type ContainerAllDevices map[string]ContainerDevices.
+type PodSingleDevice []ContainerDevices
+type PodDeviceRequests []ContainerDeviceRequests
+type PodDevices map[string]PodSingleDevice
+
+type MigTemplate struct {
+ Name string `yaml:"name"`
+ Memory int32 `yaml:"memory"`
+ Count int32 `yaml:"count"`
+}
+
+type MigTemplateUsage struct {
+ Name string `json:"name,omitempty"`
+ Memory int32 `json:"memory,omitempty"`
+ InUse bool `json:"inuse,omitempty"`
+}
-type ContainerDevices []string
+type Geometry []MigTemplate
+
+type MIGS []MigTemplateUsage
+
+type MigInUse struct {
+ Index int32
+ UsageList MIGS
+}
+
+type AllowedMigGeometries struct {
+ Models []string `yaml:"models"`
+ Geometries []Geometry `yaml:"allowedGeometries"`
+}
+
+type DeviceUsage struct {
+ ID string
+ Index uint
+ Used int32
+ Count int32
+ Usedmem int32
+ Totalmem int32
+ Totalcore int32
+ Usedcores int32
+ Mode string
+ MigTemplate []Geometry
+ MigUsage MigInUse
+ Numa int
+ Type string
+ Health bool
+ CustomInfo map[string]any
+}
+
+type DeviceInfo struct {
+ ID string `json:"id,omitempty"`
+ Index uint `json:"index,omitempty"`
+ Count int32 `json:"count,omitempty"`
+ Devmem int32 `json:"devmem,omitempty"`
+ Devcore int32 `json:"devcore,omitempty"`
+ Type string `json:"type,omitempty"`
+ Numa int `json:"numa,omitempty"`
+ Mode string `json:"mode,omitempty"`
+ MIGTemplate []Geometry `json:"migtemplate,omitempty"`
+ Health bool `json:"health,omitempty"`
+ DeviceVendor string `json:"devicevendor,omitempty"`
+ CustomInfo map[string]any `json:"custominfo,omitempty"`
+ DevicePairScore DevicePairScore `json:"devicepairscore,omitempty"`
+}
+
+type DevicePairScores []DevicePairScore
+type DevicePairScore struct {
+ ID string `json:"uuid,omitempty"`
+ Scores map[string]int `json:"score,omitempty"`
+}
+
+type NodeInfo struct {
+ ID string
+ Node *corev1.Node
+ Devices []DeviceInfo
+}
+
+type SchedulerPolicyName string
+
+const (
+ // NodeSchedulerPolicyBinpack is node use binpack scheduler policy.
+ NodeSchedulerPolicyBinpack SchedulerPolicyName = "binpack"
+ // NodeSchedulerPolicySpread is node use spread scheduler policy.
+ NodeSchedulerPolicySpread SchedulerPolicyName = "spread"
+ // GPUSchedulerPolicyBinpack is GPU use binpack scheduler.
+ GPUSchedulerPolicyBinpack SchedulerPolicyName = "binpack"
+ // GPUSchedulerPolicySpread is GPU use spread scheduler.
+ GPUSchedulerPolicySpread SchedulerPolicyName = "spread"
+ // GPUSchedulerPolicyTopology is GPU use topology scheduler.
+ GPUSchedulerPolicyTopology SchedulerPolicyName = "topology-aware"
+)
+
+const (
+ // NodeSchedulerPolicyAnnotationKey is user set Pod annotation to change this default node policy.
+ NodeSchedulerPolicyAnnotationKey = "hami.io/node-scheduler-policy"
+ // GPUSchedulerPolicyAnnotationKey is user set Pod annotation to change this default GPU policy.
+ GPUSchedulerPolicyAnnotationKey = "hami.io/gpu-scheduler-policy"
+)
-type PodDevices []ContainerDevices
+func (s SchedulerPolicyName) String() string {
+ return string(s)
+}
diff --git a/pkg/util/util.go b/pkg/util/util.go
index 85d483380..917914702 100644
--- a/pkg/util/util.go
+++ b/pkg/util/util.go
@@ -1,64 +1,551 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package util
import (
- "flag"
- "os"
- "strings"
+ "context"
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+ "github.com/Project-HAMi/HAMi/pkg/util/nodelock"
+
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ k8stypes "k8s.io/apimachinery/pkg/types"
+ "k8s.io/klog/v2"
+
+ safecast "github.com/ccoveille/go-safecast"
+)
+
+const (
+ // OneContainerMultiDeviceSplitSymbol this is when one container use multi device, use : symbol to join device info.
+ OneContainerMultiDeviceSplitSymbol = ":"
+
+ // OnePodMultiContainerSplitSymbol this is when one pod having multi container and more than one container use device, use ; symbol to join device info.
+ OnePodMultiContainerSplitSymbol = ";"
+)
- "k8s.io/klog/v2"
+var (
+ InRequestDevices map[string]string
+ SupportDevices map[string]string
+ HandshakeAnnos map[string]string
)
-func GlobalFlagSet() *flag.FlagSet {
- fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
- fs.StringVar(&ResourceName, "resource-name", "nvidia.com/gpu", "resource name")
- fs.BoolVar(&DebugMode, "debug", false, "debug mode")
- klog.InitFlags(fs)
- return fs
+func init() {
+ InRequestDevices = make(map[string]string)
+ SupportDevices = make(map[string]string)
+ HandshakeAnnos = make(map[string]string)
+}
+
+func GetNode(nodename string) (*corev1.Node, error) {
+ if nodename == "" {
+ klog.ErrorS(nil, "Node name is empty")
+ return nil, fmt.Errorf("nodename is empty")
+ }
+
+ klog.InfoS("Fetching node", "nodeName", nodename)
+ n, err := client.GetClient().CoreV1().Nodes().Get(context.Background(), nodename, metav1.GetOptions{})
+ if err != nil {
+ switch {
+ case apierrors.IsNotFound(err):
+ klog.ErrorS(err, "Node not found", "nodeName", nodename)
+ return nil, fmt.Errorf("node %s not found", nodename)
+ case apierrors.IsUnauthorized(err):
+ klog.ErrorS(err, "Unauthorized to access node", "nodeName", nodename)
+ return nil, fmt.Errorf("unauthorized to access node %s", nodename)
+ default:
+ klog.ErrorS(err, "Failed to get node", "nodeName", nodename)
+ return nil, fmt.Errorf("failed to get node %s: %v", nodename, err)
+ }
+ }
+
+ klog.InfoS("Successfully fetched node", "nodeName", nodename)
+ return n, nil
+}
+
+func GetPendingPod(ctx context.Context, node string) (*corev1.Pod, error) {
+ pod, err := GetAllocatePodByNode(ctx, node)
+ if err != nil {
+ return nil, err
+ }
+ if pod != nil {
+ return pod, nil
+ }
+ // filter pods for this node.
+ selector := fmt.Sprintf("spec.nodeName=%s", node)
+ podListOptions := metav1.ListOptions{
+ FieldSelector: selector,
+ }
+ podlist, err := client.GetClient().CoreV1().Pods("").List(ctx, podListOptions)
+ if err != nil {
+ return nil, err
+ }
+ for _, p := range podlist.Items {
+ if p.Status.Phase != corev1.PodPending {
+ continue
+ }
+ if _, ok := p.Annotations[BindTimeAnnotations]; !ok {
+ continue
+ }
+ if phase, ok := p.Annotations[DeviceBindPhase]; !ok {
+ continue
+ } else {
+ if strings.Compare(phase, DeviceBindAllocating) != 0 {
+ continue
+ }
+ }
+ if n, ok := p.Annotations[AssignedNodeAnnotations]; !ok {
+ continue
+ } else {
+ if strings.Compare(n, node) == 0 {
+ return &p, nil
+ }
+ }
+ }
+ return nil, fmt.Errorf("no binding pod found on node %s", node)
+}
+
+func GetAllocatePodByNode(ctx context.Context, nodeName string) (*corev1.Pod, error) {
+ node, err := client.GetClient().CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+ if err != nil {
+ return nil, err
+ }
+ if value, ok := node.Annotations[nodelock.NodeLockKey]; ok {
+ klog.V(2).Infof("node annotation key is %s, value is %s ", nodelock.NodeLockKey, value)
+ _, ns, name, err := nodelock.ParseNodeLock(value)
+ if err != nil {
+ return nil, err
+ }
+ if ns == "" || name == "" {
+ return nil, nil
+ }
+ return client.GetClient().CoreV1().Pods(ns).Get(ctx, name, metav1.GetOptions{})
+ }
+ return nil, nil
+}
+
+func DecodeNodeDevices(str string) ([]*DeviceInfo, error) {
+ if !strings.Contains(str, OneContainerMultiDeviceSplitSymbol) {
+ return []*DeviceInfo{}, errors.New("node annotations not decode successfully")
+ }
+ tmp := strings.Split(str, OneContainerMultiDeviceSplitSymbol)
+ var retval []*DeviceInfo
+ for _, val := range tmp {
+ if strings.Contains(val, ",") {
+ items := strings.Split(val, ",")
+ if len(items) == 7 || len(items) == 9 {
+ count, _ := strconv.ParseInt(items[1], 10, 32)
+ devmem, _ := strconv.ParseInt(items[2], 10, 32)
+ devcore, _ := strconv.ParseInt(items[3], 10, 32)
+ health, _ := strconv.ParseBool(items[6])
+ numa, _ := strconv.Atoi(items[5])
+ mode := "hami-core"
+ index := 0
+ if len(items) == 9 {
+ index, _ = strconv.Atoi(items[7])
+ mode = items[8]
+ }
+ count32, err := safecast.ToInt32(count)
+ if err != nil {
+ return []*DeviceInfo{}, errors.New("node annotations not decode successfully")
+ }
+ devmem32, err := safecast.ToInt32(devmem)
+ if err != nil {
+ return []*DeviceInfo{}, errors.New("node annotations not decode successfully")
+ }
+ devcore32, err := safecast.ToInt32(devcore)
+ if err != nil {
+ return []*DeviceInfo{}, errors.New("node annotations not decode successfully")
+ }
+ i := DeviceInfo{
+ ID: items[0],
+ Count: count32,
+ Devmem: devmem32,
+ Devcore: devcore32,
+ Type: items[4],
+ Numa: numa,
+ Health: health,
+ Mode: mode,
+ Index: uint(index),
+ }
+ retval = append(retval, &i)
+ } else {
+ return []*DeviceInfo{}, errors.New("node annotations not decode successfully")
+ }
+ }
+ }
+ return retval, nil
+}
+
+func DecodePairScores(pairScores string) (*DevicePairScores, error) {
+ devicePairScores := &DevicePairScores{}
+ if err := json.Unmarshal([]byte(pairScores), devicePairScores); err != nil {
+ return nil, err
+ }
+ return devicePairScores, nil
+}
+
+func EncodeNodeDevices(dlist []*DeviceInfo) string {
+ builder := strings.Builder{}
+ for _, val := range dlist {
+ builder.WriteString(val.ID)
+ builder.WriteString(",")
+ builder.WriteString(strconv.FormatInt(int64(val.Count), 10))
+ builder.WriteString(",")
+ builder.WriteString(strconv.Itoa(int(val.Devmem)))
+ builder.WriteString(",")
+ builder.WriteString(strconv.Itoa(int(val.Devcore)))
+ builder.WriteString(",")
+ builder.WriteString(val.Type)
+ builder.WriteString(",")
+ builder.WriteString(strconv.Itoa(val.Numa))
+ builder.WriteString(",")
+ builder.WriteString(strconv.FormatBool(val.Health))
+ builder.WriteString(",")
+ builder.WriteString(strconv.Itoa(int(val.Index)))
+ builder.WriteString(",")
+ builder.WriteString(val.Mode)
+ builder.WriteString(OneContainerMultiDeviceSplitSymbol)
+ //tmp += val.ID + "," + strconv.FormatInt(int64(val.Count), 10) + "," + strconv.Itoa(int(val.Devmem)) + "," + strconv.Itoa(int(val.Devcore)) + "," + val.Type + "," + strconv.Itoa(val.Numa) + "," + strconv.FormatBool(val.Health) + "," + strconv.Itoa(val.Index) + OneContainerMultiDeviceSplitSymbol
+ }
+ tmp := builder.String()
+ klog.V(5).Infof("Encoded node Devices: %s", tmp)
+ return tmp
+}
+
+func MarshalNodeDevices(dlist []*DeviceInfo) string {
+ data, err := json.Marshal(dlist)
+ if err != nil {
+ return ""
+ }
+ return string(data)
+}
+
+func UnMarshalNodeDevices(str string) ([]*DeviceInfo, error) {
+ var dlist []*DeviceInfo
+ err := json.Unmarshal([]byte(str), &dlist)
+ return dlist, err
}
func EncodeContainerDevices(cd ContainerDevices) string {
- return strings.Join(cd, ",")
-}
-
-func EncodePodDevices(pd PodDevices) string {
- var ss []string
- for _, cd := range pd {
- ss = append(ss, EncodeContainerDevices(cd))
- }
- return strings.Join(ss, ";")
-}
-
-func DecodeContainerDevices(str string) ContainerDevices {
- if len(str) == 0 {
- return ContainerDevices{}
- }
- return strings.Split(str, ",")
-}
-
-func DecodePodDevices(str string) PodDevices {
- if len(str) == 0 {
- return PodDevices{}
- }
- var pd PodDevices
- for _, s := range strings.Split(str, ";") {
- cd := DecodeContainerDevices(s)
- pd = append(pd, cd)
- }
- return pd
+ tmp := ""
+ for _, val := range cd {
+ tmp += val.UUID + "," + val.Type + "," + strconv.Itoa(int(val.Usedmem)) + "," + strconv.Itoa(int(val.Usedcores)) + OneContainerMultiDeviceSplitSymbol
+ }
+ klog.Infof("Encoded container Devices: %s", tmp)
+ return tmp
+ //return strings.Join(cd, ",")
+}
+
+func EncodeContainerDeviceType(cd ContainerDevices, t string) string {
+ tmp := ""
+ for _, val := range cd {
+ if strings.Compare(val.Type, t) == 0 {
+ tmp += val.UUID + "," + val.Type + "," + strconv.Itoa(int(val.Usedmem)) + "," + strconv.Itoa(int(val.Usedcores))
+ }
+ tmp += OneContainerMultiDeviceSplitSymbol
+ }
+ klog.Infof("Encoded container Certain Device type: %s->%s", t, tmp)
+ return tmp
+}
+
+func EncodePodSingleDevice(pd PodSingleDevice) string {
+ res := ""
+ for _, ctrdevs := range pd {
+ res = res + EncodeContainerDevices(ctrdevs)
+ res = res + OnePodMultiContainerSplitSymbol
+ }
+ klog.Infof("Encoded pod single devices %s", res)
+ return res
+}
+
+func EncodePodDevices(checklist map[string]string, pd PodDevices) map[string]string {
+ res := map[string]string{}
+ for devType, cd := range pd {
+ klog.Infoln("devtype=", devType)
+ res[checklist[devType]] = EncodePodSingleDevice(cd)
+ }
+ klog.Infof("Encoded pod Devices %s\n", res)
+ return res
+}
+
+func DecodeContainerDevices(str string) (ContainerDevices, error) {
+ if len(str) == 0 {
+ return ContainerDevices{}, nil
+ }
+ cd := strings.Split(str, OneContainerMultiDeviceSplitSymbol)
+ contdev := ContainerDevices{}
+ tmpdev := ContainerDevice{}
+ klog.V(5).Infof("Start to decode container device %s", str)
+ if len(str) == 0 {
+ return ContainerDevices{}, nil
+ }
+ for _, val := range cd {
+ if strings.Contains(val, ",") {
+ //fmt.Println("cd is ", val)
+ tmpstr := strings.Split(val, ",")
+ if len(tmpstr) < 4 {
+ return ContainerDevices{}, fmt.Errorf("pod annotation format error; information missing, please do not use nodeName field in task")
+ }
+ tmpdev.UUID = tmpstr[0]
+ tmpdev.Type = tmpstr[1]
+ devmem, _ := strconv.ParseInt(tmpstr[2], 10, 32)
+ tmpdev.Usedmem = int32(devmem)
+ devcores, _ := strconv.ParseInt(tmpstr[3], 10, 32)
+ tmpdev.Usedcores = int32(devcores)
+ contdev = append(contdev, tmpdev)
+ }
+ }
+ klog.V(5).Infof("Finished decoding container devices. Total devices: %d", len(contdev))
+ return contdev, nil
+}
+
+func DecodePodDevices(checklist map[string]string, annos map[string]string) (PodDevices, error) {
+ klog.V(5).Infof("checklist is [%+v], annos is [%+v]", checklist, annos)
+ if len(annos) == 0 {
+ return PodDevices{}, nil
+ }
+ pd := make(PodDevices)
+ for devID, devs := range checklist {
+ str, ok := annos[devs]
+ if !ok {
+ continue
+ }
+ pd[devID] = make(PodSingleDevice, 0)
+ for s := range strings.SplitSeq(str, OnePodMultiContainerSplitSymbol) {
+ cd, err := DecodeContainerDevices(s)
+ if err != nil {
+ return PodDevices{}, nil
+ }
+ if len(cd) == 0 {
+ continue
+ }
+ pd[devID] = append(pd[devID], cd)
+ }
+ }
+ klog.InfoS("Decoded pod annos", "poddevices", pd)
+ return pd, nil
+}
+
+func PatchNodeAnnotations(node *corev1.Node, annotations map[string]string) error {
+ type patchMetadata struct {
+ Annotations map[string]string `json:"annotations,omitempty"`
+ }
+ type patchPod struct {
+ Metadata patchMetadata `json:"metadata"`
+ //Spec patchSpec `json:"spec,omitempty"`
+ }
+
+ p := patchPod{}
+ p.Metadata.Annotations = annotations
+
+ bytes, err := json.Marshal(p)
+ if err != nil {
+ return err
+ }
+ _, err = client.GetClient().CoreV1().Nodes().
+ Patch(context.Background(), node.Name, k8stypes.StrategicMergePatchType, bytes, metav1.PatchOptions{})
+ if err != nil {
+ klog.Infoln("annotations=", annotations)
+ klog.Infof("patch node %v failed, %v", node.Name, err)
+ }
+ return err
+}
+
+func PatchPodAnnotations(pod *corev1.Pod, annotations map[string]string) error {
+ type patchMetadata struct {
+ Annotations map[string]string `json:"annotations,omitempty"`
+ Labels map[string]string `json:"labels,omitempty"`
+ }
+ type patchPod struct {
+ Metadata patchMetadata `json:"metadata"`
+ //Spec patchSpec `json:"spec,omitempty"`
+ }
+
+ p := patchPod{}
+ p.Metadata.Annotations = annotations
+ label := make(map[string]string)
+ if v, ok := annotations[AssignedNodeAnnotations]; ok && v != "" {
+ label[AssignedNodeAnnotations] = v
+ p.Metadata.Labels = label
+ }
+
+ bytes, err := json.Marshal(p)
+ if err != nil {
+ return err
+ }
+ klog.V(5).Infof("patch pod %s/%s annotation content is %s", pod.Namespace, pod.Name, string(bytes))
+ _, err = client.GetClient().CoreV1().Pods(pod.Namespace).
+ Patch(context.Background(), pod.Name, k8stypes.StrategicMergePatchType, bytes, metav1.PatchOptions{})
+ if err != nil {
+ klog.Infof("patch pod %v failed, %v", pod.Name, err)
+ }
+ return err
+}
+
+func InitKlogFlags() *flag.FlagSet {
+ // Init log flags
+ flagset := flag.NewFlagSet("klog", flag.ExitOnError)
+ klog.InitFlags(flagset)
+
+ return flagset
+}
+
+func CheckHealth(devType string, n *corev1.Node) (bool, bool) {
+ handshake := n.Annotations[HandshakeAnnos[devType]]
+ if strings.Contains(handshake, "Requesting") {
+ formertime, _ := time.Parse(time.DateTime, strings.Split(handshake, "_")[1])
+ return time.Now().Before(formertime.Add(time.Second * 60)), false
+ } else if strings.Contains(handshake, "Deleted") {
+ return true, false
+ } else {
+ return true, true
+ }
+}
+
+func MarkAnnotationsToDelete(devType string, nn string) error {
+ tmppat := make(map[string]string)
+ tmppat[devType] = "Deleted_" + time.Now().Format(time.DateTime)
+ n, err := GetNode(nn)
+ if err != nil {
+ klog.Errorln("get node failed", err.Error())
+ return err
+ }
+ return PatchNodeAnnotations(n, tmppat)
+}
+
+// Enhanced ExtractMigTemplatesFromUUID with error handling.
+func ExtractMigTemplatesFromUUID(uuid string) (int, int, error) {
+ parts := strings.Split(uuid, "[")
+ if len(parts) < 2 {
+ return -1, -1, fmt.Errorf("invalid UUID format: missing '[' delimiter")
+ }
+
+ tmp := parts[1]
+ parts = strings.Split(tmp, "]")
+ if len(parts) < 2 {
+ return -1, -1, fmt.Errorf("invalid UUID format: missing ']' delimiter")
+ }
+
+ tmp = parts[0]
+ parts = strings.Split(tmp, "-")
+ if len(parts) < 2 {
+ return -1, -1, fmt.Errorf("invalid UUID format: missing '-' delimiter")
+ }
+
+ templateIdx, err := strconv.Atoi(parts[0])
+ if err != nil {
+ return -1, -1, fmt.Errorf("invalid template index: %v", err)
+ }
+
+ pos, err := strconv.Atoi(parts[1])
+ if err != nil {
+ return -1, -1, fmt.Errorf("invalid position: %v", err)
+ }
+
+ return templateIdx, pos, nil
+}
+
+func PlatternMIG(n *MigInUse, templates []Geometry, templateIdx int) {
+ var err error
+ for _, val := range templates[templateIdx] {
+ count := 0
+ for count < int(val.Count) {
+ n.Index, err = safecast.ToInt32(templateIdx)
+ if err != nil {
+ continue
+ }
+ n.UsageList = append(n.UsageList, MigTemplateUsage{
+ Name: val.Name,
+ Memory: val.Memory,
+ InUse: false,
+ })
+ count++
+ }
+ }
+}
+
+func GetDevicesUUIDList(infos []*DeviceInfo) []string {
+ uuids := make([]string, 0)
+ for _, info := range infos {
+ uuids = append(uuids, info.ID)
+ }
+ return uuids
+}
+
+func GetGPUSchedulerPolicyByPod(defaultPolicy string, task *corev1.Pod) string {
+ userGPUPolicy := defaultPolicy
+ if task != nil && task.Annotations != nil {
+ if value, ok := task.Annotations[GPUSchedulerPolicyAnnotationKey]; ok {
+ userGPUPolicy = value
+ }
+ }
+ return userGPUPolicy
+}
+
+// ContainsTargetNonNegativeInt checks if the input string contains the target non-negative integer (including 0) as a distinct element.
+// It splits the input string by the given separator, trims whitespace from each element,
+// and verifies if any element is exactly equal to the target non-negative integer.
+// Parameters:
+//
+// input: The input string to be parsed (e.g., "0, 5, 10")
+// separator: The delimiter used to split the input string (e.g., ",", "|")
+// target: The non-negative integer to check for (must be greater than or equal to 0)
+//
+// Returns:
+//
+// bool: true if the target exists as a distinct non-negative integer element; false otherwise
+func ContainsTargetNonNegativeInt(input, separator string, target int) bool {
+ // Target must be a non-negative integer (greater than or equal to 0)
+ if target < 0 {
+ return false
+ }
+
+ // Split the input string into elements using the separator
+ elements := strings.Split(input, separator)
+
+ // Check each element
+ for _, elem := range elements {
+ // Remove leading and trailing whitespace from the element
+ trimmedElem := strings.TrimSpace(elem)
+ if trimmedElem == "" {
+ continue // Skip empty elements (e.g., from consecutive separators like ",,")
+ }
+
+ // Check for leading zeros (invalid unless the element is exactly "0")
+ if len(trimmedElem) > 1 && trimmedElem[0] == '0' {
+ continue // Skip elements with leading zeros (e.g., "01", "005")
+ }
+
+ // Convert the trimmed element to an integer
+ num, err := strconv.Atoi(trimmedElem)
+ // Check if conversion succeeded and the number matches the target (non-negative check is implicit via target validation)
+ if err == nil && num == target {
+ return true
+ }
+ }
+
+ // Target not found in any valid element
+ return false
}
diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go
index 6f5f6158f..1ab5a38e2 100644
--- a/pkg/util/util_test.go
+++ b/pkg/util/util_test.go
@@ -1,51 +1,1261 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package util
import (
- "fmt"
- "gotest.tools/v3/assert"
- "testing"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "testing"
+
+ "gotest.tools/v3/assert"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes/fake"
+
+ "github.com/Project-HAMi/HAMi/pkg/util/client"
+ "github.com/Project-HAMi/HAMi/pkg/util/nodelock"
)
+var inRequestDevices map[string]string
+
+func init() {
+ inRequestDevices = make(map[string]string)
+ inRequestDevices["NVIDIA"] = "hami.io/vgpu-devices-to-allocate"
+}
+
+func TestExtractMigTemplatesFromUUID(t *testing.T) {
+ testCases := []struct {
+ name string
+ uuid string
+ expectedTmpID int
+ expectedPos int
+ expectError bool
+ }{
+ {
+ name: "Valid UUID",
+ uuid: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313[7-9]",
+ expectedTmpID: 7,
+ expectedPos: 9,
+ expectError: false,
+ },
+ {
+ name: "Invalid UUID format - missing '[' delimiter",
+ uuid: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b32693137-9]",
+ expectedTmpID: -1,
+ expectedPos: -1,
+ expectError: true,
+ },
+ {
+ name: "Invalid UUID format - missing ']' delimiter",
+ uuid: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313[7-9",
+ expectedTmpID: -1,
+ expectedPos: -1,
+ expectError: true,
+ },
+ {
+ name: "Invalid UUID format - missing '-' delimiter",
+ uuid: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313[79]",
+ expectedTmpID: -1,
+ expectedPos: -1,
+ expectError: true,
+ },
+ {
+ name: "Invalid template index",
+ uuid: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313[a-9]",
+ expectedTmpID: -1,
+ expectedPos: -1,
+ expectError: true,
+ },
+ {
+ name: "Invalid position",
+ uuid: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313[7-b]",
+ expectedTmpID: -1,
+ expectedPos: -1,
+ expectError: true,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ tempid, pos, err := ExtractMigTemplatesFromUUID(tc.uuid)
+
+ if tc.expectError {
+ if err == nil {
+ t.Errorf("Expected an error but got none")
+ }
+ } else {
+ if err != nil {
+ t.Errorf("Did not expect an error but got: %v", err)
+ }
+ if tempid != tc.expectedTmpID || pos != tc.expectedPos {
+ t.Errorf("Expected %d:%d, got %d:%d", tc.expectedTmpID, tc.expectedPos, tempid, pos)
+ }
+ }
+ })
+ }
+}
+
func TestEmptyContainerDevicesCoding(t *testing.T) {
- cd1 := ContainerDevices{}
- s := EncodeContainerDevices(cd1)
- fmt.Println(s)
- cd2 := DecodeContainerDevices(s)
- assert.DeepEqual(t, cd1, cd2)
+ cd1 := ContainerDevices{}
+ s := EncodeContainerDevices(cd1)
+ fmt.Println(s)
+ cd2, _ := DecodeContainerDevices(s)
+ assert.DeepEqual(t, cd1, cd2)
}
func TestEmptyPodDeviceCoding(t *testing.T) {
- pd1 := PodDevices{}
- s := EncodePodDevices(pd1)
- fmt.Println(s)
- pd2 := DecodePodDevices(s)
- assert.DeepEqual(t, pd1, pd2)
+ pd1 := PodDevices{}
+ s := EncodePodDevices(inRequestDevices, pd1)
+ fmt.Println(s)
+ pd2, _ := DecodePodDevices(inRequestDevices, s)
+ assert.DeepEqual(t, pd1, pd2)
}
func TestPodDevicesCoding(t *testing.T) {
- pd1 := PodDevices{
- ContainerDevices{"1", "2"},
- ContainerDevices{},
- ContainerDevices{"3", "4"},
- }
- s := EncodePodDevices(pd1)
- fmt.Println(s)
- pd2 := DecodePodDevices(s)
- assert.DeepEqual(t, pd1, pd2)
+ tests := []struct {
+ name string
+ args PodDevices
+ }{
+ {
+ name: "one pod one container use zero device",
+ args: PodDevices{
+ "NVIDIA": PodSingleDevice{},
+ },
+ },
+ {
+ name: "one pod one container use one device",
+ args: PodDevices{
+ "NVIDIA": PodSingleDevice{
+ ContainerDevices{
+ ContainerDevice{0, "UUID1", "Type1", 1000, 30, nil},
+ },
+ },
+ },
+ },
+ {
+ name: "one pod two container, every container use one device",
+ args: PodDevices{
+ "NVIDIA": PodSingleDevice{
+ ContainerDevices{
+ ContainerDevice{0, "UUID1", "Type1", 1000, 30, nil},
+ },
+ ContainerDevices{
+ ContainerDevice{0, "UUID1", "Type1", 1000, 30, nil},
+ },
+ },
+ },
+ },
+ {
+ name: "one pod one container use two devices",
+ args: PodDevices{
+ "NVIDIA": PodSingleDevice{
+ ContainerDevices{
+ ContainerDevice{0, "UUID1", "Type1", 1000, 30, nil},
+ ContainerDevice{0, "UUID2", "Type1", 1000, 30, nil},
+ },
+ },
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := EncodePodDevices(inRequestDevices, test.args)
+ fmt.Println(s)
+ got, _ := DecodePodDevices(inRequestDevices, s)
+ assert.DeepEqual(t, test.args, got)
+ })
+ }
+}
+
+func Test_DecodePodDevices(t *testing.T) {
+ //DecodePodDevices(checklist map[string]string, annos map[string]string) (PodDevices, error)
+ InRequestDevices["NVIDIA"] = "hami.io/vgpu-devices-to-allocate"
+ SupportDevices["NVIDIA"] = "hami.io/vgpu-devices-allocated"
+ tests := []struct {
+ name string
+ args struct {
+ checklist map[string]string
+ annos map[string]string
+ }
+ want PodDevices
+ wantErr error
+ }{
+ {
+ name: "annos len is 0",
+ args: struct {
+ checklist map[string]string
+ annos map[string]string
+ }{
+ checklist: map[string]string{},
+ annos: make(map[string]string),
+ },
+ want: PodDevices{},
+ wantErr: nil,
+ },
+ {
+ name: "annos having two device",
+ args: struct {
+ checklist map[string]string
+ annos map[string]string
+ }{
+ checklist: InRequestDevices,
+ annos: map[string]string{
+ InRequestDevices["NVIDIA"]: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76,NVIDIA,500,3:;GPU-ebe7c3f7-303d-558d-435e-99a160631fe4,NVIDIA,500,3:;",
+ SupportDevices["NVIDIA"]: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76,NVIDIA,500,3:;GPU-ebe7c3f7-303d-558d-435e-99a160631fe4,NVIDIA,500,3:;",
+ },
+ },
+ want: PodDevices{
+ "NVIDIA": {
+ {
+ {
+ UUID: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76",
+ Type: "NVIDIA",
+ Usedmem: 500,
+ Usedcores: 3,
+ },
+ },
+ {
+ {
+ UUID: "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4",
+ Type: "NVIDIA",
+ Usedmem: 500,
+ Usedcores: 3,
+ },
+ },
+ },
+ },
+ wantErr: nil,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ got, gotErr := DecodePodDevices(test.args.checklist, test.args.annos)
+ assert.DeepEqual(t, test.wantErr, gotErr)
+ assert.DeepEqual(t, test.want, got)
+ })
+ }
+}
+
+func TestMarshalNodeDevices(t *testing.T) {
+ type args struct {
+ dlist []*DeviceInfo
+ }
+ tests := []struct {
+ name string
+ args args
+ want string
+ }{
+ {
+ name: "test one",
+ args: args{
+ dlist: []*DeviceInfo{
+ {
+ Index: 1,
+ ID: "id-1",
+ Count: 1,
+ Devmem: 1024,
+ Devcore: 10,
+ Type: "type",
+ Numa: 0,
+ Health: true,
+ },
+ },
+ },
+ want: "[{\"index\":1,\"id\":\"id-1\",\"count\":1,\"devmem\":1024,\"devcore\":10,\"type\":\"type\",\"numa\":0,\"health\":true}]",
+ },
+ {
+ name: "test multiple",
+ args: args{
+ dlist: []*DeviceInfo{
+ {
+ Index: 1,
+ ID: "id-1",
+ Count: 1,
+ Devmem: 1024,
+ Devcore: 10,
+ Type: "type",
+ Numa: 0,
+ Health: true,
+ },
+ {
+ Index: 2,
+ ID: "id-2",
+ Count: 2,
+ Devmem: 2048,
+ Devcore: 20,
+ Type: "type2",
+ Numa: 1,
+ Health: false,
+ },
+ },
+ },
+ want: "[{\"index\":1,\"id\":\"id-1\",\"count\":1,\"devmem\":1024,\"devcore\":10,\"type\":\"type\",\"numa\":0,\"health\":true},{\"index\":2,\"id\":\"id-2\",\"count\":2,\"devmem\":2048,\"devcore\":20,\"type\":\"type2\",\"numa\":1,\"health\":false}]",
+ },
+ {
+ name: "test empty",
+ args: args{
+ dlist: []*DeviceInfo{},
+ },
+ want: "[]",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := MarshalNodeDevices(tt.args.dlist)
+
+ var gotDeviceInfo, wantDeviceInfo []*DeviceInfo
+ // Compare the JSON contents by unmarshalling both got and want
+ err := json.Unmarshal([]byte(got), &gotDeviceInfo)
+ assert.NilError(t, err)
+
+ err = json.Unmarshal([]byte(tt.want), &wantDeviceInfo)
+ assert.NilError(t, err)
+
+ assert.DeepEqual(t, gotDeviceInfo, wantDeviceInfo)
+ })
+ }
+}
+
+func TestUnMarshalNodeDevices(t *testing.T) {
+ type args struct {
+ str string
+ }
+ tests := []struct {
+ name string
+ args args
+ want []*DeviceInfo
+ wantErr bool
+ }{
+ {
+ name: "test one",
+ args: args{
+ str: "[{\"index\":1,\"id\":\"id-1\",\"count\":1,\"devmem\":1024,\"devcore\":10,\"type\":\"type\",\"health\":true}]\n",
+ },
+ want: []*DeviceInfo{
+ {
+ Index: 1,
+ ID: "id-1",
+ Count: 1,
+ Devmem: 1024,
+ Devcore: 10,
+ Type: "type",
+ Numa: 0,
+ Health: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "test two",
+ args: args{
+ str: "[{\"index\":1,\"id\":\"id-1\",\"count\":1,\"devmem\":1024,\"devcore\":10,\"type\":\"type\",\"health\":true}," +
+ "{\"index\":2,\"id\":\"id-2\",\"count\":2,\"devmem\":4096,\"devcore\":20,\"type\":\"type2\",\"health\":false}]",
+ },
+ want: []*DeviceInfo{
+ {
+ Index: 1,
+ ID: "id-1",
+ Count: 1,
+ Devmem: 1024,
+ Devcore: 10,
+ Type: "type",
+ Numa: 0,
+ Health: true,
+ },
+ {
+ Index: 2,
+ ID: "id-2",
+ Count: 2,
+ Devmem: 4096,
+ Devcore: 20,
+ Type: "type2",
+ Numa: 0,
+ Health: false,
+ },
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := UnMarshalNodeDevices(tt.args.str)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("UnMarshalNodeDevices() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ assert.DeepEqual(t, got, tt.want)
+ })
+ }
+}
+
+func Test_DecodeNodeDevices(t *testing.T) {
+ tests := []struct {
+ name string
+ args string
+ want struct {
+ di []*DeviceInfo
+ err error
+ }
+ }{
+ {
+ name: "args is invalid",
+ args: "a",
+ want: struct {
+ di []*DeviceInfo
+ err error
+ }{
+ di: []*DeviceInfo{},
+ err: errors.New("node annotations not decode successfully"),
+ },
+ },
+ {
+ name: "str is old format",
+ args: "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4,10,7680,100,NVIDIA-Tesla P4,0,true:",
+ want: struct {
+ di []*DeviceInfo
+ err error
+ }{
+ di: []*DeviceInfo{
+ {
+ ID: "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4",
+ Index: 0,
+ Count: 10,
+ Devmem: 7680,
+ Devcore: 100,
+ Type: "NVIDIA-Tesla P4",
+ Mode: "hami-core",
+ Numa: 0,
+ Health: true,
+ },
+ },
+ err: nil,
+ },
+ },
+ {
+ name: "str is new format",
+ args: "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4,10,7680,100,NVIDIA-Tesla P4,0,true,1,hami-core:",
+ want: struct {
+ di []*DeviceInfo
+ err error
+ }{
+ di: []*DeviceInfo{
+ {
+ ID: "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4",
+ Index: 1,
+ Count: 10,
+ Devmem: 7680,
+ Devcore: 100,
+ Type: "NVIDIA-Tesla P4",
+ Mode: "hami-core",
+ Numa: 0,
+ Health: true,
+ },
+ },
+ err: nil,
+ },
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ got, err := DecodeNodeDevices(test.args)
+ assert.DeepEqual(t, test.want.di, got)
+ if err != nil {
+ assert.DeepEqual(t, test.want.err.Error(), err.Error())
+ }
+ })
+ }
+}
+
+func Test_EncodeNodeDevices(t *testing.T) {
+ tests := []struct {
+ name string
+ args []*DeviceInfo
+ want string
+ }{
+ {
+ name: "old format",
+ args: []*DeviceInfo{
+ {
+ ID: "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4",
+ Index: 0,
+ Count: 10,
+ Devmem: 7680,
+ Devcore: 100,
+ Type: "NVIDIA-Tesla P4",
+ Numa: 0,
+ Mode: "hami-core",
+ Health: true,
+ },
+ },
+ want: "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4,10,7680,100,NVIDIA-Tesla P4,0,true,0,hami-core:",
+ },
+ {
+ name: "test two",
+ args: []*DeviceInfo{
+ {
+ ID: "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4",
+ Index: 1,
+ Count: 10,
+ Devmem: 7680,
+ Devcore: 100,
+ Mode: "hami-core",
+ Type: "NVIDIA-Tesla P4",
+ Numa: 0,
+ Health: true,
+ },
+ },
+ want: "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4,10,7680,100,NVIDIA-Tesla P4,0,true,1,hami-core:",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ got := EncodeNodeDevices(test.args)
+ assert.DeepEqual(t, test.want, got)
+ })
+ }
+}
+
+func Test_CheckHealth(t *testing.T) {
+ tests := []struct {
+ name string
+ args struct {
+ devType string
+ n corev1.Node
+ }
+ want1 bool
+ want2 bool
+ }{
+ {
+ name: "Requesting state",
+ args: struct {
+ devType string
+ n corev1.Node
+ }{
+ devType: "huawei.com/Ascend910",
+ n: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ HandshakeAnnos["huawei.com/Ascend910"]: "Requesting_2128-12-02 00:00:00",
+ },
+ },
+ },
+ },
+ want1: true,
+ want2: false,
+ },
+ {
+ name: "Deleted state",
+ args: struct {
+ devType string
+ n corev1.Node
+ }{
+ devType: "huawei.com/Ascend910",
+ n: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ HandshakeAnnos["huawei.com/Ascend910"]: "Deleted",
+ },
+ },
+ },
+ },
+ want1: true,
+ want2: false,
+ },
+ {
+ name: "Unknown state",
+ args: struct {
+ devType string
+ n corev1.Node
+ }{
+ devType: "huawei.com/Ascend910",
+ n: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ HandshakeAnnos["huawei.com/Ascend910"]: "Unknown",
+ },
+ },
+ },
+ },
+ want1: true,
+ want2: true,
+ },
+ {
+ name: "Requesting state expired",
+ args: struct {
+ devType string
+ n corev1.Node
+ }{
+ devType: "huawei.com/Ascend910",
+ n: corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ HandshakeAnnos["huawei.com/Ascend910"]: "Requesting_2024-01-02 00:00:00",
+ },
+ },
+ },
+ },
+ want1: false,
+ want2: false,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result1, result2 := CheckHealth(test.args.devType, &test.args.n)
+ assert.Equal(t, result1, test.want1)
+ assert.Equal(t, result2, test.want2)
+ })
+ }
+}
+
+func TestMarkAnnotationsToDelete(t *testing.T) {
+ client.KubeClient = fake.NewSimpleClientset()
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{Name: "node-worker2"},
+ }, metav1.CreateOptions{})
+ type args struct {
+ devType string
+ nn string
+ }
+ tests := []struct {
+ name string
+ args args
+ wantErr bool
+ }{
+ {
+ name: "node not found",
+ args: args{
+ devType: "huawei.com/Ascend910",
+ nn: "node-worker1",
+ },
+ wantErr: true,
+ },
+ {
+ name: "mark annotations to delete",
+ args: args{
+ devType: "huawei.com/Ascend910",
+ nn: "node-worker2",
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if err := MarkAnnotationsToDelete(tt.args.devType, tt.args.nn); (err != nil) != tt.wantErr {
+ t.Errorf("MarkAnnotationsToDelete() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ })
+ }
+}
+
+func TestDecodeNodeDevices(t *testing.T) {
+ pairScore := "[{\"uuid\":\"GPU-0\",\"score\":{\"GPU-1\":50,\"GPU-2\":20,\"GPU-3\":20}}]"
+ _, err := DecodePairScores(pairScore)
+ assert.NilError(t, err)
+}
+
+func TestPlatternMIG(t *testing.T) {
+ tests := []struct {
+ name string
+ n *MigInUse
+ templates []Geometry
+ templateIdx int
+ want *MigInUse
+ }{
+ {
+ name: "empty template",
+ n: &MigInUse{},
+ templates: []Geometry{
+ {},
+ },
+ templateIdx: 0,
+ want: &MigInUse{
+ Index: 0,
+ UsageList: nil,
+ },
+ },
+ {
+ name: "single template with one count",
+ n: &MigInUse{},
+ templates: []Geometry{
+ {
+ {
+ Name: "1g.5gb",
+ Memory: 5,
+ Count: 1,
+ },
+ },
+ },
+ templateIdx: 0,
+ want: &MigInUse{
+ Index: 0,
+ UsageList: MIGS{
+ {
+ Name: "1g.5gb",
+ Memory: 5,
+ InUse: false,
+ },
+ },
+ },
+ },
+ {
+ name: "multiple templates with different counts",
+ n: &MigInUse{},
+ templates: []Geometry{
+ {
+ {
+ Name: "1g.5gb",
+ Memory: 5,
+ Count: 2,
+ },
+ {
+ Name: "2g.10gb",
+ Memory: 10,
+ Count: 1,
+ },
+ },
+ },
+ templateIdx: 0,
+ want: &MigInUse{
+ Index: 0,
+ UsageList: MIGS{
+ {
+ Name: "1g.5gb",
+ Memory: 5,
+ InUse: false,
+ },
+ {
+ Name: "1g.5gb",
+ Memory: 5,
+ InUse: false,
+ },
+ {
+ Name: "2g.10gb",
+ Memory: 10,
+ InUse: false,
+ },
+ },
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ PlatternMIG(tt.n, tt.templates, tt.templateIdx)
+ assert.DeepEqual(t, tt.want, tt.n)
+ })
+ }
+}
+
+func TestGetDevicesUUIDList(t *testing.T) {
+ tests := []struct {
+ name string
+ infos []*DeviceInfo
+ want []string
+ }{
+ {
+ name: "empty device list",
+ infos: []*DeviceInfo{},
+ want: []string{},
+ },
+ {
+ name: "single device",
+ infos: []*DeviceInfo{
+ {ID: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313"},
+ },
+ want: []string{"GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313"},
+ },
+ {
+ name: "multiple devices",
+ infos: []*DeviceInfo{
+ {ID: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313"},
+ {ID: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76"},
+ {ID: "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4"},
+ },
+ want: []string{
+ "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313",
+ "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76",
+ "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4",
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := GetDevicesUUIDList(tt.infos)
+ assert.DeepEqual(t, tt.want, got)
+ })
+ }
+}
+
+func TestGetPendingPod(t *testing.T) {
+ client.KubeClient = fake.NewSimpleClientset()
+ // Create test node and pod
+
+ podList := []*corev1.Pod{
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pending-pod",
+ Namespace: "default",
+ Annotations: map[string]string{
+ BindTimeAnnotations: "2024-01-01T00:00:00Z",
+ DeviceBindPhase: DeviceBindAllocating,
+ AssignedNodeAnnotations: "test-node-0",
+ },
+ },
+ Spec: corev1.PodSpec{NodeName: "test-node-0"},
+ Status: corev1.PodStatus{
+ Phase: corev1.PodPending,
+ },
+ },
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "ignore-pod-0",
+ Namespace: "default",
+ Annotations: map[string]string{},
+ },
+ Spec: corev1.PodSpec{NodeName: "test-node-0"},
+ Status: corev1.PodStatus{
+ Phase: corev1.PodFailed,
+ },
+ },
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "ignore-pod-1",
+ Namespace: "default",
+ Annotations: map[string]string{},
+ },
+ Spec: corev1.PodSpec{NodeName: "test-node-0"},
+ Status: corev1.PodStatus{
+ Phase: corev1.PodPending,
+ },
+ },
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "ignore-pod-2",
+ Namespace: "default",
+ Annotations: map[string]string{
+ BindTimeAnnotations: "2024-01-01T00:00:00Z",
+ AssignedNodeAnnotations: "test-node-0",
+ },
+ },
+ Spec: corev1.PodSpec{NodeName: "test-node-0"},
+ Status: corev1.PodStatus{
+ Phase: corev1.PodPending,
+ },
+ },
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "ignore-pod-3",
+ Namespace: "default",
+ Annotations: map[string]string{
+ BindTimeAnnotations: "2024-01-01T00:00:00Z",
+ DeviceBindPhase: "",
+ AssignedNodeAnnotations: "test-node-2",
+ },
+ },
+ Spec: corev1.PodSpec{NodeName: "test-node-2"},
+ Status: corev1.PodStatus{
+ Phase: corev1.PodPending,
+ },
+ },
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "ignore-pod-4",
+ Namespace: "default",
+ Annotations: map[string]string{
+ BindTimeAnnotations: "2024-01-01T00:00:00Z",
+ DeviceBindPhase: DeviceBindAllocating,
+ },
+ },
+ Spec: corev1.PodSpec{NodeName: "test-node-2"},
+ Status: corev1.PodStatus{
+ Phase: corev1.PodPending,
+ },
+ },
+ }
+
+ for _, pod := range podList {
+ client.KubeClient.CoreV1().Pods("default").Create(context.TODO(), pod, metav1.CreateOptions{})
+ }
+
+ node0 := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-node-0",
+ Annotations: map[string]string{},
+ },
+ }
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), node0, metav1.CreateOptions{})
+
+ allocatedPod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "allocated-pod",
+ Namespace: "default",
+ },
+ Spec: corev1.PodSpec{NodeName: "test-node-1"},
+ Status: corev1.PodStatus{
+ Phase: corev1.PodPhase(corev1.PodInitialized),
+ },
+ }
+ client.KubeClient.CoreV1().Pods("default").Create(context.TODO(), allocatedPod, metav1.CreateOptions{})
+
+ node1 := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-node-1",
+ Annotations: map[string]string{
+ nodelock.NodeLockKey: nodelock.GenerateNodeLockKeyByPod(allocatedPod),
+ },
+ },
+ }
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), node1, metav1.CreateOptions{})
+
+ pendingPod := podList[0]
+
+ tests := []struct {
+ name string
+ node string
+ wantErr bool
+ want *corev1.Pod
+ }{
+ {
+ name: "find pending pod",
+ node: "test-node-0",
+ wantErr: false,
+ want: pendingPod,
+ },
+ {
+ name: "find allocated pod",
+ node: "test-node-1",
+ wantErr: false,
+ want: allocatedPod,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := GetPendingPod(context.TODO(), tt.node)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("GetPendingPod() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if !tt.wantErr {
+ assert.Equal(t, got.Name, tt.want.Name)
+ }
+ })
+ }
+}
+
+func TestGetAllocatePodByNode(t *testing.T) {
+ client.KubeClient = fake.NewSimpleClientset()
+
+ emptyPod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "",
+ Namespace: "",
+ },
+ }
+
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pod0",
+ Namespace: "default",
+ },
+ }
+ client.KubeClient.CoreV1().Pods("default").Create(context.TODO(), pod, metav1.CreateOptions{})
+
+ emptyNode := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-node-0",
+ Annotations: map[string]string{},
+ },
+ }
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), emptyNode, metav1.CreateOptions{})
+
+ node0 := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-node-1",
+ Annotations: map[string]string{
+ nodelock.NodeLockKey: nodelock.GenerateNodeLockKeyByPod(emptyPod),
+ },
+ },
+ }
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), node0, metav1.CreateOptions{})
+
+ node1 := &corev1.Node{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-node-2",
+ Annotations: map[string]string{
+ nodelock.NodeLockKey: nodelock.GenerateNodeLockKeyByPod(pod),
+ },
+ },
+ }
+ client.KubeClient.CoreV1().Nodes().Create(context.TODO(), node1, metav1.CreateOptions{})
+
+ tests := []struct {
+ name string
+ node string
+ wantErr bool
+ want *corev1.Pod
+ }{
+ {
+ name: "node not found",
+ node: "non-existent",
+ wantErr: true,
+ want: nil,
+ },
+ {
+ name: "Missing NodeLockKey Annotation",
+ node: "test-node-0",
+ wantErr: false,
+ want: nil,
+ },
+ {
+ name: "Missing ns and name",
+ node: "test-node-1",
+ wantErr: false,
+ want: nil,
+ },
+ {
+ name: "finding allocated pod",
+ node: "test-node-2",
+ wantErr: false,
+ want: pod,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := GetAllocatePodByNode(context.TODO(), tt.node)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("GetAllocatePodByNode() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if !tt.wantErr && got != nil {
+ assert.Equal(t, got.Name, tt.want.Name)
+ }
+ })
+ }
+}
+
+func TestEncodeContainerDeviceType(t *testing.T) {
+ tests := []struct {
+ name string
+ cd ContainerDevices
+ t string
+ want string
+ }{
+ {
+ name: "empty container devices",
+ cd: ContainerDevices{},
+ t: "NVIDIA",
+ want: "",
+ },
+ {
+ name: "single matching device",
+ cd: ContainerDevices{
+ {UUID: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313", Type: "NVIDIA", Usedmem: 1000, Usedcores: 10},
+ },
+ t: "NVIDIA",
+ want: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313,NVIDIA,1000,10:",
+ },
+ {
+ name: "multiple devices with type match",
+ cd: ContainerDevices{
+ {UUID: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313", Type: "NVIDIA", Usedmem: 1000, Usedcores: 10},
+ {UUID: "GPU-8dcd427f-483b-b48f-d7e5-75fb19a52b76", Type: "AMD", Usedmem: 2000, Usedcores: 20},
+ {UUID: "GPU-ebe7c3f7-303d-558d-435e-99a160631fe4", Type: "NVIDIA", Usedmem: 3000, Usedcores: 30},
+ },
+ t: "NVIDIA",
+ want: "GPU-936619fc-f6a1-74a8-0bc6-ecf6b3269313,NVIDIA,1000,10::GPU-ebe7c3f7-303d-558d-435e-99a160631fe4,NVIDIA,3000,30:",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := EncodeContainerDeviceType(tt.cd, tt.t)
+ assert.Equal(t, got, tt.want)
+ })
+ }
+}
+
+func TestPatchPodAnnotations(t *testing.T) {
+ client.KubeClient = fake.NewSimpleClientset()
+
+ // Create test pod
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "default",
+ },
+ }
+
+ client.KubeClient.CoreV1().Pods("default").Create(context.TODO(), pod, metav1.CreateOptions{})
+
+ tests := []struct {
+ name string
+ pod *corev1.Pod
+ annotations map[string]string
+ wantErr bool
+ }{
+ {
+ name: "patch with valid annotations",
+ pod: pod,
+ annotations: map[string]string{
+ "test-key": "test-value",
+ AssignedNodeAnnotations: "node1",
+ },
+ wantErr: false,
+ },
+ {
+ name: "patch non-existent pod",
+ pod: &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "non-existent",
+ Namespace: "default",
+ },
+ },
+ annotations: map[string]string{
+ "test-key": "test-value",
+ },
+ wantErr: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ err := PatchPodAnnotations(tt.pod, tt.annotations)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PatchPodAnnotations() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ })
+ }
+}
+
+func TestContainsTargetNonNegativeInt(t *testing.T) {
+ testCases := []struct {
+ name string
+ input string
+ separator string
+ target int
+ want bool
+ }{
+ {
+ name: "target 0 with valid element '0'",
+ input: "0, 1, 2",
+ separator: ",",
+ target: 0,
+ want: true,
+ },
+ {
+ name: "target 0 with invalid leading zeros '00'",
+ input: "00, 1, 2",
+ separator: ",",
+ target: 0,
+ want: false,
+ },
+ {
+ name: "target 5 with valid element '5'",
+ input: "3, 5, 7",
+ separator: ",",
+ target: 5,
+ want: true,
+ },
+ {
+ name: "target 5 with invalid leading zero '05'",
+ input: "3, 05, 7",
+ separator: ",",
+ target: 5,
+ want: false,
+ },
+ {
+ name: "element with whitespace and valid format",
+ input: " 10 , 20, 30",
+ separator: ",",
+ target: 10,
+ want: true,
+ },
+ {
+ name: "element with whitespace and leading zeros",
+ input: " 010 , 20",
+ separator: ",",
+ target: 10,
+ want: false,
+ },
+ {
+ name: "custom separator and valid element",
+ input: "15;25;35",
+ separator: ";",
+ target: 25,
+ want: true,
+ },
+ {
+ name: "custom separator and leading zeros",
+ input: "15;025;35",
+ separator: ";",
+ target: 25,
+ want: false,
+ },
+ {
+ name: "non-integer elements",
+ input: "abc, 123, def",
+ separator: ",",
+ target: 123,
+ want: true,
+ },
+ {
+ name: "negative target (invalid)",
+ input: "5, 10, 15",
+ separator: ",",
+ target: -5,
+ want: false,
+ },
+ {
+ name: "empty input",
+ input: "",
+ separator: ",",
+ target: 0,
+ want: false,
+ },
+ {
+ name: "single element with leading zero",
+ input: "000",
+ separator: ",",
+ target: 0,
+ want: false,
+ },
+ {
+ name: "element is float (non-integer)",
+ input: "5.0, 5",
+ separator: ",",
+ target: 5,
+ want: true,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := ContainsTargetNonNegativeInt(tc.input, tc.separator, tc.target)
+ if got != tc.want {
+ t.Errorf("ContainsTargetNonNegativeInt(%q, %q, %d) = %v, want %v",
+ tc.input, tc.separator, tc.target, got, tc.want)
+ }
+ })
+ }
}
diff --git a/pkg/version/version.go b/pkg/version/version.go
index 7095c2754..93b87547a 100644
--- a/pkg/version/version.go
+++ b/pkg/version/version.go
@@ -1,37 +1,38 @@
/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
package version
import (
- "fmt"
- "github.com/spf13/cobra"
+ "fmt"
+
+ "github.com/spf13/cobra"
)
var (
- version string
- VersionCmd = &cobra.Command{
- Use: "version",
- Short: "print version",
- Run: func(cmd *cobra.Command, args []string) {
- fmt.Println(Version())
- },
- }
+ version string
+ VersionCmd = &cobra.Command{
+ Use: "version",
+ Short: "print version",
+ Run: func(cmd *cobra.Command, args []string) {
+ fmt.Println(Version())
+ },
+ }
)
func Version() string {
- return version
+ return version
}
diff --git a/pkg/version/version_test.go b/pkg/version/version_test.go
new file mode 100644
index 000000000..4981a0208
--- /dev/null
+++ b/pkg/version/version_test.go
@@ -0,0 +1,52 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package version
+
+import (
+ "bytes"
+ "io"
+ "os"
+ "testing"
+
+ "gotest.tools/v3/assert"
+)
+
+func TestVersion(t *testing.T) {
+ version = "v1.0.0.1234567890"
+ versionWant := "v1.0.0.1234567890\n"
+
+ var out bytes.Buffer
+ r, w, err := os.Pipe()
+ if err != nil {
+ t.Fatalf("os.Pipe() failed: %v", err)
+ }
+ defer r.Close()
+ originalStdout := os.Stdout
+ defer func() {
+ os.Stdout = originalStdout
+ w.Close()
+ }()
+ os.Stdout = w
+
+ VersionCmd.Run(nil, nil)
+ w.Close()
+
+ io.Copy(&out, r)
+
+ versionGet := out.String()
+ assert.Equal(t, versionWant, versionGet)
+}
diff --git a/test/e2e/node/test_node.go b/test/e2e/node/test_node.go
new file mode 100644
index 000000000..95fc3c6ad
--- /dev/null
+++ b/test/e2e/node/test_node.go
@@ -0,0 +1,117 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/onsi/ginkgo/v2"
+ "github.com/onsi/gomega"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/Project-HAMi/HAMi/test/utils"
+)
+
+var _ = ginkgo.Describe("[Node] Node E2E Tests", ginkgo.Ordered, func() {
+ var clientSet = utils.GetClientSet()
+ var nodeName string
+
+ ginkgo.BeforeAll(func() {
+ // Get all nodes in the cluster
+ nodes, err := utils.GetNodes(clientSet)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Expect(len(nodes.Items)).To(gomega.BeNumerically(">", 0), "No nodes available for testing")
+
+ // Select a node with GPU resources
+ for _, node := range nodes.Items {
+ if _, exists := node.Status.Capacity["nvidia.com/gpu"]; exists {
+ nodeName = node.Name
+ break
+ }
+ }
+ gomega.Expect(nodeName).NotTo(gomega.BeEmpty(), "No GPU node found")
+ })
+
+ ginkgo.It("verify node with labeling", func() {
+ // Add label to the node
+ ginkgo.By("Updating node " + nodeName + " by labeling " + utils.GPUNodeLabelKey + "=" + utils.GPUNodeLabelValue)
+ _, err := utils.AddNodeLabel(clientSet, nodeName, utils.GPUNodeLabelKey, utils.GPUNodeLabelValue)
+ if err != nil {
+ fmt.Printf("Failed to add label to node %s: %v\n", nodeName, err)
+ }
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ // Verify that the label was added correctly
+ ginkgo.By("Checking node " + nodeName + " label")
+ node, err := clientSet.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Expect(node.Labels[utils.GPUNodeLabelKey]).To(gomega.Equal(utils.GPUNodeLabelValue), "Label was not correctly added")
+
+ // Verify that the HamiDevicePlugin pods are running after labeling
+ ginkgo.By("Checking pods " + utils.HamiDevicePlugin + " running after labeling")
+ gomega.Eventually(func() bool {
+ pods, err := utils.GetPods(clientSet, utils.GPUNameSpace)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ for _, pod := range pods.Items {
+ if strings.Contains(pod.Name, utils.HamiDevicePlugin) {
+ err := utils.WaitForPodRunning(clientSet, utils.GPUNameSpace, pod.Name)
+ if err != nil {
+ fmt.Printf("Pod %s is not running: %v\n", pod.Name, err)
+ return false
+ }
+ }
+ }
+ return true
+ }, 300*time.Second, 10*time.Second).Should(gomega.BeTrue())
+ })
+
+ ginkgo.It("verify node after removing label", func() {
+ // Remove label from the node
+ ginkgo.By("Updating node " + nodeName + " by removing label " + utils.GPUNodeLabelKey + "=" + utils.GPUNodeLabelValue)
+ _, err := utils.RemoveNodeLabel(clientSet, nodeName, utils.GPUNodeLabelKey)
+ if err != nil {
+ fmt.Printf("Failed to remove label from node %s: %v\n", nodeName, err)
+ }
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ // Verify that the label was removed correctly
+ ginkgo.By("Checking node " + nodeName + " label")
+ node, err := clientSet.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ _, exists := node.Labels[utils.GPUNodeLabelKey]
+ gomega.Expect(exists).To(gomega.BeFalse(), "Label was not correctly removed")
+
+ // Verify that the HamiDevicePlugin pods are deleted after removing the label
+ ginkgo.By("Checking pods " + utils.HamiDevicePlugin + " deleted after removing label")
+ gomega.Eventually(func() bool {
+ pods, err := utils.GetPods(clientSet, utils.GPUNameSpace)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ for _, pod := range pods.Items {
+ if strings.Contains(pod.Name, utils.HamiDevicePlugin) {
+ fmt.Printf("Pod %s still exists\n", pod.Name)
+ return false
+ }
+ }
+ return true
+ }, 300*time.Second, 10*time.Second).Should(gomega.BeTrue())
+ })
+})
diff --git a/test/e2e/node/test_suite_test.go b/test/e2e/node/test_suite_test.go
new file mode 100644
index 000000000..eecc1601e
--- /dev/null
+++ b/test/e2e/node/test_suite_test.go
@@ -0,0 +1,35 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "flag"
+ "testing"
+
+ "github.com/onsi/ginkgo/v2"
+ "github.com/onsi/gomega"
+)
+
+func init() {
+ testing.Init()
+ flag.Parse()
+}
+
+func TestInit(t *testing.T) {
+ gomega.RegisterFailHandler(ginkgo.Fail)
+ ginkgo.RunSpecs(t, "Test workspace Service Suite")
+}
diff --git a/test/e2e/pod/test_pod.go b/test/e2e/pod/test_pod.go
new file mode 100644
index 000000000..41cdc5d7c
--- /dev/null
+++ b/test/e2e/pod/test_pod.go
@@ -0,0 +1,180 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/onsi/ginkgo/v2"
+ "github.com/onsi/gomega"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+
+ "github.com/Project-HAMi/HAMi/test/utils"
+)
+
+var _ = ginkgo.Describe("Pod E2E Tests", ginkgo.Ordered, func() {
+ const (
+ Namespace = utils.GPUNameSpace
+ NodeName = utils.GPUNode
+ NodeLabelKey = utils.GPUNodeLabelKey
+ NodeLabelValue = utils.GPUNodeLabelValue
+ DeleteTimeout = 300 * time.Second
+ DeleteInterval = 10 * time.Second
+ )
+
+ var (
+ clientSet = utils.GetClientSet()
+ newPod *corev1.Pod
+ )
+
+ ginkgo.BeforeAll(func() {
+ ginkgo.By("Adding node labeling")
+ _, err := utils.AddNodeLabel(clientSet, NodeName, NodeLabelKey, NodeLabelValue)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ })
+
+ ginkgo.AfterEach(func() {
+ ginkgo.By("Cleanup pod after each test")
+ cleanupPod(newPod, clientSet)
+ })
+
+ ginkgo.AfterAll(func() {
+ ginkgo.By("Deleting node labeling")
+ _, err := utils.RemoveNodeLabel(clientSet, NodeName, NodeLabelKey)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ })
+
+ ginkgo.It("creates a single pod with CUDA configuration", func() {
+ newPod = utils.Pod.DeepCopy()
+ newPod.Name += utils.GetRandom()
+
+ // Ensure cleanup even if the test fails
+ ginkgo.DeferCleanup(func() {
+ ginkgo.By("DeferCleanup: Deleting pod after test")
+ cleanupPod(newPod, clientSet)
+ })
+
+ createAndVerifyPod(newPod, clientSet)
+
+ ginkgo.By("Verifying GPU memory in pod by executing: " + utils.GPUExecuteNvidiaSMI)
+ output, err := utils.KubectlExecInPod(newPod.Namespace, newPod.Name, utils.GPUExecuteNvidiaSMI)
+ if err != nil {
+ fmt.Printf("nvidia-smi execution error: %v\n", err)
+ fmt.Printf("nvidia-smi output: %s\n", output)
+ }
+ gomega.Expect(err).NotTo(gomega.HaveOccurred(), "nvidia-smi command failed")
+
+ fmt.Println("nvidia-smi output:")
+ fmt.Println(string(output))
+
+ ginkgo.By("Verifying CUDA execution in pod by executing: " + utils.GPUExecuteCudaSample)
+ output, err = utils.KubectlExecInPod(newPod.Namespace, newPod.Name, utils.GPUExecuteCudaSample)
+ if err != nil {
+ fmt.Printf("CUDA sample execution error: %v\n", err)
+ fmt.Printf("CUDA sample output: %s\n", output)
+ }
+ gomega.Expect(err).NotTo(gomega.HaveOccurred(), "CUDA sample command failed")
+
+ fmt.Println("CUDA sample output:")
+ fmt.Println(string(output))
+ })
+
+ ginkgo.It("create overcommit pods", func() {
+ newPod = prepareOvercommitPod(utils.Pod.DeepCopy(), Namespace) // Pass the namespace to the helper
+
+ ginkgo.By("Creating overcommit pod in namespace " + Namespace)
+ createdPod, err := utils.CreatePod(clientSet, newPod, Namespace)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Expect(createdPod.Name).To(gomega.Equal(newPod.Name), "Pod was not created successfully")
+
+ ginkgo.By("Verifying pod is pending due to filtering")
+ gomega.Eventually(func() bool {
+ return checkPodPendingDueToFiltering(clientSet, newPod)
+ }, DeleteTimeout, DeleteInterval).Should(gomega.BeTrue())
+ })
+})
+
+func cleanupPod(pod *corev1.Pod, clientSet *kubernetes.Clientset) {
+ if podExists(pod.Namespace, pod.Name, clientSet) {
+ ginkgo.By("Deleting pod " + pod.Name + " in namespace " + pod.Namespace)
+ err := utils.DeletePod(clientSet, pod.Namespace, pod.Name)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Verifying pod " + pod.Name + " is deleted")
+ gomega.Eventually(func() bool {
+ return !podExists(pod.Namespace, pod.Name, clientSet)
+ }, 300*time.Second, 10*time.Second).Should(gomega.BeTrue())
+ }
+}
+
+func podExists(namespace, podName string, clientSet *kubernetes.Clientset) bool {
+ pod, err := clientSet.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{})
+ return err == nil && pod != nil
+}
+
+func createAndVerifyPod(pod *corev1.Pod, clientSet *kubernetes.Clientset) {
+ ginkgo.By("Creating pod " + pod.Name + " in namespace " + pod.Namespace)
+ createdPod, err := utils.CreatePod(clientSet, pod, pod.Namespace)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+ gomega.Expect(createdPod.Name).To(gomega.Equal(pod.Name), "Pod was not created successfully")
+
+ ginkgo.By("Verifying pod " + pod.Name + " is in running status")
+ err = utils.WaitForPodRunning(clientSet, pod.Namespace, pod.Name)
+ if err != nil {
+ p, _ := clientSet.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
+ fmt.Printf("Pod %s/%s status: %v\n", pod.Namespace, pod.Name, p.Status)
+ events, _ := clientSet.CoreV1().Events(pod.Namespace).List(context.TODO(), metav1.ListOptions{
+ FieldSelector: fmt.Sprintf("involvedObject.name=%s", pod.Name),
+ })
+ for _, event := range events.Items {
+ fmt.Printf("Event: %s - %s\n", event.Reason, event.Message)
+ }
+ }
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+}
+
+func prepareOvercommitPod(pod *corev1.Pod, namespace string) *corev1.Pod {
+ pod.Name += utils.GetRandom()
+ pod.Namespace = namespace // Ensure the pod's namespace is correctly set
+
+ // Modify pod spec for overcommit scenario
+ pod.Spec.Containers = append(pod.Spec.Containers, pod.Spec.Containers[0])
+ pod.Spec.Containers = append(pod.Spec.Containers, pod.Spec.Containers[0])
+ pod.Spec.Containers[1].Name = pod.Spec.Containers[0].Name + utils.GetRandom()
+ pod.Spec.Containers[2].Name = pod.Spec.Containers[0].Name + utils.GetRandom()
+
+ return pod
+}
+
+func checkPodPendingDueToFiltering(clientSet *kubernetes.Clientset, pod *corev1.Pod) bool {
+ events, err := utils.GetPodEvents(clientSet, pod.Namespace, pod.Name)
+ gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+ for _, event := range events {
+ fmt.Printf("Event: Reason=%s, Message=%s\n", event.Reason, event.Message)
+ if strings.Contains(event.Reason, utils.ErrReasonFilteringFailed) &&
+ strings.Contains(event.Message, utils.ErrMessageFilteringFailed) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/test/e2e/pod/test_suite_test.go b/test/e2e/pod/test_suite_test.go
new file mode 100644
index 000000000..bbdab8431
--- /dev/null
+++ b/test/e2e/pod/test_suite_test.go
@@ -0,0 +1,35 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "flag"
+ "testing"
+
+ "github.com/onsi/ginkgo/v2"
+ "github.com/onsi/gomega"
+)
+
+func init() {
+ testing.Init()
+ flag.Parse()
+}
+
+func TestInit(t *testing.T) {
+ gomega.RegisterFailHandler(ginkgo.Fail)
+ ginkgo.RunSpecs(t, "Test pod")
+}
diff --git a/test/e2e/test_suite_test.go b/test/e2e/test_suite_test.go
new file mode 100644
index 000000000..eb1e42a43
--- /dev/null
+++ b/test/e2e/test_suite_test.go
@@ -0,0 +1,38 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "flag"
+ "testing"
+
+ "github.com/onsi/ginkgo/v2"
+ "github.com/onsi/gomega"
+
+ "github.com/Project-HAMi/HAMi/test/utils"
+)
+
+func init() {
+ testing.Init()
+}
+
+func TestInit(t *testing.T) {
+ flag.Parse()
+ utils.DefaultKubeConfigPath()
+ gomega.RegisterFailHandler(ginkgo.Fail)
+ ginkgo.RunSpecs(t, "HAMi E2E Test Suite")
+}
diff --git a/test/utils/common.go b/test/utils/common.go
new file mode 100644
index 000000000..3b67cf6f8
--- /dev/null
+++ b/test/utils/common.go
@@ -0,0 +1,96 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "flag"
+ "fmt"
+ "math/rand"
+ "os"
+ "os/exec"
+ "strconv"
+ "time"
+
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+ "k8s.io/klog/v2"
+)
+
+var kubeConfig string
+
+func init() {
+ flag.StringVar(&kubeConfig, "kubeconfig", defaultKubeConfigPath(), "Path to the kubeConfig file")
+}
+
+func defaultKubeConfigPath() string {
+ configPath := os.Getenv("KUBE_CONF")
+ if configPath == "" {
+ klog.Fatalf("Environment variable KUBE_CONF is not set or empty. Please set it to a valid kubeconfig file path.")
+ }
+ if _, err := os.Stat(configPath); os.IsNotExist(err) {
+ klog.Fatalf("Kubeconfig file does not exist at path: %s", configPath)
+ }
+ return configPath
+}
+
+func DefaultKubeConfigPath() string {
+ configPath := os.Getenv("KUBE_CONF")
+ if configPath == "" {
+ klog.Fatalf("Environment variable KUBE_CONF is not set or empty. Please set it to a valid kubeconfig file path.")
+ }
+
+ if _, err := os.Stat(configPath); os.IsNotExist(err) {
+ klog.Fatalf("lalala Kubeconfig file does not exist at path: %s, error is %s", configPath, err)
+ }
+ return configPath
+}
+
+func GetClientSet() *kubernetes.Clientset {
+ config, err := clientcmd.BuildConfigFromFlags("", kubeConfig)
+ if err != nil {
+ klog.Fatalf("Failed to load kubeConfig: %v", err)
+ }
+
+ clientSet, err := kubernetes.NewForConfig(config)
+ if err != nil {
+ klog.Fatalf("Failed to create Kubernetes client: %v", err)
+ }
+ return clientSet
+}
+
+func GetRandom() string {
+ r := rand.New(rand.NewSource(time.Now().UnixNano()))
+ random := strconv.Itoa(r.Intn(9999))
+ return random
+}
+
+// KubectlExecInPod executes a shell command in a specified Pod using kubectl exec.
+func KubectlExecInPod(namespace, podName, command string) ([]byte, error) {
+ // Wait for the container to stabilize
+ time.Sleep(30 * time.Second)
+
+ // Build the kubectl exec command
+ cmd := exec.Command("kubectl", "exec", "-n", namespace, podName, "--", "/bin/bash", "-c", command)
+
+ // Capture the command output (both stdout and stderr)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return output, fmt.Errorf("failed to execute kubectl command: %w. Output: %s", err, output)
+ }
+
+ return output, nil
+}
diff --git a/test/utils/config.go b/test/utils/config.go
new file mode 100644
index 000000000..58f3216cf
--- /dev/null
+++ b/test/utils/config.go
@@ -0,0 +1,41 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+// test data.
+const (
+ GPUNodeLabelKey = "gpu"
+ GPUNodeLabelValue = "on"
+ GPUExecuteNvidiaSMI = "nvidia-smi"
+ GPUExecuteCudaSample = "/cuda-samples/sample"
+ GPUPodMemory = "300"
+ GPUPodMemoryUnit = "MiB"
+ GPUPodCore = "40"
+ GPUNameSpace = "hami-system"
+ GPUNode = "gpu-master"
+ GPUCudaTestPass = "Test PASSED"
+)
+
+// hami related.
+const (
+ HamiScheduler = "hami-scheduler"
+ HamiDevicePlugin = "hami-device-plugin"
+ ErrReasonFilteringFailed = "FilteringFailed"
+ ErrMessageFilteringFailed = "no available node"
+ ErrReasonFailedScheduling = "FilteringFailed"
+ ErrMessageFailedScheduling = "0/1 nodes are available"
+)
diff --git a/test/utils/event.go b/test/utils/event.go
new file mode 100644
index 000000000..05b857c99
--- /dev/null
+++ b/test/utils/event.go
@@ -0,0 +1,50 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "context"
+ "fmt"
+
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/klog/v2"
+)
+
+func GetEvents(clientSet *kubernetes.Clientset, namespace string, listOptions metav1.ListOptions) ([]v1.Event, error) {
+ events, err := clientSet.CoreV1().Events(namespace).List(context.TODO(), listOptions)
+ if err != nil {
+ return nil, err
+ }
+
+ return events.Items, nil
+}
+
+func GetPodEvents(clientSet *kubernetes.Clientset, namespace, podName string) ([]v1.Event, error) {
+ listOption := metav1.ListOptions{
+ FieldSelector: fmt.Sprintf("involvedObject.kind=Pod,involvedObject.name=%s", podName),
+ }
+
+ events, err := GetEvents(clientSet, namespace, listOption)
+ if err != nil {
+ klog.Errorf("Failed to list events for pod %s in namespace %s: %v", podName, namespace, err)
+ return nil, err
+ }
+
+ return events, nil
+}
diff --git a/test/utils/node.go b/test/utils/node.go
new file mode 100644
index 000000000..07480328e
--- /dev/null
+++ b/test/utils/node.go
@@ -0,0 +1,73 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "context"
+
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/klog/v2"
+)
+
+func GetNodes(clientSet *kubernetes.Clientset) (*v1.NodeList, error) {
+ nodes, err := clientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+ if err != nil {
+ klog.Errorf("Failed to get nodes: %v", err)
+ return nil, err
+ }
+
+ return nodes, nil
+}
+
+func UpdateNode(clientSet *kubernetes.Clientset, node *v1.Node) (*v1.Node, error) {
+ updatedNode, err := clientSet.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{})
+ if err != nil {
+ klog.Errorf("Failed to update node %s: %v", node.Name, err)
+ return nil, err
+ }
+
+ return updatedNode, nil
+}
+
+func AddNodeLabel(clientSet *kubernetes.Clientset, nodeName, labelKey, labelValue string) (*v1.Node, error) {
+ node, err := clientSet.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+ if err != nil {
+ return nil, err
+ }
+
+ if node.Labels == nil {
+ node.Labels = make(map[string]string)
+ }
+ node.Labels[labelKey] = labelValue
+
+ return UpdateNode(clientSet, node)
+}
+
+func RemoveNodeLabel(clientSet *kubernetes.Clientset, nodeName, labelKey string) (*v1.Node, error) {
+ node, err := clientSet.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+ if err != nil {
+ return nil, err
+ }
+
+ if node.Labels != nil {
+ delete(node.Labels, labelKey)
+ }
+
+ return UpdateNode(clientSet, node)
+}
diff --git a/test/utils/pod.go b/test/utils/pod.go
new file mode 100644
index 000000000..402ff1427
--- /dev/null
+++ b/test/utils/pod.go
@@ -0,0 +1,125 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/wait"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/klog/v2"
+)
+
+var Pod = &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "gpu-pod",
+ Namespace: "default",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "cuda-container",
+ Image: "nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0",
+ Command: []string{"/bin/sh"},
+ Args: []string{"-c", "sleep 86400"},
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "nvidia.com/gpu": resource.MustParse("1"),
+ "nvidia.com/gpumem": resource.MustParse(GPUPodMemory),
+ "nvidia.com/gpucores": resource.MustParse(GPUPodCore),
+ },
+ },
+ },
+ },
+ },
+}
+
+func GetPods(clientSet *kubernetes.Clientset, namespace string) (*corev1.PodList, error) {
+ pods, err := clientSet.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{})
+ if err != nil {
+ klog.Errorf("Failed to list Pods in namespace %s: %v", namespace, err)
+ return nil, err
+ }
+
+ return pods, nil
+}
+
+func CreatePod(clientSet *kubernetes.Clientset, pod *corev1.Pod, namespace string) (*corev1.Pod, error) {
+ time.Sleep(15 * time.Second)
+ createdPod, err := clientSet.CoreV1().Pods(namespace).Create(context.TODO(), pod, metav1.CreateOptions{})
+ if err != nil {
+ klog.Errorf("Failed to create Pod %s in namespace %s: %v", pod.Name, namespace, err)
+ return nil, err
+ }
+
+ return createdPod, nil
+}
+
+func DeletePod(clientSet *kubernetes.Clientset, namespace, podName string) error {
+ err := clientSet.CoreV1().Pods(namespace).Delete(context.TODO(), podName, metav1.DeleteOptions{})
+ if err != nil {
+ klog.Errorf("Failed to delete Pod %s in namespace %s: %v", podName, namespace, err)
+ return err
+ }
+ return nil
+}
+
+func WaitForPodRunning(clientSet kubernetes.Interface, namespace, podName string) error {
+ const (
+ checkInterval = 5 * time.Second // Interval for checking Pod status
+ timeout = 5 * time.Minute // Increased timeout for GPU Pods
+ )
+
+ return wait.PollUntilContextTimeout(context.TODO(), checkInterval, timeout, true, func(context.Context) (bool, error) {
+ // Fetch the Pod object from the Kubernetes API
+ pod, err := clientSet.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{})
+ if err != nil {
+ return false, fmt.Errorf("failed to get pod %s/%s: %v", namespace, podName, err)
+ }
+
+ // Print Pod status for debugging
+ fmt.Printf("Pod %s/%s status: %s\n", namespace, podName, pod.Status.Phase)
+
+ // Check if the Pod is in the Running state
+ if pod.Status.Phase == corev1.PodRunning {
+ return true, nil
+ }
+
+ // Check if the Pod is in a Failed or Unknown state
+ if pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodUnknown {
+ return false, fmt.Errorf("pod %s/%s is in failed or unknown state: %s", namespace, podName, pod.Status.Phase)
+ }
+
+ // Print Pod events for debugging
+ events, err := clientSet.CoreV1().Events(namespace).List(context.TODO(), metav1.ListOptions{
+ FieldSelector: fmt.Sprintf("involvedObject.name=%s", podName),
+ })
+ if err == nil {
+ for _, event := range events.Items {
+ fmt.Printf("Event: %s - %s\n", event.Reason, event.Message)
+ }
+ }
+
+ // If the Pod is not in Running, Failed, or Unknown state, continue waiting
+ return false, nil
+ })
+}
diff --git a/tools/plugin_cli.go b/tools/plugin_cli.go
deleted file mode 100644
index 6035cb2c3..000000000
--- a/tools/plugin_cli.go
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright © 2021 peizhaoyou
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "4pd.io/k8s-vgpu/pkg/api"
- "github.com/spf13/cobra"
- "google.golang.org/grpc"
-)
-
-var (
- runtimeSocketFlag string
- rootCmd = &cobra.Command{
- Use: "device plugin cli",
- Short: "device plugin socket cli",
- }
-
- getCmd = &cobra.Command{
- Use: "get devices",
- Short: "get devices",
- Args: cobra.ExactArgs(1),
- Run: func(cmd *cobra.Command, args []string) {
- getDevices(args[0])
- },
- }
-)
-
-func init() {
- rootCmd.Flags().SortFlags = false
- rootCmd.PersistentFlags().SortFlags = false
- rootCmd.PersistentFlags().StringVar(&runtimeSocketFlag, "socket", "/var/lib/vgpu/vgpu.sock", "device plugin socket")
-
- rootCmd.AddCommand(getCmd)
-}
-
-func getDevices(uid string) {
- ctx := context.Background()
- ctx, cancel := context.WithTimeout(ctx, time.Second*10)
- defer cancel()
- conn, err := grpc.DialContext(
- ctx,
- fmt.Sprintf("unix://%v", runtimeSocketFlag),
- grpc.WithInsecure(),
- grpc.WithBlock(),
- )
- if err != nil {
- fmt.Printf("connect device plugin error, %v\n", err)
- os.Exit(1)
- }
- client := api.NewVGPURuntimeServiceClient(conn)
- req := api.GetDeviceRequest{CtrUUID: uid}
- resp, err := client.GetDevice(ctx, &req)
- if err != nil {
- fmt.Printf("get device failed, %v\n", err)
- os.Exit(1)
- }
- fmt.Printf("%v\n", resp.String())
-}
-
-func main() {
- if err := rootCmd.Execute(); err != nil {
- fmt.Print(err)
- os.Exit(1)
- }
-}
diff --git a/version.mk b/version.mk
new file mode 100644
index 000000000..bebcb6b44
--- /dev/null
+++ b/version.mk
@@ -0,0 +1,13 @@
+GO=go
+GO111MODULE=on
+CMDS=scheduler vGPUmonitor
+DEVICES=nvidia
+OUTPUT_DIR=bin
+TARGET_ARCH=amd64
+GOLANG_IMAGE=golang:1.24.6-bullseye
+NVIDIA_IMAGE=nvidia/cuda:12.3.2-devel-ubuntu20.04
+DEST_DIR=/usr/local/vgpu/
+
+VERSION = v0.0.1
+IMG_NAME =hami
+IMG_TAG="${IMG_NAME}:${VERSION}"