Add rk3568 multi-PCIe support and board CI hardware tests

.gitignore

@@ -38,6 +38,8 @@ src/platform/__board.rs
 # systemtest
 rootfs1.zip*
 jenkins/__pycache__
+jenkins/logs/
+logs/
 jenkins-cli.jar
 tools/kconfig/.venv/
 tools/kconfig/__pycache__/

Jenkinsfile

@@ -32,6 +32,33 @@ def matrixCellDir() {
     return "${env.WORKSPACE}/.matrix/${bid.replace('/', '__')}"
 }
 
+/** Stable directory name for a BID under ``$WORKSPACE/logs/``. */
+def bidLogKey(String bid = env.BID) {
+    return (bid ?: '').replace('/', '__')
+}
+
+/** Copy per-cell logs out to ``$WORKSPACE/logs/<bid>`` before ``.matrix`` is removed. */
+def collectCellLogs() {
+    def src = "${matrixCellDir()}/logs"
+    def dest = "${env.WORKSPACE}/logs/${bidLogKey()}"
+    def logsRoot = "${env.WORKSPACE}/logs"
+    sh """
+        if [ ! -d '${src}' ]; then
+            echo "No cell logs to collect: ${src}"
+            exit 0
+        fi
+        mkdir -p '${dest}' 2>/dev/null || sudo mkdir -p '${dest}'
+        sudo chown -R "\$(id -u):\$(id -g)" '${logsRoot}' 2>/dev/null || true
+        # Source may be root-owned after board sudo ci_runner.
+        if ! cp -a '${src}/.' '${dest}/' 2>/dev/null; then
+            sudo cp -a '${src}/.' '${dest}/'
+            sudo chown -R "\$(id -u):\$(id -g)" '${dest}'
+        fi
+        echo "Collected cell logs -> ${dest}"
+        ls -la '${dest}' || true
+    """
+}
+
 /** Isolated workspace for top-level CI jobs (linter, license-checker, …). */
 def jenkinsJobDir(String name) {
     return "${env.WORKSPACE}/.jenkins/${name}"
@@ -45,6 +72,9 @@ def syncWorkspaceTo(String destDir) {
             --exclude '.jenkins/' \\
             --exclude '.matrix/' \\
             --exclude '.jenkins-matrix/' \\
+            --exclude 'logs/' \\
+            --exclude '__pycache__/' \\
+            --exclude '*.pyc' \\
             '${env.WORKSPACE}/' '${destDir}/'
     """
 }
@@ -188,7 +218,9 @@ pipeline {
         always {
             echo "=== DEBUG: Branch ${env.BRANCH_NAME} ==="
             echo "=== DEBUG: Commit ${env.GIT_COMMIT} ==="
-            deleteDir()
+            // Keep $WORKSPACE/logs (collected from cells); only tear down sandboxes.
+            sh 'sudo rm -rf .matrix 2>/dev/null || rm -rf .matrix || true'
+            archiveArtifacts artifacts: 'logs/**/*', allowEmptyArchive: true
         }
     }
 
@@ -205,6 +237,8 @@ pipeline {
         LOONGARCH64_TOOLCHAIN_PATH = '/home/light/DEMO/toolchain/loongarch_cross_tools'
         // All toolchain bins on PATH; same for every matrix cell (no per-arch selection).
         TOOLCHAIN_PATHS = "${env.RISCV_TOOLCHAIN_PATH}/bin:${env.AARCH64_TOOLCHAIN_PATH}/bin:${env.LOONGARCH64_TOOLCHAIN_PATH}/bin"
+        TFTP_DIR = '/home/light/tftp'
+        PYTHONDONTWRITEBYTECODE = '1'
     }
 
     stages {
@@ -385,8 +419,14 @@ pipeline {
                                                 "${prepareScript}"
                                         """
                                     } else if (mode == 'board') {
-                                        // Placeholder for future board artifact distribution by network.
-                                        echo "Board prepare placeholder [BID=${env.BID}]"
+                                        echo "Deploy TFTP artifacts [BID=${env.BID}, TFTP_DIR=${env.TFTP_DIR}]"
+                                        sh """
+                                            export TERM=\${TERM:-xterm}
+                                            sudo mkdir -p "${env.TFTP_DIR}"
+                                            sudo make cp ARCH=${arch} BOARD=${board} MODE=release TFTP_DIR="${env.TFTP_DIR}"
+                                            sudo cp platform/${arch}/${board}/image/dts/rk3568_limit_zone0.dtb "${env.TFTP_DIR}/"
+                                            sudo cp ${kdir}/arch/arm64/boot/Image "${env.TFTP_DIR}/"
+                                        """
                                     } else {
                                         error("jenkins/ci.yaml BID=${env.BID}: unsupported tests.mode='${mode}'")
                                     }
@@ -402,14 +442,24 @@ pipeline {
                         steps {
                             dir(matrixCellDir()) {
                                 script {
-                                    echo "Run tests via ci_runner [BID=${env.BID}]"
-                                    sh """
-                                        export TERM=\${TERM:-xterm}
-                                        ${toolchainPathShell()}
-                                        ${qemuPathShell()}
-                                        python3 jenkins/ci_runner.py \
-                                            --bid "${env.BID}"
-                                    """
+                                    def bidCfg = getBidConfig(loadCiYaml(), env.BID)
+                                    def mode = (bidCfg.tests?.mode ?: '').toString().trim()
+                                    echo "Run tests via ci_runner [BID=${env.BID}, mode=${mode}]"
+                                    if (mode == 'board') {
+                                        sh """
+                                            export TERM=\${TERM:-xterm}
+                                            sudo -E python3 jenkins/ci_runner.py \
+                                                --bid "${env.BID}"
+                                        """
+                                    } else {
+                                        sh """
+                                            export TERM=\${TERM:-xterm}
+                                            ${toolchainPathShell()}
+                                            ${qemuPathShell()}
+                                            python3 jenkins/ci_runner.py \
+                                                --bid "${env.BID}"
+                                        """
+                                    }
                                 }
                             }
                         }
@@ -418,7 +468,10 @@ pipeline {
 
                 post {
                     always {
-                        script { finishGithubCheck(matrixCheckName(), currentBuild.currentResult) }
+                        script {
+                            collectCellLogs()
+                            finishGithubCheck(matrixCheckName(), currentBuild.currentResult)
+                        }
                     }
                 }
             }

Makefile

@@ -66,13 +66,28 @@ ifeq ($(MODE), release)
   build_args += --release
 endif
 
-# color code
-COLOR_GREEN := $(shell tput setaf 2)
-COLOR_RED := $(shell tput setaf 1)
-COLOR_YELLOW := $(shell tput setaf 3)
-COLOR_BLUE := $(shell tput setaf 4)
-COLOR_BOLD := $(shell tput bold)
-COLOR_RESET := $(shell tput sgr0)
+# color code (skip tput when TERM is unset, e.g. Jenkins sh steps)
+ifdef TERM
+COLOR_GREEN := $(shell tput setaf 2 2>/dev/null)
+COLOR_RED := $(shell tput setaf 1 2>/dev/null)
+COLOR_YELLOW := $(shell tput setaf 3 2>/dev/null)
+COLOR_BLUE := $(shell tput setaf 4 2>/dev/null)
+COLOR_BOLD := $(shell tput bold 2>/dev/null)
+COLOR_RESET := $(shell tput sgr0 2>/dev/null)
+else
+COLOR_GREEN :=
+COLOR_RED :=
+COLOR_YELLOW :=
+COLOR_BLUE :=
+COLOR_BOLD :=
+COLOR_RESET :=
+endif
+
+# Defconfig / menuconfig: see tools/kconfig/kconfig_cli.py
+kconfig_python := tools/kconfig/.venv/bin/python
+
+# Defconfig / menuconfig: see tools/kconfig/kconfig_cli.py
+kconfig_python := tools/kconfig/.venv/bin/python
 
 # Defconfig / menuconfig: see tools/kconfig/kconfig_cli.py
 kconfig_python := tools/kconfig/.venv/bin/python
@@ -187,8 +202,11 @@ monitor:
 jlink-server:
 	JLinkGDBServer -select USB -if JTAG -device Cortex-A53 -port 1234
 
+TFTP_DIR ?= $(HOME)/tftp
+
 cp:
-	cp $(hvisor_bin) ~/tftp
+	@mkdir -p "$(TFTP_DIR)"
+	cp $(hvisor_bin) "$(TFTP_DIR)/"
 
 test-pre: download-test-img
 	chmod +x platform/$(ARCH)/$(BOARD)/test/runner.sh

jenkins/board_power.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+# Relay power control for board CI (socat hex frames).
+# Usage: board_power.sh off|on|cycle <port>
+#   cycle: off, wait 3s, on
+
+set -eu
+
+action=${1:?action required (off|on|cycle)}
+port=${2:?serial port required, e.g. /dev/ttyUSB1}
+
+send_frame() {
+    frame=$1
+    printf '%b' "$frame" | sudo socat - "$port"
+}
+
+case "$action" in
+    off)
+        send_frame '\xA0\x04\x00\xA4'
+        ;;
+    on)
+        send_frame '\xA0\x04\x01\xA5'
+        ;;
+    cycle)
+        send_frame '\xA0\x04\x00\xA4'
+        sleep 3
+        send_frame '\xA0\x04\x01\xA5'
+        ;;
+    *)
+        echo "usage: $0 off|on|cycle <port>" >&2
+        exit 1
+        ;;
+esac

jenkins/board_scp.sh

@@ -0,0 +1,97 @@
+#!/bin/sh
+# Stage zone1 test artifacts on the CI host for board pull via scp.
+# Zone0 boot Image， Image and rootfs2.ext4 are large and
+# persistent on the board; do not re-stage them every CI run.
+
+set -eux
+
+ARCH=${ARCH:?ARCH is required}
+BOARD=${BOARD:?BOARD is required}
+WORKSPACE_ROOT=${WORKSPACE_ROOT:-$(pwd)}
+HVISOR_TOOL_PATH=${HVISOR_TOOL_PATH:-${WORKSPACE_ROOT}/hvisor-tool}
+MODE=${MODE:-release}
+STAGING_DIR=${STAGING_DIR:-/home/light/tftp/ci_deploy}
+
+case "${HVISOR_TOOL_PATH}" in
+    /*) ;;
+    *) HVISOR_TOOL_PATH="${WORKSPACE_ROOT}/${HVISOR_TOOL_PATH}" ;;
+esac
+
+PLATFORM_DIR="${WORKSPACE_ROOT}/platform/${ARCH}/${BOARD}"
+CONFIGS_DIR="${PLATFORM_DIR}/configs"
+IMAGE_DIR="${PLATFORM_DIR}/image"
+SCRIPTS_DIR="${PLATFORM_DIR}/scripts"
+ZONE1_BOOT_SCRIPT="${SCRIPTS_DIR}/boot_zone1.sh"
+CHECK_SERIAL_SCRIPT="${WORKSPACE_ROOT}/jenkins/check_serial.sh"
+
+case "${ARCH}" in
+    x86_64) RUSTC_TARGET="x86_64-unknown-none" ;;
+    aarch64) RUSTC_TARGET="aarch64-unknown-none" ;;
+    riscv64) RUSTC_TARGET="riscv64gc-unknown-none-elf" ;;
+    *)
+        echo "error: unsupported ARCH: ${ARCH}"
+        exit 1
+        ;;
+esac
+
+BUILD_PATH="${WORKSPACE_ROOT}/target/${RUSTC_TARGET}/${MODE}"
+HVISOR_BIN="${BUILD_PATH}/hvisor.bin"
+
+if [ -n "${ZONE1_DTB:-}" ]; then
+    :
+elif [ -f "${IMAGE_DIR}/dts/rk3568_limit_zone1.dtb" ]; then
+    ZONE1_DTB="${IMAGE_DIR}/dts/rk3568_limit_zone1.dtb"
+elif [ -f "${IMAGE_DIR}/dts/zone1-linux.dtb" ]; then
+    ZONE1_DTB="${IMAGE_DIR}/dts/zone1-linux.dtb"
+else
+    ZONE1_DTB="${IMAGE_DIR}/dts/zone1-linux.dtb"
+fi
+
+echo "ARCH: ${ARCH}"
+echo "BOARD: ${BOARD}"
+echo "HVISOR_TOOL_PATH: ${HVISOR_TOOL_PATH}"
+echo "STAGING_DIR: ${STAGING_DIR}"
+
+if [ ! -f "${HVISOR_TOOL_PATH}/output/hvisor" ]; then
+    echo "error: hvisor tool binary not found: ${HVISOR_TOOL_PATH}/output/hvisor"
+    exit 1
+fi
+if [ ! -f "${HVISOR_TOOL_PATH}/output/hvisor.ko" ]; then
+    echo "error: hvisor.ko not found: ${HVISOR_TOOL_PATH}/output/hvisor.ko"
+    exit 1
+fi
+if [ ! -f "${HVISOR_BIN}" ]; then
+    echo "error: hvisor.bin not found: ${HVISOR_BIN}"
+    exit 1
+fi
+if [ ! -f "${ZONE1_BOOT_SCRIPT}" ]; then
+    echo "error: boot script not found: ${ZONE1_BOOT_SCRIPT}"
+    exit 1
+fi
+
+if [ ! -f "${ZONE1_DTB}" ] && [ -d "${IMAGE_DIR}/dts" ]; then
+    echo "zone1 dtb is missing, building from ${IMAGE_DIR}/dts"
+    make -C "${IMAGE_DIR}/dts" all || true
+fi
+
+rm -rf "${STAGING_DIR}"
+mkdir -p "${STAGING_DIR}"
+
+cp "${HVISOR_TOOL_PATH}/output/hvisor" "${HVISOR_TOOL_PATH}/output/hvisor.ko" "${STAGING_DIR}/"
+cp "${HVISOR_BIN}" "${STAGING_DIR}/"
+cp "${CONFIGS_DIR}/"* "${STAGING_DIR}/"
+cp "${ZONE1_BOOT_SCRIPT}" "${STAGING_DIR}/"
+
+if [ -f "${ZONE1_DTB}" ]; then
+    cp "${ZONE1_DTB}" "${STAGING_DIR}/"
+else
+    echo "warning: zone1 dtb unavailable, skip copying ${ZONE1_DTB}"
+fi
+
+if [ -f "${CHECK_SERIAL_SCRIPT}" ]; then
+    cp "${CHECK_SERIAL_SCRIPT}" "${STAGING_DIR}/"
+fi
+
+chmod -R a+rX "${STAGING_DIR}"
+echo "board staging completed: ${STAGING_DIR}"
+ls -la "${STAGING_DIR}"

jenkins/check_serial.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+# Non-interactive virtio-console check for inner zone (/dev/pts/x).
+# Usage: check_serial.sh <pts_dev> <log_file> [timeout_sec] [command...]
+
+set -eu
+
+pts_dev=${1:?pts device required}
+log_file=${2:?log file required}
+shift 2
+
+prompt_timeout=60
+if [ $# -gt 0 ] && [ "$1" -eq "$1" ] 2>/dev/null; then
+    prompt_timeout=$1
+    shift
+fi
+
+: > "$log_file"
+
+read_pts() {
+    timeout "${1:-2}" cat "$pts_dev" 2>/dev/null >> "$log_file" || true
+}
+
+has_console_ready() {
+    tr -d '\r' < "$log_file" | sed 's/\x1b\[[0-9;?]*[ -\/]*[@-~]//g' \
+        | grep -qE 'root@[^[:space:]]*[#$][[:space:]]*$|^[[:space:]]*#[[:space:]]*$|login:[[:space:]]*$'
+}
+
+deadline=$(( $(date +%s) + prompt_timeout ))
+while [ "$(date +%s)" -lt "$deadline" ]; do
+    read_pts 2
+    if has_console_ready; then
+        break
+    fi
+    sleep 0.2
+done
+
+if ! has_console_ready; then
+    echo "check_serial: timed out after ${prompt_timeout}s waiting for console prompt on $pts_dev" \
+        >> "$log_file"
+    exit 1
+fi
+
+if [ $# -gt 0 ]; then
+    printf '%s\r\n' "$@"
+    sleep 1
+    read_pts 10
+fi
+
+exit 0