test.yml

name: tests

on:
  push:
    branches: [main]
    # A release tag runs only the comfyui-pack job.
    tags: ["v*.*.*"]
  pull_request:
  schedule:
    # Weekly: the drift report (the drift job), what the newest mlx-vlm and
    # mlx-lm releases change in the copy under gmlx/_vendor, and the
    # comfyui-pack job against ComfyUI's newest release and master.
    - cron: "0 6 * * 1"
    # Daily: the test job on Python 3.12 and 3.13 with the nightly tests.
    - cron: "0 7 * * *"

concurrency:
  # The schedule is part of the group, so a scheduled run and a push to
  # main never cancel each other, and the daily run never cancels the weekly.
  group: tests-${{ github.ref }}-${{ github.event.schedule }}
  cancel-in-progress: true

jobs:
  changes:
    # The kinds of file a push or pull request changes, from the merge-base,
    # by scripts/changed_paths.py. A schedule counts as changing everything.
    if: github.ref_type != 'tag'
    runs-on: ubuntu-latest
    outputs:
      code: ${{ steps.diff.outputs.code }}
      pack: ${{ steps.diff.outputs.pack }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - id: diff
        env:
          BASE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'push' && github.event.before || '' }}
          HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
        run: python3 scripts/changed_paths.py --base "$BASE" --head "$HEAD_SHA" >> "$GITHUB_OUTPUT"

  test:
    needs: [changes]
    if: >-
      github.ref_type != 'tag'
      && (github.event_name != 'schedule' || github.event.schedule == '0 7 * * *')
    # macos-26, not older images: the published mlx-kquant wheels are tagged
    # macosx_26_0_arm64 (built with MACOSX_DEPLOYMENT_TARGET=26.2 for the
    # Xcode 26.5 SDK miscompile workaround) - an older runner can't install
    # them and would fall back to compiling the Metal extension from sdist.
    runs-on: macos-26
    # The 3.14 leg also runs pyright and builds the guest entry.
    timeout-minutes: ${{ matrix.python-version == '3.14' && 60 || 45 }}
    permissions:
      contents: read
      # Flake issues, and the main baseline the growth report compares with.
      issues: write
      actions: read
    strategy:
      fail-fast: false
      matrix:
        # A pull request runs 3.11, the requires-python floor, and 3.14, what
        # uv and pipx select by default. A push to main runs 3.14. The daily
        # schedule runs 3.12 and 3.13 with the nightly tests.
        python-version: ${{ fromJSON(github.event.schedule == '0 7 * * *' && '["3.12", "3.13"]' || github.event_name == 'push' && '["3.14"]' || '["3.11", "3.14"]') }}
    env:
      CODE: ${{ needs.changes.outputs.code }}
      # A docs-only change runs only the 3.14 leg: the docs tests and the
      # arch-coverage check, after the full install the docs tests need.
      # The other leg skips every step after checkout, so its check passes.
      RUN: ${{ needs.changes.outputs.code == 'true' || matrix.python-version == '3.14' }}
      DAILY: ${{ github.event.schedule == '0 7 * * *' }}
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        if: env.RUN == 'true'
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
          cache-dependency-path: pyproject.toml
      - name: Week of the year
        # Part of the Rust cache key, so a new stable release replaces the
        # cached toolchain within a week.
        id: week
        if: env.CODE == 'true'
        run: echo "week=$(date +%G-%V)" >> "$GITHUB_OUTPUT"
      - uses: actions/cache@v6
        if: env.CODE == 'true'
        with:
          path: |
            ~/.rustup/toolchains
            ~/.rustup/update-hashes
            ~/.cargo/registry/index
            ~/.cargo/registry/cache
            crates/gmlx-entry/target
          key: rust-${{ runner.os }}-${{ steps.week.outputs.week }}-${{ hashFiles('crates/gmlx-entry/rust-toolchain.toml', 'crates/gmlx-entry/Cargo.lock') }}
      - name: Rust toolchain for the guest entry
        # The channel and target match rust-toolchain.toml, which a test
        # checks. Naming them keeps an older rustup from failing on a bare
        # `rustup toolchain install`.
        if: env.CODE == 'true'
        working-directory: crates/gmlx-entry
        run: |
          rustup toolchain install stable --profile minimal --target aarch64-unknown-linux-musl
          cargo test --locked
      - name: Install
        if: env.RUN == 'true'
        run: |
          # --group needs pip 25.1 or newer.
          python -m pip install --upgrade pip
          pip install "mlx==0.32.3"
          # Wheel, not sdist. Must satisfy the pyproject floor or the
          # editable install upgrades it; tests/test_ci_pins.py gates that.
          pip install "mlx-kquant==0.5.8"
          # `all` brings the chat, voice and assistant extras, so their tests
          # run instead of skipping themselves away.
          pip install -e ".[all]"
          pip install --group test
      - name: Import canary
        if: env.RUN == 'true'
        # A broken package import must fail the build loudly. The test files
        # importorskip only true optionals, never gmlx itself.
        run: python -c "import gmlx, gmlx.commands.cli, gmlx.serve.server, gmlx.serve.patches, gmlx.spec.engine, gmlx.spec.speculative"
      - name: Wheel completeness
        if: env.RUN == 'true'
        # The editable install above can't catch packaging drops: build the
        # wheel (via sdist, so a stale build/ dir can't leak deleted modules
        # into it) and require it to ship exactly the tracked package modules,
        # plus every tracked file matching the pyproject package-data globs
        # (the vendored misaki data, the deepseek_v41 chat template, the
        # ComfyUI node pack's LICENSE and workflows).
        run: |
          python -m build
          python - <<'EOF'
          import fnmatch, glob, subprocess, sys, tomllib, zipfile
          wheel = glob.glob("dist/*.whl")[0]
          shipped = set(zipfile.ZipFile(wheel).namelist())
          tracked = subprocess.run(
              ["git", "ls-files", "gmlx/*.py"],
              capture_output=True, text=True, check=True,
          ).stdout.split()
          missing = [m for m in tracked if m not in shipped]
          if missing:
              sys.exit(f"wheel is missing tracked modules: {missing}")
          extra = [m for m in shipped if m.startswith("gmlx/")
                   and m.endswith(".py") and m not in tracked]
          if extra:
              sys.exit(f"wheel ships untracked modules (stale build/?): {extra}")
          with open("pyproject.toml", "rb") as f:
              pdata = tomllib.load(f)["tool"]["setuptools"]["package-data"]
          globs = [k.replace(".", "/") + "/" + g
                   for k, gs in pdata.items() for g in gs]
          tracked_all = subprocess.run(
              ["git", "ls-files", "gmlx"],
              capture_output=True, text=True, check=True,
          ).stdout.split()
          data = [t for t in tracked_all
                  if any(fnmatch.fnmatch(t, g) for g in globs)]
          if not data:
              sys.exit("package-data globs matched no tracked files")
          missing_data = [d for d in data if d not in shipped]
          if missing_data:
              sys.exit(f"wheel is missing package-data files: {missing_data}")
          # The ComfyUI node pack: every file, its workflows and LICENSE.
          pack = [t for t in tracked_all if t.startswith("gmlx/comfyui_pack/")]
          missing_pack = [p for p in pack if p not in shipped]
          if not pack or missing_pack:
              sys.exit(f"wheel is missing ComfyUI node pack files: {missing_pack}")
          print(f"wheel ships all {len(tracked)} tracked modules"
                f" + {len(data)} package-data files, nothing else")
          EOF
      - name: Arch coverage doc
        # docs/arch-coverage.md is generated; a new arch must not leave the
        # committed doc lying about what loads.
        if: env.RUN == 'true'
        run: python scripts/check-coverage.py --check --strict
      - name: Type check (pyright)
        # pyright checks gmlx's calls into the copied mlx-vlm and mlx-lm, whose
        # imports of mlx, transformers and the rest only this job installs, not
        # the deps-free lint job. One interpreter is enough: the checked
        # surface does not vary by Python version.
        if: env.CODE == 'true' && matrix.python-version == '3.14'
        run: |
          pip install "pyright[nodejs]==1.1.414"
          pyright
      - name: Build the guest entry
        # Once, before the tests, so their workers do not each build it.
        if: env.CODE == 'true'
        run: echo "GMLX_ENTRY_BIN=$(python scripts/build_guest_entry.py --native | tail -1)" >> "$GITHUB_ENV"
      - name: Run CPU logic tests
        if: env.RUN == 'true'
        env:
          # Hosted runners have no dependable Metal device; the conftest flips
          # the default device to CPU when this is set.
          KQUANT_FORCE_CPU: "1"
        # One worker per core. Tests that fail are rerun alone; one that then
        # passes is flaky, which warns instead of failing the step.
        run: >-
          python scripts/ci_tests.py run
          --select ${{ env.CODE == 'true' && 'full' || 'docs' }}
          ${{ matrix.python-version == '3.11' && '--skip-numerics' || '' }}
          ${{ env.DAILY == 'true' && '--nightly' || '' }}
      - name: Flake issues
        # One issue per flaky test. A push to main and the daily 3.13 leg
        # also count green runs towards closing the open ones. A fork's
        # token cannot write issues.
        if: >-
          always() && env.RUN == 'true'
          && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
        env:
          GH_TOKEN: ${{ github.token }}
          GH_REPO: ${{ github.repository }}
          EVENT: ${{ github.event_name }}
          PY: ${{ matrix.python-version }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          args=()
          if [ "$EVENT" = push ] || { [ "$DAILY" = true ] && [ "$PY" = 3.13 ]; }; then args+=(--count); fi
          if [ "$DAILY" = true ] && [ "$PY" = 3.13 ]; then args+=(--daily); fi
          python scripts/ci_tests.py issues --run-url "$RUN_URL" "${args[@]}"
      - name: Suite growth
        # The test count and test time against the newest main run that
        # uploaded a full 3.14 report.
        if: always() && env.CODE == 'true' && matrix.python-version == '3.14' && github.event_name == 'pull_request'
        env:
          GH_TOKEN: ${{ github.token }}
          GH_REPO: ${{ github.repository }}
        run: |
          for run in $(gh run list --workflow test.yml --branch main --event push --status success \
                         --limit 10 --json databaseId --jq '.[].databaseId'); do
            if gh run download "$run" --name junit-3.14 --dir baseline 2>/dev/null; then break; fi
          done
          python scripts/ci_tests.py growth --junit junit.xml --baseline baseline/junit.xml
      - uses: actions/upload-artifact@v7
        # A docs-only run's report is named apart, so it is never a baseline.
        if: always() && env.RUN == 'true'
        with:
          name: junit-${{ matrix.python-version }}${{ env.CODE != 'true' && '-docs' || '' }}
          path: junit*.xml
          if-no-files-found: ignore
      - name: No stock mlx-vlm or mlx-lm
        if: env.RUN == 'true'
        # No requirement of gmlx with all its extras installs mlx-vlm, mlx-lm
        # or mlx-embeddings, which requires a stock mlx-vlm, and the copy's
        # version files name the releases the manifest records.
        run: |
          python - <<'EOF'
          import importlib.metadata as md, json, re, sys
          found = []
          for dist in ("mlx-vlm", "mlx-lm", "mlx-embeddings"):
              try:
                  found.append(f"{dist} {md.version(dist)}")
              except md.PackageNotFoundError:
                  pass
          problems = [f"installed: {d}" for d in found]
          manifest = json.load(open("gmlx/_vendor/manifest.json"))
          files = {"mlx_vlm": "version.py", "mlx_lm": "_version.py"}
          for pkg, info in manifest["packages"].items():
              text = open(f"gmlx/_vendor/{pkg}/{files[pkg]}").read()
              m = re.search(r"__version__\s*=\s*[\"']([^\"']+)", text)
              if m is None or m.group(1) != info["version"]:
                  problems.append(f"{pkg}: version file does not say "
                                  f"{info['version']}")
          if problems:
              sys.exit("\n".join(problems))
          print("no stock mlx-vlm, mlx-lm or mlx-embeddings; the copy holds",
                ", ".join(f"{p} {i['version']}"
                          for p, i in manifest["packages"].items()))
          EOF

  guest-entry:
    # The static Linux binary `gmlx launch --container` ships, built the way
    # the release job builds it and tested as is, without a VM.
    if: github.ref_type != 'tag'
    runs-on: ubuntu-24.04-arm
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"
      - name: Rust toolchain
        working-directory: crates/gmlx-entry
        run: |
          rustup toolchain install stable --profile minimal --target aarch64-unknown-linux-musl
          cargo test --locked
      - name: Build and test the release binary
        run: |
          python scripts/build_guest_entry.py
          pip install pytest build
          GMLX_ENTRY_BIN="$PWD/gmlx/container/guest/gmlx-entry" \
            pytest -q --noconftest tests/container/test_container_entry.py
      - name: The sdist and wheel ship it
        run: |
          python -m build
          python scripts/check_guest_entry_dist.py

  stress:
    # Daily: the concurrency tests under random stalls, for the races a
    # loaded machine shows (scripts/stress_tests.py). A failing test opens
    # or reopens its flaky issue, which is the whole signal: the step never
    # fails the job, so a flake does not paint the workflow badge red.
    if: github.event.schedule == '0 7 * * *'
    runs-on: macos-26
    timeout-minutes: 45
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.14"
          cache: pip
          cache-dependency-path: pyproject.toml
      - name: Rust toolchain for the guest entry
        working-directory: crates/gmlx-entry
        run: rustup toolchain install stable --profile minimal --target aarch64-unknown-linux-musl
      - name: Install
        run: |
          python -m pip install --upgrade pip
          pip install "mlx==0.32.3"
          pip install "mlx-kquant==0.5.8"
          pip install -e ".[all]"
          pip install --group test
      - name: Build the guest entry
        run: echo "GMLX_ENTRY_BIN=$(python scripts/build_guest_entry.py --native | tail -1)" >> "$GITHUB_ENV"
      - name: Stress
        id: stress
        continue-on-error: true
        env:
          KQUANT_FORCE_CPU: "1"
        run: python scripts/stress_tests.py --preset concurrency --count 3 --minutes 30 --out stress-out
      - name: Flake issues
        if: always()
        env:
          GH_TOKEN: ${{ github.token }}
          GH_REPO: ${{ github.repository }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: python scripts/ci_tests.py issues --flaky stress-out/flaky.json --run-url "$RUN_URL"
      - uses: actions/upload-artifact@v7
        if: steps.stress.outcome == 'failure'
        with:
          name: stress-out
          path: stress-out
          if-no-files-found: ignore

  lint:
    if: github.event_name != 'schedule' && github.ref_type != 'tag'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"
      # Pinned: a new ruff's new default rules shouldn't fail an untouched repo.
      - run: pip install ruff==0.15.20
      - run: ruff check .
      - name: Copy check
        # gmlx/_vendor holds mlx-vlm and mlx-lm as copied. Every reference to
        # them must name the copy, every copied file must match its manifest
        # entry, and ruff's E9, F821 and F811 rules must find only what the
        # baseline records.
        run: |
          python scripts/vendor_upstream.py rewrite --check
          python scripts/vendor_upstream.py verify
          python scripts/vendor_upstream.py lint
      - run: python scripts/check-docs.py
      - run: python scripts/check-test-timing.py
      - name: Docs site build
        # The release workflow publishes this site. --strict fails on a link,
        # anchor or nav entry the site cannot resolve.
        run: |
          python -m pip install --upgrade pip
          pip install --group docs
          mkdocs build --strict

  drift:
    # What the newest mlx-vlm and mlx-lm releases change in the copy, as a
    # Markdown report in the run summary. It informs a port and never fails
    # the run; see docs/internals/upstream-upgrades.md.
    if: github.event.schedule == '0 6 * * 1'
    runs-on: ubuntu-latest
    continue-on-error: true
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"
      - name: Drift report
        # pipefail: a failed report fails the step, not only tee.
        run: |
          set -o pipefail
          scripts/upstream_canary.sh | tee -a "$GITHUB_STEP_SUMMARY"

  comfyui-pack:
    # The ComfyUI node pack against real ComfyUI and the stub server in
    # tests/comfyui_pack, loaded through the link that `gmlx comfyui install`
    # makes. Each run clones ComfyUI and installs torch, so the job runs on
    # the weekly schedule, on a release tag, and when a pull request changes
    # the pack, its tests or its verb. A pull request checks the ComfyUI
    # commit that the release tests pin. A tag adds the oldest ComfyUI
    # release the pack supports, as gmlx/comfyui_pack/version.py names it,
    # and the release version.py names as tested. The weekly schedule checks
    # the newest release instead and also runs ComfyUI master, which only
    # reports. tests/commands/test_comfyui.py covers the bare-interpreter
    # `python -m gmlx comfyui install` path on every test leg.
    needs: [changes]
    if: >-
      ${{ !cancelled() && (github.event.schedule == '0 6 * * 1'
      || github.ref_type == 'tag'
      || (github.event_name == 'pull_request' && needs.changes.outputs.pack == 'true')) }}
    runs-on: macos-26
    timeout-minutes: 60
    continue-on-error: ${{ matrix.comfyui == 'master' }}
    strategy:
      fail-fast: false
      matrix:
        comfyui: ${{ fromJSON(github.event.schedule == '0 6 * * 1' && '["pin", "floor", "release", "master"]' || github.ref_type == 'tag' && '["pin", "floor", "release"]' || '["pin"]') }}
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"
      - name: ComfyUI and its requirements
        env:
          PIN: 8d534945ebd53cff61e8def81757c6a6c1b9cf2d
          REPO: https://github.com/comfyanonymous/ComfyUI
        run: |
          case "${{ matrix.comfyui }}" in
            pin) ref=$PIN ;;
            floor) ref=v$(sed -n 's/^COMFYUI_FLOOR = "\(.*\)"$/\1/p' gmlx/comfyui_pack/version.py) ;;
            release)
              if [ "${{ github.event.schedule }}" = "0 6 * * 1" ]; then
                ref=$(git ls-remote --tags --refs "$REPO" 'v*' | sed 's#.*refs/tags/##' | sort -V | tail -1)
              else
                ref=v$(sed -n 's/^COMFYUI_TESTED = "\(.*\)"$/\1/p' gmlx/comfyui_pack/version.py)
              fi ;;
            master) ref=master ;;
          esac
          git init -q comfyui
          git -C comfyui fetch -q --depth 1 "$REPO" "$ref"
          git -C comfyui checkout -q FETCH_HEAD
          echo "ComfyUI $ref at $(git -C comfyui rev-parse --short HEAD)" >> "$GITHUB_STEP_SUMMARY"
          python -m venv comfyui/.venv
          comfyui/.venv/bin/pip install -q -r comfyui/requirements.txt
      - name: Install the pack with gmlx
        # The comfyui verb imports none of gmlx's dependencies, so it runs
        # from the checkout.
        run: python -m gmlx comfyui install --comfyui-dir "$PWD/comfyui"
      - name: Unit and headless suites
        run: |
          comfyui/.venv/bin/python tests/comfyui_pack/run_tests.py \
            --comfy "$PWD/comfyui" --label "${{ matrix.comfyui }}" --out "$PWD/pack-results" \
            --pack "$PWD/comfyui/custom_nodes/comfyui-gmlx"
      - name: Report
        if: always()
        run: cat pack-results/*.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true
      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: comfyui-pack-${{ matrix.comfyui }}
          path: pack-results

  gate:
    # The gate/gpu status once the test legs end, so a result pushed while
    # they ran counts without a dispatch. gate.yml also runs on its own
    # when the pull request is pushed. A red test leg still posts.
    needs: [test]
    if: ${{ !cancelled() && github.event_name == 'pull_request' }}
    # A called workflow gets no more than its caller grants.
    permissions:
      contents: read
      statuses: write
      pull-requests: read
    uses: ./.github/workflows/gate.yml