diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 8375978950..0abd22bae1 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -1,6 +1,7 @@ name: PR Validation on: + workflow_dispatch: pull_request: branches: [ "master", "spark3.5", "spark4.0", "spark4.1" ] paths-ignore: [ "**.md", "docs/**", "website/**" ] diff --git a/.github/workflows/release-notes.yml b/.github/workflows/release-notes.yml new file mode 100644 index 0000000000..0e4bffa049 --- /dev/null +++ b/.github/workflows/release-notes.yml @@ -0,0 +1,154 @@ +name: Release Notes — Publish GitHub Release + +# Automates the release-notes half of Release Guide Step 1 after public +# artifacts are available. It has historically been skipped more often than not: +# +# v1.1.3 -> Release exists +# v1.1.1 -> tag exists, NO Release object +# v1.1.3-spark4.0 / -spark4.1 / -python3.* -> no Release object +# +# The gap is not cosmetic. GitHub anchors auto-generated notes to the previous +# *Release*, not the previous tag, so v1.1.1 having no Release made v1.1.3's +# notes span all of v1.1.1 + v1.1.3. This workflow computes the previous +# primary tag itself and pins the diff base, so notes stay correct even when +# an older Release object is missing. +# +# Only the primary vX.Y.Z tag gets a Release. The -spark*/-python* tags are +# build pointers consumed by ADO pipelines, not separate products. This is +# intentionally manual: publishing a GitHub Release on tag creation would make +# the release visible before the ESRP-gated Maven and PyPI artifacts exist. + +on: + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: release-notes-${{ github.ref_name }} + cancel-in-progress: false + +jobs: + publish-release: + name: Publish GitHub Release + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Resolve version + id: v + env: + REF_NAME: ${{ github.ref_name }} + run: | + set -euo pipefail + if [[ ! "$REF_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Run this workflow with a primary vX.Y.Z tag selected, got '$REF_NAME'." + exit 1 + fi + echo "tag=$REF_NAME" >> "$GITHUB_OUTPUT" + + # A release tag must describe a commit that is actually on master. + # Tagging a stale or side branch vX.Y.Z would publish notes for code that + # was never reviewed into the mainline. + - name: Verify the tag is on master + env: + TAG: ${{ steps.v.outputs.tag }} + run: | + set -euo pipefail + git fetch --quiet origin master + if ! git merge-base --is-ancestor "refs/tags/${TAG}" origin/master; then + echo "::error::${TAG} does not point at a commit contained in master. \ + Refusing to publish a release for an off-mainline commit." + exit 1 + fi + + - name: Verify public artifacts are published + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.v.outputs.tag }} + run: | + set -euo pipefail + python3 scripts/release/verify_release.py \ + --version "${TAG#v}" \ + --skip ado,internal + + # Pick the highest primary tag strictly below this one. `sort -V` gives + # correct numeric ordering (v1.1.10 > v1.1.9), which a lexical sort does not. + - name: Determine previous release tag + id: prev + env: + TAG: ${{ steps.v.outputs.tag }} + run: | + set -euo pipefail + PREV=$(git tag --list 'v[0-9]*.[0-9]*.[0-9]*' \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V \ + | awk -v cur="$TAG" '$0 == cur {exit} {last=$0} END {print last}') + if [ -z "$PREV" ]; then + echo "No earlier release tag found — notes will cover full history." + else + echo "Previous release: $PREV" + fi + echo "prev=$PREV" >> "$GITHUB_OUTPUT" + + - name: Skip if the Release already exists + id: exists + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.v.outputs.tag }} + run: | + set -euo pipefail + if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo "Release $TAG already exists — leaving it untouched." + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Generate and publish release notes + if: steps.exists.outputs.found == 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.v.outputs.tag }} + PREV: ${{ steps.prev.outputs.prev }} + run: | + set -euo pipefail + + ARGS=(-f tag_name="$TAG" -f target_commitish="$TAG") + if [ -n "$PREV" ]; then + ARGS+=(-f previous_tag_name="$PREV") + fi + + NOTES=$(gh api "repos/${GITHUB_REPOSITORY}/releases/generate-notes" \ + -X POST "${ARGS[@]}" --jq '.body') + + VERSION="${TAG#v}" + { + echo "## Installation" + echo + echo '```bash' + echo "pip install synapseml==${VERSION}" + echo '```' + echo + echo "Maven coordinate: \`com.microsoft.azure:synapseml_2.12:${VERSION}\`" + echo + echo "| Spark | Python | Tag |" + echo "| --- | --- | --- |" + echo "| 3.5 | 3.11 | \`${TAG}-spark3.5\` |" + echo "| 4.0 | 3.12 | \`${TAG}-spark4.0\` |" + echo "| 4.1 | 3.13 | \`${TAG}-spark4.1\` |" + echo + echo "$NOTES" + } > notes.md + + gh release create "$TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --title "$TAG" \ + --notes-file notes.md \ + --verify-tag + + echo "Published release $TAG (diff base: ${PREV:-})" diff --git a/.github/workflows/release-prepare.yml b/.github/workflows/release-prepare.yml new file mode 100644 index 0000000000..01669b56dc --- /dev/null +++ b/.github/workflows/release-prepare.yml @@ -0,0 +1,321 @@ +name: Release Prepare — Version Bump PR + +# Automates Release Guide Step 1.1-1.3: bump every version string in the repo, +# regenerate the versioned docs snapshot, and open the release PR. +# +# Two recent version bumps (v1.1.0, v1.1.3) landed as unsigned direct pushes +# to master with no PR and no review. This workflow provides a repeatable, +# reviewed path and tags the exact merged commit. +# +# See also: release-tag.yml (creates derivative tags and Spark release PRs). + +on: + workflow_dispatch: + inputs: + version: + description: "New OSS version, three components (e.g. 1.1.4)" + required: true + type: string + skip_docs: + description: "Skip the versioned-docs snapshot (faster; PR will be incomplete)" + required: false + default: false + type: boolean + pull_request: + types: [closed] + branches: [master] + +permissions: {} + +jobs: + prepare: + name: Bump versions & open release PR + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' + permissions: + actions: write + contents: write + pull-requests: write + + steps: + # A release must be cut from master. Running this from a feature branch + # would open a PR that bumps versions against unreleased code. + - name: Validate ref + env: + REF: ${{ github.ref }} + run: | + set -euo pipefail + if [ "$REF" != "refs/heads/master" ]; then + echo "::error::Release prepare must run on master, got '$REF'." + exit 1 + fi + + - name: Validate version format + env: + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Version '$VERSION' must be exactly X.Y.Z. SynapseML OSS \ + does not use a fourth component; the .N super-patch belongs to SynapseML-Internal." + exit 1 + fi + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + # Refuse to re-prepare a version that is already published. Without this + # the workflow would happily open a PR that "bumps" to a shipped version. + - name: Guard against an already-released version + env: + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if git rev-parse -q --verify "refs/tags/v${VERSION}" >/dev/null; then + echo "::error::Tag v${VERSION} already exists. That version is already released." + exit 1 + fi + if git ls-remote --exit-code --heads origin "release/prepare-v${VERSION}" >/dev/null 2>&1; then + echo "::error::Branch release/prepare-v${VERSION} already exists on origin. \ + Delete it or finish the existing release PR first." + exit 1 + fi + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + + - name: Set up JDK 11 + if: ${{ !inputs.skip_docs }} + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 + with: + distribution: temurin + java-version: 11 + cache: sbt + + - name: Install sbt + if: ${{ !inputs.skip_docs }} + run: | + SBT_VERSION="$(sed -n 's/^sbt.version *= *//p' project/build.properties | tr -d ' ')" + mkdir -p "$HOME/.local/bin" + curl --fail --show-error --location --retry 3 --retry-all-errors \ + --output "$HOME/.local/bin/sbt-launch.jar" \ + "https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar" + cat > "$HOME/.local/bin/sbt" <> "$GITHUB_PATH" + + - name: Set up Node + if: ${{ !inputs.skip_docs }} + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: website/package-lock.json + + - name: Install website dependencies + if: ${{ !inputs.skip_docs }} + working-directory: website + run: npm ci + + # bump-version.py is context-anchored: it refuses to replace a bare version + # number that has no SynapseML-identifying text near it, and exits non-zero + # if any anchored occurrence of the old version survives the run. Both + # behaviours are load-bearing here, so the exit code is not suppressed. + - name: Bump version strings + id: bump + env: + VERSION: ${{ inputs.version }} + SKIP_DOCS: ${{ inputs.skip_docs }} + run: | + set -euo pipefail + ARGS=(--to "$VERSION") + if [ "$SKIP_DOCS" = "true" ]; then + ARGS+=(--skip-docs) + fi + python scripts/bump-version.py "${ARGS[@]}" + + - name: Verify the working tree actually changed + run: | + set -euo pipefail + if git diff --quiet; then + echo "::error::bump-version.py reported success but changed nothing. \ + Refusing to open an empty release PR." + exit 1 + fi + echo "Files touched: $(git diff --name-only | wc -l)" + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Commit and push + env: + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + BRANCH="release/prepare-v${VERSION}" + git checkout -b "$BRANCH" + git add -A + git commit -m "chore: Bump version to v${VERSION}" + git push origin "$BRANCH" + + - name: Open release PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ inputs.version }} + SKIP_DOCS: ${{ inputs.skip_docs }} + run: | + set -euo pipefail + BODY=$(cat <- + github.event_name == 'pull_request' && + github.event.pull_request.merged == true && + github.event.pull_request.head.repo.full_name == github.repository && + startsWith(github.event.pull_request.head.ref, 'release/prepare-v') + permissions: + actions: write + contents: write + + steps: + - name: Resolve merged release + id: release + env: + BRANCH: ${{ github.event.pull_request.head.ref }} + TITLE: ${{ github.event.pull_request.title }} + MERGED_SHA: ${{ github.event.pull_request.merge_commit_sha }} + run: | + set -euo pipefail + if [[ ! "$BRANCH" =~ ^release/prepare-v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + echo "::error::Unexpected release branch '$BRANCH'." + exit 1 + fi + VERSION="${BASH_REMATCH[1]}" + if [ "$TITLE" != "chore: Bump version to v${VERSION}" ]; then + echo "::error::Release PR title does not match branch version v${VERSION}." + exit 1 + fi + if [ -z "$MERGED_SHA" ]; then + echo "::error::pull_request.merge_commit_sha is empty." + exit 1 + fi + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "sha=$MERGED_SHA" >> "$GITHUB_OUTPUT" + + - name: Checkout merged commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ steps.release.outputs.sha }} + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify merged version and ancestry + env: + VERSION: ${{ steps.release.outputs.version }} + MERGED_SHA: ${{ steps.release.outputs.sha }} + run: | + set -euo pipefail + git fetch --no-tags origin '+refs/heads/master:refs/remotes/origin/master' + if ! git merge-base --is-ancestor "$MERGED_SHA" origin/master; then + echo "::error::Merged commit $MERGED_SHA is not contained in origin/master." + exit 1 + fi + ACTUAL=$(sed -n 's/^let version = "\([^"]*\)";/\1/p' website/docusaurus.config.js) + if [ "$ACTUAL" != "$VERSION" ]; then + echo "::error::Merged docusaurus version '$ACTUAL' does not match v${VERSION}." + exit 1 + fi + if [ ! -d "website/versioned_docs/version-${VERSION}" ] || + [ ! -f "website/versioned_sidebars/version-${VERSION}-sidebars.json" ]; then + echo "::error::Versioned docs snapshot for ${VERSION} is missing." + exit 1 + fi + if ! grep -Fxq " \"${VERSION}\"," website/versions.json; then + echo "::error::website/versions.json does not list ${VERSION}." + exit 1 + fi + + - name: Tag the exact merged commit + env: + VERSION: ${{ steps.release.outputs.version }} + MERGED_SHA: ${{ steps.release.outputs.sha }} + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + TAG="v${VERSION}" + if git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then + EXISTING=$(git rev-parse "refs/tags/$TAG^{commit}") + if [ "$EXISTING" != "$MERGED_SHA" ]; then + echo "::error::$TAG exists at $EXISTING, expected $MERGED_SHA. Refusing to move it." + exit 1 + fi + echo "$TAG already identifies the merged commit." + else + git tag "$TAG" "$MERGED_SHA" + git push origin "$TAG" + fi + + # Pushes made with GITHUB_TOKEN do not emit another workflow run. Use the + # supported workflow_dispatch exception to continue the release chain. + - name: Dispatch derivative tag orchestration + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.release.outputs.version }} + run: gh workflow run release-tag.yml --ref "v${VERSION}" + + - name: Summary + env: + VERSION: ${{ steps.release.outputs.version }} + run: | + echo "### Release v${VERSION} tagged" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Derivative tag and Spark branch orchestration was dispatched." >> "$GITHUB_STEP_SUMMARY" + echo "Next: queue ADO pipeline 17563 and complete ESRP approval." >> "$GITHUB_STEP_SUMMARY" diff --git a/scripts/bump-version.py b/scripts/bump-version.py index a175c950f0..2b3f1f6811 100755 --- a/scripts/bump-version.py +++ b/scripts/bump-version.py @@ -20,6 +20,13 @@ from pathlib import Path from typing import List, Optional, Tuple +# This script prints non-ASCII status glyphs. On a non-UTF-8 console (the +# Windows cp1252 default) that raises UnicodeEncodeError *after* files have +# already been rewritten, leaving a half-applied bump behind a non-zero exit. +for _stream in (sys.stdout, sys.stderr): + if hasattr(_stream, "reconfigure"): + _stream.reconfigure(encoding="utf-8", errors="replace") + # ── Context patterns ─────────────────────────────────────────────────────────── # SELF-ANCHORED: pattern contains SynapseML-identifying text. Safe anywhere. SELF_ANCHORED = [ @@ -53,6 +60,10 @@ "version-{V}-blue", ] +VERIFY_R_CODEGEN = ( + "core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/VerifyRCodegen.scala" +) + # LINE-ANCHORED: pattern + keyword must co-exist on the same line. LINE_ANCHORED = [ ("--version {V}", ["SynapseML"]), @@ -61,11 +72,14 @@ ('% "{V}-spark4.1"', ["synapseml"]), ("{V} version for", ["spark", "Spark"]), ("`{V}` tag", ["mmlspark"]), + # Per-module R artifact zips, e.g. synapseml-core-1.1.3.zip. Kept generic so + # a newly added module does not silently break the bump. + ("-{V}.zip", ["synapseml"]), ] # FILE-ANCHORED: pattern only safe in specific files. FILE_ANCHORED = [ - ('version = "{V}"', ["website/docusaurus.config.js"]), + ('version = "{V}"', ["website/docusaurus.config.js", VERIFY_R_CODEGEN]), ('version: "{V}"', ["website/docusaurus.config.js"]), ('const version = "{V}";', ["website/src/installArtifacts.js"]), ( @@ -118,11 +132,30 @@ "CHANGES.md", "bump-version.py", "test_bump_version.py", + "release_matrix.py", + "test_release_matrix.py", + "test_verify_release.py", + "test_bump_bbcvhd.py", + "test_release_workflows.py", + "verify_release.py", + "bump_bbcvhd.py", + "test_prev_tag.sh", "package.json", "package-lock.json", "yarn.lock", "versions.json", } +# Repo-relative posix paths. For files whose basename is too common to denylist +# safely -- the release README documents the version conventions using real +# shipped versions, which must never be rewritten, but "README.md" as a +# basename would also exclude the root README, which does need bumping. +DENYLIST_PATHS = { + ".github/workflows/release-notes.yml", + ".github/workflows/release-prepare.yml", + ".github/workflows/release-tag-spark.yml", + ".github/workflows/release-tag.yml", + "scripts/release/README.md", +} ALLOWED_EXTENSIONS = { ".md", ".sbt", @@ -195,6 +228,8 @@ def _skip_dir(name): def _skip_file(rel): if rel.name in DENYLIST_FILES: return True + if rel.as_posix() in DENYLIST_PATHS: + return True for p in rel.parts: if p in DENYLIST_DIRS: return True @@ -243,7 +278,7 @@ class FileResult: def analyze(fp, rel, content, old_v, bare_re, self_a, line_a, file_a): res = FileResult(fp, rel, content) - rel_str = str(rel) + rel_str = rel.as_posix() lines = content.split("\n") for m in bare_re.finditer(content): @@ -312,7 +347,10 @@ def _detect_version(root): sys.exit( "Error: cannot auto-detect version — website/docusaurus.config.js not found." ) - m = re.search(r'let version\s*=\s*"([^"]+)"', cfg.read_text()) + text = _read(cfg) + if text is None: + sys.exit("Error: cannot read website/docusaurus.config.js.") + m = re.search(r'let version\s*=\s*"([^"]+)"', text) if not m: sys.exit("Error: cannot parse version from website/docusaurus.config.js.") return m.group(1) @@ -497,7 +535,7 @@ def main(): sys.exit(1) # Manifest check - modified = {str(r.rel) for r in results if r.matches} + modified = {r.rel.as_posix() for r in results if r.matches} missing = sorted(EXPECTED_FILES - modified) if missing: print("\n" + "!" * 70) @@ -585,16 +623,18 @@ def main(): ) # ── Broad sweep: warn about old version in ANY text file ─────────── - modified_set = {str(r.rel) for r in changed} + modified_set = {r.rel.as_posix() for r in changed} sweep_hits = [] for dp, dn, fn in os.walk(root): dn[:] = [d for d in dn if not _skip_dir(d)] for f in fn: fp = Path(dp) / f - rel_str = str(fp.relative_to(root)) - if rel_str in modified_set or f in ( - "bump-version.py", - "test_bump_version.py", + rel = fp.relative_to(root) + rel_str = rel.as_posix() + if ( + rel_str in modified_set + or f in DENYLIST_FILES + or rel_str in DENYLIST_PATHS ): continue try: diff --git a/scripts/release/README.md b/scripts/release/README.md new file mode 100644 index 0000000000..819fb50267 --- /dev/null +++ b/scripts/release/README.md @@ -0,0 +1,162 @@ +# Release tooling + +Support scripts for the SynapseML Fabric release. They exist to remove +hand-typing and hand-checking from the release, so the remaining human work is +decision-making and approvals. + +| Script | Release Guide step | What it replaces | +| --- | --- | --- | +| `release_matrix.py` | all | Reading version conventions off a wiki page and retyping them | +| `verify_release.py` | Steps 1-3 | Manually checking tags and public/internal artifact stores | +| `bump_bbcvhd.py` | Step 4 | Hand-editing `setup.sh` + `version.txt` in BBC-VHD | + +The remaining manual gates are intentional: ESRP approval, review and merge of +the SynapseML-Internal and BBC-VHD PRs, BBC-VHD CI triage, White-Glove approval, +and release-train monitoring all require authenticated human decisions. + +## Why a matrix + +One release produces 7 git tags per repo, 6 UPack versions, 6 pip versions and +2 BBC-VHD edits, in **four different naming conventions**. The conventions are +not consistent with each other, and the inconsistency is invisible unless you +compare two real releases side by side: + +``` +OSS UPack 1.1.3-spark4-0 spark dots become dashes +Internal UPack 1.1.3-0-spark4.0 spark dots are preserved +OSS pip 1.1.3+python3.12 PEP 440 local segment +Internal pip 1.1.3.0+python3.12 super-patch, then local segment +``` + +`release_matrix.py` derives all of them from one input, including the exact +per-target Maven coordinates and ready-to-run ADO commands for both repos: + +```bash +python scripts/release/release_matrix.py --version 1.1.4 --internal-patch 0 +python scripts/release/release_matrix.py --version 1.1.4 --json # for pipelines + +# Internal-only super-patch: OSS publish stages and Maven builds are disabled +python scripts/release/release_matrix.py --version 1.1.3 \ + --internal-patch 1 --scope internal-only +``` + +The text output includes six Maven tag-build commands (definitions 17563 and +18453 for the three Spark lines) and a ready-to-run command for the live +`SynapseML-Publish-Official` pipeline (definition 35879). Publish-Official +builds the pip and UPack variants from their tags, but it does not publish the +Maven coordinates, so the six tag builds are a required, separately verified +gate. The current Publish-Official pipeline accepts a base version, Internal +patch, and per-target booleans; it derives the tag refs itself. That is newer +than the wiki example that asks for refs to be typed individually. + +The Internal companion helper consumes the matrix JSON directly, so its branch +pins and tags cannot drift from this source of truth: + +```bash +python scripts/release/release_matrix.py --version 1.1.4 --json > release-plan.json +python ../SynapseML-Internal/scripts/release/prepare_release.py \ + --plan release-plan.json --target master --write +``` + +Azure Artifacts versions are immutable, so re-publishing after a bad build +needs a counter. OSS and Internal are separate packages and are rebuilt +independently, so the counters are independent: + +```bash +# reproduces the real v1.1.1 BBC-VHD state exactly +python scripts/release/release_matrix.py --version 1.1.1 --targets spark4.0 \ + --upack-iteration spark4.0=1 +``` + +Pipeline 35879 exposes one OSS and one Internal rebuild variable per run. A +plan therefore requires every selected target to use the same counter; split +targets with different counters into separate commands. The emitted +`--variables SYNAPSEML_PATCH_VERSION=...` and +`SYNAPSEML_INTERNAL_PATCH_VERSION=...` arguments are part of the publish +command rather than documentation-only expected values. + +## Verifying a release + +`verify_release.py` checks GitHub and Internal tags, the user-facing +`synapseml_`, release-guide `synapseml-core_`, and +`synapseml-internal_` coordinates on the Maven CDN, the PyPI package, +and every selected Synapse-Conda and UPack artifact against the matrix. It +exits non-zero if anything is missing or if a source cannot be read. This is +worth running even when the publish pipeline reports success: its pip and +UPack publish tasks use `continueOnError: true`, so a green pipeline does not +by itself prove the artifacts exist. + +```bash +python scripts/release/verify_release.py --version 1.1.3 --internal-patch 0 + +# Historical/rebuilt packages use the same independent counters as the matrix: +python scripts/release/verify_release.py --version 1.1.1 \ + --targets spark4.0 --upack-iteration spark4.0=1 +``` + +The full v1.1.3 replay intentionally reports one missing artifact: +`synapseml-internal_2.13:1.1.3.0-spark4.0`. That historical gap is why +Internal Maven is now a required row instead of being inferred from tags or a +green pip/UPack build. The OSS-only replay remains complete with +`--skip ado,internal`. + +Internal checks use `ADO_TOKEN` when set, or the active Azure CLI login: + +```bash +export ADO_TOKEN="$(az account get-access-token \ + --resource 499b84ac-1321-427f-aa17-267ca6975798 --query accessToken -o tsv)" +``` + +Set `GH_TOKEN` to raise the GitHub API rate limit. Use +`--skip ado,internal` for the OSS-only precondition that gates GitHub Release +publication. + +`--skip` values can select a source, artifact family, or release scope: +`github` skips OSS tags; `ado` skips Internal tags and every ADO-backed +artifact; `upack` and `pip` skip those artifact families; `internal` skips +Internal tags, Maven, UPacks, and wheels while retaining OSS checks; and +`public` skips the OSS Maven CDN and PyPI publication gates. Multiple values +are combined. + +## Updating BBC-VHD + +```bash +python scripts/release/bump_bbcvhd.py --repo ../BBC-VHD --version 1.1.4 \ + --internal-patch 0 --target spark4.0 --dry-run +``` + +Writes only three lines: the two version variables in `setup.sh`, and a +one-patch bump of `version.txt` (a VHD component revision, unrelated to the +SynapseML version, bumped to force an image rebuild). Re-running with identical +package versions is rejected so an accidental retry cannot silently increment +the component revision; use a package rebuild counter or `--force-revision` for +an intentional image-only rebuild. + +## GitHub workflow sequence + +1. Run **Release Prepare** on `master`. It bumps versions, rebuilds and + snapshots docs, opens the reviewed PR, and dispatches branch validation. +2. When that exact release PR merges, the workflow tags its merged commit and + dispatches the existing derivative-tag/Spark-branch orchestrator. +3. Complete the SynapseML-Internal branch bumps and tags from the same matrix, + then queue every Maven tag-build command emitted by the matrix. +4. After all six Maven coordinates exist, run pipeline 35879 using the matrix + output and verify every selected pip and UPack artifact. +5. Run **Release Notes** with the primary tag selected. It refuses to publish + until the public Maven and PyPI artifacts exist. +6. Use `bump_bbcvhd.py`, run BBC-VHD CI, and complete White-Glove and train + monitoring from Release Guide Steps 4-5. + +## Tests + +```bash +pytest scripts/release/test_release_matrix.py \ + scripts/release/test_verify_release.py \ + scripts/release/test_bump_bbcvhd.py \ + scripts/release/test_release_workflows.py +bash scripts/release/test_prev_tag.sh # needs a full clone with tags +``` + +Expected values are transcribed from live v1.1.1 and v1.1.3 data rather than +from documentation, so a failure means the tooling has drifted from what was +actually shipped. diff --git a/scripts/release/bump_bbcvhd.py b/scripts/release/bump_bbcvhd.py new file mode 100644 index 0000000000..14ebf93d99 --- /dev/null +++ b/scripts/release/bump_bbcvhd.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Apply a SynapseML release to a BBC-VHD component directory (Release Guide Step 4). + +Step 4 is hand-edited today, and it is the single most error-prone edit in the +release. Each component's setup.sh carries two version strings that look almost +identical but follow *different* mangling rules: + + SYNAPSEML_VERSION=1.1.1-spark4-0-1 # spark dot -> dash, rebuild counter + SYNAPSEML_INTERNAL_VERSION=1.1.1-0-spark4.0 # spark dot PRESERVED, no counter + +Getting either wrong produces a VHD that fails at image-build time, hours later +and far from the typo. This script derives both from release_matrix, so the two +conventions are applied by the same code that the tests pin against production. + +version.txt is a VHD component revision, unrelated to the SynapseML version; it +is bumped by exactly one patch to force the image to rebuild. + +Usage: + python bump_bbcvhd.py --repo --version 1.1.4 \\ + --internal-patch 0 --target spark4.0 +""" + +import argparse +import os +import re +import sys +from pathlib import Path +from typing import List, Optional, Tuple + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from release_matrix import TARGETS_BY_KEY, build_plan # noqa: E402 + +# BBC-VHD names component directories without the dot: spark4.0 -> spark40. +COMPONENT_DIR = {"master": "spark35", "spark4.0": "spark40", "spark4.1": "spark41"} + +OSS_VAR = "SYNAPSEML_VERSION" +INTERNAL_VAR = "SYNAPSEML_INTERNAL_VERSION" + + +def read_text_exact(path: Path) -> str: + """Read UTF-8 without universal-newline conversion.""" + with path.open("r", encoding="utf-8", newline="") as stream: + return stream.read() + + +def write_text_exact(path: Path, text: str) -> None: + """Write UTF-8 while preserving the newline bytes already present in text.""" + with path.open("w", encoding="utf-8", newline="") as stream: + stream.write(text) + + +def bump_component_revision(text: str) -> Tuple[str, str, str]: + """Bump the trailing patch of a version.txt revision (1.4.26 -> 1.4.27).""" + m = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(\r?\n)?", text) + if not m: + raise ValueError( + f"version.txt must contain only an X.Y.Z revision and optional newline, " + f"found {text!r}" + ) + old = f"{m.group(1)}.{m.group(2)}.{m.group(3)}" + new = f"{m.group(1)}.{m.group(2)}.{int(m.group(3)) + 1}" + return new + (m.group(4) or ""), old, new + + +def set_shell_var(text: str, var: str, value: str) -> Tuple[str, Optional[str]]: + """Replace `VAR=...` on its own line. Returns (new_text, old_value).""" + pat = re.compile( + rf"^(?P{re.escape(var)}=)(?P[^\r\n]*)(?P\r?)$", + re.MULTILINE, + ) + found = pat.search(text) + if not found: + return text, None + old = found.group("val") + # A single anchored assignment is expected. Two would mean the later one + # silently wins at runtime while this script rewrites only the first. + if len(pat.findall(text)) > 1: + raise ValueError(f"{var} is assigned more than once; refusing to guess") + return ( + pat.sub( + lambda match: (f"{match.group('lead')}{value}{match.group('cr')}"), + text, + count=1, + ), + old, + ) + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description="Apply a SynapseML release to BBC-VHD.") + p.add_argument("--repo", required=True, type=Path, help="BBC-VHD checkout root") + p.add_argument("--version", required=True, help="OSS version, e.g. 1.1.4") + p.add_argument("--internal-patch", default="0", help="Internal super-patch digit") + p.add_argument( + "--target", + required=True, + choices=sorted(COMPONENT_DIR), + help="Which spark component to update", + ) + p.add_argument("--upack-iteration", type=int, default=0, help="OSS rebuild counter") + p.add_argument( + "--internal-upack-iteration", + type=int, + default=0, + help="Internal rebuild counter", + ) + p.add_argument( + "--force-revision", + action="store_true", + help="Bump version.txt even when both package versions already match", + ) + p.add_argument("--dry-run", action="store_true") + args = p.parse_args(argv) + + if args.target not in TARGETS_BY_KEY: + print(f"error: unknown target {args.target}", file=sys.stderr) + return 2 + + try: + plan = build_plan( + args.version, + args.internal_patch, + [args.target], + {args.target: args.upack_iteration} if args.upack_iteration else None, + ( + {args.target: args.internal_upack_iteration} + if args.internal_upack_iteration + else None + ), + "internal-only" if args.internal_patch != "0" else "full", + ) + except ValueError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + tp = plan.targets[0] + comp = args.repo / "Components" / "MMLSpark" / COMPONENT_DIR[args.target] + setup_sh, version_txt = comp / "setup.sh", comp / "version.txt" + + for f in (setup_sh, version_txt): + if not f.is_file(): + print( + f"error: {f} not found. Is --repo a BBC-VHD checkout?", file=sys.stderr + ) + return 2 + + original_setup_text = read_text_exact(setup_sh) + setup_text = original_setup_text + try: + setup_text, old_oss = set_shell_var(setup_text, OSS_VAR, tp.oss_upack_version) + setup_text, old_int = set_shell_var( + setup_text, INTERNAL_VAR, tp.internal_upack_version + ) + except ValueError as e: + print(f"error: {setup_sh}: {e}", file=sys.stderr) + return 2 + + # Both assignments must exist. A missing one means the component layout + # changed and a silent no-op would ship the previous release's artifacts. + for var, old in ((OSS_VAR, old_oss), (INTERNAL_VAR, old_int)): + if old is None: + print(f"error: {var} not found in {setup_sh}", file=sys.stderr) + return 2 + + packages_changed = ( + old_oss != tp.oss_upack_version or old_int != tp.internal_upack_version + ) + if not packages_changed and not args.force_revision: + print( + "error: BBC-VHD already references the requested package versions; " + "refusing to bump version.txt again. Use a rebuild counter or " + "--force-revision for an intentional image-only rebuild.", + file=sys.stderr, + ) + return 2 + + original_version_text = read_text_exact(version_txt) + try: + version_text, old_rev, new_rev = bump_component_revision(original_version_text) + except ValueError as e: + print(f"error: {version_txt}: {e}", file=sys.stderr) + return 2 + + label = "[DRY RUN] " if args.dry_run else "" + print(f"{label}{comp.relative_to(args.repo).as_posix()}") + print(f" {OSS_VAR} {old_oss} -> {tp.oss_upack_version}") + print(f" {INTERNAL_VAR} {old_int} -> {tp.internal_upack_version}") + print(f" version.txt {old_rev} -> {new_rev}") + + if args.dry_run: + return 0 + + def restore_originals() -> List[str]: + failures = [] + for path, text in ( + (setup_sh, original_setup_text), + (version_txt, original_version_text), + ): + try: + write_text_exact(path, text) + except OSError as error: + failures.append(f"{path}: {error}") + return failures + + try: + write_text_exact(setup_sh, setup_text) + write_text_exact(version_txt, version_text) + except OSError as error: + rollback_failures = restore_originals() + print( + f"error: BBC-VHD update failed and was rolled back: {error}", + file=sys.stderr, + ) + for failure in rollback_failures: + print(f"error: rollback failed for {failure}", file=sys.stderr) + return 1 + + # Post-condition: re-read and confirm. The whole point of this script is to + # remove doubt about what landed in the file. + try: + check = read_text_exact(setup_sh) + written_version = read_text_exact(version_txt) + except OSError as error: + rollback_failures = restore_originals() + print( + f"error: could not verify BBC-VHD update; changes were rolled back: " + f"{error}", + file=sys.stderr, + ) + for failure in rollback_failures: + print(f"error: rollback failed for {failure}", file=sys.stderr) + return 1 + + postcondition_error = None + for var, want in ( + (OSS_VAR, tp.oss_upack_version), + (INTERNAL_VAR, tp.internal_upack_version), + ): + if not re.search( + rf"^{re.escape(var)}={re.escape(want)}\r?$", + check, + re.MULTILINE, + ): + postcondition_error = var + break + if written_version != version_text: + postcondition_error = "version.txt" + if postcondition_error: + rollback_failures = restore_originals() + print( + f"error: post-condition failed for {postcondition_error}; " + "changes were rolled back", + file=sys.stderr, + ) + for failure in rollback_failures: + print(f"error: rollback failed for {failure}", file=sys.stderr) + return 1 + print(" verified on disk") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/release_matrix.py b/scripts/release/release_matrix.py new file mode 100644 index 0000000000..d9de371ad3 --- /dev/null +++ b/scripts/release/release_matrix.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +""" +SynapseML Release Matrix - the single source of truth for a release. + +Given one decision (the OSS version) and one optional decision (the internal +super-patch), this module derives EVERY downstream identifier a release needs: +git tags in both repos, Universal Package versions, pip wheel versions, and the +BBC-VHD variable values. + +Why this exists +--------------- +These identifiers are NOT consistently derivable by eye. Verified against the +live feeds for v1.1.1/v1.1.3: + + * OSS UPack mangles dots to dashes: synapseml 1.1.3-spark4-0 + * Internal UPack preserves dots: synapseml_internal 1.1.3-0-spark4.0 + * Pip uses a PEP 440 local segment: synapseml 1.1.3+python3.12 + * Internal pip folds the super-patch in: synapseml-internal 1.1.3.0+python3.12 + * master carries THREE tags: v1.1.3, v1.1.3-spark3.5, v1.1.3-python3.11 + +Every one of those asymmetries has caused, or can cause, a hand-typed mistake. +Encode them once, here, and have every other tool read from this. + +Usage: + python scripts/release/release_matrix.py --version 1.1.4 + python scripts/release/release_matrix.py --version 1.1.4 --json + python scripts/release/release_matrix.py --version 1.1.4 --targets master,spark4.0 +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, asdict, field +from typing import Dict, List, Optional, Union + +OSS_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$") +PATCH_RE = re.compile(r"^(0|[1-9]\d*)$") + +UPACK_FEED = "BBC-VHD_PublicPackages" +PIP_FEED = "Synapse-Conda" +ADO_ORG = "https://msdata.visualstudio.com" +ADO_PROJECT = "A365" +OSS_MAVEN_PIPELINE_ID = 17563 +INTERNAL_MAVEN_PIPELINE_ID = 18453 +PUBLISH_PIPELINE_ID = 35879 +RELEASE_SCOPES = ("full", "internal-only") + +PipelineValue = Union[str, bool] + + +@dataclass(frozen=True) +class Target: + """One Spark/Python build target and the branch that produces it.""" + + key: str + branch: str + spark: str + python: str + scala: str + base_branch: Optional[str] + # master is the anchor: it alone carries the bare `vX.Y.Z` tag. + is_anchor: bool = False + + +TARGETS: List[Target] = [ + Target("master", "master", "3.5", "3.11", "2.12", None, is_anchor=True), + Target("spark4.0", "spark4.0", "4.0", "3.12", "2.13", "master"), + Target("spark4.1", "spark4.1", "4.1", "3.13", "2.13", "spark4.0"), +] + +TARGETS_BY_KEY = {t.key: t for t in TARGETS} + + +def parse_iterations(raw: str, flag: str) -> Dict[str, int]: + """Parse a comma-separated KEY=N rebuild-counter argument.""" + out: Dict[str, int] = {} + for item in (value.strip() for value in raw.split(",") if value.strip()): + if "=" not in item: + raise ValueError(f"{flag} expects KEY=N, got {item!r}") + key, _, number = item.partition("=") + key = key.strip() + number = number.strip() + if not key: + raise ValueError( + f"{flag} expects KEY=N with a non-empty target, got {item!r}" + ) + if key in out: + raise ValueError(f"{flag} repeats target {key!r}") + if not re.fullmatch(r"[1-9]\d*", number): + raise ValueError( + f"iteration for {key!r} must be a positive integer, got {number!r}" + ) + out[key] = int(number) + return out + + +def _upack_oss_suffix(target: Target) -> str: + """OSS UPack suffix. Spark dots become dashes: 4.0 -> spark4-0.""" + if target.is_anchor: + return "" + return "-spark" + target.spark.replace(".", "-") + + +def _upack_internal_suffix(target: Target) -> str: + """Internal UPack suffix. Spark dots are PRESERVED: 4.0 -> spark4.0.""" + if target.is_anchor: + return "" + return f"-spark{target.spark}" + + +@dataclass +class TargetPlan: + key: str + branch: str + base_branch: Optional[str] + spark: str + python: str + scala: str + oss_tags: List[str] + internal_tags: List[str] + oss_maven_tag: str + internal_maven_tag: str + oss_maven_version: str + internal_maven_version: str + oss_upack_version: str + internal_upack_version: str + oss_pip_version: str + internal_pip_version: str + + +@dataclass +class ReleasePlan: + oss_version: str + internal_version: str + internal_patch: str + scope: str + ado_org: str = ADO_ORG + ado_project: str = ADO_PROJECT + upack_feed: str = UPACK_FEED + pip_feed: str = PIP_FEED + oss_maven_pipeline_id: int = OSS_MAVEN_PIPELINE_ID + internal_maven_pipeline_id: int = INTERNAL_MAVEN_PIPELINE_ID + publish_pipeline_id: int = PUBLISH_PIPELINE_ID + publish_parameters: Dict[str, PipelineValue] = field(default_factory=dict) + publish_variables: Dict[str, str] = field(default_factory=dict) + targets: List[TargetPlan] = field(default_factory=list) + + @property + def all_oss_tags(self) -> List[str]: + return [t for tp in self.targets for t in tp.oss_tags] + + @property + def all_internal_tags(self) -> List[str]: + return [t for tp in self.targets for t in tp.internal_tags] + + +def build_plan( + oss_version: str, + internal_patch: str = "0", + target_keys: Optional[List[str]] = None, + upack_iteration: Optional[Dict[str, int]] = None, + internal_upack_iteration: Optional[Dict[str, int]] = None, + scope: str = "full", +) -> ReleasePlan: + """Derive the full release plan. + + `upack_iteration` maps a target key to a rebuild counter. Azure Artifacts + UPack versions are immutable per version string, so a re-publish after a bad + build must append `-N`. This is the `-1` in the real `1.1.1-spark4-0-1`. + + OSS and Internal are published as separate packages and are rebuilt + independently, so they carry independent counters. Production proves it: + v1.1.1 shipped `synapseml=1.1.1-spark4-0-1` alongside + `synapseml_internal=1.1.1-0-spark4.0` (no counter). + """ + if not isinstance(oss_version, str) or not OSS_VERSION_RE.fullmatch(oss_version): + raise ValueError(f"OSS version must be X.Y.Z (got {oss_version!r})") + if not isinstance(internal_patch, str) or not PATCH_RE.fullmatch(internal_patch): + raise ValueError( + "internal patch must be a non-negative integer without leading zeroes " + f"(got {internal_patch!r})" + ) + if scope not in RELEASE_SCOPES: + raise ValueError(f"scope must be one of {RELEASE_SCOPES} (got {scope!r})") + if scope == "full" and internal_patch != "0": + raise ValueError( + "a nonzero Internal patch is an Internal-only hotfix; " + "use --scope internal-only" + ) + if scope == "internal-only" and internal_patch == "0": + raise ValueError("--scope internal-only requires a nonzero --internal-patch") + + keys = target_keys or [t.key for t in TARGETS] + unknown = [k for k in keys if k not in TARGETS_BY_KEY] + if unknown: + raise ValueError(f"unknown target(s): {unknown}. Known: {list(TARGETS_BY_KEY)}") + if len(keys) != len(set(keys)): + raise ValueError(f"targets must be unique (got {keys!r})") + selected = set(keys) + + upack_iteration = upack_iteration or {} + internal_upack_iteration = internal_upack_iteration or {} + for label, iterations in ( + ("OSS UPack", upack_iteration), + ("Internal UPack", internal_upack_iteration), + ): + unknown_iterations = sorted(set(iterations) - set(keys)) + if unknown_iterations: + raise ValueError( + f"{label} iteration has unselected or unknown target(s): " + f"{unknown_iterations}" + ) + invalid_iterations = { + key: value + for key, value in iterations.items() + if isinstance(value, bool) or not isinstance(value, int) or value < 1 + } + if invalid_iterations: + raise ValueError( + f"{label} iterations must be positive integers " + f"(got {invalid_iterations!r})" + ) + if iterations and set(iterations) != selected: + raise ValueError( + f"{label} rebuild counters must cover every selected target. " + "Pipeline 35879 accepts one global counter, so run targets with " + "different counters as separate plans." + ) + if len(set(iterations.values())) > 1: + raise ValueError( + f"{label} rebuild counters must have one value per pipeline run. " + "Run targets with different counters as separate plans." + ) + + internal_version = f"{oss_version}.{internal_patch}" + oss_counter = next(iter(upack_iteration.values()), None) + internal_counter = next(iter(internal_upack_iteration.values()), None) + build_oss = scope == "full" + publish_variables = {} + if oss_counter and build_oss: + publish_variables["SYNAPSEML_PATCH_VERSION"] = str(oss_counter) + if internal_counter: + publish_variables["SYNAPSEML_INTERNAL_PATCH_VERSION"] = str(internal_counter) + plan = ReleasePlan( + oss_version=oss_version, + internal_version=internal_version, + internal_patch=internal_patch, + scope=scope, + publish_variables=publish_variables, + publish_parameters={ + "synapseml_version": oss_version, + "internal_patch_version": internal_patch, + "build_synapseml_pip_py311": build_oss and "master" in selected, + "build_synapseml_pip_py312": build_oss and "spark4.0" in selected, + "build_synapseml_pip_py313": build_oss and "spark4.1" in selected, + "build_synapseml_upack_default": build_oss and "master" in selected, + "build_synapseml_upack_spark4": build_oss and "spark4.0" in selected, + "build_synapseml_upack_spark41": build_oss and "spark4.1" in selected, + "build_internal_pip_py311": "master" in selected, + "build_internal_pip_py312": "spark4.0" in selected, + "build_internal_pip_py313": "spark4.1" in selected, + "build_internal_upack_default": "master" in selected, + "build_internal_upack_spark4": "spark4.0" in selected, + "build_internal_upack_spark41": "spark4.1" in selected, + }, + ) + + for key in keys: + t = TARGETS_BY_KEY[key] + v, iv = oss_version, internal_version + oss_maven_version = v if t.is_anchor else f"{v}-spark{t.spark}" + internal_maven_version = iv if t.is_anchor else f"{iv}-spark{t.spark}" + + oss_tags = [f"v{v}-spark{t.spark}", f"v{v}-python{t.python}"] + internal_tags = [f"v{iv}-spark{t.spark}", f"v{iv}-python{t.python}"] + if t.is_anchor: + oss_tags.insert(0, f"v{v}") + internal_tags.insert(0, f"v{iv}") + + it = upack_iteration.get(key) + iter_suffix = f"-{it}" if it else "" + it_int = internal_upack_iteration.get(key) + iter_suffix_internal = f"-{it_int}" if it_int else "" + + plan.targets.append( + TargetPlan( + key=t.key, + branch=t.branch, + base_branch=t.base_branch, + spark=t.spark, + python=t.python, + scala=t.scala, + oss_tags=oss_tags, + internal_tags=internal_tags, + oss_maven_tag=f"v{oss_maven_version}", + internal_maven_tag=f"v{internal_maven_version}", + oss_maven_version=oss_maven_version, + internal_maven_version=internal_maven_version, + oss_upack_version=f"{v}{_upack_oss_suffix(t)}{iter_suffix}", + internal_upack_version=( + f"{v}-{internal_patch}{_upack_internal_suffix(t)}{iter_suffix_internal}" + ), + oss_pip_version=f"{v}+python{t.python}", + internal_pip_version=f"{iv}+python{t.python}", + ) + ) + return plan + + +def render_text(plan: ReleasePlan) -> str: + out: List[str] = [] + out.append( + f"SynapseML release plan OSS v{plan.oss_version} " + f"Internal v{plan.internal_version} scope={plan.scope}" + ) + out.append("") + out.append("GIT TAGS") + for tp in plan.targets: + out.append( + f" [{tp.key}] branch={tp.branch} spark={tp.spark} " + f"python={tp.python} scala={tp.scala}" + ) + oss_label = "create" if plan.scope == "full" else "required existing" + out.append( + f" github/microsoft/SynapseML ({oss_label}) : " + f"{', '.join(tp.oss_tags)}" + ) + out.append( + f" ado/SynapseML-Internal (create) : " + f"{', '.join(tp.internal_tags)}" + ) + out.append("") + out.append("MAVEN TAG BUILDS") + out.append( + " Queue every selected row. These builds publish the Maven coordinates; " + "Publish-Official does not." + ) + for tp in plan.targets: + if plan.scope == "full": + out.append( + f" [{tp.key}] OSS com.microsoft.azure:synapseml_{tp.scala}:" + f"{tp.oss_maven_version}" + ) + out.append( + f" az pipelines run --id {plan.oss_maven_pipeline_id} " + f"--org {plan.ado_org} --project {plan.ado_project} " + f"--branch refs/tags/{tp.oss_maven_tag}" + ) + out.append( + f" [{tp.key}] Internal com.microsoft.azure:synapseml-internal_{tp.scala}:" + f"{tp.internal_maven_version}" + ) + out.append( + f" az pipelines run --id {plan.internal_maven_pipeline_id} " + f"--org {plan.ado_org} --project {plan.ado_project} " + f"--branch refs/tags/{tp.internal_maven_tag}" + ) + out.append("") + out.append( + f"ADO PUBLISH PIPELINE {plan.publish_pipeline_id} " + f"({plan.ado_org}/{plan.ado_project})" + ) + for name, value in plan.publish_parameters.items(): + rendered = str(value).lower() if isinstance(value, bool) else value + out.append(f" {name}={rendered}") + out.append("") + out.append("COPY-PASTE QUEUE COMMAND") + parameters = [] + for name, value in plan.publish_parameters.items(): + rendered = str(value).lower() if isinstance(value, bool) else value + parameters.append(f"{name}={rendered}") + publish_command = ( + f" az pipelines run --id {plan.publish_pipeline_id} " + f"--org {plan.ado_org} --project {plan.ado_project} --parameters " + + " ".join(parameters) + ) + if plan.publish_variables: + publish_command += " --variables " + " ".join( + f"{name}={value}" for name, value in plan.publish_variables.items() + ) + out.append(publish_command) + out.append("") + out.append(f"UPACK ({plan.upack_feed})") + for tp in plan.targets: + out.append( + f" [{tp.key}] synapseml={tp.oss_upack_version} synapseml_internal={tp.internal_upack_version}" + ) + out.append("") + out.append(f"PIP ({plan.pip_feed})") + for tp in plan.targets: + out.append( + f" [{tp.key}] synapseml={tp.oss_pip_version} synapseml-internal={tp.internal_pip_version}" + ) + out.append("") + out.append("BBC-VHD setup.sh values") + for tp in plan.targets: + comp = "spark35" if tp.key == "master" else "spark" + tp.spark.replace(".", "") + out.append(f" Components/MMLSpark/{comp}/setup.sh") + out.append(f" SYNAPSEML_VERSION={tp.oss_upack_version}") + out.append(f" SYNAPSEML_INTERNAL_VERSION={tp.internal_upack_version}") + return "\n".join(out) + + +def main(argv: Optional[List[str]] = None) -> int: + p = argparse.ArgumentParser(description="Derive the SynapseML release matrix.") + p.add_argument("--version", required=True, help="OSS version, e.g. 1.1.4") + p.add_argument( + "--internal-patch", default="0", help="Internal super-patch digit (default 0)" + ) + p.add_argument( + "--targets", default="", help="Comma-separated subset, e.g. master,spark4.0" + ) + p.add_argument( + "--upack-iteration", + default="", + metavar="KEY=N", + help="OSS UPack rebuild counters, e.g. spark4.0=1. Repeat with commas. " + "Azure Artifacts versions are immutable, so a re-publish needs -N.", + ) + p.add_argument( + "--internal-upack-iteration", + default="", + metavar="KEY=N", + help="Internal UPack rebuild counters. Independent of --upack-iteration, " + "because the two packages are published and rebuilt separately.", + ) + p.add_argument( + "--scope", + choices=RELEASE_SCOPES, + default="full", + help="full release or a nonzero Internal-only super-patch", + ) + p.add_argument("--json", action="store_true", help="Emit JSON instead of text") + args = p.parse_args(argv) + + keys = [k.strip() for k in args.targets.split(",") if k.strip()] or None + + try: + iterations = parse_iterations(args.upack_iteration, "--upack-iteration") + internal_iterations = parse_iterations( + args.internal_upack_iteration, "--internal-upack-iteration" + ) + plan = build_plan( + args.version, + args.internal_patch, + keys, + iterations, + internal_iterations, + args.scope, + ) + except ValueError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + print(json.dumps(asdict(plan), indent=2) if args.json else render_text(plan)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/test_bump_bbcvhd.py b/scripts/release/test_bump_bbcvhd.py new file mode 100644 index 0000000000..e959c0110b --- /dev/null +++ b/scripts/release/test_bump_bbcvhd.py @@ -0,0 +1,184 @@ +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import bump_bbcvhd as bump # noqa: E402 + + +def make_component( + tmp_path: Path, + *, + newline="\n", + oss="1.1.1-spark4-0-1", + internal="1.1.1-0-spark4.0", + revision="1.4.26", +): + component = tmp_path / "Components" / "MMLSpark" / "spark40" + component.mkdir(parents=True) + setup = ( + "#!/bin/bash" + + newline + + f"{bump.OSS_VAR}={oss}" + + newline + + f"{bump.INTERNAL_VAR}={internal}" + + newline + + "echo ready" + + newline + ) + bump.write_text_exact(component / "setup.sh", setup) + bump.write_text_exact(component / "version.txt", revision + newline) + return component + + +@pytest.mark.parametrize("newline", ["", "\n", "\r\n"]) +def test_bump_component_revision_preserves_trailing_newline(newline): + updated, old, new = bump.bump_component_revision("1.4.26" + newline) + assert (old, new) == ("1.4.26", "1.4.27") + assert updated == "1.4.27" + newline + + +@pytest.mark.parametrize("invalid", [" 1.4.26\n", "1.4.26 \n", "1.4\n", "\n1.4.26\n"]) +def test_bump_component_revision_rejects_non_bare_content(invalid): + with pytest.raises(ValueError): + bump.bump_component_revision(invalid) + + +def test_set_shell_var_preserves_crlf(): + text = "A=1\r\nSYNAPSEML_VERSION=old\r\nB=2\r\n" + updated, old = bump.set_shell_var(text, bump.OSS_VAR, "new") + assert old == "old" + assert updated == "A=1\r\nSYNAPSEML_VERSION=new\r\nB=2\r\n" + + +def test_set_shell_var_rejects_duplicate_assignments(): + text = "SYNAPSEML_VERSION=one\nSYNAPSEML_VERSION=two\n" + with pytest.raises(ValueError): + bump.set_shell_var(text, bump.OSS_VAR, "new") + + +def test_main_updates_component_and_preserves_crlf(tmp_path): + component = make_component(tmp_path, newline="\r\n") + + result = bump.main( + [ + "--repo", + str(tmp_path), + "--version", + "1.1.3", + "--target", + "spark4.0", + ] + ) + + assert result == 0 + setup = bump.read_text_exact(component / "setup.sh") + assert f"{bump.OSS_VAR}=1.1.3-spark4-0\r\n" in setup + assert f"{bump.INTERNAL_VAR}=1.1.3-0-spark4.0\r\n" in setup + assert "\n" not in setup.replace("\r\n", "") + assert bump.read_text_exact(component / "version.txt") == "1.4.27\r\n" + + +def test_dry_run_does_not_write(tmp_path): + component = make_component(tmp_path) + before_setup = bump.read_text_exact(component / "setup.sh") + before_revision = bump.read_text_exact(component / "version.txt") + + result = bump.main( + [ + "--repo", + str(tmp_path), + "--version", + "1.1.3", + "--target", + "spark4.0", + "--dry-run", + ] + ) + + assert result == 0 + assert bump.read_text_exact(component / "setup.sh") == before_setup + assert bump.read_text_exact(component / "version.txt") == before_revision + + +def test_repeated_release_does_not_bump_component_revision(tmp_path, capsys): + component = make_component( + tmp_path, + oss="1.1.3-spark4-0", + internal="1.1.3-0-spark4.0", + ) + + result = bump.main( + [ + "--repo", + str(tmp_path), + "--version", + "1.1.3", + "--target", + "spark4.0", + ] + ) + + assert result == 2 + assert bump.read_text_exact(component / "version.txt") == "1.4.26\n" + assert "already references" in capsys.readouterr().err + + +def test_force_revision_allows_intentional_image_rebuild(tmp_path): + component = make_component( + tmp_path, + oss="1.1.3-spark4-0", + internal="1.1.3-0-spark4.0", + ) + + result = bump.main( + [ + "--repo", + str(tmp_path), + "--version", + "1.1.3", + "--target", + "spark4.0", + "--force-revision", + ] + ) + + assert result == 0 + assert bump.read_text_exact(component / "version.txt") == "1.4.27\n" + + +def test_write_failure_rolls_back_both_files(tmp_path, monkeypatch, capsys): + component = make_component(tmp_path) + setup_path = component / "setup.sh" + version_path = component / "version.txt" + original_setup = bump.read_text_exact(setup_path) + original_version = bump.read_text_exact(version_path) + real_write = bump.write_text_exact + calls = [] + + def fail_second_write(path, text): + calls.append(path) + if len(calls) == 2: + raise OSError("simulated version write failure") + real_write(path, text) + + monkeypatch.setattr(bump, "write_text_exact", fail_second_write) + result = bump.main( + [ + "--repo", + str(tmp_path), + "--version", + "1.1.3", + "--target", + "spark4.0", + ] + ) + + assert result == 1 + assert bump.read_text_exact(setup_path) == original_setup + assert bump.read_text_exact(version_path) == original_version + assert "was rolled back" in capsys.readouterr().err diff --git a/scripts/release/test_prev_tag.sh b/scripts/release/test_prev_tag.sh new file mode 100755 index 0000000000..558f853d84 --- /dev/null +++ b/scripts/release/test_prev_tag.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# Validates the "previous primary release tag" algorithm used by +# .github/workflows/release-notes.yml against the repository's real tag list. +set -euo pipefail + +if python3 -c 'import sys; raise SystemExit(sys.version_info.major != 3)' >/dev/null 2>&1; then + PYTHON_BIN=python3 +elif python -c 'import sys; raise SystemExit(sys.version_info.major != 3)' >/dev/null 2>&1; then + PYTHON_BIN=python +else + echo "FAIL Python 3 is required to sort semantic release tags" >&2 + exit 1 +fi + +primary_tags() { + # macOS/BSD sort has no GNU -V mode, so use the release tooling's Python 3. + git tag --list 'v[0-9]*.[0-9]*.[0-9]*' \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + | "$PYTHON_BIN" -c 'import sys +tags = [line.strip() for line in sys.stdin if line.strip()] +tags.sort(key=lambda tag: tuple(map(int, tag[1:].split(".")))) +sys.stdout.writelines(f"{tag}\n" for tag in tags)' +} + +prev_tag() { + local cur="$1" + primary_tags \ + | awk -v cur="$cur" '$0 == cur {found=1} !found {last=$0} END {print last}' +} + +fail=0 +check() { + local tag="$1" want="$2" got + got="$(prev_tag "$tag")" + if [ "$got" = "$want" ]; then + printf 'PASS %-12s prev=%s\n' "$tag" "${got:-}" + else + printf 'FAIL %-12s want=%s got=%s\n' "$tag" "${want:-}" "${got:-}" + fail=1 + fi +} + +# Expectations transcribed from the live tag list. +check v1.1.3 v1.1.1 # v1.1.2 was abandoned; must skip the gap +check v1.1.1 v1.1.0 +check v1.1.0 v1.0.15 +check v1.0.15 v1.0.14 +check v1.0.14 v1.0.13 +check v1.0.10 v1.0.9 # numeric, not lexical: v1.0.10 must follow v1.0.9 + +# The repository predates the current naming examples. Whatever the oldest +# primary semantic-version tag is, it must have no predecessor. +OLDEST=$(primary_tags | sed -n '1p') +check "$OLDEST" "" + +# Suffixed tags must never be selected as a predecessor. +if prev_tag v1.1.3 | grep -q -- '-'; then + echo "FAIL suffixed tag leaked into predecessor selection" + fail=1 +else + echo "PASS suffixed tags excluded" +fi + +exit "$fail" diff --git a/scripts/release/test_release_matrix.py b/scripts/release/test_release_matrix.py new file mode 100644 index 0000000000..76f42b3084 --- /dev/null +++ b/scripts/release/test_release_matrix.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Tests for release_matrix. Expected values are transcribed from the LIVE +v1.1.3 / v1.1.1 releases (github tags + BBC-VHD_PublicPackages + Synapse-Conda), +so a regression here means the matrix has drifted from reality.""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from release_matrix import build_plan, parse_iterations, render_text # noqa: E402 + + +def _by_key(plan): + return {tp.key: tp for tp in plan.targets} + + +def test_rejects_bad_versions(): + for bad in ["1.1", "v1.1.3", "1.1.3.0", "abc", ""]: + with pytest.raises(ValueError): + build_plan(bad) + + +def test_rejects_non_numeric_internal_patch(): + with pytest.raises(ValueError): + build_plan("1.1.3", internal_patch="x") + + +@pytest.mark.parametrize("patch", ["00", "01", "-1", "²"]) +def test_rejects_non_canonical_internal_patch(patch): + with pytest.raises(ValueError): + build_plan("1.1.3", internal_patch=patch) + + +def test_rejects_unknown_target(): + with pytest.raises(ValueError): + build_plan("1.1.3", target_keys=["spark9.9"]) + + +def test_rejects_duplicate_targets(): + with pytest.raises(ValueError): + build_plan("1.1.3", target_keys=["master", "master"]) + + +@pytest.mark.parametrize( + "kwargs", + [ + {"upack_iteration": {"spark9.9": 1}}, + {"upack_iteration": {"spark4.0": 0}}, + {"upack_iteration": {"spark4.0": -1}}, + {"internal_upack_iteration": {"spark4.0": True}}, + ], +) +def test_rejects_invalid_rebuild_iterations(kwargs): + with pytest.raises(ValueError): + build_plan("1.1.3", **kwargs) + + +def test_rejects_iteration_for_unselected_target(): + with pytest.raises(ValueError): + build_plan( + "1.1.3", + target_keys=["master"], + upack_iteration={"spark4.0": 1}, + ) + + +def test_parse_iterations_normalizes_key_and_value_whitespace(): + assert parse_iterations( + " spark4.0 = 1, master= 2 ", + "--upack-iteration", + ) == {"spark4.0": 1, "master": 2} + + +@pytest.mark.parametrize("raw", ["=1", " = 1"]) +def test_parse_iterations_rejects_empty_target(raw): + with pytest.raises(ValueError, match="non-empty target"): + parse_iterations(raw, "--upack-iteration") + + +def test_master_carries_three_oss_tags(): + """Verified live: v1.1.3, v1.1.3-spark3.5 and v1.1.3-python3.11 all point + at commit a833941704b5. A release that creates only two of them is broken.""" + m = _by_key(build_plan("1.1.3"))["master"] + assert m.oss_tags == ["v1.1.3", "v1.1.3-spark3.5", "v1.1.3-python3.11"] + assert m.internal_tags == ["v1.1.3.0", "v1.1.3.0-spark3.5", "v1.1.3.0-python3.11"] + + +def test_non_anchor_targets_have_no_bare_tag(): + t = _by_key(build_plan("1.1.3"))["spark4.0"] + assert t.oss_tags == ["v1.1.3-spark4.0", "v1.1.3-python3.12"] + assert "v1.1.3" not in t.oss_tags + + +def test_upack_dot_dash_asymmetry_is_preserved(): + """The single most error-prone fact in the whole release: + OSS UPack mangles the dot, internal UPack does not.""" + t = _by_key(build_plan("1.1.3"))["spark4.0"] + assert t.oss_upack_version == "1.1.3-spark4-0" + assert t.internal_upack_version == "1.1.3-0-spark4.0" + + t41 = _by_key(build_plan("1.1.3"))["spark4.1"] + assert t41.oss_upack_version == "1.1.3-spark4-1" + assert t41.internal_upack_version == "1.1.3-0-spark4.1" + + +def test_master_upack_has_no_spark_suffix(): + m = _by_key(build_plan("1.1.3"))["master"] + assert m.oss_upack_version == "1.1.3" + assert m.internal_upack_version == "1.1.3-0" + + +def test_pip_uses_pep440_local_segment(): + m = _by_key(build_plan("1.1.3")) + assert m["master"].oss_pip_version == "1.1.3+python3.11" + assert m["spark4.0"].oss_pip_version == "1.1.3+python3.12" + assert m["spark4.1"].internal_pip_version == "1.1.3.0+python3.13" + + +def test_maven_coordinates_follow_release_tags(): + m = _by_key(build_plan("1.1.3")) + assert m["master"].scala == "2.12" + assert m["master"].oss_maven_version == "1.1.3" + assert m["master"].internal_maven_version == "1.1.3.0" + assert m["master"].oss_maven_tag == "v1.1.3" + assert m["master"].internal_maven_tag == "v1.1.3.0" + assert m["spark4.0"].scala == "2.13" + assert m["spark4.0"].oss_maven_version == "1.1.3-spark4.0" + assert m["spark4.0"].internal_maven_version == "1.1.3.0-spark4.0" + assert m["spark4.1"].oss_maven_version == "1.1.3-spark4.1" + + +def test_text_plan_emits_every_selected_maven_build(): + plan = build_plan("1.1.4", target_keys=["master", "spark4.1"]) + text = render_text(plan) + + assert text.count("az pipelines run --id 17563") == 2 + assert text.count("az pipelines run --id 18453") == 2 + assert "--branch refs/tags/v1.1.4" in text + assert "--branch refs/tags/v1.1.4.0" in text + assert "--branch refs/tags/v1.1.4-spark4.1" in text + assert "--branch refs/tags/v1.1.4.0-spark4.1" in text + + +def test_internal_superpatch_flows_everywhere(): + """v1.1.3.1 was a real internal-only hotfix: UPack 1.1.3-1, pip 1.1.3.1+python3.11.""" + m = _by_key(build_plan("1.1.3", internal_patch="1", scope="internal-only"))[ + "master" + ] + assert m.internal_tags[0] == "v1.1.3.1" + assert m.internal_upack_version == "1.1.3-1" + assert m.internal_pip_version == "1.1.3.1+python3.11" + assert ( + m.oss_upack_version == "1.1.3" + ), "OSS artifacts must not move on an internal-only hotfix" + + +def test_upack_rebuild_iteration_suffix(): + """1.1.1-spark4-0-1 exists in the live feed: a republish after a bad build.""" + m = _by_key( + build_plan( + "1.1.1", + target_keys=["spark4.0"], + upack_iteration={"spark4.0": 1}, + ) + )["spark4.0"] + assert m.oss_upack_version == "1.1.1-spark4-0-1" + assert m.internal_upack_version == "1.1.1-0-spark4.0", ( + "OSS and Internal are separate packages with independent rebuild " + "counters; an OSS republish must not renumber the Internal package" + ) + + +def test_internal_rebuild_iteration_is_independent(): + m = _by_key( + build_plan( + "1.1.1", + target_keys=["spark4.0"], + internal_upack_iteration={"spark4.0": 2}, + ) + )["spark4.0"] + assert m.oss_upack_version == "1.1.1-spark4-0" + assert m.internal_upack_version == "1.1.1-0-spark4.0-2" + + +def test_reproduces_production_bbcvhd_spark40_setup_sh(): + """Byte-for-byte round-trip against the live BBC-VHD dev/spark40 file: + + SYNAPSEML_VERSION=1.1.1-spark4-0-1 + SYNAPSEML_INTERNAL_VERSION=1.1.1-0-spark4.0 + + Note the asymmetry that makes hand-editing this file so error-prone: the + OSS package mangles the spark dot to a dash and carries a rebuild counter, + while the Internal package preserves the dot and carries none. + """ + m = _by_key( + build_plan( + "1.1.1", + internal_patch="0", + target_keys=["spark4.0"], + upack_iteration={"spark4.0": 1}, + ) + )["spark4.0"] + assert m.oss_upack_version == "1.1.1-spark4-0-1" + assert m.internal_upack_version == "1.1.1-0-spark4.0" + + +def test_target_subset_is_respected(): + plan = build_plan("1.1.4", target_keys=["master", "spark4.0"]) + assert [tp.key for tp in plan.targets] == ["master", "spark4.0"] + + +def test_publish_parameters_enable_exact_selected_targets(): + plan = build_plan("1.1.4", target_keys=["master", "spark4.1"]) + assert plan.publish_pipeline_id == 35879 + assert plan.publish_parameters["synapseml_version"] == "1.1.4" + assert plan.publish_parameters["internal_patch_version"] == "0" + assert plan.publish_parameters["build_synapseml_pip_py311"] is True + assert plan.publish_parameters["build_synapseml_upack_default"] is True + assert plan.publish_parameters["build_synapseml_pip_py312"] is False + assert plan.publish_parameters["build_synapseml_upack_spark4"] is False + assert plan.publish_parameters["build_synapseml_pip_py313"] is True + assert plan.publish_parameters["build_synapseml_upack_spark41"] is True + assert plan.publish_parameters["build_internal_pip_py313"] is True + assert plan.publish_parameters["build_internal_upack_spark41"] is True + + +def test_internal_only_hotfix_never_republishes_oss(): + plan = build_plan( + "1.1.3", + internal_patch="1", + target_keys=["master"], + scope="internal-only", + ) + text = render_text(plan) + + assert plan.scope == "internal-only" + assert not any( + value + for name, value in plan.publish_parameters.items() + if name.startswith("build_synapseml_") + ) + assert plan.publish_parameters["build_internal_pip_py311"] is True + assert plan.publish_parameters["build_internal_upack_default"] is True + assert "az pipelines run --id 17563" not in text + assert text.count("az pipelines run --id 18453") == 1 + + +def test_internal_only_hotfix_preserves_but_does_not_republish_oss_rebuild(): + plan = build_plan( + "1.1.3", + internal_patch="1", + target_keys=["master"], + upack_iteration={"master": 1}, + scope="internal-only", + ) + + assert plan.targets[0].oss_upack_version == "1.1.3-1" + assert "SYNAPSEML_PATCH_VERSION" not in plan.publish_variables + + +@pytest.mark.parametrize( + "patch,scope", + [("1", "full"), ("0", "internal-only"), ("0", "unknown")], +) +def test_rejects_unsafe_release_scope(patch, scope): + with pytest.raises(ValueError): + build_plan("1.1.3", internal_patch=patch, scope=scope) + + +def test_rebuild_counter_reaches_publish_pipeline(): + plan = build_plan( + "1.1.1", + target_keys=["spark4.0"], + upack_iteration={"spark4.0": 1}, + internal_upack_iteration={"spark4.0": 2}, + ) + text = render_text(plan) + + assert plan.publish_variables == { + "SYNAPSEML_PATCH_VERSION": "1", + "SYNAPSEML_INTERNAL_PATCH_VERSION": "2", + } + assert ( + "--variables SYNAPSEML_PATCH_VERSION=1 " + "SYNAPSEML_INTERNAL_PATCH_VERSION=2" in text + ) + + +def test_rejects_per_target_counters_that_one_pipeline_cannot_express(): + with pytest.raises(ValueError, match="cover every selected target"): + build_plan("1.1.1", upack_iteration={"spark4.0": 1}) + with pytest.raises(ValueError, match="one value per pipeline run"): + build_plan( + "1.1.1", + upack_iteration={"master": 1, "spark4.0": 2, "spark4.1": 1}, + ) + + +def test_base_branch_chain_matches_rebase_order(): + b = {tp.key: tp.base_branch for tp in build_plan("1.1.4").targets} + assert b == {"master": None, "spark4.0": "master", "spark4.1": "spark4.0"} + + +def test_all_tag_helpers_are_unique_and_complete(): + plan = build_plan("1.1.4") + assert len(plan.all_oss_tags) == len(set(plan.all_oss_tags)) == 7 + assert len(plan.all_internal_tags) == len(set(plan.all_internal_tags)) == 7 + + +@pytest.mark.parametrize( + "args", + [ + ["--version", "1.1.4", "--upack-iteration", "spark4.0=0"], + ["--version", "1.1.4", "--upack-iteration", "spark4.0=²"], + [ + "--version", + "1.1.4", + "--upack-iteration", + "spark4.0=1,spark4.0=2", + ], + ], +) +def test_cli_rejects_invalid_iterations(args): + from release_matrix import main + + assert main(args) == 2 diff --git a/scripts/release/test_release_workflows.py b/scripts/release/test_release_workflows.py new file mode 100644 index 0000000000..4672732559 --- /dev/null +++ b/scripts/release/test_release_workflows.py @@ -0,0 +1,43 @@ +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS = ROOT / ".github" / "workflows" + + +def read_workflow(name): + return (WORKFLOWS / name).read_text(encoding="utf-8") + + +def test_release_notes_is_manual_and_artifact_gated(): + workflow = read_workflow("release-notes.yml") + trigger = workflow.split("permissions:", 1)[0] + assert "\n workflow_dispatch:" in trigger + assert "\n push:" not in trigger + assert "--skip ado,internal" in workflow + assert "--targets master" not in workflow + assert "python3 scripts/release/verify_release.py" in workflow + assert 'target_commitish="$TAG"' in workflow + + +def test_release_prepare_tags_merged_commit_and_dispatches_orchestrator(): + workflow = read_workflow("release-prepare.yml") + assert "\n pull_request:" in workflow + assert "github.event.pull_request.merge_commit_sha" in workflow + assert "website/versioned_docs/version-${VERSION}" in workflow + assert 'git tag "$TAG" "$MERGED_SHA"' in workflow + assert 'gh workflow run release-tag.yml --ref "v${VERSION}"' in workflow + assert "gh workflow run release-notes.yml" not in workflow + assert ( + "curl --fail --show-error --location --retry 3 --retry-all-errors" in workflow + ) + + +def test_generated_release_pr_receives_dispatched_validation(): + prepare = read_workflow("release-prepare.yml") + validation = read_workflow("pr-validation.yml") + assert "gh workflow run pr-validation.yml" in prepare + assert "gh workflow run website-deploy.yml" in prepare + assert "\n workflow_dispatch:" in validation.split("jobs:", 1)[0] diff --git a/scripts/release/test_verify_release.py b/scripts/release/test_verify_release.py new file mode 100644 index 0000000000..513adda1c4 --- /dev/null +++ b/scripts/release/test_verify_release.py @@ -0,0 +1,522 @@ +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json +import os +import sys +import urllib.error +import urllib.parse + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import verify_release as verify # noqa: E402 + + +class FakeResponse: + def __init__(self, body, headers=None): + self._body = body + self.headers = headers or {} + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self): + return json.dumps(self._body).encode("utf-8") + + +class AlwaysPresentChecker: + def __init__(self, *_args, **_kwargs): + pass + + def github_tag(self, _tag): + return verify.OK, "github-commit" + + def public_maven(self, _module, _scala, _version): + return verify.OK + + def internal_maven(self, _scala, _version): + return verify.OK + + def public_pypi(self, _version): + return verify.OK + + def ado_tag(self, _tag): + return verify.OK, "ado-commit" + + def upack(self, _package, _version, internal=False): + return verify.OK + + def pip(self, _package, _version, internal=False): + return verify.OK + + +@pytest.mark.parametrize( + "platform,command_type,use_shell", + [ + ("win32", str, True), + ("linux", list, False), + ], +) +def test_ado_token_uses_platform_appropriate_command( + monkeypatch, platform, command_type, use_shell +): + captured = {} + + def run(command, **kwargs): + captured["command"] = command + captured["kwargs"] = kwargs + return verify.subprocess.CompletedProcess(command, 0, "token\n", "") + + monkeypatch.setattr(verify.sys, "platform", platform) + monkeypatch.setattr(verify.subprocess, "run", run) + + assert verify._get_ado_token(None) == "token" + assert isinstance(captured["command"], command_type) + assert captured["kwargs"]["shell"] is use_shell + if use_shell: + assert "az account get-access-token" in captured["command"] + else: + assert captured["command"][:3] == ["az", "account", "get-access-token"] + + +def test_ado_token_reports_missing_azure_cli(monkeypatch): + def missing_cli(*_args, **_kwargs): + raise FileNotFoundError("az not found") + + monkeypatch.setattr(verify.subprocess, "run", missing_cli) + + with pytest.raises(RuntimeError, match="set ADO_TOKEN"): + verify._get_ado_token(None) + + +def test_json_get_parses_successful_response(monkeypatch): + monkeypatch.setattr( + verify.urllib.request, + "urlopen", + lambda *_args, **_kwargs: FakeResponse({"value": ["ok"]}), + ) + assert verify._json_get("https://example", {}) == {"value": ["ok"]} + + +def test_json_get_returns_none_only_for_not_found(monkeypatch): + def not_found(*_args, **_kwargs): + raise urllib.error.HTTPError("https://example", 404, "missing", {}, None) + + monkeypatch.setattr(verify.urllib.request, "urlopen", not_found) + assert verify._json_get("https://example", {}) is None + + +@pytest.mark.parametrize( + "error", + [ + urllib.error.HTTPError("https://example", 500, "failed", {}, None), + urllib.error.URLError("network unavailable"), + ], +) +def test_json_get_surfaces_service_and_network_failures(monkeypatch, error): + def fail(*_args, **_kwargs): + raise error + + monkeypatch.setattr(verify.urllib.request, "urlopen", fail) + with pytest.raises(RuntimeError): + verify._json_get("https://example", {}) + + +def test_url_exists_uses_head_without_downloading_body(monkeypatch): + methods = [] + + def open_url(request, **_kwargs): + methods.append(request.get_method()) + return FakeResponse({}) + + monkeypatch.setattr(verify.urllib.request, "urlopen", open_url) + + assert verify._url_exists("https://example/artifact.jar", {}) + assert methods == ["HEAD"] + + +@pytest.mark.parametrize("head_status", [405, 501]) +def test_url_exists_falls_back_to_get_when_head_is_unsupported( + monkeypatch, head_status +): + methods = [] + + def open_url(request, **_kwargs): + method = request.get_method() + methods.append(method) + if method == "HEAD": + raise urllib.error.HTTPError( + request.full_url, head_status, "unsupported", {}, None + ) + return FakeResponse({}) + + monkeypatch.setattr(verify.urllib.request, "urlopen", open_url) + + assert verify._url_exists("https://example/artifact.jar", {}) + assert methods == ["HEAD", "GET"] + + +def test_url_exists_returns_false_for_missing_head(monkeypatch): + methods = [] + + def not_found(request, **_kwargs): + methods.append(request.get_method()) + raise urllib.error.HTTPError(request.full_url, 404, "missing", {}, None) + + monkeypatch.setattr(verify.urllib.request, "urlopen", not_found) + + assert not verify._url_exists("https://example/missing.jar", {}) + assert methods == ["HEAD"] + + +def test_url_exists_returns_false_when_fallback_get_is_missing(monkeypatch): + methods = [] + + def open_url(request, **_kwargs): + method = request.get_method() + methods.append(method) + status = 405 if method == "HEAD" else 404 + raise urllib.error.HTTPError(request.full_url, status, "unavailable", {}, None) + + monkeypatch.setattr(verify.urllib.request, "urlopen", open_url) + + assert not verify._url_exists("https://example/missing.jar", {}) + assert methods == ["HEAD", "GET"] + + +def test_checker_skips_ado_login_when_all_ado_checks_are_skipped(monkeypatch): + def fail_if_called(_token): + raise AssertionError("ADO login should not be requested") + + monkeypatch.setattr(verify, "_get_ado_token", fail_if_called) + checker = verify.Checker(None, None, ["ado"]) + assert checker._ado_headers is None + + +def test_feed_lookup_filters_exact_package_and_follows_continuation(monkeypatch): + calls = [] + + def fake_page(url, _headers): + calls.append(url) + if len(calls) == 1: + return ( + { + "value": [ + { + "name": "unrelated", + "versions": [{"version": "9.9.9"}], + } + ] + }, + {"x-ms-continuationtoken": "next page"}, + ) + return ( + { + "value": [ + { + "name": "synapseml", + "versions": [ + {"version": "1.1.3+python3.11"}, + {"version": "1.1.3+python3.12"}, + ], + } + ] + }, + {}, + ) + + monkeypatch.setattr(verify, "_get_ado_token", lambda _token: "token") + monkeypatch.setattr(verify, "_json_get_page", fake_page) + checker = verify.Checker("token", None, []) + + assert checker._feed_versions("Synapse-Conda", "pypi", "synapseml") == [ + "1.1.3+python3.11", + "1.1.3+python3.12", + ] + first_query = urllib.parse.parse_qs(urllib.parse.urlsplit(calls[0]).query) + second_query = urllib.parse.parse_qs(urllib.parse.urlsplit(calls[1]).query) + assert first_query["packageNameQuery"] == ["synapseml"] + assert first_query["includeAllVersions"] == ["true"] + assert first_query["api-version"] == ["7.1-preview.1"] + assert second_query["continuationToken"] == ["next page"] + + checker._feed_versions("Synapse-Conda", "pypi", "synapseml") + assert len(calls) == 2 + + +def test_run_checks_public_and_internal_maven_and_pypi(monkeypatch): + monkeypatch.setattr(verify, "Checker", AlwaysPresentChecker) + rows, complete = verify.run("1.1.3", "0", ["master"], None, None, []) + + assert complete + assert [row["name"] for row in rows if row["kind"] == "maven"] == [ + "synapseml_2.12", + "synapseml-core_2.12", + "synapseml-cognitive_2.12", + "synapseml-deep-learning_2.12", + "synapseml-lightgbm_2.12", + "synapseml-opencv_2.12", + "synapseml-vw_2.12", + "synapseml-internal_2.12", + ] + assert any(row["kind"] == "pypi" for row in rows) + + +def test_missing_public_install_coordinate_fails_release(monkeypatch): + class MissingInstallCoordinateChecker(AlwaysPresentChecker): + def public_maven(self, module, _scala, _version): + return verify.MISSING if module == "synapseml" else verify.OK + + monkeypatch.setattr(verify, "Checker", MissingInstallCoordinateChecker) + + rows, complete = verify.run("1.1.3", "0", ["master"], None, None, []) + + assert not complete + assert [row["name"] for row in rows if row["status"] == verify.MISSING] == [ + "synapseml_2.12" + ] + + +def test_run_applies_upack_rebuild_counters(monkeypatch): + monkeypatch.setattr(verify, "Checker", AlwaysPresentChecker) + rows, complete = verify.run( + "1.1.1", + "0", + ["spark4.0"], + None, + None, + [], + {"spark4.0": 1}, + ) + + assert complete + assert any( + row["kind"] == "upack" + and row["name"] == "synapseml" + and row["identifier"] == "1.1.1-spark4-0-1" + for row in rows + ) + + +def test_internal_skip_omits_only_internal_ado_artifacts(monkeypatch): + feed_calls = [] + + monkeypatch.setattr(verify, "_get_ado_token", lambda _token: "token") + monkeypatch.setattr( + verify, + "_json_get", + lambda url, _headers: ( + {"info": {"version": "1.1.3"}} + if url.startswith(verify.PYPI_BASE) + else {"object": {"type": "commit", "sha": "github-commit"}} + ), + ) + monkeypatch.setattr(verify, "_url_exists", lambda _url, _headers: True) + + def no_versions(_checker, feed, protocol, package): + feed_calls.append((feed, protocol, package)) + return [] + + monkeypatch.setattr(verify.Checker, "_feed_versions", no_versions) + + rows, complete = verify.run( + "1.1.3", + "0", + ["master"], + None, + None, + ["internal"], + ) + + assert not complete + assert feed_calls == [ + ("BBC-VHD_PublicPackages", "upack", "synapseml"), + ("Synapse-Conda", "pypi", "synapseml"), + ] + internal_rows = [ + row + for row in rows + if row["name"].startswith("ado/") + or row["name"].startswith("synapseml-internal_") + or row["name"] in {"synapseml_internal", "synapseml-internal"} + ] + assert internal_rows + assert all(row["status"] == verify.SKIPPED for row in internal_rows) + oss_feed_rows = [ + row + for row in rows + if row["kind"] in {"upack", "pip"} and row["name"] == "synapseml" + ] + assert all(row["status"] == verify.MISSING for row in oss_feed_rows) + + +def test_main_rejects_unknown_skip_without_network(capsys): + assert verify.main(["--version", "1.1.3", "--skip", "typo"]) == 2 + assert "unknown --skip" in capsys.readouterr().err + + +def test_skip_help_defines_internal_and_public_scopes(capsys): + with pytest.raises(SystemExit) as exc: + verify.main(["--help"]) + + assert exc.value.code == 0 + help_text = " ".join(capsys.readouterr().out.split()) + assert "internal (Internal tags, Maven, UPacks, and wheels)" in help_text + assert "public (OSS Maven CDN and PyPI)" in help_text + + +def test_public_pypi_requires_the_requested_version(monkeypatch): + monkeypatch.setattr( + verify, + "_json_get", + lambda _url, _headers: {"info": {"version": "1.1.2"}}, + ) + checker = verify.Checker(None, None, ["ado"]) + assert checker.public_pypi("1.1.3") == verify.MISSING + + +def test_public_maven_uses_release_specific_coordinate(monkeypatch): + requested = [] + + def exists(url, headers): + requested.append((url, headers)) + return True + + monkeypatch.setattr(verify, "_url_exists", exists) + checker = verify.Checker(None, "github-token", ["ado"]) + assert checker.public_maven("synapseml", "2.13", "1.1.3-spark4.0") == verify.OK + assert checker.public_maven("synapseml-core", "2.13", "1.1.3-spark4.0") == verify.OK + assert requested == [ + ( + "https://mmlspark.azureedge.net/maven/com/microsoft/azure/" + "synapseml_2.13/1.1.3-spark4.0/" + "synapseml_2.13-1.1.3-spark4.0.pom", + {"User-Agent": "synapseml-release-verify"}, + ), + ( + "https://mmlspark.azureedge.net/maven/com/microsoft/azure/" + "synapseml_2.13/1.1.3-spark4.0/" + "synapseml_2.13-1.1.3-spark4.0.jar", + {"User-Agent": "synapseml-release-verify"}, + ), + ( + "https://mmlspark.azureedge.net/maven/com/microsoft/azure/" + "synapseml-core_2.13/1.1.3-spark4.0/" + "synapseml-core_2.13-1.1.3-spark4.0.pom", + {"User-Agent": "synapseml-release-verify"}, + ), + ( + "https://mmlspark.azureedge.net/maven/com/microsoft/azure/" + "synapseml-core_2.13/1.1.3-spark4.0/" + "synapseml-core_2.13-1.1.3-spark4.0.jar", + {"User-Agent": "synapseml-release-verify"}, + ), + ( + "https://mmlspark.azureedge.net/maven/com/microsoft/azure/" + "synapseml-core_2.13/1.1.3-spark4.0/" + "synapseml-core_2.13-1.1.3-spark4.0-tests.jar", + {"User-Agent": "synapseml-release-verify"}, + ), + ] + + +def test_internal_maven_uses_release_specific_coordinate(monkeypatch): + requested = [] + + def exists(url, headers): + requested.append((url, headers)) + return True + + monkeypatch.setattr(verify, "_url_exists", exists) + checker = verify.Checker(None, None, ["ado"]) + + assert checker.internal_maven("2.13", "1.1.3.0-spark4.1") == verify.OK + assert requested == [ + ( + "https://mmlspark.azureedge.net/maven/com/microsoft/azure/" + "synapseml-internal_2.13/1.1.3.0-spark4.1/" + "synapseml-internal_2.13-1.1.3.0-spark4.1.pom", + {"User-Agent": "synapseml-release-verify"}, + ), + ( + "https://mmlspark.azureedge.net/maven/com/microsoft/azure/" + "synapseml-internal_2.13/1.1.3.0-spark4.1/" + "synapseml-internal_2.13-1.1.3.0-spark4.1.jar", + {"User-Agent": "synapseml-release-verify"}, + ), + ] + + +def test_tag_family_must_share_one_commit(monkeypatch): + class MismatchedTagChecker(AlwaysPresentChecker): + def github_tag(self, tag): + return verify.OK, tag + + monkeypatch.setattr(verify, "Checker", MismatchedTagChecker) + + rows, complete = verify.run("1.1.3", "0", ["master"], None, None, []) + + assert not complete + assert [ + row + for row in rows + if row["kind"] == "tag-set" and row["status"] == verify.MISSING + ] == [ + { + "kind": "tag-set", + "target": "master", + "name": "github/microsoft/SynapseML/same-commit", + "identifier": ("v1.1.3, v1.1.3-spark3.5, v1.1.3-python3.11"), + "status": verify.MISSING, + } + ] + + +def test_github_tag_peels_annotated_tag(monkeypatch): + responses = { + "https://api.github.com/repos/microsoft/SynapseML/git/ref/tags/v1.1.3": { + "object": { + "type": "tag", + "sha": "tag-object", + "url": "https://api.github.com/tag-object", + } + }, + "https://api.github.com/tag-object": { + "object": {"type": "commit", "sha": "release-commit"} + }, + } + monkeypatch.setattr( + verify, + "_json_get", + lambda url, _headers: responses[url], + ) + + checker = verify.Checker(None, None, ["ado"]) + assert checker.github_tag("v1.1.3") == (verify.OK, "release-commit") + + +def test_ado_tag_requests_and_uses_peeled_commit(monkeypatch): + requested = [] + + def get(url, _headers): + requested.append(url) + return { + "value": [ + { + "name": "refs/tags/v1.1.3.0", + "objectId": "annotated-tag-object", + "peeledObjectId": "release-commit", + } + ] + } + + monkeypatch.setattr(verify, "_get_ado_token", lambda _token: "token") + monkeypatch.setattr(verify, "_json_get", get) + + checker = verify.Checker("token", None, []) + assert checker.ado_tag("v1.1.3.0") == (verify.OK, "release-commit") + assert "peelTags=true" in requested[0] diff --git a/scripts/release/verify_release.py b/scripts/release/verify_release.py new file mode 100644 index 0000000000..f3b81e55f5 --- /dev/null +++ b/scripts/release/verify_release.py @@ -0,0 +1,567 @@ +#!/usr/bin/env python3 +# Copyright (C) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +""" +Verify a SynapseML release end-to-end against live sources of truth. + +Automates the artifact checks in Release Guide Steps 1-3, and doubles as a +regression test for `release_matrix.py`: run it against an already-shipped +version and every non-skipped row must be PRESENT. + +NOTE: the wiki currently tells you to run `az artifacts universal show`. +That command does not exist in the Azure CLI. This uses the Azure Artifacts +REST API instead, which works. + +Auth: internal checks use ADO_TOKEN when set, otherwise the script shells out to + az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 +GitHub checks use GH_TOKEN when set and otherwise use the unauthenticated API. + +Usage: + python scripts/release/verify_release.py --version 1.1.3 + python scripts/release/verify_release.py --version 1.1.4 --internal-patch 0 --json + python scripts/release/verify_release.py --version 1.1.4 --skip ado,internal + python scripts/release/verify_release.py --version 1.1.4 --skip internal +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request +from typing import Dict, List, Optional, Tuple + +sys.path.insert(0, __file__.rsplit("/", 1)[0].rsplit("\\", 1)[0]) +from release_matrix import ( # noqa: E402 + ADO_ORG, + ADO_PROJECT, + build_plan, + parse_iterations, +) + +ADO_RESOURCE = "499b84ac-1321-427f-aa17-267ca6975798" +ORG_SHORT = "msdata" +GITHUB_REPO = "microsoft/SynapseML" +INTERNAL_REPO = "SynapseML-Internal" +MAVEN_BASE = "https://mmlspark.azureedge.net/maven" +PUBLIC_MAVEN_MODULES = ( + "synapseml", + "synapseml-core", + "synapseml-cognitive", + "synapseml-deep-learning", + "synapseml-lightgbm", + "synapseml-opencv", + "synapseml-vw", +) +INTERNAL_MAVEN_MODULE = "synapseml-internal" +PYPI_BASE = "https://pypi.org/pypi" +SKIP_CHOICES = {"github", "ado", "upack", "pip", "internal", "public"} + +OK, MISSING, SKIPPED = "PRESENT", "MISSING", "SKIPPED" + + +def _get_ado_token(explicit: Optional[str]) -> str: + if explicit: + return explicit + command = [ + "az", + "account", + "get-access-token", + "--resource", + ADO_RESOURCE, + "--query", + "accessToken", + "-o", + "tsv", + ] + use_shell = sys.platform == "win32" + try: + out = subprocess.run( + subprocess.list2cmdline(command) if use_shell else command, + capture_output=True, + text=True, + shell=use_shell, + ) + except OSError as e: + raise RuntimeError( + "could not run Azure CLI 'az'; install Azure CLI and sign in, " + f"or set ADO_TOKEN: {e}" + ) from e + if out.returncode != 0: + raise RuntimeError(f"could not get ADO token: {out.stderr.strip()}") + token = out.stdout.strip() + if not token: + raise RuntimeError("Azure CLI returned an empty ADO token") + return token + + +def _json_get_page(url: str, headers: Dict[str, str]) -> Tuple[Optional[dict], object]: + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=60) as r: + return json.loads(r.read().decode("utf-8")), r.headers + except urllib.error.HTTPError as e: + if e.code == 404: + return None, {} + raise RuntimeError(f"HTTP {e.code} for {url}") from e + except urllib.error.URLError as e: + raise RuntimeError(f"request failed for {url}: {e.reason}") from e + + +def _json_get(url: str, headers: Dict[str, str]) -> Optional[dict]: + data, _ = _json_get_page(url, headers) + return data + + +def _url_exists(url: str, headers: Dict[str, str]) -> bool: + for method in ("HEAD", "GET"): + req = urllib.request.Request(url, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=60): + return True + except urllib.error.HTTPError as e: + if e.code == 404: + return False + if method == "HEAD" and e.code in {405, 501}: + continue + raise RuntimeError(f"HTTP {e.code} for {url}") from e + except urllib.error.URLError as e: + raise RuntimeError(f"request failed for {url}: {e.reason}") from e + raise AssertionError("GET fallback did not return a result") + + +def _with_query(url: str, **updates: str) -> str: + parts = urllib.parse.urlsplit(url) + query = dict(urllib.parse.parse_qsl(parts.query, keep_blank_values=True)) + query.update(updates) + return urllib.parse.urlunsplit( + ( + parts.scheme, + parts.netloc, + parts.path, + urllib.parse.urlencode(query), + parts.fragment, + ) + ) + + +class Checker: + def __init__(self, token: Optional[str], gh_token: Optional[str], skip: List[str]): + self.skip = set(skip) + self._ado_headers = None + needs_ado = ( + "ado" not in self.skip + and not { + "upack", + "pip", + "internal", + } + <= self.skip + ) + if needs_ado: + self._ado_headers = {"Authorization": f"Bearer {_get_ado_token(token)}"} + self._gh_headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "synapseml-release-verify", + } + self._public_headers = {"User-Agent": "synapseml-release-verify"} + if gh_token: + self._gh_headers["Authorization"] = f"Bearer {gh_token}" + self._pkg_cache: Dict[Tuple[str, str, str], List[str]] = {} + + # --- git tags --------------------------------------------------------- + def github_tag(self, tag: str) -> Tuple[str, Optional[str]]: + if "github" in self.skip: + return SKIPPED, None + encoded_tag = urllib.parse.quote(tag, safe="") + url = f"https://api.github.com/repos/{GITHUB_REPO}/git/ref/tags/{encoded_tag}" + data = _json_get(url, self._gh_headers) + if data is None: + return MISSING, None + + obj = data.get("object") + for _ in range(5): + if not isinstance(obj, dict): + raise RuntimeError(f"GitHub tag {tag} has no object") + object_type = obj.get("type") + sha = obj.get("sha") + if object_type == "commit" and isinstance(sha, str) and sha: + return OK, sha + if object_type != "tag" or not isinstance(obj.get("url"), str): + raise RuntimeError( + f"GitHub tag {tag} has unsupported object type {object_type!r}" + ) + data = _json_get(obj["url"], self._gh_headers) + if data is None: + raise RuntimeError(f"annotated GitHub tag object not found for {tag}") + obj = data.get("object") + raise RuntimeError(f"GitHub tag {tag} has more than five annotation layers") + + def _maven( + self, module: str, scala: str, version: str, require_tests: bool = False + ) -> str: + artifact = f"{module}_{scala}" + escaped_version = urllib.parse.quote(version, safe="") + base = ( + f"{MAVEN_BASE}/com/microsoft/azure/{artifact}/{escaped_version}/" + f"{artifact}-{escaped_version}" + ) + files = [f"{base}.pom", f"{base}.jar"] + if require_tests: + files.append(f"{base}-tests.jar") + return ( + OK + if all(_url_exists(url, self._public_headers) for url in files) + else MISSING + ) + + def public_maven(self, module: str, scala: str, version: str) -> str: + if "public" in self.skip: + return SKIPPED + return self._maven( + module, + scala, + version, + require_tests=module == "synapseml-core", + ) + + def internal_maven(self, scala: str, version: str) -> str: + if "internal" in self.skip: + return SKIPPED + return self._maven(INTERNAL_MAVEN_MODULE, scala, version) + + def public_pypi(self, version: str) -> str: + if "public" in self.skip: + return SKIPPED + url = f"{PYPI_BASE}/synapseml/{urllib.parse.quote(version, safe='')}/json" + data = _json_get(url, self._public_headers) + published = data and data.get("info", {}).get("version") == version + return OK if published else MISSING + + def ado_tag(self, tag: str) -> Tuple[str, Optional[str]]: + if "ado" in self.skip or "internal" in self.skip: + return SKIPPED, None + url = ( + f"https://dev.azure.com/{ORG_SHORT}/{ADO_PROJECT}/_apis/git/repositories/" + f"{INTERNAL_REPO}/refs?filter=tags/{tag}&peelTags=true&api-version=7.1" + ) + data = _json_get(url, self._ado_headers) + if data is None: + raise RuntimeError(f"SynapseML-Internal refs endpoint not found: {url}") + refs = data.get("value") + if not isinstance(refs, list): + raise RuntimeError("SynapseML-Internal refs response has no value list") + wanted = f"refs/tags/{tag}" + found = next((value for value in refs if value.get("name") == wanted), None) + if found is None: + return MISSING, None + peeled = found.get("peeledObjectId") + commit = ( + peeled + if isinstance(peeled, str) and peeled.strip("0") + else found.get("objectId") + ) + if not isinstance(commit, str) or not commit: + raise RuntimeError(f"SynapseML-Internal tag {tag} has no object ID") + return OK, commit + + # --- artifact feeds --------------------------------------------------- + def _feed_versions(self, feed: str, protocol: str, package: str) -> List[str]: + normalized = package.lower() + key = (feed, protocol.lower(), normalized) + if key in self._pkg_cache: + return self._pkg_cache[key] + + url = ( + f"https://feeds.dev.azure.com/{ORG_SHORT}/{ADO_PROJECT}/_apis/packaging/" + f"Feeds/{urllib.parse.quote(feed, safe='')}/packages" + ) + url = _with_query( + url, + **{ + "protocolType": protocol, + "packageNameQuery": package, + "includeAllVersions": "true", + "api-version": "7.1-preview.1", + }, + ) + + versions: List[str] = [] + while url: + data, headers = _json_get_page(url, self._ado_headers) + if data is None: + raise RuntimeError(f"Azure Artifacts feed not found: {feed}") + packages = data.get("value") + if not isinstance(packages, list): + raise RuntimeError( + f"Azure Artifacts response for {feed}/{package} has no value list" + ) + for item in packages: + if item.get("name", "").lower() == normalized: + versions.extend( + version["version"] + for version in item.get("versions", []) + if "version" in version + ) + continuation = headers.get("x-ms-continuationtoken") + url = ( + _with_query(url, continuationToken=continuation) if continuation else "" + ) + + self._pkg_cache[key] = versions + return versions + + def upack(self, package: str, version: str, internal: bool = False) -> str: + if ( + "upack" in self.skip + or "ado" in self.skip + or (internal and "internal" in self.skip) + ): + return SKIPPED + return ( + OK + if version + in self._feed_versions("BBC-VHD_PublicPackages", "upack", package) + else MISSING + ) + + def pip(self, package: str, version: str, internal: bool = False) -> str: + if ( + "pip" in self.skip + or "ado" in self.skip + or (internal and "internal" in self.skip) + ): + return SKIPPED + # Azure Artifacts normalises pypi names: synapseml_internal -> synapseml-internal + return ( + OK + if version + in self._feed_versions("Synapse-Conda", "pypi", package.replace("_", "-")) + else MISSING + ) + + +def run( + version: str, + internal_patch: str, + target_keys, + token, + gh_token, + skip, + upack_iteration=None, + internal_upack_iteration=None, +) -> Tuple[List[dict], bool]: + plan = build_plan( + version, + internal_patch, + target_keys, + upack_iteration, + internal_upack_iteration, + "internal-only" if internal_patch != "0" else "full", + ) + c = Checker(token, gh_token, skip) + rows: List[dict] = [] + + def add(kind, target, name, ident, status): + rows.append( + { + "kind": kind, + "target": target, + "name": name, + "identifier": ident, + "status": status, + } + ) + + def add_tag_family(target, name, tags, check): + results = [] + for tag in tags: + status, commit = check(tag) + add("git-tag", target, name, tag, status) + results.append((status, commit)) + + if all(status == SKIPPED for status, _ in results): + consistency = SKIPPED + elif all(status == OK and commit for status, commit in results): + consistency = OK if len({commit for _, commit in results}) == 1 else MISSING + else: + consistency = MISSING + add( + "tag-set", + target, + name + "/same-commit", + ", ".join(tags), + consistency, + ) + + for tp in plan.targets: + add_tag_family( + tp.key, + "github/" + GITHUB_REPO, + tp.oss_tags, + c.github_tag, + ) + for module in PUBLIC_MAVEN_MODULES: + artifact = f"{module}_{tp.scala}" + add( + "maven", + tp.key, + artifact, + tp.oss_maven_version, + c.public_maven(module, tp.scala, tp.oss_maven_version), + ) + add( + "maven", + tp.key, + f"{INTERNAL_MAVEN_MODULE}_{tp.scala}", + tp.internal_maven_version, + c.internal_maven(tp.scala, tp.internal_maven_version), + ) + if tp.key == "master": + add( + "pypi", + tp.key, + "pypi/synapseml", + plan.oss_version, + c.public_pypi(plan.oss_version), + ) + add_tag_family( + tp.key, + "ado/" + INTERNAL_REPO, + tp.internal_tags, + c.ado_tag, + ) + add( + "upack", + tp.key, + "synapseml", + tp.oss_upack_version, + c.upack("synapseml", tp.oss_upack_version), + ) + add( + "upack", + tp.key, + "synapseml_internal", + tp.internal_upack_version, + c.upack( + "synapseml_internal", + tp.internal_upack_version, + internal=True, + ), + ) + add( + "pip", + tp.key, + "synapseml", + tp.oss_pip_version, + c.pip("synapseml", tp.oss_pip_version), + ) + add( + "pip", + tp.key, + "synapseml-internal", + tp.internal_pip_version, + c.pip( + "synapseml_internal", + tp.internal_pip_version, + internal=True, + ), + ) + + ok = not any(r["status"] == MISSING for r in rows) + return rows, ok + + +def main(argv=None) -> int: + p = argparse.ArgumentParser( + description="Verify a SynapseML release's artifacts and tags." + ) + p.add_argument("--version", required=True) + p.add_argument("--internal-patch", default="0") + p.add_argument("--targets", default="") + p.add_argument( + "--upack-iteration", + default="", + metavar="KEY=N", + help="OSS UPack rebuild counters, e.g. spark4.0=1", + ) + p.add_argument( + "--internal-upack-iteration", + default="", + metavar="KEY=N", + help="Internal UPack rebuild counters, e.g. spark4.0=1", + ) + p.add_argument( + "--skip", + default="", + help=( + "Comma-separated checks to skip: github (OSS tags), " + "ado (all ADO-backed checks), upack (all UPacks), " + "pip (all Synapse-Conda wheels), internal " + "(Internal tags, Maven, UPacks, and wheels), " + "public (OSS Maven CDN and PyPI)" + ), + ) + p.add_argument("--json", action="store_true") + args = p.parse_args(argv) + + keys = [k.strip() for k in args.targets.split(",") if k.strip()] or None + skip = [s.strip() for s in args.skip.split(",") if s.strip()] + unknown_skip = sorted(set(skip) - SKIP_CHOICES) + if unknown_skip: + print( + f"error: unknown --skip value(s): {unknown_skip}; " + f"known: {sorted(SKIP_CHOICES)}", + file=sys.stderr, + ) + return 2 + + try: + upack_iteration = parse_iterations(args.upack_iteration, "--upack-iteration") + internal_upack_iteration = parse_iterations( + args.internal_upack_iteration, + "--internal-upack-iteration", + ) + rows, ok = run( + args.version, + args.internal_patch, + keys, + os.environ.get("ADO_TOKEN"), + os.environ.get("GH_TOKEN"), + skip, + upack_iteration, + internal_upack_iteration, + ) + except (ValueError, RuntimeError) as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + if args.json: + print( + json.dumps( + {"version": args.version, "complete": ok, "rows": rows}, indent=2 + ) + ) + else: + print( + f"{'STATUS':<8} {'KIND':<8} {'TARGET':<9} {'PACKAGE/REPO':<30} IDENTIFIER" + ) + for r in rows: + print( + f"{r['status']:<8} {r['kind']:<8} {r['target']:<9} {r['name']:<30} {r['identifier']}" + ) + n_missing = sum(1 for r in rows if r["status"] == MISSING) + print("") + print( + f"{len(rows)} checks, {n_missing} missing -> {'COMPLETE' if ok else 'INCOMPLETE'}" + ) + + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_bump_version.py b/scripts/test_bump_version.py index 3b7d11967e..8ca5c3a8c2 100644 --- a/scripts/test_bump_version.py +++ b/scripts/test_bump_version.py @@ -467,6 +467,10 @@ def test_denylist_dir_in_path(self): def test_versioned_docs_in_path(self): assert _skip_file(Path("versioned_docs/v1/intro.md")) + @pytest.mark.parametrize("path", sorted(bump.DENYLIST_PATHS)) + def test_denylist_repo_relative_path(self, path): + assert _skip_file(Path(path)) + @pytest.mark.parametrize("ext", sorted(ALLOWED_EXTENSIONS)) def test_all_allowed_extensions(self, ext): assert not _skip_file(Path(f"test{ext}")) @@ -990,7 +994,7 @@ def live_results(self): docusaurus = REPO_ROOT / "website" / "docusaurus.config.js" if not docusaurus.exists(): pytest.skip("Not running inside SynapseML repo") - content = docusaurus.read_text() + content = docusaurus.read_text(encoding="utf-8") m = re.search(r'let version\s*=\s*"([^"]+)"', content) assert m, "Cannot detect version from docusaurus.config.js" old_v = m.group(1) @@ -1005,9 +1009,9 @@ def live_results(self): continue r = analyze(fp, rel, c, old_v, bare_re, self_a, line_a, file_a) if r.matches: - results[str(rel)] = len(r.matches) + results[rel.as_posix()] = len(r.matches) if r.unanchored: - unanchored[str(rel)] = r.unanchored + unanchored[rel.as_posix()] = r.unanchored return { "files": results, "total_files": len(results), diff --git a/tools/ci/tests/test_pipeline_yaml.py b/tools/ci/tests/test_pipeline_yaml.py index 3dc8f491d7..75ce515a38 100644 --- a/tools/ci/tests/test_pipeline_yaml.py +++ b/tools/ci/tests/test_pipeline_yaml.py @@ -281,6 +281,11 @@ def test_prewarm_job_present(): } +def test_release_tags_require_explicit_maven_pipeline_queue(): + data = yaml.safe_load(_pipeline_text()) + assert "tags" not in data["trigger"] + + def test_databricks_e2e_uses_fail_open_pr_impact_detection(): assert DATABRICKS_IMPACT.exists() data = yaml.safe_load(_pipeline_text()) @@ -330,6 +335,17 @@ def test_databricks_e2e_uses_fail_open_pr_impact_detection(): assert any(step.get("displayName") == "Publish Test Results" for step in steps) +def test_fabric_e2e_skips_untrusted_fork_builds(): + data = yaml.safe_load(_pipeline_text()) + jobs = {j.get("job"): j for j in _jobs(data["jobs"])} + condition = jobs["FabricE2E"]["condition"] + fork_guard = ( + r"ne\(\s*variables\[['\"]System\.PullRequest\.IsFork['\"]\]," + + r"\s*['\"]True['\"]\s*\)" + ) + assert re.search(fork_guard, condition) + + def test_fabric_e2e_cleans_stale_artifacts_before_running_tests(): data = yaml.safe_load(_pipeline_text()) jobs = {j.get("job"): j for j in _jobs(data["jobs"])}