diff --git a/.github/workflows/release-4-publish-release.yml b/.github/workflows/release-4-publish-release.yml index a0725670115..22b750f16d6 100644 --- a/.github/workflows/release-4-publish-release.yml +++ b/.github/workflows/release-4-publish-release.yml @@ -186,160 +186,6 @@ jobs: | Staging Repository ID | \`${staging_repo_id}\` | EOT - - name: Copy distribution from SVN dev to release space - env: - SVN_USERNAME: ${{ secrets.POLARIS_SVN_DEV_USERNAME }} # zizmor: ignore[secrets-outside-env] - SVN_PASSWORD: ${{ secrets.POLARIS_SVN_DEV_PASSWORD }} # zizmor: ignore[secrets-outside-env] - run: | - echo "::add-mask::$SVN_PASSWORD" - - source "${LIBS_DIR}/_constants.sh" - source "${LIBS_DIR}/_exec.sh" - - # Define source and destination URLs - dev_artifacts_url="${APACHE_DIST_URL}/dev/polaris/${version_without_rc}" - release_artifacts_url="${APACHE_DIST_URL}/release/polaris/${version_without_rc}" - - dev_helm_url="${APACHE_DIST_URL}/dev/polaris/helm-chart/${version_without_rc}" - release_helm_url="${APACHE_DIST_URL}/release/polaris/helm-chart/${version_without_rc}" - - dev_python_client_url="${APACHE_DIST_URL}/dev/polaris/python-client/${version_without_rc}" - release_python_client_url="${APACHE_DIST_URL}/release/polaris/python-client/${version_without_rc}" - - exec_process svn mv --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ - "${dev_artifacts_url}" "${release_artifacts_url}" \ - -m "Release Apache Polaris ${version_without_rc}" - - exec_process svn mv --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ - "${dev_helm_url}" "${release_helm_url}" \ - -m "Release Apache Polaris Helm chart ${version_without_rc}" - - exec_process svn mv --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ - "${dev_python_client_url}" "${release_python_client_url}" \ - -m "Release Apache Polaris Python client ${version_without_rc}" - - cat <> $GITHUB_STEP_SUMMARY - ## Distribution - Artifacts, Helm chart, and Python client moved from dist dev to dist release - EOT - - - name: Clean up old releases from dist release repository - env: - SVN_USERNAME: ${{ secrets.POLARIS_SVN_DEV_USERNAME }} # zizmor: ignore[secrets-outside-env] - SVN_PASSWORD: ${{ secrets.POLARIS_SVN_DEV_PASSWORD }} # zizmor: ignore[secrets-outside-env] - run: | - echo "::add-mask::$SVN_PASSWORD" - - source "${LIBS_DIR}/_constants.sh" - source "${LIBS_DIR}/_exec.sh" - - release_artifacts_url="${APACHE_DIST_URL}/release/polaris" - release_helm_url="${APACHE_DIST_URL}/release/polaris/helm-chart" - - # List and remove old release artifact versions - old_versions=$(svn list --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive "${release_artifacts_url}" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+(-incubating)?/$' | sed 's|/$||' | grep -v "^${version_without_rc}$" || true) - - if [[ -n "${old_versions}" ]]; then - echo "## Old Releases Cleanup" >> $GITHUB_STEP_SUMMARY - echo "Removing old release versions:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "${old_versions}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - for old_version in ${old_versions}; do - exec_process svn rm --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ - "${release_artifacts_url}/${old_version}" \ - -m "Remove old release ${old_version} (superseded by ${version_without_rc})" - done - else - echo "## Old Releases Cleanup" >> $GITHUB_STEP_SUMMARY - echo "No old release versions found to remove" >> $GITHUB_STEP_SUMMARY - fi - - # List and remove old Helm chart versions - old_helm_versions=$(svn list --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive "${release_helm_url}" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+(-incubating)?/$' | sed 's|/$||' | grep -v "^${version_without_rc}$" || true) - - if [[ -n "${old_helm_versions}" ]]; then - echo "" >> $GITHUB_STEP_SUMMARY - echo "Removing old Helm chart versions:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "${old_helm_versions}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - for old_helm_version in ${old_helm_versions}; do - exec_process svn rm --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ - "${release_helm_url}/${old_helm_version}" \ - -m "Remove old Helm chart ${old_helm_version} (superseded by ${version_without_rc})" - done - else - echo "" >> $GITHUB_STEP_SUMMARY - echo "No old Helm chart versions found to remove" >> $GITHUB_STEP_SUMMARY - fi - - # List and remove old Python client versions - release_python_client_url="${APACHE_DIST_URL}/release/polaris/python-client" - old_python_client_versions=$(svn list --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive "${release_python_client_url}" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+(-incubating)?/$' | sed 's|/$||' | grep -v "^${version_without_rc}$" || true) - - if [[ -n "${old_python_client_versions}" ]]; then - echo "" >> $GITHUB_STEP_SUMMARY - echo "Removing old Python client versions:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "${old_python_client_versions}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - for old_python_client_version in ${old_python_client_versions}; do - exec_process svn rm --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ - "${release_python_client_url}/${old_python_client_version}" \ - -m "Remove old Python client ${old_python_client_version} (superseded by ${version_without_rc})" - done - else - echo "" >> $GITHUB_STEP_SUMMARY - echo "No old Python client versions found to remove" >> $GITHUB_STEP_SUMMARY - fi - - - name: Transfer Helm index and artifacthub-repo.yml from dev to release - env: - SVN_USERNAME: ${{ secrets.POLARIS_SVN_DEV_USERNAME }} # zizmor: ignore[secrets-outside-env] - SVN_PASSWORD: ${{ secrets.POLARIS_SVN_DEV_PASSWORD }} # zizmor: ignore[secrets-outside-env] - run: | - echo "::add-mask::$SVN_PASSWORD" - - source "${LIBS_DIR}/_constants.sh" - source "${LIBS_DIR}/_exec.sh" - - # Define source and destination URLs - dev_helm_chart_url="${APACHE_DIST_URL}/dev/polaris/helm-chart" - release_helm_chart_url="${APACHE_DIST_URL}/release/polaris/helm-chart" - - # Remove any existing index.yaml / artifacthub-repo.yml at the release location before the move. - # svn mv to a URL fails with "already exists" if the destination path is present, which is the - # case on every release after the first one has put these files under release/polaris/helm-chart/. - existing_release_files=$(svn list --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive "${release_helm_chart_url}" | grep -E '^(index\.yaml|artifacthub-repo\.yml)$' || true) - - if [[ -n "${existing_release_files}" ]]; then - files_to_remove=() - for f in ${existing_release_files}; do - files_to_remove+=("${release_helm_chart_url}/${f}") - done - exec_process svn rm --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ - "${files_to_remove[@]}" \ - -m "Remove previous Helm index and artifacthub-repo.yml (superseded by ${version_without_rc})" - fi - - # Move index.yaml and artifacthub-repo.yml from dev to release - # The index was already properly generated in workflow 3 with relative URLs for the current - # release and absolute URLs to archive.apache.org for previous releases, so we just transfer it as-is - exec_process svn mv --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ - "${dev_helm_chart_url}/index.yaml" \ - "${dev_helm_chart_url}/artifacthub-repo.yml" \ - "${release_helm_chart_url}/" \ - -m "Transfer Helm index and artifacthub-repo.yml for ${version_without_rc} release" - - cat <> $GITHUB_STEP_SUMMARY - ## Helm Repository Metadata - Helm index and artifacthub-repo.yml transferred from dist dev to dist release - EOT - - name: Create final release tag and push to Git repository run: | source "${LIBS_DIR}/_exec.sh" @@ -420,17 +266,20 @@ jobs: artifacts_dir="${RELEASEY_DIR}/release-artifacts" exec_process mkdir -p "${artifacts_dir}" - # Download artifacts from Apache dist release space - release_artifacts_url="${APACHE_DIST_URL}/release/polaris/${version_without_rc}" - release_helm_url="${APACHE_DIST_URL}/release/polaris/helm-chart/${version_without_rc}" + # Download artifacts from the Apache dist dev area. + # The artifacts remain in dist dev after this workflow completes so that a PMC + # member can perform the dist dev → dist release `svn mv` manually (the credentials + # used by this workflow do not have write access to the release area). + dev_artifacts_url="${APACHE_DIST_URL}/dev/polaris/${version_without_rc}" + dev_helm_url="${APACHE_DIST_URL}/dev/polaris/helm-chart/${version_without_rc}" - # Download main artifacts + # Download main artifacts (non-destructive: artifacts stay in dist dev) exec_process svn export --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ - "${release_artifacts_url}" "${artifacts_dir}/artifacts" + "${dev_artifacts_url}" "${artifacts_dir}/artifacts" - # Download Helm chart artifacts + # Download Helm chart artifacts (non-destructive: artifacts stay in dist dev) exec_process svn export --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ - "${release_helm_url}" "${artifacts_dir}/helm" + "${dev_helm_url}" "${artifacts_dir}/helm" # Prepare the content of the Github Release # ******************************************************************************** @@ -500,15 +349,17 @@ jobs: | Component | Status | | --- | --- | - | Distribution artifacts | ✅ Moved to release space | - | Helm chart and index | ✅ Updated in release space | - | Old releases cleanup | ✅ Removed from dist release | | Git release tag | ✅ Created and pushed | | GitHub release | ✅ Created with artifacts attached | | Docker images | ✅ Published to Docker Hub | | Nexus repository | ✅ Released | | Version | \`${version_without_rc}\` | | Final release tag | \`${final_release_tag}\` | + + ⚠️ Manual steps required on \`dist.apache.org\` (the release artifacts remain in dist dev \`dev/polaris\` for these operations): + - \`svn mv\` distribution artifacts, Helm chart, and Python client from dist dev \`dev/polaris\` to dist release \`release/polaris\`. + - \`svn mv\` the Helm \`index.yaml\` and \`artifacthub-repo.yml\` from dist dev to \`release/polaris/helm-chart/\`. + - \`svn rm\` old release versions from \`release/polaris\` that have been superseded by \`${version_without_rc}\`. EOT publish-python-client: @@ -539,7 +390,7 @@ jobs: sudo apt-get update sudo apt-get install -y subversion - - name: Download Python wheel from Apache dist release repository + - name: Download Python wheel from Apache dist dev repository env: SVN_USERNAME: ${{ secrets.POLARIS_SVN_DEV_USERNAME }} # zizmor: ignore[secrets-outside-env] SVN_PASSWORD: ${{ secrets.POLARIS_SVN_DEV_PASSWORD }} # zizmor: ignore[secrets-outside-env] @@ -549,10 +400,12 @@ jobs: source "${LIBS_DIR}/_constants.sh" source "${LIBS_DIR}/_exec.sh" - release_python_client_url="${APACHE_DIST_URL}/release/polaris/python-client/${version_without_rc}" + # Download from dist dev. The wheel remains in dist dev so that a PMC member can + # perform the dist dev → dist release `svn mv` manually after this workflow. + dev_python_client_url="${APACHE_DIST_URL}/dev/polaris/python-client/${version_without_rc}" exec_process svn export --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ - "${release_python_client_url}" dist/ + "${dev_python_client_url}" dist/ - name: Publish Python client release to PyPI uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/releasey/README.md b/releasey/README.md index 9dc1ed826bc..b5a17f92859 100644 --- a/releasey/README.md +++ b/releasey/README.md @@ -46,10 +46,13 @@ The release automation is implemented through the following GitHub workflows: - Builds Docker images for server and admin tool - Builds Helm charts and stages them to dist dev repository 4. **[Publish Release](../.github/workflows/release-4-publish-release.yml)** - Finalizes the release: - - Copies distribution from dist dev to dist release space - - Creates a final release tag and GitHub release + - Creates a final release tag and GitHub release (artifacts downloaded from dist dev) - Publishes Docker images to Docker Hub - Releases the candidate repository on Nexus + - Note: moving artifacts from `dist dev` to `dist release` and cleaning up superseded + versions in `dist release` are performed manually by a PMC member after this workflow, + because the workflow credentials do not have write access to the release area. See the + semi-automated release guide for the exact `svn` commands. ## Directory Structure diff --git a/releasey/release-process-flowchart.mmd b/releasey/release-process-flowchart.mmd index cf18bd5cf38..e254c47e0ca 100644 --- a/releasey/release-process-flowchart.mmd +++ b/releasey/release-process-flowchart.mmd @@ -26,14 +26,16 @@ flowchart TD E --> H["👤 Release Manager

• Start vote thread on dev mailing list
• Wait for the vote to pass"] - H --> I["🤖 GitHub Workflow
🔧 Manual trigger from release manager

• Copy distribution from dist dev to dist release space
• Create a final release tag to Git repository
• Create the GitHub release
• Publish Docker images to DockerHub
• Release the candidate repository on Nexus"] + H --> I["🤖 GitHub Workflow
🔧 Manual trigger from release manager

• Create a final release tag to Git repository
• Create the GitHub release (artifacts downloaded from dist dev)
• Publish Docker images to DockerHub
• Release the candidate repository on Nexus"] - I --> J["👤 Release Manager

• Update documentation - TBD - Process yet to be determined"] + I --> L["👤 Release Manager

• svn mv distribution, Helm chart, and Python client from dist dev to dist release
• svn mv Helm index.yaml and artifacthub-repo.yml from dist dev to dist release
• svn rm old release versions superseded by this release"] - I --> K["👤 Release Manager

• Send announcement e-mail to dev mailing list
• Send announcement to announce Apache mailing list"] + L --> J["👤 Release Manager

• Update documentation - TBD - Process yet to be determined"] + + L --> K["👤 Release Manager

• Send announcement e-mail to dev mailing list
• Send announcement to announce Apache mailing list"] %% Class definition for left-aligned text classDef leftAlign text-align:left; %% Apply left-align class to all nodes - class B,C,D,E,H,I,J,K leftAlign; + class B,C,D,E,H,I,J,K,L leftAlign; diff --git a/releasey/release-process-flowchart.png b/releasey/release-process-flowchart.png index e5814512d14..c71d1d7350c 100644 Binary files a/releasey/release-process-flowchart.png and b/releasey/release-process-flowchart.png differ diff --git a/site/content/community/release-guides/semi-automated-release-guide.md b/site/content/community/release-guides/semi-automated-release-guide.md index 55408fe21c7..25e0a44f784 100644 --- a/site/content/community/release-guides/semi-automated-release-guide.md +++ b/site/content/community/release-guides/semi-automated-release-guide.md @@ -164,17 +164,61 @@ We will proceed with publishing the approved artifacts and sending out the annou ## Publish the release The final workflow to run is [`Release - 4 - Publish Release After Vote Success`](https://github.com/apache/polaris/actions/workflows/release-4-publish-release.yml). This workflow will: * Verify that the release branch HEAD matches the last RC tag -* Copy artifacts from the dist dev to the dist release SVN repository -* Update the Helm index in dist release repository accordingly * Create a final release tag * Rebuild and publish Docker images to Docker Hub -* Create a Github release with the release artifacts +* Create a Github release with the release artifacts (downloaded from dist dev) * Release the candidate repository on Apache Nexus This workflow can only be run from the `release/[major].[minor].x` branch for which a vote has passed. The workflow verifies that no commits have been added to the release branch since the last RC was created. It also requires the Nexus staging repository id (`orgapachepolaris-`) that was created by the previous workflow. ![Screenshot of the fourth release workflow for 1.3.0-incubating](/img/release-guides/github-workflow-4.png "Screenshot of the fourth release workflow for 1.3.0-incubating") +### Manual steps on dist.apache.org + +The workflow does not move artifacts from the dist dev to the dist release SVN repository, because the credentials used by the workflow do not have write access to the release area. A PMC member must run the following commands manually after the workflow completes. + +Move the release artifacts, Helm chart, and Python client from `dev/polaris` to `release/polaris`: + +``` +svn mv https://dist.apache.org/repos/dist/dev/polaris/[major].[minor].[patch] \ + https://dist.apache.org/repos/dist/release/polaris/[major].[minor].[patch] \ + -m "Release Apache Polaris [major].[minor].[patch]" + +svn mv https://dist.apache.org/repos/dist/dev/polaris/helm-chart/[major].[minor].[patch] \ + https://dist.apache.org/repos/dist/release/polaris/helm-chart/[major].[minor].[patch] \ + -m "Release Apache Polaris Helm chart [major].[minor].[patch]" + +svn mv https://dist.apache.org/repos/dist/dev/polaris/python-client/[major].[minor].[patch] \ + https://dist.apache.org/repos/dist/release/polaris/python-client/[major].[minor].[patch] \ + -m "Release Apache Polaris Python client [major].[minor].[patch]" +``` + +Transfer the Helm index and Artifact Hub metadata from dist dev to dist release. If `index.yaml` and `artifacthub-repo.yml` already exist under `release/polaris/helm-chart/` (which is the case after the first release), remove them first or `svn mv` will fail with "already exists": + +``` +svn rm https://dist.apache.org/repos/dist/release/polaris/helm-chart/index.yaml \ + https://dist.apache.org/repos/dist/release/polaris/helm-chart/artifacthub-repo.yml \ + -m "Remove previous Helm index and artifacthub-repo.yml (superseded by [major].[minor].[patch])" + +svn mv https://dist.apache.org/repos/dist/dev/polaris/helm-chart/index.yaml \ + https://dist.apache.org/repos/dist/dev/polaris/helm-chart/artifacthub-repo.yml \ + https://dist.apache.org/repos/dist/release/polaris/helm-chart/ \ + -m "Transfer Helm index and artifacthub-repo.yml for [major].[minor].[patch] release" +``` + +Finally, remove old release versions from `release/polaris`, `release/polaris/helm-chart`, and `release/polaris/python-client` that are superseded by the new release (ASF policy requires keeping only the current release in the `release/` area; older versions are archived automatically). For each old version directory listed by `svn list`: + +``` +svn rm https://dist.apache.org/repos/dist/release/polaris/[old-version] \ + -m "Remove old release [old-version] (superseded by [major].[minor].[patch])" + +svn rm https://dist.apache.org/repos/dist/release/polaris/helm-chart/[old-version] \ + -m "Remove old Helm chart [old-version] (superseded by [major].[minor].[patch])" + +svn rm https://dist.apache.org/repos/dist/release/polaris/python-client/[old-version] \ + -m "Remove old Python client [old-version] (superseded by [major].[minor].[patch])" +``` + ## Publish docs These steps have not been automated yet.