From 5da6b878ce4e67e2229b20c6e485aed5069b192e Mon Sep 17 00:00:00 2001 From: Oguz Kocer Date: Wed, 15 Jul 2026 03:33:33 -0400 Subject: [PATCH 1/2] =?UTF-8?q?Add=20a=20promote-to-production=20flow=20(b?= =?UTF-8?q?eta=20=E2=86=92=20production)=20to=20the=20Play=20Store?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the gather_production_candidate and promote_to_production lanes, the promote-production.yml pipeline, and its command scripts. Promotes the current beta build to the production track as a draft via the Play API for WordPress and Jetpack, gated by a Buildkite confirmation block step. Shared beta-promotion helpers and constants are renamed for track parity. --- .../commands/gather-production-candidate.sh | 15 + .buildkite/commands/promote-to-production.sh | 32 ++ .buildkite/promote-production.yml | 20 ++ fastlane/lanes/promote.rb | 321 ++++++++++++++++-- 4 files changed, 353 insertions(+), 35 deletions(-) create mode 100755 .buildkite/commands/gather-production-candidate.sh create mode 100755 .buildkite/commands/promote-to-production.sh create mode 100644 .buildkite/promote-production.yml diff --git a/.buildkite/commands/gather-production-candidate.sh b/.buildkite/commands/gather-production-candidate.sh new file mode 100755 index 000000000000..c69820a398e0 --- /dev/null +++ b/.buildkite/commands/gather-production-candidate.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -eu + +# Determines the current beta build, opens the "confirm the release" block step, and posts to +# Slack. No build — just gems + secrets. + +echo "--- :rubygems: Setting up Gems" +install_gems + +echo "--- :closed_lock_with_key: Installing Secrets" +bundle exec fastlane run configure_apply + +echo "--- :android: Gathering the production candidate and opening the block step" +bundle exec fastlane gather_production_candidate diff --git a/.buildkite/commands/promote-to-production.sh b/.buildkite/commands/promote-to-production.sh new file mode 100755 index 000000000000..0a82bd77afe8 --- /dev/null +++ b/.buildkite/commands/promote-to-production.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -eu + +# Releases the build gathered by the preceding step to the production track, once a developer has +# confirmed the block step. The version code is passed as the first argument; the Yes/No +# confirmation is read from Buildkite meta-data. No build — just gems + secrets. + +VERSION_CODE="${1:-}" + +# `production_ready_to_release` must stay in sync with PRODUCTION_CONFIRM_META_DATA_KEY in +# fastlane/lanes/promote.rb, which is the key the block-step confirmation field writes. +READY="$(buildkite-agent meta-data get "production_ready_to_release" --default "no")" + +if [[ "${READY}" != "yes" ]]; then + echo "--- :no_entry_sign: Release not confirmed (production_ready_to_release=${READY}); skipping." + exit 0 +fi + +if [[ -z "${VERSION_CODE}" ]]; then + echo "+++ :x: No version code was provided to release." + exit 1 +fi + +echo "--- :rubygems: Setting up Gems" +install_gems + +echo "--- :closed_lock_with_key: Installing Secrets" +bundle exec fastlane run configure_apply + +echo "--- :rocket: Releasing ${VERSION_CODE} to the production track" +bundle exec fastlane promote_to_production version_code:"${VERSION_CODE}" diff --git a/.buildkite/promote-production.yml b/.buildkite/promote-production.yml new file mode 100644 index 000000000000..962def6b425b --- /dev/null +++ b/.buildkite/promote-production.yml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +--- + +# Production release — run manually via the `PIPELINE` env var. Determines the current beta build, +# opens a block step to confirm the release, and releases it to the production track. + +agents: + queue: "android" + +steps: + - label: ":android: Gather build to release to production" + command: ".buildkite/commands/gather-production-candidate.sh" + plugins: [$CI_TOOLKIT] + +# Slack backstop for any failed build — its unique value is failures that abort before the lane runs +# (gems/secrets) and can't self-report; on a lane failure it may also accompany the lane's own +# message. Channel intentionally stays the test channel for now (see send_slack_message). +notify: + - slack: "#test-wpmobile-slack-integration" + if: build.state == "failed" diff --git a/fastlane/lanes/promote.rb b/fastlane/lanes/promote.rb index 09e3bc063e7e..10fe35f01396 100644 --- a/fastlane/lanes/promote.rb +++ b/fastlane/lanes/promote.rb @@ -4,24 +4,38 @@ require 'net/http' require 'yaml' -# Promotes an already-uploaded build to the Play Store `beta` track without rebuilding. -# `gather_beta_candidates` lists the promotable builds and opens a Buildkite block step for a -# developer to pick one; `promote_to_beta` promotes the chosen version code for both apps. +# Promotes already-uploaded builds between Play Store tracks without rebuilding. # -# WordPress and Jetpack share a version code from the same build, so one pick promotes both. +# Beta (internal → beta): `gather_beta_candidates` lists the promotable builds and opens a Buildkite +# block step for a developer to pick one; `promote_to_beta` promotes the chosen version code. +# +# Production (beta → production): `gather_production_candidate` takes the current beta build and opens +# a confirmation block step; `promote_to_production` promotes it to production. +# +# WordPress and Jetpack share a version code from the same build, so one action promotes both apps. BETA_TRACK = 'beta' +PRODUCTION_TRACK = 'production' +# Beta promotion (picker). # The most candidates to offer in the picker. -PROMOTION_CANDIDATE_LIMIT = 12 - +BETA_CANDIDATE_LIMIT = 12 # The block step writes the chosen version code here; the promote step reads it back. # NOTE: `.buildkite/commands/promote-to-beta.sh` reads this same key as a bare string literal # (`meta-data get "beta_build_to_promote"`) — keep the two in sync. -PROMOTION_META_DATA_KEY = 'beta_build_to_promote' +BETA_META_DATA_KEY = 'beta_build_to_promote' # Matched via the Buildkite API to find the block step's job and build its unblock URL. -PROMOTION_BLOCK_LABEL = ':android: Promote to beta' -PROMOTION_BLOCK_STEP_KEY = 'promote_to_beta_block' +BETA_BLOCK_LABEL = ':android: Promote to beta' +BETA_BLOCK_STEP_KEY = 'promote_to_beta_block' + +# Production release (confirm). +# The confirmation field's Yes/No value is written here; the release step reads it back. +# NOTE: `.buildkite/commands/promote-to-production.sh` reads this same key as a bare string literal +# (`meta-data get "production_ready_to_release"`) — keep the two in sync. +PRODUCTION_CONFIRM_META_DATA_KEY = 'production_ready_to_release' +# Matched via the Buildkite API to find the block step's job and build its unblock URL. +PRODUCTION_BLOCK_LABEL = ':android: Promote to production' +PRODUCTION_BLOCK_STEP_KEY = 'promote_to_production_block' # Written verbatim into the generated steps so `buildkite-agent pipeline upload` interpolates it. CI_TOOLKIT_PLUGIN_REF = '$CI_TOOLKIT' @@ -55,9 +69,12 @@ next end - write_promotion_steps_file(candidates: candidates) + write_beta_promotion_steps_file(candidates: candidates) upload_promotion_steps - post_candidates_to_slack(candidates: candidates, pick_url: promotion_unblock_dialog_url) + post_beta_candidates_to_slack( + candidates: candidates, + pick_url: promotion_unblock_dialog_url(block_step_key: BETA_BLOCK_STEP_KEY, block_label: BETA_BLOCK_LABEL) + ) UI.success("Prepared #{candidates.count} promotion candidate(s) and opened the block step.") rescue StandardError => e @@ -85,7 +102,7 @@ results = distribute_to_beta(version_code: version_code) - post_promotion_result_to_slack(version_code: version_code, results: results) + post_beta_result_to_slack(version_code: version_code, results: results) result_posted = true failed = results.reject { |_, result| result[:ok] }.keys @@ -97,17 +114,80 @@ raise end + # Determines the production release candidate — the current beta build, when it's ahead of + # production — and opens a confirmation block step for a developer to approve the release. + # + # @called_by CI (`.buildkite/commands/gather-production-candidate.sh`) + desc 'Gather the production release candidate and open the confirmation block step' + lane :gather_production_candidate do + ensure_promotion_on_trunk! + # Fail loudly up front if Slack isn't configured (see gather_beta_candidates). + get_required_env('SLACK_WEBHOOK') + + version_code = production_promotion_candidate + + if version_code.nil? + notify_slack(':android: *Production release* — the beta build is not ahead of production. Nothing to release.') + UI.important('No production release candidate; skipping block step generation.') + next + end + + write_production_release_steps_file(version_code: version_code) + upload_promotion_steps + post_production_candidate_to_slack( + version_code: version_code, + confirm_url: promotion_unblock_dialog_url(block_step_key: PRODUCTION_BLOCK_STEP_KEY, block_label: PRODUCTION_BLOCK_LABEL) + ) + + UI.success("Prepared production release candidate #{version_code} and opened the block step.") + rescue StandardError => e + notify_slack(":x: *Production release* failed before the confirmation could open — #{e.message}") + raise + end + + # Promotes the confirmed build to the `production` track for both WordPress and Jetpack. + # + # @param [String] version_code The version code to promote, e.g. `270084231`. + # @called_by CI (`.buildkite/commands/promote-to-production.sh`) + desc 'Promote an existing build to the production track (WordPress + Jetpack)' + lane :promote_to_production do |version_code: nil| + # Set once the per-app result has been posted, so the rescue doesn't double-report it. + result_posted = false + ensure_promotion_on_trunk! + # Fail loudly up front if Slack isn't configured (see gather_beta_candidates). + get_required_env('SLACK_WEBHOOK') + + version_code = version_code.to_s.strip + UI.user_error!('`version_code` is required, e.g. `version_code:270084231`') if version_code.empty? + UI.user_error!("`version_code` must be an integer, got #{version_code.inspect}") unless version_code.match?(/\A\d+\z/) + + UI.important("Promoting version code #{version_code} to the production track for WordPress and Jetpack") + + results = distribute_to_production(version_code: version_code) + + post_production_result_to_slack(version_code: version_code, results: results) + result_posted = true + + failed = results.reject { |_, result| result[:ok] }.keys + UI.user_error!("Production release failed for: #{failed.join(', ')}") unless failed.empty? + + UI.success("Promoted #{version_code} to production for: #{results.keys.join(', ')}") + rescue StandardError => e + notify_slack(":x: *Production release* failed — #{e.message}") unless result_posted + raise + end + ################################################# # Candidate discovery ################################################# # The promotable version codes: present in both apps' Play libraries and above the current beta - # release, newest first, capped to PROMOTION_CANDIDATE_LIMIT. + # release, newest first, capped to BETA_CANDIDATE_LIMIT. def beta_promotion_candidates apps = %i[wordpress jetpack] combined_floor = apps.filter_map do |app| - codes = beta_track_version_codes(package_name: APP_SPECIFIC_VALUES[app][:package_name]) + codes = track_version_codes(package_name: APP_SPECIFIC_VALUES[app][:package_name], track: BETA_TRACK) UI.message("#{app}: beta track version codes = #{codes.sort.inspect}") codes.max end.max @@ -120,20 +200,50 @@ def beta_promotion_candidates end common = available_per_app.reduce(:&) || [] - common.select { |code| combined_floor.nil? || code > combined_floor }.max(PROMOTION_CANDIDATE_LIMIT) + common.select { |code| combined_floor.nil? || code > combined_floor }.max(BETA_CANDIDATE_LIMIT) + end + + # The single production release candidate: the version code both apps share on the `beta` track, + # when it's ahead of production. Returns nil when there's nothing to release (no beta build, or + # beta not ahead of production). + def production_promotion_candidate + beta_code = common_track_version_code(track: BETA_TRACK, label: 'beta') + return nil if beta_code.nil? + + production_code = common_track_version_code(track: PRODUCTION_TRACK, label: 'production') + return nil if production_code && beta_code <= production_code + + beta_code + end + + # The version code both apps share on the given track (they promote together, so it must match), or + # nil when neither app has a release there. Raises when the two apps disagree — a mismatch means an + # earlier promotion only half-completed and must be reconciled by hand before releasing. + def common_track_version_code(track:, label:) + codes = %i[wordpress jetpack].map do |app| + code = track_version_codes(package_name: APP_SPECIFIC_VALUES[app][:package_name], track: track).max + UI.message("#{app}: #{label} version code = #{code.inspect}") + code + end + return nil if codes.all?(&:nil?) + + unless codes.uniq.one? + UI.user_error!("WordPress and Jetpack #{label} version codes differ (#{codes.inspect}); reconcile by hand before releasing.") + end + codes.first end - # Reads the version codes currently on the given package's `beta` track, as an array of integers. + # Reads the version codes currently on the given package's track, as an array of integers. # Returns an empty array only when the track legitimately has no releases; a lookup error raises. - def beta_track_version_codes(package_name:) + def track_version_codes(package_name:, track:) google_play_track_version_codes( package_name: package_name, - track: BETA_TRACK, + track: track, json_key: UPLOAD_TO_PLAY_STORE_JSON_KEY ) rescue StandardError => e - # Raise rather than return []: an errored lookup must not read as an empty beta track. - UI.user_error!("Unable to read the beta track version codes for #{package_name}: #{e.message}") + # Raise rather than return []: an errored lookup must not read as an empty track. + UI.user_error!("Unable to read the #{track} track version codes for #{package_name}: #{e.message}") end # Play intermittently 400s a call with "Google Api Error: ... This Edit has been deleted." Neither @@ -177,7 +287,7 @@ def available_aab_version_codes(package_name:) end Array(codes).compact.map(&:to_i) rescue StandardError => e - # Raise rather than return [], as in beta_track_version_codes. + # Raise rather than return [], as in track_version_codes. UI.user_error!("Unable to list the available AAB version codes for #{package_name}: #{e.message}") end @@ -243,25 +353,82 @@ def promote_version_code_to_beta(package_name:, version_code:) end end + # Promotes a version code to production for each app, returning a per-app `{ ok:, error: }` result. + # A failure for one app doesn't stop the other. + def distribute_to_production(version_code:) + %i[wordpress jetpack].to_h do |app| + result = + begin + promote_version_code_to_production( + package_name: APP_SPECIFIC_VALUES[app][:package_name], + version_code: version_code + ) + { ok: true } + rescue StandardError => e + UI.error("Failed to promote #{app} (#{version_code}): #{e.message}") + { ok: false, error: e.message } + end + [app, result] + end + end + + # Creates a `production` release referencing an already-uploaded version code, via the Play API — + # the same approach as promote_version_code_to_beta (see there for why upload_to_play_store can't). + def promote_version_code_to_production(package_name:, version_code:) + require 'supply' + require 'supply/options' + + Supply.config = FastlaneCore::Configuration.create( + Supply::Options.available_options, + { json_key: UPLOAD_TO_PLAY_STORE_JSON_KEY, package_name: package_name, track: PRODUCTION_TRACK } + ) + + with_play_edit_retries("Promoting #{version_code} to production for #{package_name}") do + client = Supply::Client.make_from_config + client.begin_edit(package_name: package_name) + + committed = false + begin + release = AndroidPublisher::TrackRelease.new( + # TODO: swap `draft` for the staged rollout below once the full production flow is ready; + # until then it ships as a draft a human starts from the Play Console. + status: 'draft', + # status: 'inProgress', + # user_fraction: 0.001, + version_codes: [Integer(version_code), *PLAY_STORE_VERSION_CODES_TO_RETAIN] + ) + track = client.tracks(PRODUCTION_TRACK).first || AndroidPublisher::Track.new(track: PRODUCTION_TRACK) + track.releases = [release] + + client.update_track(PRODUCTION_TRACK, track) + client.commit_current_edit! + committed = true + ensure + # Discard the edit if we bailed before committing (a committed edit can't be aborted). + client.abort_current_edit unless committed + end + end + end + ################################################# # Buildkite block step ################################################# - def write_promotion_steps_file(candidates:) + def write_beta_promotion_steps_file(candidates:) options = candidates.map { |code| { 'label' => code.to_s, 'value' => code.to_s } } steps = { 'steps' => [ { - 'block' => PROMOTION_BLOCK_LABEL, - 'key' => PROMOTION_BLOCK_STEP_KEY, + 'block' => BETA_BLOCK_LABEL, + 'key' => BETA_BLOCK_STEP_KEY, 'prompt' => 'Choose the build to release to beta testers. This promotes the matching WordPress and Jetpack builds.', # Keep the build "running" (not green) while it waits for a human. 'blocked_state' => 'running', 'fields' => [ { 'select' => 'Build to promote', - 'key' => PROMOTION_META_DATA_KEY, + 'key' => BETA_META_DATA_KEY, # Required, no default: an un-actioned unblock can't silently promote a build. 'required' => true, 'options' => options @@ -283,6 +450,46 @@ def write_promotion_steps_file(candidates:) UI.message("Wrote promotion steps for #{candidates.count} build(s) to #{PROMOTION_STEPS_FILE}") end + # Writes the confirmation block step + the release step. There's no picker — the candidate is baked + # into the release command; the block step only gates on a Yes/No confirmation. + def write_production_release_steps_file(version_code:) + steps = { + 'steps' => [ + { + 'block' => PRODUCTION_BLOCK_LABEL, + 'key' => PRODUCTION_BLOCK_STEP_KEY, + 'prompt' => "Release build #{version_code} to production? This releases the matching WordPress and Jetpack builds.", + # Keep the build "running" (not green) while it waits for a human. + 'blocked_state' => 'running', + 'fields' => [ + { + 'select' => 'Release to production?', + 'key' => PRODUCTION_CONFIRM_META_DATA_KEY, + # Required, no default: an un-actioned unblock can't silently release a build. + 'required' => true, + 'options' => [ + { 'label' => 'Yes', 'value' => 'yes' }, + { 'label' => 'No', 'value' => 'no' } + ] + } + ] + }, + { + 'label' => ':rocket: Release build to production', + # The candidate rides along as an argument; the confirmation Yes/No comes from meta-data. + 'command' => ".buildkite/commands/promote-to-production.sh #{version_code}", + 'plugins' => [CI_TOOLKIT_PLUGIN_REF], + 'agents' => { 'queue' => 'android' } + } + ] + } + + FileUtils.mkdir_p(File.dirname(PROMOTION_STEPS_FILE)) + # `line_width: -1` keeps each label on one line (no YAML folding). + File.write(PROMOTION_STEPS_FILE, steps.to_yaml(line_width: -1)) + UI.message("Wrote production release steps for build #{version_code} to #{PROMOTION_STEPS_FILE}") + end + # Source `shared-pipeline-vars` first so `$CI_TOOLKIT` is interpolated. def upload_promotion_steps sh('bash', '-c', "cd '#{PROJECT_ROOT_FOLDER}' && source .buildkite/shared-pipeline-vars && buildkite-agent pipeline upload '#{PROMOTION_STEPS_RELATIVE_PATH}'") @@ -294,7 +501,7 @@ def upload_promotion_steps # Posts the candidate list, linking to the unblock dialog when the job id resolved, otherwise to # the build so a degraded link isn't mistaken for the picker. - def post_candidates_to_slack(candidates:, pick_url: nil) + def post_beta_candidates_to_slack(candidates:, pick_url: nil) build_url = ENV.fetch('BUILDKITE_BUILD_URL', nil) candidate_lines = candidates.map { |code| "• `#{code}`" } @@ -318,8 +525,8 @@ def post_candidates_to_slack(candidates:, pick_url: nil) ) end - # Posts the per-app outcome of a promotion. - def post_promotion_result_to_slack(version_code:, results:) + # Posts the per-app outcome of a beta promotion. + def post_beta_result_to_slack(version_code:, results:) status_lines = results.map do |app, result| next "• #{app}: :x: #{result[:error]}" unless result[:ok] @@ -338,29 +545,73 @@ def post_promotion_result_to_slack(version_code:, results:) ) end + # Posts the candidate, linking to the unblock dialog when the job id resolved, otherwise to the + # build so a degraded link isn't mistaken for the confirmation. + def post_production_candidate_to_slack(version_code:, confirm_url: nil) + build_url = ENV.fetch('BUILDKITE_BUILD_URL', nil) + + confirm_line = + if confirm_url + "\n\n:point_right: <#{confirm_url}|Confirm the production release>" + elsif build_url + "\n\n:warning: Couldn't deep-link to the confirmation — open <#{build_url}|the build> and unblock the release step." + else + '' + end + + notify_slack( + <<~MSG + :android: *Production release* — confirm releasing build `#{version_code}` to production (WordPress + Jetpack).#{confirm_line} + MSG + ) + end + + # Posts the per-app outcome of a production release. + def post_production_result_to_slack(version_code:, results:) + status_lines = results.map do |app, result| + next "• #{app}: :x: #{result[:error]}" unless result[:ok] + + "• #{app}: :white_check_mark: draft created on production" + end + + all_ok = results.values.all? { |result| result[:ok] } + header = all_ok ? ':rocket: *Production draft created*' : ':warning: *Production release finished with errors*' + + notify_slack( + <<~MSG + #{header} — `#{version_code}` + + #{status_lines.join("\n")} + MSG + ) + end + ################################################# # Buildkite deep link ################################################# # The block step's unblock-dialog URL, or nil (callers fall back to the build URL). - def promotion_unblock_dialog_url + def promotion_unblock_dialog_url(block_step_key:, block_label:) org = ENV.fetch('BUILDKITE_ORGANIZATION_SLUG', BUILDKITE_ORGANIZATION) pipeline = ENV.fetch('BUILDKITE_PIPELINE_SLUG', BUILDKITE_PIPELINE) build_number = ENV.fetch('BUILDKITE_BUILD_NUMBER', nil) return nil if build_number.nil? - job_id = block_step_job_id(org: org, pipeline: pipeline, build_number: build_number) + job_id = block_step_job_id( + org: org, pipeline: pipeline, build_number: build_number, + block_step_key: block_step_key, block_label: block_label + ) return nil if job_id.nil? "https://buildkite.com/organizations/#{org}/pipelines/#{pipeline}/builds/#{build_number}/jobs/#{job_id}/unblock_dialog" end # Polls the build for the just-uploaded block step's job id (it takes a moment to register). - def block_step_job_id(org:, pipeline:, build_number:) + def block_step_job_id(org:, pipeline:, build_number:, block_step_key:, block_label:) uri = URI("https://api.buildkite.com/v2/organizations/#{org}/pipelines/#{pipeline}/builds/#{build_number}") 5.times do |attempt| - job = find_promotion_block_job(uri) + job = find_promotion_block_job(uri, block_step_key: block_step_key, block_label: block_label) return job['id'] if job sleep(2) unless attempt == 4 @@ -373,15 +624,15 @@ def block_step_job_id(org:, pipeline:, build_number:) nil end - def find_promotion_block_job(uri) + def find_promotion_block_job(uri, block_step_key:, block_label:) response = buildkite_api_get(uri) return nil unless response.is_a?(Net::HTTPSuccess) manual_jobs = JSON.parse(response.body).fetch('jobs', []).select { |job| job['type'] == 'manual' } # Prefer the stable step key; fall back to the label only if the API doesn't surface it. - manual_jobs.find { |job| job['step_key'] == PROMOTION_BLOCK_STEP_KEY } || - manual_jobs.find { |job| job['label'].to_s.include?(PROMOTION_BLOCK_LABEL) } + manual_jobs.find { |job| job['step_key'] == block_step_key } || + manual_jobs.find { |job| job['label'].to_s.include?(block_label) } end def buildkite_api_get(uri) From a25c5673fec1f1a3fa010681505ed4410dd175e4 Mon Sep 17 00:00:00 2001 From: Oguz Kocer Date: Wed, 15 Jul 2026 03:40:41 -0400 Subject: [PATCH 2/2] Use modifier unless in common_track_version_code (Style/IfUnlessModifier) --- fastlane/lanes/promote.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fastlane/lanes/promote.rb b/fastlane/lanes/promote.rb index 10fe35f01396..81b282b86bfe 100644 --- a/fastlane/lanes/promote.rb +++ b/fastlane/lanes/promote.rb @@ -227,9 +227,7 @@ def common_track_version_code(track:, label:) end return nil if codes.all?(&:nil?) - unless codes.uniq.one? - UI.user_error!("WordPress and Jetpack #{label} version codes differ (#{codes.inspect}); reconcile by hand before releasing.") - end + UI.user_error!("WordPress and Jetpack #{label} version codes differ (#{codes.inspect}); reconcile by hand before releasing.") unless codes.uniq.one? codes.first end