From 81d2bdd819af710ea1be1959d6b6389472eeaa97 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 6 May 2026 15:30:42 -0500 Subject: [PATCH 01/14] GH-3547: Add semi-automated release pipeline for Apache Parquet Java Adds a release automation framework modeled after Apache Polaris, adapted for Parquet's Maven-based build. Replaces the manual maven-release-plugin workflow with explicit, scriptable steps that support both CI (GitHub Actions) and local execution, with dry-run by default. Scripts (release/bin/): - prepare-rc.sh: full pre-vote flow (branch, version, tag, Nexus, SVN, GitHub pre-release, vote email) - publish-release.sh: full post-vote flow (SVN promotion, final tag, Nexus release, GitHub release, version bump, announce email) - cancel-rc.sh: rollback a failed RC (Nexus drop, SVN cleanup) Shared libraries (release/libs/): - _constants.sh, _log.sh, _exec.sh, _version.sh - _github.sh, _nexus.sh, _maven.sh GitHub Actions workflows: - release-prepare-rc.yml, release-publish.yml, release-cancel-rc.yml - ci-release-scripts.yml (bats unit tests on PR/push) Includes 85 bats unit tests covering all shared libraries. --- .github/workflows/ci-release-scripts.yml | 59 ++++ .github/workflows/release-cancel-rc.yml | 74 +++++ .github/workflows/release-prepare-rc.yml | 87 +++++ .github/workflows/release-publish.yml | 93 ++++++ release/bin/cancel-rc.sh | 172 ++++++++++ release/bin/prepare-rc.sh | 395 +++++++++++++++++++++++ release/bin/publish-release.sh | 379 ++++++++++++++++++++++ release/libs/_constants.sh | 41 +++ release/libs/_exec.sh | 92 ++++++ release/libs/_github.sh | 77 +++++ release/libs/_log.sh | 67 ++++ release/libs/_maven.sh | 104 ++++++ release/libs/_nexus.sh | 111 +++++++ release/libs/_version.sh | 132 ++++++++ release/tests/constants.bats | 83 +++++ release/tests/exec.bats | 101 ++++++ release/tests/github.bats | 112 +++++++ release/tests/log.bats | 88 +++++ release/tests/maven.bats | 117 +++++++ release/tests/nexus.bats | 108 +++++++ release/tests/test_helper/common.bash | 36 +++ release/tests/version.bats | 332 +++++++++++++++++++ 22 files changed, 2860 insertions(+) create mode 100644 .github/workflows/ci-release-scripts.yml create mode 100644 .github/workflows/release-cancel-rc.yml create mode 100644 .github/workflows/release-prepare-rc.yml create mode 100644 .github/workflows/release-publish.yml create mode 100755 release/bin/cancel-rc.sh create mode 100755 release/bin/prepare-rc.sh create mode 100755 release/bin/publish-release.sh create mode 100644 release/libs/_constants.sh create mode 100644 release/libs/_exec.sh create mode 100644 release/libs/_github.sh create mode 100644 release/libs/_log.sh create mode 100644 release/libs/_maven.sh create mode 100644 release/libs/_nexus.sh create mode 100644 release/libs/_version.sh create mode 100644 release/tests/constants.bats create mode 100644 release/tests/exec.bats create mode 100644 release/tests/github.bats create mode 100644 release/tests/log.bats create mode 100644 release/tests/maven.bats create mode 100644 release/tests/nexus.bats create mode 100644 release/tests/test_helper/common.bash create mode 100644 release/tests/version.bats diff --git a/.github/workflows/ci-release-scripts.yml b/.github/workflows/ci-release-scripts.yml new file mode 100644 index 0000000000..0ffcad84b8 --- /dev/null +++ b/.github/workflows/ci-release-scripts.yml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Test Release Scripts + +on: + pull_request: + paths: + - 'release/**' + push: + branches: + - master + paths: + - 'release/**' + +jobs: + bats: + name: Release Script Unit Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install bats-core + run: | + sudo apt-get update + sudo apt-get install -y bats + + - name: Run bats tests + run: bats release/tests/*.bats + + - name: Verify scripts are executable + run: | + for script in release/bin/*.sh; do + if [[ ! -x "$script" ]]; then + echo "ERROR: $script is not executable" + exit 1 + fi + done + echo "All scripts are executable" diff --git a/.github/workflows/release-cancel-rc.yml b/.github/workflows/release-cancel-rc.yml new file mode 100644 index 0000000000..ff51173608 --- /dev/null +++ b/.github/workflows/release-cancel-rc.yml @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Release - Cancel RC + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g., 1.18.0)' + required: true + type: string + rc_number: + description: 'RC number to cancel (e.g., 0)' + required: true + type: string + staging_repo_id: + description: 'Nexus staging repository ID to drop (e.g., orgapacheparquet-1234)' + required: true + type: string + dry_run: + description: 'Dry run mode (no actual changes)' + required: false + type: boolean + default: true + +jobs: + cancel-rc: + name: Cancel Release Candidate + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y subversion + + - name: Cancel Release Candidate + env: + DRY_RUN: ${{ inputs.dry_run && '1' || '0' }} + NEXUS_USERNAME: ${{ secrets.PARQUET_NEXUS_USER }} + NEXUS_PASSWORD: ${{ secrets.PARQUET_NEXUS_PASSWORD }} + SVN_USERNAME: ${{ secrets.PARQUET_SVN_DEV_USERNAME }} + SVN_PASSWORD: ${{ secrets.PARQUET_SVN_DEV_PASSWORD }} + INPUT_VERSION: ${{ inputs.version }} + INPUT_RC_NUMBER: ${{ inputs.rc_number }} + INPUT_STAGING_REPO_ID: ${{ inputs.staging_repo_id }} + run: | + ./release/bin/cancel-rc.sh \ + "${INPUT_VERSION}" \ + "${INPUT_RC_NUMBER}" \ + "${INPUT_STAGING_REPO_ID}" diff --git a/.github/workflows/release-prepare-rc.yml b/.github/workflows/release-prepare-rc.yml new file mode 100644 index 0000000000..47d5e42acb --- /dev/null +++ b/.github/workflows/release-prepare-rc.yml @@ -0,0 +1,87 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Release - Prepare RC + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g., 1.18.0)' + required: true + type: string + rc_number: + description: 'RC number override (leave empty for auto-detect)' + required: false + type: string + default: '' + dry_run: + description: 'Dry run mode (no actual changes)' + required: false + type: boolean + default: true + +jobs: + prepare-rc: + name: Prepare Release Candidate + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '11' + + - name: Import GPG key and configure Git signing + env: + GPG_PRIVATE_KEY: ${{ secrets.PARQUET_GPG_PRIVATE_KEY }} + run: | + echo "${GPG_PRIVATE_KEY}" | gpg --batch --import + KEY_ID=$(gpg --list-keys --with-colons | grep '^fpr' | head -1 | cut -d: -f10) + git config --global user.signingkey "${KEY_ID}" + git config --global commit.gpgsign true + + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y subversion + + - name: Prepare Release Candidate + env: + DRY_RUN: ${{ inputs.dry_run && '1' || '0' }} + NEXUS_USERNAME: ${{ secrets.PARQUET_NEXUS_USER }} + NEXUS_PASSWORD: ${{ secrets.PARQUET_NEXUS_PASSWORD }} + SVN_USERNAME: ${{ secrets.PARQUET_SVN_DEV_USERNAME }} + SVN_PASSWORD: ${{ secrets.PARQUET_SVN_DEV_PASSWORD }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUT_VERSION: ${{ inputs.version }} + INPUT_RC_NUMBER: ${{ inputs.rc_number }} + run: | + args=("${INPUT_VERSION}") + if [[ -n "${INPUT_RC_NUMBER}" ]]; then + args+=(--rc "${INPUT_RC_NUMBER}") + fi + ./release/bin/prepare-rc.sh "${args[@]}" diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml new file mode 100644 index 0000000000..a909343d16 --- /dev/null +++ b/.github/workflows/release-publish.yml @@ -0,0 +1,93 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Release - Publish After Vote + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g., 1.18.0)' + required: true + type: string + rc_number: + description: 'RC number that passed the vote (leave empty to auto-detect latest)' + required: false + type: string + default: '' + staging_repo_id: + description: 'Nexus staging repository ID (e.g., orgapacheparquet-1234)' + required: true + type: string + next_dev_version: + description: 'Next development version without -SNAPSHOT (e.g., 1.18.1 or 1.19.0)' + required: true + type: string + dry_run: + description: 'Dry run mode (no actual changes)' + required: false + type: boolean + default: true + +jobs: + publish-release: + name: Publish Release + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '11' + + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y subversion + + - name: Configure Git + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Publish Release + env: + DRY_RUN: ${{ inputs.dry_run && '1' || '0' }} + NEXUS_USERNAME: ${{ secrets.PARQUET_NEXUS_USER }} + NEXUS_PASSWORD: ${{ secrets.PARQUET_NEXUS_PASSWORD }} + SVN_USERNAME: ${{ secrets.PARQUET_SVN_DEV_USERNAME }} + SVN_PASSWORD: ${{ secrets.PARQUET_SVN_DEV_PASSWORD }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUT_VERSION: ${{ inputs.version }} + INPUT_RC_NUMBER: ${{ inputs.rc_number }} + INPUT_STAGING_REPO_ID: ${{ inputs.staging_repo_id }} + INPUT_NEXT_DEV_VERSION: ${{ inputs.next_dev_version }} + run: | + args=("${INPUT_VERSION}" "${INPUT_STAGING_REPO_ID}" "${INPUT_NEXT_DEV_VERSION}") + if [[ -n "${INPUT_RC_NUMBER}" ]]; then + args+=(--rc "${INPUT_RC_NUMBER}") + fi + ./release/bin/publish-release.sh "${args[@]}" diff --git a/release/bin/cancel-rc.sh b/release/bin/cancel-rc.sh new file mode 100755 index 0000000000..26f3ab8647 --- /dev/null +++ b/release/bin/cancel-rc.sh @@ -0,0 +1,172 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIBS_DIR="${SCRIPT_DIR}/../libs" + +source "${LIBS_DIR}/_constants.sh" +source "${LIBS_DIR}/_log.sh" +source "${LIBS_DIR}/_exec.sh" +source "${LIBS_DIR}/_version.sh" +source "${LIBS_DIR}/_nexus.sh" + +# --------------------------------------------------------------------------- +# Usage +# --------------------------------------------------------------------------- +function usage { + cat < + +Cancel a release candidate after a failed vote. + +Arguments: + version Release version (e.g., 1.18.0) + rc-num RC number to cancel (e.g., 0) + staging-repo-id Nexus staging repository ID (e.g., orgapacheparquet-1234) + +Environment variables: + DRY_RUN Set to 0 for real execution (default: 1) + NEXUS_USERNAME Apache Nexus username + NEXUS_PASSWORD Apache Nexus password + SVN_USERNAME SVN username for dist.apache.org + SVN_PASSWORD SVN password + +Example: + DRY_RUN=1 $0 1.18.0 0 orgapacheparquet-1234 +EOF + exit "${1:-0}" +} + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +if [[ $# -lt 3 ]]; then + print_error "Expected 3 arguments, got $#" + usage 1 +fi + +version="$1" +rc_num="$2" +staging_repo_id="$3" + +# --------------------------------------------------------------------------- +# Validate inputs +# --------------------------------------------------------------------------- +step_summary "## Release Candidate Cancellation" +step_summary "" + +if [[ ${DRY_RUN:-1} -eq 1 ]]; then + step_summary "> **DRY RUN** -- no changes will be made" + step_summary "" +fi + +if ! validate_and_extract_version "${version}"; then + print_error "Invalid version format: '${version}'" + exit 1 +fi + +if ! [[ "${rc_num}" =~ ^[0-9]+$ ]]; then + print_error "Invalid RC number: '${rc_num}'. Expected a non-negative integer." + exit 1 +fi + +if ! [[ "${staging_repo_id}" =~ ^[a-zA-Z][a-zA-Z0-9._-]*$ ]]; then + print_error "Invalid staging repository ID: '${staging_repo_id}'. Expected alphanumeric with dots/hyphens (e.g., orgapacheparquet-1234)." + exit 1 +fi + +rc_tag="${TAG_PREFIX}${version}-rc${rc_num}" + +step_summary "| Parameter | Value |" +step_summary "| --- | --- |" +step_summary "| Version | \`${version}\` |" +step_summary "| RC tag | \`${rc_tag}\` |" +step_summary "| Staging repo | \`${staging_repo_id}\` |" + +# --------------------------------------------------------------------------- +# Step 1: Drop Nexus staging repo +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Nexus Cleanup" + +nexus_drop_staging_repo "${staging_repo_id}" "Cancel Apache Parquet ${version} RC${rc_num}" + +step_summary "Dropped staging repository \`${staging_repo_id}\`" + +# --------------------------------------------------------------------------- +# Step 2: Delete SVN artifacts from dist/dev +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### SVN Cleanup" + +dev_url="${APACHE_DIST_URL}${APACHE_DIST_DEV_PATH}/${rc_tag}" + +if [[ ${DRY_RUN:-1} -ne 1 ]]; then + if svn ls --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive "${dev_url}" >/dev/null 2>&1; then + exec_process svn rm \ + --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + "${dev_url}" \ + -m "Cancel Apache Parquet ${version} RC${rc_num}" + step_summary "Deleted \`${dev_url}\`" + else + print_warning "SVN directory not found: ${dev_url}" + step_summary "Directory not found at \`${dev_url}\` (may already be deleted)" + fi +else + print_command "Dry-run, WOULD delete ${dev_url}" + step_summary "Would delete \`${dev_url}\` (dry-run)" +fi + +# --------------------------------------------------------------------------- +# Step 3: Generate vote failure email +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Vote Failure Email" +step_summary "" +step_summary '```' +step_summary "Subject: [RESULT][VOTE] Release Apache Parquet ${version} RC${rc_num}" +step_summary "" +step_summary "Hello everyone," +step_summary "" +step_summary "Thanks to all who participated in the vote for Release Apache Parquet ${version} (rc${rc_num})." +step_summary "" +step_summary "The vote failed due to [REASON - TO BE FILLED BY RELEASE MANAGER]." +step_summary "" +step_summary "A new release candidate will be proposed soon once the issues are addressed." +step_summary "" +step_summary "Thanks," +step_summary '```' + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +step_summary "" +step_summary "---" +step_summary "### Summary" +step_summary "" +step_summary "| Step | Status |" +step_summary "| --- | --- |" +step_summary "| Nexus staging repo | dropped |" +step_summary "| SVN dist/dev | deleted |" +step_summary "| Failure email | generated |" + +print_success "Release candidate ${rc_tag} cancelled successfully." diff --git a/release/bin/prepare-rc.sh b/release/bin/prepare-rc.sh new file mode 100755 index 0000000000..c19f14b28a --- /dev/null +++ b/release/bin/prepare-rc.sh @@ -0,0 +1,395 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIBS_DIR="${SCRIPT_DIR}/../libs" + +source "${LIBS_DIR}/_constants.sh" +source "${LIBS_DIR}/_log.sh" +source "${LIBS_DIR}/_exec.sh" +source "${LIBS_DIR}/_version.sh" +source "${LIBS_DIR}/_github.sh" +source "${LIBS_DIR}/_nexus.sh" +source "${LIBS_DIR}/_maven.sh" + +trap 'rm -f .release-settings.xml' EXIT + +# --------------------------------------------------------------------------- +# Usage +# --------------------------------------------------------------------------- +function usage { + cat < [OPTIONS] + +Prepare a release candidate for Apache Parquet Java. + +Arguments: + version Release version (e.g., 1.18.0) + +Options: + --rc Override RC number (default: auto-detect) + --skip-branch-creation Do not create the release branch + --help Show this help + +Environment variables: + DRY_RUN Set to 0 for real execution (default: 1) + NEXUS_USERNAME Apache Nexus username + NEXUS_PASSWORD Apache Nexus password + SVN_USERNAME SVN username for dist.apache.org + SVN_PASSWORD SVN password + GITHUB_TOKEN GitHub token for CI checks and release creation + +Example: + DRY_RUN=1 $0 1.18.0 + DRY_RUN=0 $0 1.18.0 --rc 2 +EOF + exit "${1:-0}" +} + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +version="" +rc_override="" +skip_branch=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --rc) + if [[ -z "${2:-}" ]]; then + print_error "--rc requires a value" + usage 1 + fi + rc_override="$2" + if [[ ! "${rc_override}" =~ ^[0-9]+$ ]]; then + print_error "--rc value must be a non-negative integer, got: '${rc_override}'" + exit 1 + fi + shift 2 + ;; + --skip-branch-creation) + skip_branch=true + shift + ;; + --help|-h) + usage 0 + ;; + -*) + print_error "Unknown option: $1" + usage 1 + ;; + *) + if [[ -z "${version}" ]]; then + version="$1" + else + print_error "Unexpected argument: $1" + usage 1 + fi + shift + ;; + esac +done + +if [[ -z "${version}" ]]; then + print_error "Version is required" + usage 1 +fi + +# --------------------------------------------------------------------------- +# Step 0: Validate inputs +# --------------------------------------------------------------------------- +step_summary "## Release Candidate Preparation" +step_summary "" + +if [[ ${DRY_RUN:-1} -eq 1 ]]; then + step_summary "> **DRY RUN** -- no changes will be made" + step_summary "" +fi + +if ! validate_and_extract_version "${version}"; then + print_error "Invalid version format: '${version}'. Expected: X.Y.Z" + exit 1 +fi + +step_summary "| Parameter | Value |" +step_summary "| --- | --- |" +step_summary "| Version | \`${version}\` |" + +# Check prerequisites +if ! command -v gpg &>/dev/null; then + print_warning "gpg not found -- GPG signing will fail" +fi +if ! command -v svn &>/dev/null; then + print_warning "svn not found -- SVN staging will fail" +fi +if [[ ! -x ./mvnw ]]; then + print_error "mvnw not found in current directory. Run from the repo root." + exit 1 +fi + +# --------------------------------------------------------------------------- +# Step 1: Create release branch (idempotent) +# --------------------------------------------------------------------------- +release_branch="${BRANCH_PREFIX}${major}.${minor}.x" +step_summary "| Release branch | \`${release_branch}\` |" + +if [[ "${skip_branch}" == "true" ]]; then + print_info "Skipping branch creation (--skip-branch-creation)" +elif git show-ref --verify --quiet "refs/remotes/origin/${release_branch}" 2>/dev/null; then + print_info "Release branch ${release_branch} already exists, skipping creation" +else + print_info "Creating release branch ${release_branch} from master..." + exec_process git branch "${release_branch}" origin/master + exec_process git push origin "${release_branch}" --set-upstream + step_summary "" + step_summary "Created release branch \`${release_branch}\`" +fi + +# Switch to the release branch +if [[ "$(git branch --show-current)" != "${release_branch}" ]]; then + print_info "Switching to ${release_branch}..." + exec_process git checkout "${release_branch}" +fi + +# --------------------------------------------------------------------------- +# Step 2: Auto-detect RC number +# --------------------------------------------------------------------------- +if [[ -n "${rc_override}" ]]; then + rc_number="${rc_override}" + print_info "Using RC override: rc${rc_number}" +else + find_next_rc_number "${version}" + print_info "Auto-detected next RC: rc${rc_number}" +fi + +rc_tag="${TAG_PREFIX}${version}-rc${rc_number}" +step_summary "| RC number | \`${rc_number}\` |" +step_summary "| RC tag | \`${rc_tag}\` |" + +# Check if tag already exists +if git rev-parse "${rc_tag}" >/dev/null 2>&1; then + print_error "Tag ${rc_tag} already exists. Use --rc to specify a different RC number." + exit 1 +fi + +# --------------------------------------------------------------------------- +# Step 3: Verify CI checks +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### CI Verification" + +current_commit=$(git rev-parse HEAD) +step_summary "| Commit | \`${current_commit}\` |" + +if ! check_github_checks_passed "${current_commit}"; then + print_error "CI checks are not passing. Fix CI before creating an RC." + step_summary "CI checks: **FAILED**" + exit 1 +fi +step_summary "CI checks: **PASSED**" + +# --------------------------------------------------------------------------- +# Step 4: Set POM versions (only on rc0 or if version doesn't match) +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Version Update" + +current_pom_version=$(get_current_pom_version || echo "unknown") +print_info "Current POM version: ${current_pom_version}" + +if [[ "${current_pom_version}" == "${version}" ]]; then + print_info "POM version already set to ${version}, skipping version update" + step_summary "POM version already at \`${version}\`, no update needed" +else + print_info "Setting POM version to ${version}..." + set_pom_version "${version}" + step_summary "Updated POM version: \`${current_pom_version}\` -> \`${version}\`" + + # Commit version changes + exec_process git add -A + exec_process git commit -m "Set version to ${version} for release" +fi + +# --------------------------------------------------------------------------- +# Step 5: Create RC tag and push +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Tag and Push" + +exec_process git tag -a "${rc_tag}" -m "Apache Parquet ${version} RC${rc_number}" +exec_process git push origin "${release_branch}" +exec_process git push origin "${rc_tag}" + +tag_commit=$(git rev-parse HEAD) +step_summary "Created tag \`${rc_tag}\` at \`${tag_commit}\`" + +# --------------------------------------------------------------------------- +# Step 6: Deploy to Nexus +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Nexus Deployment" + +settings_file=".release-settings.xml" +generate_maven_settings "${settings_file}" + +maven_deploy "${settings_file}" + +step_summary "Deployed artifacts to Apache Nexus staging" + +# Find and close the staging repo +nexus_find_open_staging_repo "org.apache.parquet" +nexus_close_staging_repo "${staging_repo_id}" "Apache Parquet ${version} RC${rc_number}" + +step_summary "Closed staging repository: \`${staging_repo_id}\`" + +maven_cleanup_settings "${settings_file}" + +# --------------------------------------------------------------------------- +# Step 7: Build source tarball +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Source Tarball" + +tarball_name="${TAG_PREFIX}${version}.tar.gz" + +if [[ ${DRY_RUN:-1} -ne 1 ]]; then + release_hash=$(git rev-list -1 "${rc_tag}") +else + release_hash=$(git rev-parse HEAD) +fi + +print_info "Building source tarball from ${rc_tag} (${release_hash})..." + +exec_process git archive "${release_hash}" --prefix "${TAG_PREFIX}${version}/" -o "${tarball_name}" +exec_process gpg --armor --output "${tarball_name}.asc" --detach-sig "${tarball_name}" +calculate_sha512 "${tarball_name}" + +step_summary "Built \`${tarball_name}\` from \`${release_hash}\`" + +# --------------------------------------------------------------------------- +# Step 8: Stage to SVN dist/dev +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### SVN Staging" + +svn_dir="${APACHE_DIST_URL}${APACHE_DIST_DEV_PATH}" +rc_svn_dir="${rc_tag}" + +if [[ ${DRY_RUN:-1} -ne 1 ]]; then + if [[ -d tmp/ ]]; then + rm -rf tmp/ + fi + + exec_process_with_retries 5 60 "tmp" \ + svn co --depth=empty --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + "${svn_dir}" tmp + + mkdir -p "tmp/${rc_svn_dir}" + cp "${tarball_name}" "${tarball_name}.asc" "${tarball_name}.sha512" "tmp/${rc_svn_dir}/" + + (cd tmp && exec_process svn add "${rc_svn_dir}") + (cd tmp && exec_process svn ci \ + --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + -m "Apache Parquet ${version} RC${rc_number}") + + rm -rf tmp +else + print_command "Dry-run, WOULD stage to ${svn_dir}/${rc_svn_dir}" +fi + +step_summary "Staged source tarball to \`${svn_dir}/${rc_svn_dir}\`" + +# --------------------------------------------------------------------------- +# Step 9: Create GitHub pre-release +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### GitHub Pre-Release" + +if [[ -n "${GITHUB_TOKEN:-}" ]]; then + exec_process gh release create "${rc_tag}" \ + --title "Apache Parquet ${version} RC${rc_number}" \ + --prerelease \ + --generate-notes \ + --target "${tag_commit}" + step_summary "Created GitHub pre-release for \`${rc_tag}\`" +else + print_warning "GITHUB_TOKEN not set, skipping GitHub pre-release creation" + step_summary "Skipped GitHub pre-release (no GITHUB_TOKEN)" +fi + +# --------------------------------------------------------------------------- +# Step 10: Generate vote email +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Vote Email" +step_summary "" +step_summary '```' +step_summary "Subject: [VOTE] Release Apache Parquet ${version} RC${rc_number}" +step_summary "" +step_summary "Hi everyone," +step_summary "" +step_summary "I propose the following RC to be released as official Apache Parquet ${version} release." +step_summary "" +step_summary "The commit id is ${tag_commit}" +step_summary "* This corresponds to the tag: ${rc_tag}" +step_summary "* https://github.com/apache/parquet-java/tree/${tag_commit}" +step_summary "" +step_summary "The release tarball, signature, and checksums are here:" +step_summary "* https://dist.apache.org/repos/dist/dev/parquet/${rc_tag}" +step_summary "" +step_summary "You can find the KEYS file here:" +step_summary "* https://downloads.apache.org/parquet/KEYS" +step_summary "" +step_summary "You can find the changelog here:" +step_summary "https://github.com/apache/parquet-java/releases/tag/${rc_tag}" +step_summary "" +step_summary "Binary artifacts are staged in Nexus here:" +step_summary "* ${NEXUS_STAGING_GROUP_URL}" +step_summary "* Staging repository ID: ${staging_repo_id:-UNKNOWN}" +step_summary "" +step_summary "Please download, verify, and test." +step_summary "" +step_summary "Please vote in the next 72 hours." +step_summary "" +step_summary "[ ] +1 Release this as Apache Parquet ${version}" +step_summary "[ ] +0" +step_summary "[ ] -1 Do not release this because..." +step_summary '```' + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +step_summary "" +step_summary "---" +step_summary "### Summary" +step_summary "" +step_summary "| Step | Status |" +step_summary "| --- | --- |" +step_summary "| Release branch | \`${release_branch}\` |" +step_summary "| RC tag | \`${rc_tag}\` |" +step_summary "| Nexus staging repo | \`${staging_repo_id:-UNKNOWN}\` |" +step_summary "| Source tarball | \`${tarball_name}\` |" +step_summary "| SVN dist/dev | staged |" +step_summary "| GitHub pre-release | created |" +step_summary "| Vote email | generated |" + +print_success "Release candidate ${rc_tag} prepared successfully!" diff --git a/release/bin/publish-release.sh b/release/bin/publish-release.sh new file mode 100755 index 0000000000..1f9d232f76 --- /dev/null +++ b/release/bin/publish-release.sh @@ -0,0 +1,379 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIBS_DIR="${SCRIPT_DIR}/../libs" + +source "${LIBS_DIR}/_constants.sh" +source "${LIBS_DIR}/_log.sh" +source "${LIBS_DIR}/_exec.sh" +source "${LIBS_DIR}/_version.sh" +source "${LIBS_DIR}/_nexus.sh" +source "${LIBS_DIR}/_maven.sh" + +# --------------------------------------------------------------------------- +# Usage +# --------------------------------------------------------------------------- +function usage { + cat < [--rc ] + +Publish a release after the vote passes. + +Arguments: + version Release version (e.g., 1.18.0) + staging-repo-id Nexus staging repository ID (e.g., orgapacheparquet-1234) + next-dev-version Next development version without -SNAPSHOT (e.g., 1.18.1 or 1.19.0) + +Options: + --rc RC number that passed the vote (default: auto-detect latest) + --help Show this help + +Environment variables: + DRY_RUN Set to 0 for real execution (default: 1) + NEXUS_USERNAME Apache Nexus username + NEXUS_PASSWORD Apache Nexus password + SVN_USERNAME SVN username for dist.apache.org + SVN_PASSWORD SVN password + GITHUB_TOKEN GitHub token for release creation + +Example: + DRY_RUN=1 $0 1.18.0 orgapacheparquet-1234 1.18.1 + DRY_RUN=1 $0 1.18.0 orgapacheparquet-1234 1.18.1 --rc 2 +EOF + exit "${1:-0}" +} + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +version="" +staging_repo_id="" +next_dev_version="" +rc_num="" +positional=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --rc) + if [[ -z "${2:-}" ]]; then + print_error "--rc requires a value" + usage 1 + fi + rc_num="$2" + shift 2 + ;; + --help|-h) + usage 0 + ;; + -*) + print_error "Unknown option: $1" + usage 1 + ;; + *) + positional+=("$1") + shift + ;; + esac +done + +if [[ ${#positional[@]} -lt 3 ]]; then + print_error "Expected 3 positional arguments (version, staging-repo-id, next-dev-version), got ${#positional[@]}" + usage 1 +fi + +version="${positional[0]}" +staging_repo_id="${positional[1]}" +next_dev_version="${positional[2]}" + +# --------------------------------------------------------------------------- +# Validate inputs +# --------------------------------------------------------------------------- +step_summary "## Release Publication" +step_summary "" + +if [[ ${DRY_RUN:-1} -eq 1 ]]; then + step_summary "> **DRY RUN** -- no changes will be made" + step_summary "" +fi + +if ! validate_and_extract_version "${version}"; then + print_error "Invalid version format: '${version}'" + exit 1 +fi + +if ! [[ "${next_dev_version}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + print_error "Invalid next development version format: '${next_dev_version}'. Expected: X.Y.Z" + exit 1 +fi + +if ! [[ "${staging_repo_id}" =~ ^[a-zA-Z][a-zA-Z0-9._-]*$ ]]; then + print_error "Invalid staging repository ID: '${staging_repo_id}'. Expected alphanumeric with dots/hyphens (e.g., orgapacheparquet-1234)." + exit 1 +fi + +if [[ -z "${rc_num}" ]]; then + print_info "No RC number specified, auto-detecting latest RC for ${version}..." + if ! find_latest_rc_number "${version}"; then + exit 1 + fi + rc_num="${latest_rc_number}" + print_info "Auto-detected latest RC: rc${rc_num}" +else + if ! [[ "${rc_num}" =~ ^[0-9]+$ ]]; then + print_error "Invalid RC number: '${rc_num}'. Expected a non-negative integer." + exit 1 + fi + + if find_latest_rc_number "${version}" 2>/dev/null; then + if [[ "${rc_num}" -ne "${latest_rc_number}" ]]; then + print_error "RC${rc_num} is not the latest RC for ${version}. Latest is rc${latest_rc_number}." + print_error "Publishing an older RC is likely a mistake. If intentional, delete the newer RC tags first." + exit 1 + fi + fi +fi + +rc_tag="${TAG_PREFIX}${version}-rc${rc_num}" +final_tag="${TAG_PREFIX}${version}" + +if ! git rev-parse "${rc_tag}" >/dev/null 2>&1; then + print_error "RC tag ${rc_tag} does not exist" + exit 1 +fi + +rc_commit=$(git rev-list -1 "${rc_tag}") +current_commit=$(git rev-parse HEAD) + +if [[ "${current_commit}" != "${rc_commit}" ]]; then + print_error "Current HEAD (${current_commit}) does not match RC tag ${rc_tag} (${rc_commit})" + print_error "The release branch has commits beyond the voted RC. Either reset the branch or create a new RC." + exit 1 +fi + +if git rev-parse "${final_tag}" >/dev/null 2>&1; then + print_error "Final release tag ${final_tag} already exists" + exit 1 +fi + +step_summary "| Parameter | Value |" +step_summary "| --- | --- |" +step_summary "| Version | \`${version}\` |" +step_summary "| RC tag | \`${rc_tag}\` |" +step_summary "| Final tag | \`${final_tag}\` |" +step_summary "| Staging repo | \`${staging_repo_id}\` |" +step_summary "| Next dev version | \`${next_dev_version}-SNAPSHOT\` |" +step_summary "| Commit | \`${rc_commit}\` |" + +# --------------------------------------------------------------------------- +# Step 1: Move SVN artifacts from dist/dev to dist/release +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### SVN Promotion" + +dev_url="${APACHE_DIST_URL}${APACHE_DIST_DEV_PATH}/${rc_tag}" +release_url="${APACHE_DIST_URL}${APACHE_DIST_RELEASE_PATH}/${TAG_PREFIX}${version}" + +exec_process svn mv \ + --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + "${dev_url}" "${release_url}" \ + -m "Release Apache Parquet ${version}" + +step_summary "Moved \`${dev_url}\` -> \`${release_url}\`" + +# --------------------------------------------------------------------------- +# Step 2: Clean up old releases from dist/release +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Old Release Cleanup" + +if [[ ${DRY_RUN:-1} -ne 1 ]]; then + release_base_url="${APACHE_DIST_URL}${APACHE_DIST_RELEASE_PATH}" + svn_listing="" + if ! svn_listing=$(svn list --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + "${release_base_url}" 2>&1); then + print_error "Failed to list SVN releases at ${release_base_url}: ${svn_listing}" + exit 1 + fi + + old_versions=$(echo "${svn_listing}" | grep -E "^${TAG_PREFIX}[0-9]" | sed 's|/$||' | grep -v "${TAG_PREFIX}${version}$" || true) + + if [[ -n "${old_versions}" ]]; then + step_summary "Removing old releases:" + while IFS= read -r old_dir; do + [[ -z "${old_dir}" ]] && continue + exec_process svn rm \ + --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + "${release_base_url}/${old_dir}" \ + -m "Remove old release ${old_dir} (superseded by ${version})" + step_summary "- Removed \`${old_dir}\`" + done <<< "${old_versions}" + else + step_summary "No old releases to clean up" + fi +else + print_command "Dry-run, WOULD clean up old releases from dist/release" + step_summary "Would clean up old releases (dry-run)" +fi + +# --------------------------------------------------------------------------- +# Step 3: Create final release tag +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Release Tag" + +exec_process git tag -a "${final_tag}" "${rc_commit}" -m "Release Apache Parquet ${version}" +exec_process git push origin "${final_tag}" + +step_summary "Created tag \`${final_tag}\` at \`${rc_commit}\`" + +# --------------------------------------------------------------------------- +# Step 4: Release Nexus staging repo +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Nexus Release" + +nexus_release_staging_repo "${staging_repo_id}" "Apache Parquet ${version}" + +step_summary "Released staging repository \`${staging_repo_id}\` to Maven Central" + +# --------------------------------------------------------------------------- +# Step 5: Create GitHub Release +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### GitHub Release" + +if [[ -n "${GITHUB_TOKEN:-}" ]]; then + # If a pre-release exists for the RC tag, update it; otherwise create a new release + if gh release view "${rc_tag}" &>/dev/null 2>&1; then + print_info "Found existing pre-release for ${rc_tag}" + fi + + exec_process gh release create "${final_tag}" \ + --title "Apache Parquet ${version}" \ + --generate-notes \ + --latest \ + --target "${rc_commit}" + + step_summary "Created GitHub release for \`${final_tag}\`" +else + print_warning "GITHUB_TOKEN not set, skipping GitHub release creation" + step_summary "Skipped GitHub release (no GITHUB_TOKEN)" +fi + +# --------------------------------------------------------------------------- +# Step 6: Bump to next development version +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Version Bump" + +next_snapshot="${next_dev_version}-SNAPSHOT" +print_info "Bumping version to ${next_snapshot}..." + +set_pom_version "${next_snapshot}" + +# Update previous.version property +exec_process ./mvnw -pl . versions:set-property \ + -Dproperty=previous.version -DnewVersion="${version}" \ + --batch-mode -q + +exec_process git add -A +exec_process git commit -m "Prepare for next development iteration (${next_snapshot})" +exec_process git push origin HEAD + +step_summary "Bumped version to \`${next_snapshot}\`, set \`previous.version=${version}\`" + +# --------------------------------------------------------------------------- +# Step 7: Generate announce email +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Announce Email" +step_summary "" +step_summary '```' +step_summary "Subject: [ANNOUNCE] Apache Parquet ${version}" +step_summary "" +step_summary "I'm pleased to announce the release of Apache Parquet ${version}!" +step_summary "" +step_summary "Parquet is a general-purpose columnar file format for nested data. It uses" +step_summary "space-efficient encodings and a compressed and splittable structure for" +step_summary "processing frameworks like Hadoop." +step_summary "" +step_summary "Changes are listed at: https://github.com/apache/parquet-java/releases/tag/${final_tag}" +step_summary "" +step_summary "This release can be downloaded from: https://parquet.apache.org/downloads/" +step_summary "" +step_summary "Java artifacts are available from Maven Central." +step_summary "" +step_summary "Thanks to everyone for contributing!" +step_summary '```' + +# --------------------------------------------------------------------------- +# Step 8: Reminder -- update parquet.apache.org +# --------------------------------------------------------------------------- +release_date=$(date +%Y-%m-%d) +step_summary "" +step_summary "### Manual Follow-up: Update parquet.apache.org" +step_summary "" +step_summary "Create a release blog post PR against \`apache/parquet-site\`." +step_summary "Add a new file \`content/en/blog/parquet-java/parquet-java-${version}.md\` with:" +step_summary "" +step_summary '```markdown' +step_summary "---" +step_summary "title: \"Apache Parquet Java ${version}\"" +step_summary "date: ${release_date}" +step_summary "summary: \"Release notes for Apache Parquet Java ${version}\"" +step_summary "---" +step_summary "" +step_summary "Apache Parquet Java ${version} has been released." +step_summary "" +step_summary "For the full list of changes, see the" +step_summary "[release notes](https://github.com/apache/parquet-java/releases/tag/${final_tag})." +step_summary "" +step_summary "Java artifacts are available from" +step_summary "[Maven Central](https://search.maven.org/search?q=g:org.apache.parquet%20AND%20v:${version})." +step_summary "" +step_summary "Source and binary downloads are available from the" +step_summary "[Apache downloads page](https://parquet.apache.org/downloads/)." +step_summary '```' +step_summary "" +step_summary "Submit the PR against the \`staging\` branch of" +step_summary "[\`apache/parquet-site\`](https://github.com/apache/parquet-site)." + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +step_summary "" +step_summary "---" +step_summary "### Summary" +step_summary "" +step_summary "| Step | Status |" +step_summary "| --- | --- |" +step_summary "| SVN promotion | done |" +step_summary "| Old release cleanup | done |" +step_summary "| Final release tag | \`${final_tag}\` |" +step_summary "| Nexus release | \`${staging_repo_id}\` released |" +step_summary "| GitHub release | created |" +step_summary "| Version bump | \`${next_snapshot}\` |" +step_summary "| Announce email | generated |" +step_summary "| Site update | **manual** -- see template above |" + +print_success "Release ${version} published successfully!" diff --git a/release/libs/_constants.sh b/release/libs/_constants.sh new file mode 100644 index 0000000000..16926c17ca --- /dev/null +++ b/release/libs/_constants.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[[ -n "${_CONSTANTS_LOADED:-}" ]] && return 0 2>/dev/null || true +_CONSTANTS_LOADED=1 + +TAG_PREFIX="apache-parquet-" +BRANCH_PREFIX="parquet-" + +APACHE_DIST_URL=${APACHE_DIST_URL:-"https://dist.apache.org/repos/dist"} +APACHE_DIST_DEV_PATH="/dev/parquet" +APACHE_DIST_RELEASE_PATH="/release/parquet" + +NEXUS_BASE_URL=${NEXUS_BASE_URL:-"https://repository.apache.org/service/local"} +NEXUS_STAGING_GROUP_URL="https://repository.apache.org/content/groups/staging/org/apache/parquet/" + +DRY_RUN=${DRY_RUN:-1} + +VERSION_REGEX="([0-9]+)\.([0-9]+)\.([0-9]+)" +VERSION_REGEX_GIT_TAG="^${TAG_PREFIX}${VERSION_REGEX}-rc([0-9]+)$" +VERSION_REGEX_FINAL_TAG="^${TAG_PREFIX}${VERSION_REGEX}$" +BRANCH_VERSION_REGEX="^parquet-([0-9]+)\.([0-9]+)\.x$" + +GITHUB_REPO=${GITHUB_REPOSITORY:-"apache/parquet-java"} diff --git a/release/libs/_exec.sh b/release/libs/_exec.sh new file mode 100644 index 0000000000..3b75e38df1 --- /dev/null +++ b/release/libs/_exec.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[[ -n "${_EXEC_LOADED:-}" ]] && return 0 2>/dev/null || true +_EXEC_LOADED=1 + +LIBS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$LIBS_DIR/_constants.sh" +source "$LIBS_DIR/_log.sh" + +function _redact_secrets { + local cmd_str="$*" + local secret_var + for secret_var in NEXUS_PASSWORD NEXUS_USERNAME SVN_PASSWORD SVN_USERNAME GITHUB_TOKEN; do + local secret_val="${!secret_var:-}" + if [[ -n "${secret_val}" ]]; then + cmd_str="${cmd_str//${secret_val}/***}" + fi + done + echo "${cmd_str}" +} + +function exec_process { + local redacted + redacted=$(_redact_secrets "$@") + if [[ ${DRY_RUN:-1} -ne 1 ]]; then + print_command "Executing '${redacted}'" + "$@" + else + print_command "Dry-run, WOULD execute '${redacted}'" + fi +} + +function exec_process_with_retries { + if [[ $# -lt 4 ]]; then + echo "ERROR: exec_process_with_retries requires: max_attempts sleep_duration cleanup_path command [args...]" + exit 1 + fi + + local max_attempts="${1}" + local sleep_duration="${2}" + local cleanup_path="${3}" + shift 3 + + local attempt=1 + while true; do + if exec_process "$@"; then + break + fi + if [[ $attempt -ge $max_attempts ]]; then + echo "ERROR: Command failed after ${max_attempts} attempts: ${*}" + exit 1 + fi + echo "WARNING: Command failed (attempt ${attempt}/${max_attempts}), retrying in ${sleep_duration} seconds..." + if [[ -n "${cleanup_path}" && -e "${cleanup_path}" ]]; then + rm -rf "${cleanup_path}" + fi + sleep "${sleep_duration}" + ((attempt++)) + done +} + +function calculate_sha512 { + local source_file="$1" + local source_dir source_base + source_dir="$(dirname "${source_file}")" + source_base="$(basename "${source_file}")" + local target_file="${source_file}.sha512" + if [[ ${DRY_RUN:-1} -ne 1 ]]; then + (cd "${source_dir}" && shasum -a 512 "${source_base}") > "${target_file}" + else + print_command "Dry-run, WOULD run: cd ${source_dir} && shasum -a 512 ${source_base} > ${target_file}" + fi +} diff --git a/release/libs/_github.sh b/release/libs/_github.sh new file mode 100644 index 0000000000..88348ad092 --- /dev/null +++ b/release/libs/_github.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[[ -n "${_GITHUB_LOADED:-}" ]] && return 0 2>/dev/null || true +_GITHUB_LOADED=1 + +LIBS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${LIBS_DIR}/_log.sh" +source "${LIBS_DIR}/_constants.sh" +source "${LIBS_DIR}/_exec.sh" + +function check_github_checks_passed() { + local commit_sha="$1" + + print_info "Checking GitHub CI status for commit ${commit_sha}..." + + if [[ -z "${GITHUB_TOKEN:-}" ]]; then + print_warning "GITHUB_TOKEN not set, skipping CI check verification" + return 0 + fi + + if [[ ${DRY_RUN:-1} -eq 1 ]]; then + print_info "DRY_RUN is enabled, skipping GitHub check verification" + return 0 + fi + + local repo_info="${GITHUB_REPO}" + + local num_incomplete + if ! num_incomplete=$(gh api "repos/${repo_info}/commits/${commit_sha}/check-runs" \ + --jq '[.check_runs[] | select(.status != "completed")] | length'); then + print_error "Failed to fetch GitHub check runs for commit ${commit_sha}" + return 1 + fi + + if [[ ${num_incomplete} -ne 0 ]]; then + print_error "Found ${num_incomplete} still-running GitHub checks for commit ${commit_sha}" + gh api "repos/${repo_info}/commits/${commit_sha}/check-runs" \ + --jq '.check_runs[] | select(.status != "completed") | " - \(.name): \(.status)"' >&2 + return 1 + fi + + local num_failed + if ! num_failed=$(gh api "repos/${repo_info}/commits/${commit_sha}/check-runs" \ + --jq '[.check_runs[] | select(.conclusion != "success" and .conclusion != "skipped")] | length'); then + print_error "Failed to fetch GitHub check runs for commit ${commit_sha}" + return 1 + fi + + if [[ ${num_failed} -ne 0 ]]; then + print_error "Found ${num_failed} failed GitHub checks for commit ${commit_sha}" + gh api "repos/${repo_info}/commits/${commit_sha}/check-runs" \ + --jq '.check_runs[] | select(.conclusion != "success" and .conclusion != "skipped") | " - \(.name): \(.conclusion)"' >&2 + return 1 + fi + + print_info "All GitHub checks passed for commit ${commit_sha}" + return 0 +} diff --git a/release/libs/_log.sh b/release/libs/_log.sh new file mode 100644 index 0000000000..7be4cf27d5 --- /dev/null +++ b/release/libs/_log.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[[ -n "${_LOG_LOADED:-}" ]] && return 0 2>/dev/null || true +_LOG_LOADED=1 + +if [[ -t 2 ]] && + [[ "${NO_COLOR:-}" != "1" ]] && + [[ "${TERM:-}" != "dumb" ]] && + command -v tput >/dev/null; then + RED=${RED:-$(tput setaf 1)} + GREEN=${GREEN:-$(tput setaf 2)} + YELLOW=${YELLOW:-$(tput bold; tput setaf 3)} + BLUE=${BLUE:-$(tput setaf 4)} + RESET=${RESET:-$(tput sgr0)} +else + RED=${RED:-''} + GREEN=${GREEN:-''} + YELLOW=${YELLOW:-''} + BLUE=${BLUE:-''} + RESET=${RESET:-''} +fi + +function print_error() { + echo -e "${RED}ERROR: $*${RESET}" >&2 +} + +function print_warning() { + echo -e "${YELLOW}WARNING: $*${RESET}" >&2 +} + +function print_info() { + echo "INFO: $*" >&2 +} + +function print_success() { + echo -e "${GREEN}SUCCESS: $*${RESET}" >&2 +} + +function print_command() { + echo -e "${BLUE}DEBUG: $*${RESET}" >&2 +} + +function step_summary() { + local msg="$1" + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + echo "$msg" >> "$GITHUB_STEP_SUMMARY" + fi + echo "$msg" +} diff --git a/release/libs/_maven.sh b/release/libs/_maven.sh new file mode 100644 index 0000000000..38a91838d5 --- /dev/null +++ b/release/libs/_maven.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[[ -n "${_MAVEN_LOADED:-}" ]] && return 0 2>/dev/null || true +_MAVEN_LOADED=1 + +LIBS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${LIBS_DIR}/_constants.sh" +source "${LIBS_DIR}/_exec.sh" + +function _xml_escape { + local str="$1" + str="${str//&/&}" + str="${str///>}" + str="${str//\"/"}" + str="${str//\'/'}" + echo "${str}" +} + +function generate_maven_settings { + local settings_file="${1:-.release-settings.xml}" + + if [[ -z "${NEXUS_USERNAME:-}" || -z "${NEXUS_PASSWORD:-}" ]]; then + print_warning "NEXUS_USERNAME or NEXUS_PASSWORD not set; Maven deploy may fail" + fi + + local esc_username esc_password + esc_username=$(_xml_escape "${NEXUS_USERNAME:-}") + esc_password=$(_xml_escape "${NEXUS_PASSWORD:-}") + + ( + umask 077 + cat > "${settings_file}" < + + + + apache.releases.https + ${esc_username} + ${esc_password} + + + + + gpg-release + + true + + + + + gpg-release + + +EOF + ) + + print_info "Generated Maven settings at ${settings_file} (mode 600)" +} + +function maven_deploy { + local settings_file="${1:-.release-settings.xml}" + + if [[ ! -f "${settings_file}" ]]; then + print_info "Generating Maven settings..." + generate_maven_settings "${settings_file}" + fi + + exec_process ./mvnw deploy \ + -Papache-release \ + -DskipTests \ + -Darguments=-DskipTests \ + --settings "${settings_file}" \ + --batch-mode +} + +function maven_cleanup_settings { + local settings_file="${1:-.release-settings.xml}" + if [[ -f "${settings_file}" ]]; then + rm -f "${settings_file}" + print_info "Cleaned up Maven settings file" + fi +} diff --git a/release/libs/_nexus.sh b/release/libs/_nexus.sh new file mode 100644 index 0000000000..7f591d1e07 --- /dev/null +++ b/release/libs/_nexus.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[[ -n "${_NEXUS_LOADED:-}" ]] && return 0 2>/dev/null || true +_NEXUS_LOADED=1 + +LIBS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${LIBS_DIR}/_constants.sh" +source "${LIBS_DIR}/_exec.sh" + +function _nexus_bulk_action { + local action="$1" + local repo_id="$2" + local description="$3" + + local url="${NEXUS_BASE_URL}/staging/bulk/${action}" + local payload + payload=$(jq -n --arg id "${repo_id}" --arg desc "${description}" \ + '{"data": {"stagedRepositoryIds": [$id], "description": $desc}}') + + print_info "Nexus ${action}: repo_id=${repo_id}" + + if [[ ${DRY_RUN:-1} -ne 1 ]]; then + print_command "Executing 'curl --fail -X POST ${url}' (credentials via stdin)" + curl --fail --silent --show-error \ + -K <(printf 'user = "%s:%s"\n' "${NEXUS_USERNAME}" "${NEXUS_PASSWORD}") \ + -H "Content-Type: application/json" \ + -d "${payload}" \ + "${url}" + else + print_command "Dry-run, WOULD POST to ${url} with payload for repo ${repo_id}" + fi +} + +function nexus_close_staging_repo { + local repo_id="$1" + local description="${2:-Closing staging repository}" + _nexus_bulk_action "close" "${repo_id}" "${description}" +} + +function nexus_release_staging_repo { + local repo_id="$1" + local description="${2:-Releasing staging repository}" + _nexus_bulk_action "promote" "${repo_id}" "${description}" +} + +function nexus_drop_staging_repo { + local repo_id="$1" + local description="${2:-Dropping staging repository}" + _nexus_bulk_action "drop" "${repo_id}" "${description}" +} + +function nexus_find_open_staging_repo { + local profile_name="${1:-org.apache.parquet}" + + print_info "Searching for open staging repository for ${profile_name}..." + + if [[ ${DRY_RUN:-1} -eq 1 ]]; then + print_command "Dry-run, WOULD search Nexus for open staging repo" + staging_repo_id="DRY-RUN-REPO-ID" + return 0 + fi + + local response + if ! response=$(curl --fail --silent --show-error \ + -K <(printf 'user = "%s:%s"\n' "${NEXUS_USERNAME}" "${NEXUS_PASSWORD}") \ + "${NEXUS_BASE_URL}/staging/profile_repositories"); then + print_error "Failed to query Nexus staging repositories" + return 1 + fi + + staging_repo_id=$(echo "${response}" | \ + NEXUS_PROFILE_NAME="${profile_name}" python3 -c " +import sys, os, xml.etree.ElementTree as ET +profile = os.environ['NEXUS_PROFILE_NAME'] +tree = ET.parse(sys.stdin) +for repo in tree.findall('.//stagingProfileRepository'): + repo_type = repo.find('type') + repo_id = repo.find('repositoryId') + if repo_type is not None and repo_type.text == 'open' and repo_id is not None: + if profile.replace('.', '') in (repo_id.text or ''): + print(repo_id.text) + break +" 2>/dev/null) + + if [[ -z "${staging_repo_id}" ]]; then + print_error "No open staging repository found for ${profile_name}" + return 1 + fi + + print_info "Found staging repository: ${staging_repo_id}" + return 0 +} diff --git a/release/libs/_version.sh b/release/libs/_version.sh new file mode 100644 index 0000000000..2b2a4b31ad --- /dev/null +++ b/release/libs/_version.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[[ -n "${_VERSION_LOADED:-}" ]] && return 0 2>/dev/null || true +_VERSION_LOADED=1 + +LIBS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$LIBS_DIR/_constants.sh" +source "$LIBS_DIR/_exec.sh" + +function validate_and_extract_version { + local version="$1" + if [[ ! ${version} =~ ^${VERSION_REGEX}$ ]]; then + return 1 + fi + major="${BASH_REMATCH[1]}" + minor="${BASH_REMATCH[2]}" + patch="${BASH_REMATCH[3]}" + version_without_rc="${major}.${minor}.${patch}" + return 0 +} + +function validate_and_extract_git_tag_version { + local tag="$1" + if [[ ! ${tag} =~ ${VERSION_REGEX_GIT_TAG} ]]; then + return 1 + fi + major="${BASH_REMATCH[1]}" + minor="${BASH_REMATCH[2]}" + patch="${BASH_REMATCH[3]}" + rc_number="${BASH_REMATCH[4]}" + version_without_rc="${major}.${minor}.${patch}" + return 0 +} + +function validate_and_extract_branch_version { + local branch="$1" + if [[ ! ${branch} =~ ${BRANCH_VERSION_REGEX} ]]; then + return 1 + fi + major="${BASH_REMATCH[1]}" + minor="${BASH_REMATCH[2]}" + return 0 +} + +function find_next_rc_number { + local version_without_rc="$1" + local tag_pattern="${TAG_PREFIX}${version_without_rc}-rc*" + local existing_tags + existing_tags=$(git tag -l "${tag_pattern}" | sort -V) + + if [[ -z "${existing_tags}" ]]; then + rc_number=0 + else + local highest_rc + highest_rc=$(echo "${existing_tags}" | sed "s/${TAG_PREFIX}${version_without_rc}-rc//" | sort -n | tail -1) + rc_number=$((highest_rc + 1)) + fi + return 0 +} + +function find_latest_rc_number { + local version_without_rc="$1" + local tag_pattern="${TAG_PREFIX}${version_without_rc}-rc*" + local existing_tags + existing_tags=$(git tag -l "${tag_pattern}" | sort -V) + + if [[ -z "${existing_tags}" ]]; then + print_error "No RC tags found for version ${version_without_rc}" + return 1 + fi + + latest_rc_number=$(echo "${existing_tags}" | sed "s/${TAG_PREFIX}${version_without_rc}-rc//" | sort -n | tail -1) + return 0 +} + +function find_next_patch_number { + local major="$1" + local minor="$2" + local rc_tag_pattern="${TAG_PREFIX}${major}.${minor}.*-rc*" + local existing_rc_tags + existing_rc_tags=$(git tag -l "${rc_tag_pattern}" | sort -V) + + if [[ -z "${existing_rc_tags}" ]]; then + patch=0 + else + local highest_patch=-1 + while IFS= read -r tag; do + if [[ ${tag} =~ ${TAG_PREFIX}${major}\.${minor}\.([0-9]+)-rc[0-9]+ ]]; then + local current_patch="${BASH_REMATCH[1]}" + if [[ ${current_patch} -gt ${highest_patch} ]]; then + highest_patch=${current_patch} + fi + fi + done <<< "${existing_rc_tags}" + + local final_tag="${TAG_PREFIX}${major}.${minor}.${highest_patch}" + if git rev-parse "${final_tag}" >/dev/null 2>&1; then + patch=$((highest_patch + 1)) + else + patch=${highest_patch} + fi + fi + return 0 +} + +function set_pom_version { + local version="$1" + exec_process ./mvnw versions:set -DnewVersion="${version}" -DgenerateBackupPoms=false --batch-mode -q +} + +function get_current_pom_version { + ./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout 2>/dev/null +} diff --git a/release/tests/constants.bats b/release/tests/constants.bats new file mode 100644 index 0000000000..fe0c7be10d --- /dev/null +++ b/release/tests/constants.bats @@ -0,0 +1,83 @@ +#!/usr/bin/env bats +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +setup() { + load test_helper/common + source "${LIBS_DIR}/_constants.sh" +} + +@test "TAG_PREFIX is apache-parquet-" { + [ "$TAG_PREFIX" = "apache-parquet-" ] +} + +@test "BRANCH_PREFIX is parquet-" { + [ "$BRANCH_PREFIX" = "parquet-" ] +} + +@test "APACHE_DIST_URL has correct default" { + [ "$APACHE_DIST_URL" = "https://dist.apache.org/repos/dist" ] +} + +@test "APACHE_DIST_DEV_PATH is /dev/parquet" { + [ "$APACHE_DIST_DEV_PATH" = "/dev/parquet" ] +} + +@test "APACHE_DIST_RELEASE_PATH is /release/parquet" { + [ "$APACHE_DIST_RELEASE_PATH" = "/release/parquet" ] +} + +@test "NEXUS_BASE_URL has correct default" { + [ "$NEXUS_BASE_URL" = "https://repository.apache.org/service/local" ] +} + +@test "DRY_RUN defaults to 1" { + [ "$DRY_RUN" = "1" ] +} + +@test "VERSION_REGEX matches semver components" { + [[ "1.18.0" =~ ^${VERSION_REGEX}$ ]] + [ "${BASH_REMATCH[1]}" = "1" ] + [ "${BASH_REMATCH[2]}" = "18" ] + [ "${BASH_REMATCH[3]}" = "0" ] +} + +@test "VERSION_REGEX_GIT_TAG matches RC tag" { + [[ "apache-parquet-1.18.0-rc3" =~ ${VERSION_REGEX_GIT_TAG} ]] + [ "${BASH_REMATCH[1]}" = "1" ] + [ "${BASH_REMATCH[4]}" = "3" ] +} + +@test "VERSION_REGEX_GIT_TAG rejects final tag" { + [[ ! "apache-parquet-1.18.0" =~ ${VERSION_REGEX_GIT_TAG} ]] +} + +@test "BRANCH_VERSION_REGEX matches parquet-1.18.x" { + [[ "parquet-1.18.x" =~ ${BRANCH_VERSION_REGEX} ]] + [ "${BASH_REMATCH[1]}" = "1" ] + [ "${BASH_REMATCH[2]}" = "18" ] +} + +@test "BRANCH_VERSION_REGEX rejects parquet-1.18.0" { + [[ ! "parquet-1.18.0" =~ ${BRANCH_VERSION_REGEX} ]] +} + +@test "GITHUB_REPO has correct default" { + [ "$GITHUB_REPO" = "apache/parquet-java" ] +} diff --git a/release/tests/exec.bats b/release/tests/exec.bats new file mode 100644 index 0000000000..4b794022f2 --- /dev/null +++ b/release/tests/exec.bats @@ -0,0 +1,101 @@ +#!/usr/bin/env bats +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +setup() { + load test_helper/common + source "${LIBS_DIR}/_exec.sh" +} + +# ---- exec_process ---- + +@test "exec_process: dry-run prints but does not execute" { + DRY_RUN=1 + run exec_process echo "should not appear as direct output" + [ "$status" -eq 0 ] + [[ "$output" == *"Dry-run, WOULD execute"* ]] + [[ "$output" == *"echo"* ]] +} + +@test "exec_process: real run executes command" { + DRY_RUN=0 + run exec_process echo "hello from exec" + [ "$status" -eq 0 ] + [[ "$output" == *"hello from exec"* ]] +} + +@test "exec_process: real run preserves exit code" { + DRY_RUN=0 + run exec_process false + [ "$status" -ne 0 ] +} + +# ---- exec_process_with_retries ---- + +@test "exec_process_with_retries: succeeds on first attempt" { + DRY_RUN=0 + run exec_process_with_retries 3 0 "" echo "ok" + [ "$status" -eq 0 ] + [[ "$output" == *"ok"* ]] +} + +@test "exec_process_with_retries: fails after max attempts" { + DRY_RUN=0 + run exec_process_with_retries 2 0 "" false + [ "$status" -ne 0 ] + [[ "$output" == *"failed after 2 attempts"* ]] +} + +@test "exec_process_with_retries: requires at least 4 args" { + DRY_RUN=0 + run exec_process_with_retries 3 0 + [ "$status" -ne 0 ] +} + +# ---- calculate_sha512 ---- + +@test "calculate_sha512: creates checksum file in real mode" { + DRY_RUN=0 + local tmpfile + tmpfile=$(mktemp) + echo "test content" > "$tmpfile" + + calculate_sha512 "$tmpfile" + + [ -f "${tmpfile}.sha512" ] + # The checksum file should contain the filename + local basename + basename=$(basename "$tmpfile") + [[ "$(cat "${tmpfile}.sha512")" == *"${basename}"* ]] + + rm -f "$tmpfile" "${tmpfile}.sha512" +} + +@test "calculate_sha512: dry-run does not create file" { + DRY_RUN=1 + local tmpfile + tmpfile=$(mktemp) + echo "test content" > "$tmpfile" + + run calculate_sha512 "$tmpfile" + [ "$status" -eq 0 ] + [ ! -f "${tmpfile}.sha512" ] + + rm -f "$tmpfile" +} diff --git a/release/tests/github.bats b/release/tests/github.bats new file mode 100644 index 0000000000..65e10d5676 --- /dev/null +++ b/release/tests/github.bats @@ -0,0 +1,112 @@ +#!/usr/bin/env bats +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +setup() { + load test_helper/common + source "${LIBS_DIR}/_github.sh" +} + +# ---- check_github_checks_passed ---- + +@test "check_github_checks_passed: skips when GITHUB_TOKEN not set" { + unset GITHUB_TOKEN + run check_github_checks_passed "abc123" + [ "$status" -eq 0 ] + [[ "$output" == *"GITHUB_TOKEN not set"* ]] +} + +@test "check_github_checks_passed: skips in dry-run mode" { + export GITHUB_TOKEN="fake-token" + DRY_RUN=1 + run check_github_checks_passed "abc123" + [ "$status" -eq 0 ] + [[ "$output" == *"DRY_RUN"* ]] +} + +@test "check_github_checks_passed: succeeds when all checks completed and passed" { + export GITHUB_TOKEN="fake-token" + DRY_RUN=0 + + gh() { + echo "0" + return 0 + } + export -f gh + + run check_github_checks_passed "abc123" + [ "$status" -eq 0 ] + [[ "$output" == *"All GitHub checks passed"* ]] +} + +@test "check_github_checks_passed: fails when checks are still running" { + export GITHUB_TOKEN="fake-token" + DRY_RUN=0 + + gh() { + if [[ "$*" == *"status"* && "$*" == *"length"* ]]; then + echo "1" + elif [[ "$*" == *"status"* ]]; then + echo " - CI Hadoop 3: in_progress" + else + echo "0" + fi + return 0 + } + export -f gh + + run check_github_checks_passed "abc123" + [ "$status" -eq 1 ] + [[ "$output" == *"still-running"* ]] +} + +@test "check_github_checks_passed: fails when checks have failed conclusions" { + export GITHUB_TOKEN="fake-token" + DRY_RUN=0 + + gh() { + if [[ "$*" == *"status"* && "$*" == *"length"* ]]; then + echo "0" + elif [[ "$*" == *"conclusion"* && "$*" == *"length"* ]]; then + echo "2" + elif [[ "$*" == *"conclusion"* ]]; then + echo " - CI Hadoop 3: failure" + else + echo "0" + fi + return 0 + } + export -f gh + + run check_github_checks_passed "abc123" + [ "$status" -eq 1 ] + [[ "$output" == *"failed GitHub checks"* ]] +} + +@test "check_github_checks_passed: fails when gh api errors" { + export GITHUB_TOKEN="fake-token" + DRY_RUN=0 + + gh() { return 1; } + export -f gh + + run check_github_checks_passed "abc123" + [ "$status" -eq 1 ] + [[ "$output" == *"Failed to fetch"* ]] +} diff --git a/release/tests/log.bats b/release/tests/log.bats new file mode 100644 index 0000000000..c4ffb8b851 --- /dev/null +++ b/release/tests/log.bats @@ -0,0 +1,88 @@ +#!/usr/bin/env bats +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +setup() { + load test_helper/common + unset _LOG_LOADED + source "${LIBS_DIR}/_log.sh" + TEST_TMPDIR=$(mktemp -d) +} + +teardown() { + rm -rf "${TEST_TMPDIR}" +} + +# ---- print_error ---- + +@test "print_error: writes to stderr" { + run print_error "test error" + [ "$status" -eq 0 ] + [[ "$output" == *"ERROR: test error"* ]] +} + +# ---- print_warning ---- + +@test "print_warning: writes to stderr" { + run print_warning "test warning" + [ "$status" -eq 0 ] + [[ "$output" == *"WARNING: test warning"* ]] +} + +# ---- print_info ---- + +@test "print_info: writes to stderr" { + run print_info "test info" + [ "$status" -eq 0 ] + [[ "$output" == *"INFO: test info"* ]] +} + +# ---- step_summary ---- + +@test "step_summary: writes to stdout when GITHUB_STEP_SUMMARY unset" { + unset GITHUB_STEP_SUMMARY + run step_summary "hello" + [ "$status" -eq 0 ] + [ "$output" = "hello" ] +} + +@test "step_summary: writes to file when GITHUB_STEP_SUMMARY is set" { + export GITHUB_STEP_SUMMARY="${TEST_TMPDIR}/summary.md" + step_summary "line one" + step_summary "line two" + [ -f "${GITHUB_STEP_SUMMARY}" ] + [[ "$(cat "${GITHUB_STEP_SUMMARY}")" == *"line one"* ]] + [[ "$(cat "${GITHUB_STEP_SUMMARY}")" == *"line two"* ]] +} + +@test "step_summary: appends to existing GITHUB_STEP_SUMMARY file" { + export GITHUB_STEP_SUMMARY="${TEST_TMPDIR}/summary.md" + echo "existing" > "${GITHUB_STEP_SUMMARY}" + step_summary "new line" + [[ "$(cat "${GITHUB_STEP_SUMMARY}")" == *"existing"* ]] + [[ "$(cat "${GITHUB_STEP_SUMMARY}")" == *"new line"* ]] +} + +# ---- color suppression ---- + +@test "NO_COLOR suppresses colors" { + export NO_COLOR=1 + [ -z "${RED}" ] || [ "${RED}" = "" ] + [ -z "${GREEN}" ] || [ "${GREEN}" = "" ] +} diff --git a/release/tests/maven.bats b/release/tests/maven.bats new file mode 100644 index 0000000000..bb34f2b74f --- /dev/null +++ b/release/tests/maven.bats @@ -0,0 +1,117 @@ +#!/usr/bin/env bats +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +setup() { + load test_helper/common + source "${LIBS_DIR}/_maven.sh" + export NEXUS_USERNAME="testuser" + export NEXUS_PASSWORD="testpass" + TEST_TMPDIR=$(mktemp -d) +} + +teardown() { + rm -rf "${TEST_TMPDIR}" +} + +# ---- generate_maven_settings ---- + +@test "generate_maven_settings: creates settings file" { + local settings_file="${TEST_TMPDIR}/settings.xml" + generate_maven_settings "${settings_file}" + [ -f "${settings_file}" ] +} + +@test "generate_maven_settings: includes server ID" { + local settings_file="${TEST_TMPDIR}/settings.xml" + generate_maven_settings "${settings_file}" + [[ "$(cat "${settings_file}")" == *"apache.releases.https"* ]] +} + +@test "generate_maven_settings: includes credentials" { + local settings_file="${TEST_TMPDIR}/settings.xml" + generate_maven_settings "${settings_file}" + local content + content=$(cat "${settings_file}") + [[ "$content" == *"testuser"* ]] + [[ "$content" == *"testpass"* ]] +} + +@test "generate_maven_settings: enables GPG agent" { + local settings_file="${TEST_TMPDIR}/settings.xml" + generate_maven_settings "${settings_file}" + [[ "$(cat "${settings_file}")" == *"gpg.useagent"* ]] + [[ "$(cat "${settings_file}")" == *"true"* ]] +} + +@test "generate_maven_settings: produces valid XML" { + local settings_file="${TEST_TMPDIR}/settings.xml" + generate_maven_settings "${settings_file}" + # Check that xmllint can parse it, if available + if command -v xmllint &>/dev/null; then + run xmllint --noout "${settings_file}" + [ "$status" -eq 0 ] + else + # Fallback: check basic XML structure + [[ "$(head -1 "${settings_file}")" == *""* ]] + fi +} + +# ---- maven_deploy ---- + +@test "maven_deploy: dry-run does not execute mvnw" { + DRY_RUN=1 + local settings_file="${TEST_TMPDIR}/settings.xml" + generate_maven_settings "${settings_file}" + run maven_deploy "${settings_file}" + [ "$status" -eq 0 ] + [[ "$output" == *"Dry-run"* ]] + [[ "$output" == *"deploy"* ]] +} + +@test "maven_deploy: includes apache-release profile in dry-run output" { + DRY_RUN=1 + local settings_file="${TEST_TMPDIR}/settings.xml" + generate_maven_settings "${settings_file}" + run maven_deploy "${settings_file}" + [[ "$output" == *"apache-release"* ]] +} + +@test "maven_deploy: includes skipTests in dry-run output" { + DRY_RUN=1 + local settings_file="${TEST_TMPDIR}/settings.xml" + generate_maven_settings "${settings_file}" + run maven_deploy "${settings_file}" + [[ "$output" == *"skipTests"* ]] +} + +# ---- maven_cleanup_settings ---- + +@test "maven_cleanup_settings: removes settings file" { + local settings_file="${TEST_TMPDIR}/settings.xml" + echo "test" > "${settings_file}" + maven_cleanup_settings "${settings_file}" + [ ! -f "${settings_file}" ] +} + +@test "maven_cleanup_settings: no error if file missing" { + run maven_cleanup_settings "${TEST_TMPDIR}/nonexistent.xml" + [ "$status" -eq 0 ] +} diff --git a/release/tests/nexus.bats b/release/tests/nexus.bats new file mode 100644 index 0000000000..dd66c457a7 --- /dev/null +++ b/release/tests/nexus.bats @@ -0,0 +1,108 @@ +#!/usr/bin/env bats +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +setup() { + load test_helper/common + source "${LIBS_DIR}/_nexus.sh" + export NEXUS_USERNAME="testuser" + export NEXUS_PASSWORD="testpass" +} + +# ---- nexus_close_staging_repo ---- + +@test "nexus_close_staging_repo: dry-run does not call curl" { + DRY_RUN=1 + run nexus_close_staging_repo "orgapacheparquet-1234" "test close" + [ "$status" -eq 0 ] + [[ "$output" == *"Dry-run"* ]] + [[ "$output" == *"close"* ]] +} + +@test "nexus_close_staging_repo: constructs correct URL in dry-run" { + DRY_RUN=1 + run nexus_close_staging_repo "orgapacheparquet-1234" + [[ "$output" == *"staging/bulk/close"* ]] +} + +# ---- nexus_release_staging_repo ---- + +@test "nexus_release_staging_repo: dry-run includes promote URL" { + DRY_RUN=1 + run nexus_release_staging_repo "orgapacheparquet-1234" + [[ "$output" == *"staging/bulk/promote"* ]] +} + +# ---- nexus_drop_staging_repo ---- + +@test "nexus_drop_staging_repo: dry-run includes drop URL" { + DRY_RUN=1 + run nexus_drop_staging_repo "orgapacheparquet-1234" + [[ "$output" == *"staging/bulk/drop"* ]] +} + +# ---- _nexus_bulk_action ---- + +@test "_nexus_bulk_action: includes repo ID in payload" { + DRY_RUN=1 + run _nexus_bulk_action "close" "orgapacheparquet-5678" "test" + [[ "$output" == *"orgapacheparquet-5678"* ]] +} + +@test "_nexus_bulk_action: uses correct base URL" { + DRY_RUN=1 + NEXUS_BASE_URL="https://example.com/nexus" + run _nexus_bulk_action "drop" "repo-123" "test" + [[ "$output" == *"example.com/nexus/staging/bulk/drop"* ]] +} + +# ---- nexus_find_open_staging_repo ---- + +@test "nexus_find_open_staging_repo: dry-run returns placeholder" { + DRY_RUN=1 + nexus_find_open_staging_repo "org.apache.parquet" + [ "$staging_repo_id" = "DRY-RUN-REPO-ID" ] +} + +# ---- real-mode tests with mocked curl ---- + +@test "nexus_close_staging_repo: real mode calls curl with correct args" { + DRY_RUN=0 + curl() { + echo "CURL_ARGS: $*" >&2 + return 0 + } + export -f curl + run nexus_close_staging_repo "orgapacheparquet-9999" "test desc" + [ "$status" -eq 0 ] + [[ "$output" == *"staging/bulk/close"* ]] + [[ "$output" == *"orgapacheparquet-9999"* ]] +} + +@test "nexus_drop_staging_repo: real mode calls curl with auth" { + DRY_RUN=0 + curl() { + echo "CURL_ARGS: $*" >&2 + return 0 + } + export -f curl + run nexus_drop_staging_repo "orgapacheparquet-1111" + [ "$status" -eq 0 ] + [[ "$output" == *"staging/bulk/drop"* ]] +} diff --git a/release/tests/test_helper/common.bash b/release/tests/test_helper/common.bash new file mode 100644 index 0000000000..70385e1374 --- /dev/null +++ b/release/tests/test_helper/common.bash @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +LIBS_DIR="${BATS_TEST_DIRNAME}/../libs" + +# Reset include guards so libraries can be re-sourced in each test +unset _CONSTANTS_LOADED _LOG_LOADED _EXEC_LOADED _VERSION_LOADED +unset _GITHUB_LOADED _NEXUS_LOADED _MAVEN_LOADED + +# Reset global variables that library functions set +_reset_version_vars() { + unset major minor patch rc_number version_without_rc staging_repo_id +} + +# Suppress colored output in tests +export NO_COLOR=1 + +# Default to dry-run in tests +export DRY_RUN=1 diff --git a/release/tests/version.bats b/release/tests/version.bats new file mode 100644 index 0000000000..18f505c252 --- /dev/null +++ b/release/tests/version.bats @@ -0,0 +1,332 @@ +#!/usr/bin/env bats +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +setup() { + load test_helper/common + _reset_version_vars + source "${LIBS_DIR}/_version.sh" +} + +# ---- validate_and_extract_version ---- + +@test "validate_and_extract_version: accepts 1.18.0" { + run validate_and_extract_version "1.18.0" + [ "$status" -eq 0 ] +} + +@test "validate_and_extract_version: extracts major.minor.patch" { + validate_and_extract_version "1.18.0" + [ "$major" = "1" ] + [ "$minor" = "18" ] + [ "$patch" = "0" ] + [ "$version_without_rc" = "1.18.0" ] +} + +@test "validate_and_extract_version: accepts 2.0.10" { + validate_and_extract_version "2.0.10" + [ "$major" = "2" ] + [ "$minor" = "0" ] + [ "$patch" = "10" ] +} + +@test "validate_and_extract_version: rejects missing patch" { + run validate_and_extract_version "1.18" + [ "$status" -eq 1 ] +} + +@test "validate_and_extract_version: rejects empty string" { + run validate_and_extract_version "" + [ "$status" -eq 1 ] +} + +@test "validate_and_extract_version: rejects alpha characters" { + run validate_and_extract_version "1.18.0-beta" + [ "$status" -eq 1 ] +} + +@test "validate_and_extract_version: rejects SNAPSHOT suffix" { + run validate_and_extract_version "1.18.0-SNAPSHOT" + [ "$status" -eq 1 ] +} + +# ---- validate_and_extract_git_tag_version ---- + +@test "validate_and_extract_git_tag_version: parses apache-parquet-1.18.0-rc0" { + validate_and_extract_git_tag_version "apache-parquet-1.18.0-rc0" + [ "$major" = "1" ] + [ "$minor" = "18" ] + [ "$patch" = "0" ] + [ "$rc_number" = "0" ] + [ "$version_without_rc" = "1.18.0" ] +} + +@test "validate_and_extract_git_tag_version: parses apache-parquet-2.1.3-rc12" { + validate_and_extract_git_tag_version "apache-parquet-2.1.3-rc12" + [ "$major" = "2" ] + [ "$minor" = "1" ] + [ "$patch" = "3" ] + [ "$rc_number" = "12" ] +} + +@test "validate_and_extract_git_tag_version: rejects tag without rc suffix" { + run validate_and_extract_git_tag_version "apache-parquet-1.18.0" + [ "$status" -eq 1 ] +} + +@test "validate_and_extract_git_tag_version: rejects wrong prefix" { + run validate_and_extract_git_tag_version "apache-polaris-1.18.0-rc0" + [ "$status" -eq 1 ] +} + +@test "validate_and_extract_git_tag_version: rejects bare version" { + run validate_and_extract_git_tag_version "1.18.0-rc0" + [ "$status" -eq 1 ] +} + +# ---- validate_and_extract_branch_version ---- + +@test "validate_and_extract_branch_version: parses parquet-1.18.x" { + validate_and_extract_branch_version "parquet-1.18.x" + [ "$major" = "1" ] + [ "$minor" = "18" ] +} + +@test "validate_and_extract_branch_version: parses parquet-2.0.x" { + validate_and_extract_branch_version "parquet-2.0.x" + [ "$major" = "2" ] + [ "$minor" = "0" ] +} + +@test "validate_and_extract_branch_version: rejects release/ prefix" { + run validate_and_extract_branch_version "release/1.18.x" + [ "$status" -eq 1 ] +} + +@test "validate_and_extract_branch_version: rejects full version" { + run validate_and_extract_branch_version "parquet-1.18.0" + [ "$status" -eq 1 ] +} + +@test "validate_and_extract_branch_version: rejects master" { + run validate_and_extract_branch_version "master" + [ "$status" -eq 1 ] +} + +# ---- find_next_rc_number ---- + +@test "find_next_rc_number: returns 0 when no tags exist" { + git() { echo ""; } + export -f git + find_next_rc_number "1.18.0" + [ "$rc_number" = "0" ] +} + +@test "find_next_rc_number: returns 1 after rc0" { + git() { + if [[ "$1" == "tag" && "$2" == "-l" ]]; then + echo "apache-parquet-1.18.0-rc0" + fi + } + export -f git + find_next_rc_number "1.18.0" + [ "$rc_number" = "1" ] +} + +@test "find_next_rc_number: returns 3 after rc0, rc1, rc2" { + git() { + if [[ "$1" == "tag" && "$2" == "-l" ]]; then + printf "apache-parquet-1.18.0-rc0\napache-parquet-1.18.0-rc1\napache-parquet-1.18.0-rc2\n" + fi + } + export -f git + find_next_rc_number "1.18.0" + [ "$rc_number" = "3" ] +} + +@test "find_next_rc_number: handles gap in rc numbers" { + git() { + if [[ "$1" == "tag" && "$2" == "-l" ]]; then + printf "apache-parquet-1.18.0-rc0\napache-parquet-1.18.0-rc5\n" + fi + } + export -f git + find_next_rc_number "1.18.0" + [ "$rc_number" = "6" ] +} + +@test "find_next_rc_number: ignores tags for other versions" { + git() { + if [[ "$1" == "tag" && "$2" == "-l" ]]; then + echo "" + fi + } + export -f git + find_next_rc_number "1.19.0" + [ "$rc_number" = "0" ] +} + +# ---- find_next_patch_number ---- + +@test "find_next_patch_number: returns 0 when no tags exist" { + git() { + case "$1" in + tag) echo "" ;; + rev-parse) return 1 ;; + esac + } + export -f git + find_next_patch_number "1" "18" + [ "$patch" = "0" ] +} + +@test "find_next_patch_number: returns 0 when only rc tags exist for patch 0" { + git() { + case "$1" in + tag) printf "apache-parquet-1.18.0-rc0\napache-parquet-1.18.0-rc1\n" ;; + rev-parse) return 1 ;; + esac + } + export -f git + find_next_patch_number "1" "18" + [ "$patch" = "0" ] +} + +@test "find_next_patch_number: returns 1 when patch 0 has final release" { + git() { + case "$1" in + tag) printf "apache-parquet-1.18.0-rc0\n" ;; + rev-parse) + if [[ "$2" == "apache-parquet-1.18.0" ]]; then + echo "abc123" + return 0 + fi + return 1 + ;; + esac + } + export -f git + find_next_patch_number "1" "18" + [ "$patch" = "1" ] +} + +@test "find_next_patch_number: returns 2 when patches 0 and 1 have final releases" { + git() { + case "$1" in + tag) printf "apache-parquet-1.16.0-rc0\napache-parquet-1.16.1-rc0\n" ;; + rev-parse) + case "$2" in + "apache-parquet-1.16.0"|"apache-parquet-1.16.1") + echo "abc123" + return 0 + ;; + esac + return 1 + ;; + esac + } + export -f git + find_next_patch_number "1" "16" + [ "$patch" = "2" ] +} + +# ---- set_pom_version ---- + +@test "set_pom_version: passes correct args to mvnw in dry-run" { + DRY_RUN=1 + run set_pom_version "1.18.0" + [ "$status" -eq 0 ] + [[ "$output" == *"versions:set"* ]] + [[ "$output" == *"-DnewVersion=1.18.0"* ]] + [[ "$output" == *"-DgenerateBackupPoms=false"* ]] +} + +@test "set_pom_version: calls mvnw with correct args in real mode" { + DRY_RUN=0 + local captured_args="" + mvnw() { captured_args="$*"; } + # Create a fake mvnw wrapper that the function calls via ./mvnw + function mock_mvnw_wrapper { + echo "$@" > "${BATS_TEST_TMPDIR}/mvnw_args" + } + # Override exec_process to capture command + exec_process() { echo "EXEC: $*"; } + export -f exec_process + run set_pom_version "2.0.0-SNAPSHOT" + [ "$status" -eq 0 ] + [[ "$output" == *"2.0.0-SNAPSHOT"* ]] +} + +# ---- find_latest_rc_number ---- + +@test "find_latest_rc_number: returns highest RC number" { + cd "$(mktemp -d)" + git init -q + git commit --allow-empty -m "init" -q + git tag "apache-parquet-1.18.0-rc0" + git tag "apache-parquet-1.18.0-rc1" + git tag "apache-parquet-1.18.0-rc2" + find_latest_rc_number "1.18.0" + [ "$latest_rc_number" = "2" ] +} + +@test "find_latest_rc_number: returns 0 when only rc0 exists" { + cd "$(mktemp -d)" + git init -q + git commit --allow-empty -m "init" -q + git tag "apache-parquet-2.0.0-rc0" + find_latest_rc_number "2.0.0" + [ "$latest_rc_number" = "0" ] +} + +@test "find_latest_rc_number: fails when no RC tags exist" { + cd "$(mktemp -d)" + git init -q + git commit --allow-empty -m "init" -q + run find_latest_rc_number "9.9.9" + [ "$status" -eq 1 ] + [[ "$output" == *"No RC tags found"* ]] +} + +# ---- get_current_pom_version ---- + +@test "get_current_pom_version: calls mvnw help:evaluate" { + # Mock ./mvnw + function fake_mvnw { + if [[ "$*" == *"help:evaluate"* ]]; then + echo "1.18.0-SNAPSHOT" + fi + } + # Temporarily create a fake mvnw in a temp dir + local tmpbin + tmpbin=$(mktemp -d) + cat > "${tmpbin}/mvnw" << 'SCRIPT' +#!/bin/bash +if [[ "$*" == *"help:evaluate"* ]]; then + echo "1.18.0-SNAPSHOT" +fi +SCRIPT + chmod +x "${tmpbin}/mvnw" + # Run from tmpbin so ./mvnw resolves + cd "${tmpbin}" + local result + result=$(get_current_pom_version) + [ "$result" = "1.18.0-SNAPSHOT" ] + rm -rf "${tmpbin}" +} From 4ebedf3baed6902737592319b35e1f4198a57a9a Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 6 May 2026 16:21:41 -0500 Subject: [PATCH 02/14] Auto-compute next development version in publish-release Remove the next_dev_version input from publish-release.sh and the workflow. The next version is always the current patch incremented by one (e.g. 1.18.0 -> 1.18.1-SNAPSHOT), since the release branch only produces patches for that major.minor. --- .github/workflows/release-publish.yml | 7 +------ release/bin/publish-release.sh | 21 +++++++++------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index a909343d16..cdf1f4b79b 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -35,10 +35,6 @@ on: description: 'Nexus staging repository ID (e.g., orgapacheparquet-1234)' required: true type: string - next_dev_version: - description: 'Next development version without -SNAPSHOT (e.g., 1.18.1 or 1.19.0)' - required: true - type: string dry_run: description: 'Dry run mode (no actual changes)' required: false @@ -84,9 +80,8 @@ jobs: INPUT_VERSION: ${{ inputs.version }} INPUT_RC_NUMBER: ${{ inputs.rc_number }} INPUT_STAGING_REPO_ID: ${{ inputs.staging_repo_id }} - INPUT_NEXT_DEV_VERSION: ${{ inputs.next_dev_version }} run: | - args=("${INPUT_VERSION}" "${INPUT_STAGING_REPO_ID}" "${INPUT_NEXT_DEV_VERSION}") + args=("${INPUT_VERSION}" "${INPUT_STAGING_REPO_ID}") if [[ -n "${INPUT_RC_NUMBER}" ]]; then args+=(--rc "${INPUT_RC_NUMBER}") fi diff --git a/release/bin/publish-release.sh b/release/bin/publish-release.sh index 1f9d232f76..d9a95cecc4 100755 --- a/release/bin/publish-release.sh +++ b/release/bin/publish-release.sh @@ -35,19 +35,21 @@ source "${LIBS_DIR}/_maven.sh" # --------------------------------------------------------------------------- function usage { cat < [--rc ] +Usage: $0 [--rc ] Publish a release after the vote passes. Arguments: version Release version (e.g., 1.18.0) staging-repo-id Nexus staging repository ID (e.g., orgapacheparquet-1234) - next-dev-version Next development version without -SNAPSHOT (e.g., 1.18.1 or 1.19.0) Options: --rc RC number that passed the vote (default: auto-detect latest) --help Show this help +The next development version is auto-computed by incrementing the patch +version (e.g., 1.18.0 -> 1.18.1-SNAPSHOT). + Environment variables: DRY_RUN Set to 0 for real execution (default: 1) NEXUS_USERNAME Apache Nexus username @@ -57,8 +59,8 @@ Environment variables: GITHUB_TOKEN GitHub token for release creation Example: - DRY_RUN=1 $0 1.18.0 orgapacheparquet-1234 1.18.1 - DRY_RUN=1 $0 1.18.0 orgapacheparquet-1234 1.18.1 --rc 2 + DRY_RUN=1 $0 1.18.0 orgapacheparquet-1234 + DRY_RUN=1 $0 1.18.0 orgapacheparquet-1234 --rc 2 EOF exit "${1:-0}" } @@ -68,7 +70,6 @@ EOF # --------------------------------------------------------------------------- version="" staging_repo_id="" -next_dev_version="" rc_num="" positional=() @@ -96,14 +97,13 @@ while [[ $# -gt 0 ]]; do esac done -if [[ ${#positional[@]} -lt 3 ]]; then - print_error "Expected 3 positional arguments (version, staging-repo-id, next-dev-version), got ${#positional[@]}" +if [[ ${#positional[@]} -lt 2 ]]; then + print_error "Expected 2 positional arguments (version, staging-repo-id), got ${#positional[@]}" usage 1 fi version="${positional[0]}" staging_repo_id="${positional[1]}" -next_dev_version="${positional[2]}" # --------------------------------------------------------------------------- # Validate inputs @@ -121,10 +121,7 @@ if ! validate_and_extract_version "${version}"; then exit 1 fi -if ! [[ "${next_dev_version}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - print_error "Invalid next development version format: '${next_dev_version}'. Expected: X.Y.Z" - exit 1 -fi +next_dev_version="${major}.${minor}.$(( patch + 1 ))" if ! [[ "${staging_repo_id}" =~ ^[a-zA-Z][a-zA-Z0-9._-]*$ ]]; then print_error "Invalid staging repository ID: '${staging_repo_id}'. Expected alphanumeric with dots/hyphens (e.g., orgapacheparquet-1234)." From 7eb3b2c0d79e06646fdd19017e7dff8f6ff274fa Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 6 May 2026 16:32:54 -0500 Subject: [PATCH 03/14] Use env var references in Maven settings instead of real credentials The settings.xml now contains ${env.NEXUS_USERNAME} and ${env.NEXUS_PASSWORD} instead of the actual secret values. Maven resolves these from environment variables at build time, so the file itself contains no secrets and cannot be exfiltrated. --- release/libs/_maven.sh | 27 +++++---------------------- release/tests/maven.bats | 8 +++++--- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/release/libs/_maven.sh b/release/libs/_maven.sh index 38a91838d5..ddb916adc4 100644 --- a/release/libs/_maven.sh +++ b/release/libs/_maven.sh @@ -26,16 +26,6 @@ LIBS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${LIBS_DIR}/_constants.sh" source "${LIBS_DIR}/_exec.sh" -function _xml_escape { - local str="$1" - str="${str//&/&}" - str="${str///>}" - str="${str//\"/"}" - str="${str//\'/'}" - echo "${str}" -} - function generate_maven_settings { local settings_file="${1:-.release-settings.xml}" @@ -43,13 +33,7 @@ function generate_maven_settings { print_warning "NEXUS_USERNAME or NEXUS_PASSWORD not set; Maven deploy may fail" fi - local esc_username esc_password - esc_username=$(_xml_escape "${NEXUS_USERNAME:-}") - esc_password=$(_xml_escape "${NEXUS_PASSWORD:-}") - - ( - umask 077 - cat > "${settings_file}" < "${settings_file}" <<'SETTINGS_EOF' apache.releases.https - ${esc_username} - ${esc_password} + ${env.NEXUS_USERNAME} + ${env.NEXUS_PASSWORD} @@ -73,10 +57,9 @@ function generate_maven_settings { gpg-release -EOF - ) +SETTINGS_EOF - print_info "Generated Maven settings at ${settings_file} (mode 600)" + print_info "Generated Maven settings at ${settings_file} (credentials resolved from env vars at build time)" } function maven_deploy { diff --git a/release/tests/maven.bats b/release/tests/maven.bats index bb34f2b74f..9dff0e6a9a 100644 --- a/release/tests/maven.bats +++ b/release/tests/maven.bats @@ -44,13 +44,15 @@ teardown() { [[ "$(cat "${settings_file}")" == *"apache.releases.https"* ]] } -@test "generate_maven_settings: includes credentials" { +@test "generate_maven_settings: uses env var references instead of real credentials" { local settings_file="${TEST_TMPDIR}/settings.xml" generate_maven_settings "${settings_file}" local content content=$(cat "${settings_file}") - [[ "$content" == *"testuser"* ]] - [[ "$content" == *"testpass"* ]] + [[ "$content" == *'${env.NEXUS_USERNAME}'* ]] + [[ "$content" == *'${env.NEXUS_PASSWORD}'* ]] + [[ "$content" != *"testuser"* ]] + [[ "$content" != *"testpass"* ]] } @test "generate_maven_settings: enables GPG agent" { From 1524fe43f8cc1548a685397c59bcc84ae4ccfb60 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 6 May 2026 16:37:36 -0500 Subject: [PATCH 04/14] Strict regex filtering for RC tag lookups The glob-based git tag -l "...-rc*" could match malformed tags like "-rc10extra" or "-rc-foo". Add a _filter_rc_tags helper that applies a strict ^...-rc[0-9]+$ regex after the glob, so only well-formed RC tags are considered when auto-detecting RC numbers. --- release/libs/_version.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/release/libs/_version.sh b/release/libs/_version.sh index 2b2a4b31ad..2a1cc85685 100644 --- a/release/libs/_version.sh +++ b/release/libs/_version.sh @@ -61,11 +61,16 @@ function validate_and_extract_branch_version { return 0 } +function _filter_rc_tags { + local version_without_rc="$1" + local exact_pattern="^${TAG_PREFIX}${version_without_rc}-rc[0-9]+$" + git tag -l "${TAG_PREFIX}${version_without_rc}-rc*" | grep -E "${exact_pattern}" +} + function find_next_rc_number { local version_without_rc="$1" - local tag_pattern="${TAG_PREFIX}${version_without_rc}-rc*" local existing_tags - existing_tags=$(git tag -l "${tag_pattern}" | sort -V) + existing_tags=$(_filter_rc_tags "${version_without_rc}" || true) if [[ -z "${existing_tags}" ]]; then rc_number=0 @@ -79,9 +84,8 @@ function find_next_rc_number { function find_latest_rc_number { local version_without_rc="$1" - local tag_pattern="${TAG_PREFIX}${version_without_rc}-rc*" local existing_tags - existing_tags=$(git tag -l "${tag_pattern}" | sort -V) + existing_tags=$(_filter_rc_tags "${version_without_rc}" || true) if [[ -z "${existing_tags}" ]]; then print_error "No RC tags found for version ${version_without_rc}" From 4d52d06b3ab8e2573d78ae38b8e43d4489fec873 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 6 May 2026 16:41:38 -0500 Subject: [PATCH 05/14] Fail if GITHUB_TOKEN is missing during real CI check verification Previously, a missing GITHUB_TOKEN silently skipped CI verification and returned success, allowing a release to proceed even if CI was red. Now it fails unless running in dry-run mode. --- release/libs/_github.sh | 10 +++++----- release/tests/github.bats | 13 +++++++++++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/release/libs/_github.sh b/release/libs/_github.sh index 88348ad092..050aa305a2 100644 --- a/release/libs/_github.sh +++ b/release/libs/_github.sh @@ -32,16 +32,16 @@ function check_github_checks_passed() { print_info "Checking GitHub CI status for commit ${commit_sha}..." - if [[ -z "${GITHUB_TOKEN:-}" ]]; then - print_warning "GITHUB_TOKEN not set, skipping CI check verification" - return 0 - fi - if [[ ${DRY_RUN:-1} -eq 1 ]]; then print_info "DRY_RUN is enabled, skipping GitHub check verification" return 0 fi + if [[ -z "${GITHUB_TOKEN:-}" ]]; then + print_error "GITHUB_TOKEN is required to verify CI checks" + return 1 + fi + local repo_info="${GITHUB_REPO}" local num_incomplete diff --git a/release/tests/github.bats b/release/tests/github.bats index 65e10d5676..a1500136b1 100644 --- a/release/tests/github.bats +++ b/release/tests/github.bats @@ -25,11 +25,20 @@ setup() { # ---- check_github_checks_passed ---- -@test "check_github_checks_passed: skips when GITHUB_TOKEN not set" { +@test "check_github_checks_passed: fails when GITHUB_TOKEN not set" { unset GITHUB_TOKEN + DRY_RUN=0 + run check_github_checks_passed "abc123" + [ "$status" -eq 1 ] + [[ "$output" == *"GITHUB_TOKEN is required"* ]] +} + +@test "check_github_checks_passed: skips in dry-run even without GITHUB_TOKEN" { + unset GITHUB_TOKEN + DRY_RUN=1 run check_github_checks_passed "abc123" [ "$status" -eq 0 ] - [[ "$output" == *"GITHUB_TOKEN not set"* ]] + [[ "$output" == *"DRY_RUN"* ]] } @test "check_github_checks_passed: skips in dry-run mode" { From 7dbabbffcc8f4f2d4556dedd5573a3c07b1acfe2 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 6 May 2026 16:53:47 -0500 Subject: [PATCH 06/14] Fix CI: configure git identity for bats tests The find_latest_rc_number tests create temporary git repos and run git commit, which requires user.name and user.email to be set. The GitHub Actions runner has no default git identity. --- .github/workflows/ci-release-scripts.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-release-scripts.yml b/.github/workflows/ci-release-scripts.yml index 0ffcad84b8..78812a997d 100644 --- a/.github/workflows/ci-release-scripts.yml +++ b/.github/workflows/ci-release-scripts.yml @@ -45,6 +45,11 @@ jobs: sudo apt-get update sudo apt-get install -y bats + - name: Configure Git identity for tests + run: | + git config --global user.name "CI" + git config --global user.email "ci@test" + - name: Run bats tests run: bats release/tests/*.bats From a7f8d72f654e7ba03a330ed0dd304ceefd05e7d8 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Thu, 7 May 2026 09:05:53 -0500 Subject: [PATCH 07/14] Fix: add git user.name/email to prepare-rc workflow The GPG step configured signing but never set user.name or user.email, so git commit would fail on the CI runner. --- .github/workflows/release-prepare-rc.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-prepare-rc.yml b/.github/workflows/release-prepare-rc.yml index 47d5e42acb..5fd11cb656 100644 --- a/.github/workflows/release-prepare-rc.yml +++ b/.github/workflows/release-prepare-rc.yml @@ -57,12 +57,14 @@ jobs: distribution: temurin java-version: '11' - - name: Import GPG key and configure Git signing + - name: Import GPG key and configure Git env: GPG_PRIVATE_KEY: ${{ secrets.PARQUET_GPG_PRIVATE_KEY }} run: | echo "${GPG_PRIVATE_KEY}" | gpg --batch --import KEY_ID=$(gpg --list-keys --with-colons | grep '^fpr' | head -1 | cut -d: -f10) + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.signingkey "${KEY_ID}" git config --global commit.gpgsign true From 4356ea18e62a1ae5acf621e2f2ef3e4d103ad3c0 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Thu, 7 May 2026 15:38:38 -0500 Subject: [PATCH 08/14] Add fork guard to release workflows Skip release jobs when run outside the apache/* namespace. Prevents forkers from hitting missing-secret failures when clicking "Run workflow" in their fork's GitHub UI. Not strictly required for security (secrets don't propagate to forks), but fails fast and obviously instead of partway through. --- .github/workflows/release-cancel-rc.yml | 1 + .github/workflows/release-prepare-rc.yml | 1 + .github/workflows/release-publish.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/release-cancel-rc.yml b/.github/workflows/release-cancel-rc.yml index ff51173608..6d413545a8 100644 --- a/.github/workflows/release-cancel-rc.yml +++ b/.github/workflows/release-cancel-rc.yml @@ -42,6 +42,7 @@ on: jobs: cancel-rc: + if: github.repository_owner == 'apache' name: Cancel Release Candidate runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/release-prepare-rc.yml b/.github/workflows/release-prepare-rc.yml index 5fd11cb656..f0d103a497 100644 --- a/.github/workflows/release-prepare-rc.yml +++ b/.github/workflows/release-prepare-rc.yml @@ -39,6 +39,7 @@ on: jobs: prepare-rc: + if: github.repository_owner == 'apache' name: Prepare Release Candidate runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index cdf1f4b79b..73c2e9eb8a 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -43,6 +43,7 @@ on: jobs: publish-release: + if: github.repository_owner == 'apache' name: Publish Release runs-on: ubuntu-latest permissions: From 2b898b092c2f5aefc3f0cb48ae1e2159ee5999e2 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Thu, 7 May 2026 16:48:44 -0500 Subject: [PATCH 09/14] Verify staging repository before destructive Nexus actions Before publish-release.sh promotes a staging repo to Maven Central and before cancel-rc.sh drops one, perform a four-layer check on the provided staging_repo_id: 1. Profile == org.apache.parquet (catches wrong-project IDs) 2. State == closed (catches already-released, already-dropped, never-closed) 3. Artifact present: parquet-common-.pom in the repo (catches wrong-version IDs and empty repos) 4. Description contains "Apache Parquet RC" (catches wrong-RC of the same version line) The first three are unambiguous facts about the repo and hard-fail the script. The description is a free-text field editable in the Nexus UI, so a mismatch hard-fails by default but can be bypassed with --allow-description-mismatch (also exposed as a workflow input) for recovery scenarios. This closes a gap that exists in both Polaris's and the original Parquet manual procedure: "type the staging repo ID into a form field" replaces the Nexus-UI human eyeball check without a verification step in between. --- .github/workflows/release-cancel-rc.yml | 15 +- .github/workflows/release-publish.yml | 9 ++ release/bin/cancel-rc.sh | 59 +++++++- release/bin/publish-release.sh | 26 +++- release/libs/_constants.sh | 5 + release/libs/_nexus.sh | 105 +++++++++++++- release/tests/nexus.bats | 183 ++++++++++++++++++++++++ 7 files changed, 390 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release-cancel-rc.yml b/.github/workflows/release-cancel-rc.yml index 6d413545a8..d97d7894be 100644 --- a/.github/workflows/release-cancel-rc.yml +++ b/.github/workflows/release-cancel-rc.yml @@ -34,6 +34,11 @@ on: description: 'Nexus staging repository ID to drop (e.g., orgapacheparquet-1234)' required: true type: string + allow_description_mismatch: + description: 'Bypass the staging-repo description check (recovery only)' + required: false + type: boolean + default: false dry_run: description: 'Dry run mode (no actual changes)' required: false @@ -68,8 +73,10 @@ jobs: INPUT_VERSION: ${{ inputs.version }} INPUT_RC_NUMBER: ${{ inputs.rc_number }} INPUT_STAGING_REPO_ID: ${{ inputs.staging_repo_id }} + INPUT_ALLOW_DESCRIPTION_MISMATCH: ${{ inputs.allow_description_mismatch && '1' || '0' }} run: | - ./release/bin/cancel-rc.sh \ - "${INPUT_VERSION}" \ - "${INPUT_RC_NUMBER}" \ - "${INPUT_STAGING_REPO_ID}" + args=("${INPUT_VERSION}" "${INPUT_RC_NUMBER}" "${INPUT_STAGING_REPO_ID}") + if [[ "${INPUT_ALLOW_DESCRIPTION_MISMATCH}" == "1" ]]; then + args+=(--allow-description-mismatch) + fi + ./release/bin/cancel-rc.sh "${args[@]}" diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 73c2e9eb8a..4c416b2165 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -35,6 +35,11 @@ on: description: 'Nexus staging repository ID (e.g., orgapacheparquet-1234)' required: true type: string + allow_description_mismatch: + description: 'Bypass the staging-repo description check (recovery only)' + required: false + type: boolean + default: false dry_run: description: 'Dry run mode (no actual changes)' required: false @@ -81,9 +86,13 @@ jobs: INPUT_VERSION: ${{ inputs.version }} INPUT_RC_NUMBER: ${{ inputs.rc_number }} INPUT_STAGING_REPO_ID: ${{ inputs.staging_repo_id }} + INPUT_ALLOW_DESCRIPTION_MISMATCH: ${{ inputs.allow_description_mismatch && '1' || '0' }} run: | args=("${INPUT_VERSION}" "${INPUT_STAGING_REPO_ID}") if [[ -n "${INPUT_RC_NUMBER}" ]]; then args+=(--rc "${INPUT_RC_NUMBER}") fi + if [[ "${INPUT_ALLOW_DESCRIPTION_MISMATCH}" == "1" ]]; then + args+=(--allow-description-mismatch) + fi ./release/bin/publish-release.sh "${args[@]}" diff --git a/release/bin/cancel-rc.sh b/release/bin/cancel-rc.sh index 26f3ab8647..45788ad201 100755 --- a/release/bin/cancel-rc.sh +++ b/release/bin/cancel-rc.sh @@ -34,7 +34,7 @@ source "${LIBS_DIR}/_nexus.sh" # --------------------------------------------------------------------------- function usage { cat < +Usage: $0 [--allow-description-mismatch] Cancel a release candidate after a failed vote. @@ -43,6 +43,15 @@ Arguments: rc-num RC number to cancel (e.g., 0) staging-repo-id Nexus staging repository ID (e.g., orgapacheparquet-1234) +Options: + --allow-description-mismatch + Bypass the staging-repo description check (for recovery scenarios) + +Before dropping the staging repo, this script verifies that it belongs +to org.apache.parquet, is in 'closed' state, contains +${NEXUS_VERIFY_ARTIFACT_ID:-parquet-common}-.pom, and has a +description matching "Apache Parquet RC". + Environment variables: DRY_RUN Set to 0 for real execution (default: 1) NEXUS_USERNAME Apache Nexus username @@ -59,14 +68,40 @@ EOF # --------------------------------------------------------------------------- # Parse arguments # --------------------------------------------------------------------------- -if [[ $# -lt 3 ]]; then - print_error "Expected 3 arguments, got $#" +version="" +rc_num="" +staging_repo_id="" +allow_description_mismatch=0 +positional=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --allow-description-mismatch) + allow_description_mismatch=1 + shift + ;; + --help|-h) + usage 0 + ;; + -*) + print_error "Unknown option: $1" + usage 1 + ;; + *) + positional+=("$1") + shift + ;; + esac +done + +if [[ ${#positional[@]} -lt 3 ]]; then + print_error "Expected 3 positional arguments (version, rc-num, staging-repo-id), got ${#positional[@]}" usage 1 fi -version="$1" -rc_num="$2" -staging_repo_id="$3" +version="${positional[0]}" +rc_num="${positional[1]}" +staging_repo_id="${positional[2]}" # --------------------------------------------------------------------------- # Validate inputs @@ -102,6 +137,18 @@ step_summary "| Version | \`${version}\` |" step_summary "| RC tag | \`${rc_tag}\` |" step_summary "| Staging repo | \`${staging_repo_id}\` |" +# --------------------------------------------------------------------------- +# Step 0: Verify staging repository before any destructive action +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Staging Repository Verification" + +if ! nexus_verify_staging_repo "${staging_repo_id}" "${version}" "${rc_num}" "${allow_description_mismatch}"; then + step_summary "Staging repository verification: **FAILED**" + exit 1 +fi +step_summary "Staging repository \`${staging_repo_id}\` verified" + # --------------------------------------------------------------------------- # Step 1: Drop Nexus staging repo # --------------------------------------------------------------------------- diff --git a/release/bin/publish-release.sh b/release/bin/publish-release.sh index d9a95cecc4..061054df99 100755 --- a/release/bin/publish-release.sh +++ b/release/bin/publish-release.sh @@ -35,7 +35,7 @@ source "${LIBS_DIR}/_maven.sh" # --------------------------------------------------------------------------- function usage { cat < [--rc ] +Usage: $0 [--rc ] [--allow-description-mismatch] Publish a release after the vote passes. @@ -45,8 +45,15 @@ Arguments: Options: --rc RC number that passed the vote (default: auto-detect latest) + --allow-description-mismatch + Bypass the staging-repo description check (for recovery scenarios) --help Show this help +Before any destructive action, this script verifies that the staging repo +belongs to org.apache.parquet, is in 'closed' state, contains +${NEXUS_VERIFY_ARTIFACT_ID:-parquet-common}-.pom, and has a +description matching "Apache Parquet RC". + The next development version is auto-computed by incrementing the patch version (e.g., 1.18.0 -> 1.18.1-SNAPSHOT). @@ -71,6 +78,7 @@ EOF version="" staging_repo_id="" rc_num="" +allow_description_mismatch=0 positional=() while [[ $# -gt 0 ]]; do @@ -83,6 +91,10 @@ while [[ $# -gt 0 ]]; do rc_num="$2" shift 2 ;; + --allow-description-mismatch) + allow_description_mismatch=1 + shift + ;; --help|-h) usage 0 ;; @@ -181,6 +193,18 @@ step_summary "| Staging repo | \`${staging_repo_id}\` |" step_summary "| Next dev version | \`${next_dev_version}-SNAPSHOT\` |" step_summary "| Commit | \`${rc_commit}\` |" +# --------------------------------------------------------------------------- +# Step 0: Verify staging repository before any destructive action +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Staging Repository Verification" + +if ! nexus_verify_staging_repo "${staging_repo_id}" "${version}" "${rc_num}" "${allow_description_mismatch}"; then + step_summary "Staging repository verification: **FAILED**" + exit 1 +fi +step_summary "Staging repository \`${staging_repo_id}\` verified" + # --------------------------------------------------------------------------- # Step 1: Move SVN artifacts from dist/dev to dist/release # --------------------------------------------------------------------------- diff --git a/release/libs/_constants.sh b/release/libs/_constants.sh index 16926c17ca..2536427154 100644 --- a/release/libs/_constants.sh +++ b/release/libs/_constants.sh @@ -29,8 +29,13 @@ APACHE_DIST_DEV_PATH="/dev/parquet" APACHE_DIST_RELEASE_PATH="/release/parquet" NEXUS_BASE_URL=${NEXUS_BASE_URL:-"https://repository.apache.org/service/local"} +NEXUS_CONTENT_BASE_URL=${NEXUS_CONTENT_BASE_URL:-"https://repository.apache.org/content/repositories"} NEXUS_STAGING_GROUP_URL="https://repository.apache.org/content/groups/staging/org/apache/parquet/" +NEXUS_PROFILE_NAME="org.apache.parquet" +NEXUS_VERIFY_GROUP_PATH="org/apache/parquet" +NEXUS_VERIFY_ARTIFACT_ID="parquet-common" + DRY_RUN=${DRY_RUN:-1} VERSION_REGEX="([0-9]+)\.([0-9]+)\.([0-9]+)" diff --git a/release/libs/_nexus.sh b/release/libs/_nexus.sh index 7f591d1e07..20c43a0f3f 100644 --- a/release/libs/_nexus.sh +++ b/release/libs/_nexus.sh @@ -68,8 +68,111 @@ function nexus_drop_staging_repo { _nexus_bulk_action "drop" "${repo_id}" "${description}" } +function nexus_get_staging_repo_metadata { + local repo_id="$1" + local url="${NEXUS_BASE_URL}/staging/repository/${repo_id}" + + nexus_repo_profile="" + nexus_repo_state="" + nexus_repo_description="" + + if [[ ${DRY_RUN:-1} -eq 1 ]]; then + print_command "Dry-run, WOULD GET ${url} (skipping metadata fetch)" + nexus_repo_profile="${NEXUS_PROFILE_NAME}" + nexus_repo_state="closed" + nexus_repo_description="DRY_RUN_DESCRIPTION" + return 0 + fi + + local response + if ! response=$(curl --fail --silent --show-error \ + -K <(printf 'user = "%s:%s"\n' "${NEXUS_USERNAME}" "${NEXUS_PASSWORD}") \ + -H "Accept: application/json" \ + "${url}"); then + print_error "Failed to fetch staging repository metadata for ${repo_id}" + return 1 + fi + + nexus_repo_profile=$(echo "${response}" | jq -r '.profileName // ""') + nexus_repo_state=$(echo "${response}" | jq -r '.type // ""') + nexus_repo_description=$(echo "${response}" | jq -r '.description // ""') + + if [[ -z "${nexus_repo_profile}" || -z "${nexus_repo_state}" ]]; then + print_error "Unable to parse staging repository metadata for ${repo_id}" + return 1 + fi + + return 0 +} + +function nexus_check_staging_artifact { + local repo_id="$1" + local version="$2" + local artifact_url="${NEXUS_CONTENT_BASE_URL}/${repo_id}/${NEXUS_VERIFY_GROUP_PATH}/${NEXUS_VERIFY_ARTIFACT_ID}/${version}/${NEXUS_VERIFY_ARTIFACT_ID}-${version}.pom" + + if [[ ${DRY_RUN:-1} -eq 1 ]]; then + print_command "Dry-run, WOULD HEAD ${artifact_url}" + return 0 + fi + + if ! curl --fail --silent --show-error --head \ + -K <(printf 'user = "%s:%s"\n' "${NEXUS_USERNAME}" "${NEXUS_PASSWORD}") \ + "${artifact_url}" >/dev/null; then + print_error "Expected artifact not found in staging repo: ${artifact_url}" + return 1 + fi + + return 0 +} + +function nexus_verify_staging_repo { + local repo_id="$1" + local version="$2" + local rc_num="$3" + local allow_description_mismatch="${4:-0}" + + local expected_description="Apache Parquet ${version} RC${rc_num}" + + print_info "Verifying staging repository ${repo_id}..." + + if ! nexus_get_staging_repo_metadata "${repo_id}"; then + return 1 + fi + + if [[ "${nexus_repo_profile}" != "${NEXUS_PROFILE_NAME}" ]]; then + print_error "Profile mismatch: expected '${NEXUS_PROFILE_NAME}', got '${nexus_repo_profile}'" + print_error "This staging repo does not belong to Apache Parquet." + return 1 + fi + + if [[ "${nexus_repo_state}" != "closed" ]]; then + print_error "Unexpected state: expected 'closed', got '${nexus_repo_state}'" + print_error "Staging repo must be closed (not open/released/dropped) before this action." + return 1 + fi + + if ! nexus_check_staging_artifact "${repo_id}" "${version}"; then + print_error "Verification failed: ${NEXUS_VERIFY_ARTIFACT_ID}-${version}.pom not found in staging repo." + print_error "This staging repo does not appear to contain ${version} artifacts." + return 1 + fi + + if [[ ${DRY_RUN:-1} -ne 1 && "${nexus_repo_description}" != *"${expected_description}"* ]]; then + print_warning "Description mismatch: expected to contain '${expected_description}'" + print_warning "Actual description: '${nexus_repo_description}'" + if [[ "${allow_description_mismatch}" != "1" ]]; then + print_error "Refusing to proceed. Re-run with --allow-description-mismatch to bypass." + return 1 + fi + print_warning "Continuing despite description mismatch (--allow-description-mismatch)." + fi + + print_info "Staging repository ${repo_id} verified." + return 0 +} + function nexus_find_open_staging_repo { - local profile_name="${1:-org.apache.parquet}" + local profile_name="${1:-${NEXUS_PROFILE_NAME}}" print_info "Searching for open staging repository for ${profile_name}..." diff --git a/release/tests/nexus.bats b/release/tests/nexus.bats index dd66c457a7..10d3d39ba4 100644 --- a/release/tests/nexus.bats +++ b/release/tests/nexus.bats @@ -106,3 +106,186 @@ setup() { [ "$status" -eq 0 ] [[ "$output" == *"staging/bulk/drop"* ]] } + +# ---- nexus_get_staging_repo_metadata ---- + +@test "nexus_get_staging_repo_metadata: dry-run sets placeholder values" { + DRY_RUN=1 + nexus_get_staging_repo_metadata "orgapacheparquet-1234" + [ "${nexus_repo_profile}" = "org.apache.parquet" ] + [ "${nexus_repo_state}" = "closed" ] + [ -n "${nexus_repo_description}" ] +} + +@test "nexus_get_staging_repo_metadata: parses real-mode JSON response" { + DRY_RUN=0 + curl() { + cat <<'JSON' +{ + "profileName": "org.apache.parquet", + "type": "closed", + "description": "Apache Parquet 1.18.0 RC0" +} +JSON + return 0 + } + export -f curl + nexus_get_staging_repo_metadata "orgapacheparquet-1234" + [ "${nexus_repo_profile}" = "org.apache.parquet" ] + [ "${nexus_repo_state}" = "closed" ] + [ "${nexus_repo_description}" = "Apache Parquet 1.18.0 RC0" ] +} + +@test "nexus_get_staging_repo_metadata: fails when curl fails" { + DRY_RUN=0 + curl() { return 22; } + export -f curl + run nexus_get_staging_repo_metadata "orgapacheparquet-1234" + [ "$status" -eq 1 ] + [[ "$output" == *"Failed to fetch"* ]] +} + +# ---- nexus_check_staging_artifact ---- + +@test "nexus_check_staging_artifact: dry-run skips check" { + DRY_RUN=1 + run nexus_check_staging_artifact "orgapacheparquet-1234" "1.18.0" + [ "$status" -eq 0 ] + [[ "$output" == *"Dry-run"* ]] + [[ "$output" == *"parquet-common-1.18.0.pom"* ]] +} + +@test "nexus_check_staging_artifact: succeeds when artifact present" { + DRY_RUN=0 + curl() { return 0; } + export -f curl + run nexus_check_staging_artifact "orgapacheparquet-1234" "1.18.0" + [ "$status" -eq 0 ] +} + +@test "nexus_check_staging_artifact: fails when artifact missing" { + DRY_RUN=0 + curl() { return 22; } + export -f curl + run nexus_check_staging_artifact "orgapacheparquet-1234" "1.18.0" + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] +} + +# ---- nexus_verify_staging_repo ---- + +@test "nexus_verify_staging_repo: dry-run passes without making real calls" { + DRY_RUN=1 + run nexus_verify_staging_repo "orgapacheparquet-1234" "1.18.0" "0" + [ "$status" -eq 0 ] + [[ "$output" == *"verified"* ]] +} + +@test "nexus_verify_staging_repo: rejects wrong profile" { + DRY_RUN=0 + curl() { + cat <<'JSON' +{"profileName": "org.apache.iceberg", "type": "closed", "description": "Apache Parquet 1.18.0 RC0"} +JSON + return 0 + } + export -f curl + run nexus_verify_staging_repo "orgapacheparquet-1234" "1.18.0" "0" + [ "$status" -eq 1 ] + [[ "$output" == *"Profile mismatch"* ]] +} + +@test "nexus_verify_staging_repo: rejects non-closed state" { + DRY_RUN=0 + curl() { + cat <<'JSON' +{"profileName": "org.apache.parquet", "type": "open", "description": "Apache Parquet 1.18.0 RC0"} +JSON + return 0 + } + export -f curl + run nexus_verify_staging_repo "orgapacheparquet-1234" "1.18.0" "0" + [ "$status" -eq 1 ] + [[ "$output" == *"Unexpected state"* ]] +} + +@test "nexus_verify_staging_repo: rejects released state" { + DRY_RUN=0 + curl() { + cat <<'JSON' +{"profileName": "org.apache.parquet", "type": "released", "description": "Apache Parquet 1.18.0 RC0"} +JSON + return 0 + } + export -f curl + run nexus_verify_staging_repo "orgapacheparquet-1234" "1.18.0" "0" + [ "$status" -eq 1 ] + [[ "$output" == *"Unexpected state"* ]] +} + +@test "nexus_verify_staging_repo: rejects when artifact missing" { + DRY_RUN=0 + curl() { + if [[ "$*" == *"staging/repository"* ]]; then + cat <<'JSON' +{"profileName": "org.apache.parquet", "type": "closed", "description": "Apache Parquet 1.18.0 RC0"} +JSON + return 0 + fi + return 22 + } + export -f curl + run nexus_verify_staging_repo "orgapacheparquet-1234" "1.18.0" "0" + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] +} + +@test "nexus_verify_staging_repo: rejects description mismatch by default" { + DRY_RUN=0 + curl() { + if [[ "$*" == *"staging/repository"* ]]; then + cat <<'JSON' +{"profileName": "org.apache.parquet", "type": "closed", "description": "Apache Parquet 1.18.0 RC1"} +JSON + fi + return 0 + } + export -f curl + run nexus_verify_staging_repo "orgapacheparquet-1234" "1.18.0" "0" + [ "$status" -eq 1 ] + [[ "$output" == *"Description mismatch"* ]] + [[ "$output" == *"--allow-description-mismatch"* ]] +} + +@test "nexus_verify_staging_repo: allows description mismatch with flag" { + DRY_RUN=0 + curl() { + if [[ "$*" == *"staging/repository"* ]]; then + cat <<'JSON' +{"profileName": "org.apache.parquet", "type": "closed", "description": "Apache Parquet 1.18.0 RC1"} +JSON + fi + return 0 + } + export -f curl + run nexus_verify_staging_repo "orgapacheparquet-1234" "1.18.0" "0" "1" + [ "$status" -eq 0 ] + [[ "$output" == *"Description mismatch"* ]] + [[ "$output" == *"Continuing despite"* ]] +} + +@test "nexus_verify_staging_repo: passes when everything matches" { + DRY_RUN=0 + curl() { + if [[ "$*" == *"staging/repository"* ]]; then + cat <<'JSON' +{"profileName": "org.apache.parquet", "type": "closed", "description": "Apache Parquet 1.18.0 RC0"} +JSON + fi + return 0 + } + export -f curl + run nexus_verify_staging_repo "orgapacheparquet-1234" "1.18.0" "0" + [ "$status" -eq 0 ] + [[ "$output" == *"verified"* ]] +} From f27be0c3b737aeaa412f0d565f2a14d3f219a54b Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Fri, 8 May 2026 14:41:15 -0500 Subject: [PATCH 10/14] Add per-version concurrency guards to release workflows Prevents two release workflows from racing on the same version: prepare-rc, publish, and cancel-rc all share group `release-${version}` with cancel-in-progress=false. Different versions can still run in parallel. --- .github/workflows/release-cancel-rc.yml | 4 ++++ .github/workflows/release-prepare-rc.yml | 4 ++++ .github/workflows/release-publish.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/release-cancel-rc.yml b/.github/workflows/release-cancel-rc.yml index d97d7894be..833c12b0f9 100644 --- a/.github/workflows/release-cancel-rc.yml +++ b/.github/workflows/release-cancel-rc.yml @@ -45,6 +45,10 @@ on: type: boolean default: true +concurrency: + group: release-${{ inputs.version }} + cancel-in-progress: false + jobs: cancel-rc: if: github.repository_owner == 'apache' diff --git a/.github/workflows/release-prepare-rc.yml b/.github/workflows/release-prepare-rc.yml index f0d103a497..ac79678f01 100644 --- a/.github/workflows/release-prepare-rc.yml +++ b/.github/workflows/release-prepare-rc.yml @@ -37,6 +37,10 @@ on: type: boolean default: true +concurrency: + group: release-${{ inputs.version }} + cancel-in-progress: false + jobs: prepare-rc: if: github.repository_owner == 'apache' diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 4c416b2165..daad9d908d 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -46,6 +46,10 @@ on: type: boolean default: true +concurrency: + group: release-${{ inputs.version }} + cancel-in-progress: false + jobs: publish-release: if: github.repository_owner == 'apache' From 35f756be0084d872e76b51a4402fc1c244afb0f5 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Fri, 8 May 2026 15:39:40 -0500 Subject: [PATCH 11/14] Address review: clean up dead code and harden _redact_secrets Apply quick wins from the multi-agent review: - Remove stale `source _maven.sh` from publish-release.sh (no maven_* function is ever called from this script) - Delete dead `validate_and_extract_git_tag_version` and `find_next_patch_number` (no callers; the latter also had an unanchored regex that the strict-filter commit was meant to address) - Fix `_redact_secrets` glob-pattern bug: secrets containing `*`, `?`, `[`, or `\` were treated as patterns by `${var//pattern/string}` and could fail to redact. Switch to a literal-string split-and-rejoin using `${rest%%"$secret"*}` (quoting forces literal match) - Drop unused `VERSION_REGEX_GIT_TAG` and `VERSION_REGEX_FINAL_TAG` constants and rewrite `BRANCH_VERSION_REGEX` to use `${BRANCH_PREFIX}` - Add 6 tests for `_redact_secrets` covering glob metacharacters, backslashes, multiple occurrences, and multiple secrets --- release/bin/publish-release.sh | 1 - release/libs/_constants.sh | 4 +- release/libs/_exec.sh | 17 ++++-- release/libs/_version.sh | 43 --------------- release/tests/constants.bats | 10 ---- release/tests/exec.bats | 53 ++++++++++++++++++ release/tests/version.bats | 98 ---------------------------------- 7 files changed, 68 insertions(+), 158 deletions(-) diff --git a/release/bin/publish-release.sh b/release/bin/publish-release.sh index 061054df99..09b9720f1d 100755 --- a/release/bin/publish-release.sh +++ b/release/bin/publish-release.sh @@ -28,7 +28,6 @@ source "${LIBS_DIR}/_log.sh" source "${LIBS_DIR}/_exec.sh" source "${LIBS_DIR}/_version.sh" source "${LIBS_DIR}/_nexus.sh" -source "${LIBS_DIR}/_maven.sh" # --------------------------------------------------------------------------- # Usage diff --git a/release/libs/_constants.sh b/release/libs/_constants.sh index 2536427154..409d5da07f 100644 --- a/release/libs/_constants.sh +++ b/release/libs/_constants.sh @@ -39,8 +39,6 @@ NEXUS_VERIFY_ARTIFACT_ID="parquet-common" DRY_RUN=${DRY_RUN:-1} VERSION_REGEX="([0-9]+)\.([0-9]+)\.([0-9]+)" -VERSION_REGEX_GIT_TAG="^${TAG_PREFIX}${VERSION_REGEX}-rc([0-9]+)$" -VERSION_REGEX_FINAL_TAG="^${TAG_PREFIX}${VERSION_REGEX}$" -BRANCH_VERSION_REGEX="^parquet-([0-9]+)\.([0-9]+)\.x$" +BRANCH_VERSION_REGEX="^${BRANCH_PREFIX}([0-9]+)\.([0-9]+)\.x$" GITHUB_REPO=${GITHUB_REPOSITORY:-"apache/parquet-java"} diff --git a/release/libs/_exec.sh b/release/libs/_exec.sh index 3b75e38df1..7920c0c5b0 100644 --- a/release/libs/_exec.sh +++ b/release/libs/_exec.sh @@ -31,9 +31,20 @@ function _redact_secrets { local secret_var for secret_var in NEXUS_PASSWORD NEXUS_USERNAME SVN_PASSWORD SVN_USERNAME GITHUB_TOKEN; do local secret_val="${!secret_var:-}" - if [[ -n "${secret_val}" ]]; then - cmd_str="${cmd_str//${secret_val}/***}" - fi + [[ -z "${secret_val}" ]] && continue + # Literal-string replace via split-and-rejoin: quoting "$secret_val" + # inside ${var%%pattern} forces the inner expansion to match + # literally (bash documents this for parameter-expansion patterns), + # which avoids treating glob metacharacters (* ? [ \) in the secret + # as pattern syntax. The plain ${var//pattern/string} form does not + # offer any escape mechanism. + local rest="${cmd_str}" + local result="" + while [[ "${rest}" == *"${secret_val}"* ]]; do + result+="${rest%%"${secret_val}"*}***" + rest="${rest#*"${secret_val}"}" + done + cmd_str="${result}${rest}" done echo "${cmd_str}" } diff --git a/release/libs/_version.sh b/release/libs/_version.sh index 2a1cc85685..61ada8d760 100644 --- a/release/libs/_version.sh +++ b/release/libs/_version.sh @@ -38,19 +38,6 @@ function validate_and_extract_version { return 0 } -function validate_and_extract_git_tag_version { - local tag="$1" - if [[ ! ${tag} =~ ${VERSION_REGEX_GIT_TAG} ]]; then - return 1 - fi - major="${BASH_REMATCH[1]}" - minor="${BASH_REMATCH[2]}" - patch="${BASH_REMATCH[3]}" - rc_number="${BASH_REMATCH[4]}" - version_without_rc="${major}.${minor}.${patch}" - return 0 -} - function validate_and_extract_branch_version { local branch="$1" if [[ ! ${branch} =~ ${BRANCH_VERSION_REGEX} ]]; then @@ -96,36 +83,6 @@ function find_latest_rc_number { return 0 } -function find_next_patch_number { - local major="$1" - local minor="$2" - local rc_tag_pattern="${TAG_PREFIX}${major}.${minor}.*-rc*" - local existing_rc_tags - existing_rc_tags=$(git tag -l "${rc_tag_pattern}" | sort -V) - - if [[ -z "${existing_rc_tags}" ]]; then - patch=0 - else - local highest_patch=-1 - while IFS= read -r tag; do - if [[ ${tag} =~ ${TAG_PREFIX}${major}\.${minor}\.([0-9]+)-rc[0-9]+ ]]; then - local current_patch="${BASH_REMATCH[1]}" - if [[ ${current_patch} -gt ${highest_patch} ]]; then - highest_patch=${current_patch} - fi - fi - done <<< "${existing_rc_tags}" - - local final_tag="${TAG_PREFIX}${major}.${minor}.${highest_patch}" - if git rev-parse "${final_tag}" >/dev/null 2>&1; then - patch=$((highest_patch + 1)) - else - patch=${highest_patch} - fi - fi - return 0 -} - function set_pom_version { local version="$1" exec_process ./mvnw versions:set -DnewVersion="${version}" -DgenerateBackupPoms=false --batch-mode -q diff --git a/release/tests/constants.bats b/release/tests/constants.bats index fe0c7be10d..1011af5e01 100644 --- a/release/tests/constants.bats +++ b/release/tests/constants.bats @@ -58,16 +58,6 @@ setup() { [ "${BASH_REMATCH[3]}" = "0" ] } -@test "VERSION_REGEX_GIT_TAG matches RC tag" { - [[ "apache-parquet-1.18.0-rc3" =~ ${VERSION_REGEX_GIT_TAG} ]] - [ "${BASH_REMATCH[1]}" = "1" ] - [ "${BASH_REMATCH[4]}" = "3" ] -} - -@test "VERSION_REGEX_GIT_TAG rejects final tag" { - [[ ! "apache-parquet-1.18.0" =~ ${VERSION_REGEX_GIT_TAG} ]] -} - @test "BRANCH_VERSION_REGEX matches parquet-1.18.x" { [[ "parquet-1.18.x" =~ ${BRANCH_VERSION_REGEX} ]] [ "${BASH_REMATCH[1]}" = "1" ] diff --git a/release/tests/exec.bats b/release/tests/exec.bats index 4b794022f2..c5f3e66e04 100644 --- a/release/tests/exec.bats +++ b/release/tests/exec.bats @@ -23,6 +23,59 @@ setup() { source "${LIBS_DIR}/_exec.sh" } +# ---- _redact_secrets ---- + +@test "_redact_secrets: replaces NEXUS_PASSWORD with ***" { + NEXUS_PASSWORD="hunter2" + run _redact_secrets curl -u user:hunter2 https://example.com + [ "$status" -eq 0 ] + [[ "$output" != *"hunter2"* ]] + [[ "$output" == *"***"* ]] +} + +@test "_redact_secrets: skips empty secrets" { + unset NEXUS_PASSWORD NEXUS_USERNAME SVN_PASSWORD SVN_USERNAME GITHUB_TOKEN + run _redact_secrets echo "no secrets here" + [ "$status" -eq 0 ] + [ "$output" = "echo no secrets here" ] +} + +@test "_redact_secrets: handles secrets with glob metacharacters" { + NEXUS_PASSWORD='ab*cd?ef[g]' + run _redact_secrets 'user:ab*cd?ef[g] more' + [ "$status" -eq 0 ] + [[ "$output" != *'ab*cd'* ]] + [[ "$output" == *'***'* ]] +} + +@test "_redact_secrets: secret with backslash is matched literally" { + NEXUS_PASSWORD='abc\def' + run _redact_secrets 'pw=abc\def end' + [ "$status" -eq 0 ] + [[ "$output" != *'abc\def'* ]] + [[ "$output" == *'***'* ]] +} + +@test "_redact_secrets: replaces every occurrence" { + GITHUB_TOKEN="tok123" + run _redact_secrets "Authorization: tok123 Bearer tok123 etc" + [ "$status" -eq 0 ] + [[ "$output" != *"tok123"* ]] + # Two occurrences should both be redacted + local count + count=$(echo "$output" | grep -o "\*\*\*" | wc -l) + [ "$count" -eq 2 ] +} + +@test "_redact_secrets: redacts multiple different secrets" { + NEXUS_USERNAME="alice" + NEXUS_PASSWORD="hunter2" + run _redact_secrets "user=alice pass=hunter2" + [ "$status" -eq 0 ] + [[ "$output" != *"alice"* ]] + [[ "$output" != *"hunter2"* ]] +} + # ---- exec_process ---- @test "exec_process: dry-run prints but does not execute" { diff --git a/release/tests/version.bats b/release/tests/version.bats index 18f505c252..9554fd0dbb 100644 --- a/release/tests/version.bats +++ b/release/tests/version.bats @@ -66,40 +66,6 @@ setup() { [ "$status" -eq 1 ] } -# ---- validate_and_extract_git_tag_version ---- - -@test "validate_and_extract_git_tag_version: parses apache-parquet-1.18.0-rc0" { - validate_and_extract_git_tag_version "apache-parquet-1.18.0-rc0" - [ "$major" = "1" ] - [ "$minor" = "18" ] - [ "$patch" = "0" ] - [ "$rc_number" = "0" ] - [ "$version_without_rc" = "1.18.0" ] -} - -@test "validate_and_extract_git_tag_version: parses apache-parquet-2.1.3-rc12" { - validate_and_extract_git_tag_version "apache-parquet-2.1.3-rc12" - [ "$major" = "2" ] - [ "$minor" = "1" ] - [ "$patch" = "3" ] - [ "$rc_number" = "12" ] -} - -@test "validate_and_extract_git_tag_version: rejects tag without rc suffix" { - run validate_and_extract_git_tag_version "apache-parquet-1.18.0" - [ "$status" -eq 1 ] -} - -@test "validate_and_extract_git_tag_version: rejects wrong prefix" { - run validate_and_extract_git_tag_version "apache-polaris-1.18.0-rc0" - [ "$status" -eq 1 ] -} - -@test "validate_and_extract_git_tag_version: rejects bare version" { - run validate_and_extract_git_tag_version "1.18.0-rc0" - [ "$status" -eq 1 ] -} - # ---- validate_and_extract_branch_version ---- @test "validate_and_extract_branch_version: parses parquet-1.18.x" { @@ -182,70 +148,6 @@ setup() { [ "$rc_number" = "0" ] } -# ---- find_next_patch_number ---- - -@test "find_next_patch_number: returns 0 when no tags exist" { - git() { - case "$1" in - tag) echo "" ;; - rev-parse) return 1 ;; - esac - } - export -f git - find_next_patch_number "1" "18" - [ "$patch" = "0" ] -} - -@test "find_next_patch_number: returns 0 when only rc tags exist for patch 0" { - git() { - case "$1" in - tag) printf "apache-parquet-1.18.0-rc0\napache-parquet-1.18.0-rc1\n" ;; - rev-parse) return 1 ;; - esac - } - export -f git - find_next_patch_number "1" "18" - [ "$patch" = "0" ] -} - -@test "find_next_patch_number: returns 1 when patch 0 has final release" { - git() { - case "$1" in - tag) printf "apache-parquet-1.18.0-rc0\n" ;; - rev-parse) - if [[ "$2" == "apache-parquet-1.18.0" ]]; then - echo "abc123" - return 0 - fi - return 1 - ;; - esac - } - export -f git - find_next_patch_number "1" "18" - [ "$patch" = "1" ] -} - -@test "find_next_patch_number: returns 2 when patches 0 and 1 have final releases" { - git() { - case "$1" in - tag) printf "apache-parquet-1.16.0-rc0\napache-parquet-1.16.1-rc0\n" ;; - rev-parse) - case "$2" in - "apache-parquet-1.16.0"|"apache-parquet-1.16.1") - echo "abc123" - return 0 - ;; - esac - return 1 - ;; - esac - } - export -f git - find_next_patch_number "1" "16" - [ "$patch" = "2" ] -} - # ---- set_pom_version ---- @test "set_pom_version: passes correct args to mvnw in dry-run" { From 778472ea39d09d807f6fe9dbf7c242ecf20f8706 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Fri, 8 May 2026 15:43:46 -0500 Subject: [PATCH 12/14] Address review: tighten release-time safety guards Three correctness fixes from the multi-agent review: 1. publish-release.sh: refuse to push the next-dev-version commit unless we are on a release branch matching the version we just published. The HEAD-vs-RC guard alone did not catch the case where prepare-rc was run with --skip-branch-creation against master and master itself was the RC commit. 2. nexus_find_open_staging_repo: filter on the explicit element instead of doing a substring match on the repository ID, and reject when more than one open repo exists for the profile (operator must clean up the leftover before we can pick safely). Also surface XML-parse errors instead of silently returning empty. 3. prepare-rc.sh: re-verify via nexus_check_staging_artifact that the open repo we are about to close actually contains the version we just deployed. Closing the wrong repo would corrupt a concurrent release on a different version. Plus a documentation fix: 4. prepare-rc.sh CI step summary now states explicitly that the RC tag sits on a child commit of the CI-verified commit, not on the verified commit itself. The version-bump commit is a mechanical change, but the previous wording ("CI checks: PASSED" right before creating an unverified tag) was misleading. Add 6 tests for the new nexus_find_open_staging_repo behavior covering the happy path, no-open-repo, multiple-open-repos, curl failure, and malformed XML. --- release/bin/prepare-rc.sh | 26 +++++++-- release/bin/publish-release.sh | 21 ++++++- release/libs/_nexus.sh | 50 +++++++++++++---- release/tests/nexus.bats | 100 +++++++++++++++++++++++++++++++++ 4 files changed, 180 insertions(+), 17 deletions(-) diff --git a/release/bin/prepare-rc.sh b/release/bin/prepare-rc.sh index c19f14b28a..172356fc4a 100755 --- a/release/bin/prepare-rc.sh +++ b/release/bin/prepare-rc.sh @@ -198,14 +198,20 @@ step_summary "" step_summary "### CI Verification" current_commit=$(git rev-parse HEAD) -step_summary "| Commit | \`${current_commit}\` |" +step_summary "| Verified commit | \`${current_commit}\` |" if ! check_github_checks_passed "${current_commit}"; then print_error "CI checks are not passing. Fix CI before creating an RC." step_summary "CI checks: **FAILED**" exit 1 fi -step_summary "CI checks: **PASSED**" +step_summary "CI checks on \`${current_commit}\`: **PASSED**" +step_summary "" +step_summary "> **Note:** the RC tag is placed on a child commit (\"Set" +step_summary "> version to ${version} for release\") that is created in" +step_summary "> the next step. That commit is not directly CI-verified;" +step_summary "> it is a mechanical version bump on top of the verified" +step_summary "> commit above." # --------------------------------------------------------------------------- # Step 4: Set POM versions (only on rc0 or if version doesn't match) @@ -255,8 +261,20 @@ maven_deploy "${settings_file}" step_summary "Deployed artifacts to Apache Nexus staging" -# Find and close the staging repo -nexus_find_open_staging_repo "org.apache.parquet" +# Find and close the staging repo. Verify it actually contains the +# artifacts we just deployed before closing -- another concurrent run +# could have created the only open repo we see, and closing the wrong +# one would corrupt that release. +nexus_find_open_staging_repo "${NEXUS_PROFILE_NAME}" + +if [[ ${DRY_RUN:-1} -ne 1 ]]; then + if ! nexus_check_staging_artifact "${staging_repo_id}" "${version}"; then + print_error "Open staging repo ${staging_repo_id} does not contain ${version} artifacts." + print_error "Refusing to close it -- another release may be in flight. Investigate Nexus before retrying." + exit 1 + fi +fi + nexus_close_staging_repo "${staging_repo_id}" "Apache Parquet ${version} RC${rc_number}" step_summary "Closed staging repository: \`${staging_repo_id}\`" diff --git a/release/bin/publish-release.sh b/release/bin/publish-release.sh index 09b9720f1d..0517c8458f 100755 --- a/release/bin/publish-release.sh +++ b/release/bin/publish-release.sh @@ -318,9 +318,28 @@ exec_process ./mvnw -pl . versions:set-property \ exec_process git add -A exec_process git commit -m "Prepare for next development iteration (${next_snapshot})" + +# Refuse to push the version bump to anything that doesn't look like a +# release branch. Otherwise a workflow run dispatched against master +# (e.g. after `prepare-rc.sh --skip-branch-creation`) would push the +# next-dev-version commit straight to master. +current_branch=$(git rev-parse --abbrev-ref HEAD) +expected_branch="${BRANCH_PREFIX}${major}.${minor}.x" +if [[ "${current_branch}" != "${expected_branch}" ]]; then + print_error "Refusing to push version bump: current branch is '${current_branch}'" + print_error "Expected to be on release branch '${expected_branch}' for version ${version}." + print_error "Run the publish workflow with that branch selected, or check out '${expected_branch}' locally." + exit 1 +fi +# Also confirm the format independently in case someone has named a +# non-release branch to look like one. +if ! validate_and_extract_branch_version "${current_branch}"; then + print_error "Branch '${current_branch}' is not a valid release branch name." + exit 1 +fi exec_process git push origin HEAD -step_summary "Bumped version to \`${next_snapshot}\`, set \`previous.version=${version}\`" +step_summary "Bumped version to \`${next_snapshot}\` on branch \`${current_branch}\`, set \`previous.version=${version}\`" # --------------------------------------------------------------------------- # Step 7: Generate announce email diff --git a/release/libs/_nexus.sh b/release/libs/_nexus.sh index 20c43a0f3f..1dd3f1c4e7 100644 --- a/release/libs/_nexus.sh +++ b/release/libs/_nexus.sh @@ -190,25 +190,51 @@ function nexus_find_open_staging_repo { return 1 fi - staging_repo_id=$(echo "${response}" | \ - NEXUS_PROFILE_NAME="${profile_name}" python3 -c " + # Filter on exactly (not on a substring of ), + # and reject the result if multiple open repos exist for the profile — + # Nexus permits this but it indicates a leftover from a previous failed + # run that the operator must clean up before we can pick the right one. + local matches + matches=$(echo "${response}" | \ + NEXUS_PROFILE_NAME_VAR="${profile_name}" python3 -c " import sys, os, xml.etree.ElementTree as ET -profile = os.environ['NEXUS_PROFILE_NAME'] -tree = ET.parse(sys.stdin) +profile = os.environ['NEXUS_PROFILE_NAME_VAR'] +try: + tree = ET.parse(sys.stdin) +except ET.ParseError as e: + print('PARSE_ERROR:' + str(e), file=sys.stderr) + sys.exit(2) for repo in tree.findall('.//stagingProfileRepository'): repo_type = repo.find('type') + repo_profile = repo.find('profileName') repo_id = repo.find('repositoryId') - if repo_type is not None and repo_type.text == 'open' and repo_id is not None: - if profile.replace('.', '') in (repo_id.text or ''): - print(repo_id.text) - break -" 2>/dev/null) - - if [[ -z "${staging_repo_id}" ]]; then - print_error "No open staging repository found for ${profile_name}" + if repo_type is None or repo_id is None or repo_profile is None: + continue + if repo_type.text == 'open' and repo_profile.text == profile: + print(repo_id.text) +" 2>&1) + local py_status=$? + + if [[ ${py_status} -ne 0 ]]; then + print_error "Failed to parse Nexus staging response: ${matches}" return 1 fi + local match_count + match_count=$(echo -n "${matches}" | grep -c . || true) + + if [[ ${match_count} -eq 0 ]]; then + print_error "No open staging repository found for profile '${profile_name}'" + return 1 + fi + if [[ ${match_count} -gt 1 ]]; then + print_error "Multiple open staging repositories found for profile '${profile_name}':" + echo "${matches}" | while read -r line; do print_error " - ${line}"; done + print_error "Please drop or close the stale repos in Nexus and re-run." + return 1 + fi + + staging_repo_id=$(echo "${matches}" | head -n 1) print_info "Found staging repository: ${staging_repo_id}" return 0 } diff --git a/release/tests/nexus.bats b/release/tests/nexus.bats index 10d3d39ba4..6ac108b513 100644 --- a/release/tests/nexus.bats +++ b/release/tests/nexus.bats @@ -80,6 +80,106 @@ setup() { [ "$staging_repo_id" = "DRY-RUN-REPO-ID" ] } +@test "nexus_find_open_staging_repo: real mode picks the open repo for the profile" { + DRY_RUN=0 + curl() { + cat <<'XML' + + + + org.apache.iceberg + open + orgapacheiceberg-7777 + + + org.apache.parquet + closed + orgapacheparquet-1111 + + + org.apache.parquet + open + orgapacheparquet-2222 + + + +XML + return 0 + } + export -f curl + nexus_find_open_staging_repo "org.apache.parquet" + [ "${staging_repo_id}" = "orgapacheparquet-2222" ] +} + +@test "nexus_find_open_staging_repo: rejects when no open repo for profile" { + DRY_RUN=0 + curl() { + cat <<'XML' + + + + org.apache.parquet + closed + orgapacheparquet-1111 + + + +XML + return 0 + } + export -f curl + run nexus_find_open_staging_repo "org.apache.parquet" + [ "$status" -eq 1 ] + [[ "$output" == *"No open staging repository"* ]] +} + +@test "nexus_find_open_staging_repo: rejects when profile has multiple open repos" { + DRY_RUN=0 + curl() { + cat <<'XML' + + + + org.apache.parquet + open + orgapacheparquet-1111 + + + org.apache.parquet + open + orgapacheparquet-2222 + + + +XML + return 0 + } + export -f curl + run nexus_find_open_staging_repo "org.apache.parquet" + [ "$status" -eq 1 ] + [[ "$output" == *"Multiple open staging repositories"* ]] + [[ "$output" == *"orgapacheparquet-1111"* ]] + [[ "$output" == *"orgapacheparquet-2222"* ]] +} + +@test "nexus_find_open_staging_repo: rejects when curl fails" { + DRY_RUN=0 + curl() { return 22; } + export -f curl + run nexus_find_open_staging_repo "org.apache.parquet" + [ "$status" -eq 1 ] + [[ "$output" == *"Failed to query"* ]] +} + +@test "nexus_find_open_staging_repo: rejects malformed XML" { + DRY_RUN=0 + curl() { echo " Date: Fri, 8 May 2026 15:48:39 -0500 Subject: [PATCH 13/14] Address review: extract _svn.sh shared library Every other external system this PR talks to (Nexus, GitHub, Maven) already has a dedicated lib + bats coverage. SVN was the exception: each bin script open-coded `svn co/cp/add/ci/mv/rm/ls` with the same credentials and dry-run boilerplate. Extract release/libs/_svn.sh exposing: - svn_stage_rc -- checkout+add+commit - svn_promote_rc_to_release -- svn mv dev -> release - svn_remove_rc -- cancel-time cleanup - svn_list_old_releases -- enumerate stale dirs - svn_remove_release -- delete a stale release Migrate prepare-rc.sh, publish-release.sh, and cancel-rc.sh to use the new library. Side benefits: - prepare-rc.sh no longer clobbers a stray `tmp/` at the repo root; svn_stage_rc uses `mktemp -d -t parquet-release-svn-XXXXXX` and cleans it up via a RETURN trap. - The credentials / --non-interactive boilerplate now lives in one place; future redaction or auth-method changes are a single edit. Add release/tests/svn.bats with 14 tests covering dry-run, real-mode mocked-svn, missing-path skip, list filtering, and stale-listing failures. test_helper/common.bash resets the new _SVN_LOADED guard. --- release/bin/cancel-rc.sh | 18 +-- release/bin/prepare-rc.sh | 29 +--- release/bin/publish-release.sh | 24 +--- release/libs/_svn.sh | 161 ++++++++++++++++++++++ release/tests/svn.bats | 187 ++++++++++++++++++++++++++ release/tests/test_helper/common.bash | 2 +- 6 files changed, 361 insertions(+), 60 deletions(-) create mode 100644 release/libs/_svn.sh create mode 100644 release/tests/svn.bats diff --git a/release/bin/cancel-rc.sh b/release/bin/cancel-rc.sh index 45788ad201..37b12ffd5c 100755 --- a/release/bin/cancel-rc.sh +++ b/release/bin/cancel-rc.sh @@ -28,6 +28,7 @@ source "${LIBS_DIR}/_log.sh" source "${LIBS_DIR}/_exec.sh" source "${LIBS_DIR}/_version.sh" source "${LIBS_DIR}/_nexus.sh" +source "${LIBS_DIR}/_svn.sh" # --------------------------------------------------------------------------- # Usage @@ -167,21 +168,8 @@ step_summary "### SVN Cleanup" dev_url="${APACHE_DIST_URL}${APACHE_DIST_DEV_PATH}/${rc_tag}" -if [[ ${DRY_RUN:-1} -ne 1 ]]; then - if svn ls --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive "${dev_url}" >/dev/null 2>&1; then - exec_process svn rm \ - --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ - "${dev_url}" \ - -m "Cancel Apache Parquet ${version} RC${rc_num}" - step_summary "Deleted \`${dev_url}\`" - else - print_warning "SVN directory not found: ${dev_url}" - step_summary "Directory not found at \`${dev_url}\` (may already be deleted)" - fi -else - print_command "Dry-run, WOULD delete ${dev_url}" - step_summary "Would delete \`${dev_url}\` (dry-run)" -fi +svn_remove_rc "${version}" "${rc_num}" +step_summary "Removed \`${dev_url}\`" # --------------------------------------------------------------------------- # Step 3: Generate vote failure email diff --git a/release/bin/prepare-rc.sh b/release/bin/prepare-rc.sh index 172356fc4a..8ae29f6fc1 100755 --- a/release/bin/prepare-rc.sh +++ b/release/bin/prepare-rc.sh @@ -30,6 +30,7 @@ source "${LIBS_DIR}/_version.sh" source "${LIBS_DIR}/_github.sh" source "${LIBS_DIR}/_nexus.sh" source "${LIBS_DIR}/_maven.sh" +source "${LIBS_DIR}/_svn.sh" trap 'rm -f .release-settings.xml' EXIT @@ -309,32 +310,10 @@ step_summary "Built \`${tarball_name}\` from \`${release_hash}\`" step_summary "" step_summary "### SVN Staging" -svn_dir="${APACHE_DIST_URL}${APACHE_DIST_DEV_PATH}" -rc_svn_dir="${rc_tag}" +svn_stage_rc "${version}" "${rc_number}" \ + "${tarball_name}" "${tarball_name}.asc" "${tarball_name}.sha512" -if [[ ${DRY_RUN:-1} -ne 1 ]]; then - if [[ -d tmp/ ]]; then - rm -rf tmp/ - fi - - exec_process_with_retries 5 60 "tmp" \ - svn co --depth=empty --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ - "${svn_dir}" tmp - - mkdir -p "tmp/${rc_svn_dir}" - cp "${tarball_name}" "${tarball_name}.asc" "${tarball_name}.sha512" "tmp/${rc_svn_dir}/" - - (cd tmp && exec_process svn add "${rc_svn_dir}") - (cd tmp && exec_process svn ci \ - --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ - -m "Apache Parquet ${version} RC${rc_number}") - - rm -rf tmp -else - print_command "Dry-run, WOULD stage to ${svn_dir}/${rc_svn_dir}" -fi - -step_summary "Staged source tarball to \`${svn_dir}/${rc_svn_dir}\`" +step_summary "Staged source tarball to \`${APACHE_DIST_URL}${APACHE_DIST_DEV_PATH}/${rc_tag}\`" # --------------------------------------------------------------------------- # Step 9: Create GitHub pre-release diff --git a/release/bin/publish-release.sh b/release/bin/publish-release.sh index 0517c8458f..c34a9db25c 100755 --- a/release/bin/publish-release.sh +++ b/release/bin/publish-release.sh @@ -28,6 +28,7 @@ source "${LIBS_DIR}/_log.sh" source "${LIBS_DIR}/_exec.sh" source "${LIBS_DIR}/_version.sh" source "${LIBS_DIR}/_nexus.sh" +source "${LIBS_DIR}/_svn.sh" # --------------------------------------------------------------------------- # Usage @@ -210,15 +211,9 @@ step_summary "Staging repository \`${staging_repo_id}\` verified" step_summary "" step_summary "### SVN Promotion" -dev_url="${APACHE_DIST_URL}${APACHE_DIST_DEV_PATH}/${rc_tag}" -release_url="${APACHE_DIST_URL}${APACHE_DIST_RELEASE_PATH}/${TAG_PREFIX}${version}" +svn_promote_rc_to_release "${version}" "${rc_num}" -exec_process svn mv \ - --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ - "${dev_url}" "${release_url}" \ - -m "Release Apache Parquet ${version}" - -step_summary "Moved \`${dev_url}\` -> \`${release_url}\`" +step_summary "Promoted \`${rc_tag}\` -> \`${final_tag}\` on dist.apache.org" # --------------------------------------------------------------------------- # Step 2: Clean up old releases from dist/release @@ -227,24 +222,15 @@ step_summary "" step_summary "### Old Release Cleanup" if [[ ${DRY_RUN:-1} -ne 1 ]]; then - release_base_url="${APACHE_DIST_URL}${APACHE_DIST_RELEASE_PATH}" - svn_listing="" - if ! svn_listing=$(svn list --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ - "${release_base_url}" 2>&1); then - print_error "Failed to list SVN releases at ${release_base_url}: ${svn_listing}" + if ! old_versions=$(svn_list_old_releases "${version}"); then exit 1 fi - old_versions=$(echo "${svn_listing}" | grep -E "^${TAG_PREFIX}[0-9]" | sed 's|/$||' | grep -v "${TAG_PREFIX}${version}$" || true) - if [[ -n "${old_versions}" ]]; then step_summary "Removing old releases:" while IFS= read -r old_dir; do [[ -z "${old_dir}" ]] && continue - exec_process svn rm \ - --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ - "${release_base_url}/${old_dir}" \ - -m "Remove old release ${old_dir} (superseded by ${version})" + svn_remove_release "${old_dir}" "${version}" step_summary "- Removed \`${old_dir}\`" done <<< "${old_versions}" else diff --git a/release/libs/_svn.sh b/release/libs/_svn.sh new file mode 100644 index 0000000000..b9b8cc6509 --- /dev/null +++ b/release/libs/_svn.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Helpers for talking to dist.apache.org via svn. +# +# Required env vars: SVN_USERNAME, SVN_PASSWORD. +# All operations honor DRY_RUN through exec_process / explicit guards. + +[[ -n "${_SVN_LOADED:-}" ]] && return 0 2>/dev/null || true +_SVN_LOADED=1 + +LIBS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${LIBS_DIR}/_constants.sh" +source "${LIBS_DIR}/_log.sh" +source "${LIBS_DIR}/_exec.sh" + +# Allow ${SVN_USERNAME}/${SVN_PASSWORD} to be expanded under `set -u` +# even before the operator has provided real credentials -- their +# absence is fine in dry-run; in real mode svn itself will reject the +# request. +: "${SVN_USERNAME:=}" +: "${SVN_PASSWORD:=}" + +# svn_stage_rc +# Stages the given files under dist/dev/parquet// via svn. +# Uses a per-call mktemp checkout dir; cleans it up on success. +function svn_stage_rc { + local version="$1" + local rc_num="$2" + shift 2 + local files=("$@") + + local rc_tag="${TAG_PREFIX}${version}-rc${rc_num}" + local dev_url="${APACHE_DIST_URL}${APACHE_DIST_DEV_PATH}" + local commit_msg="Apache Parquet ${version} RC${rc_num}" + + if [[ ${#files[@]} -eq 0 ]]; then + print_error "svn_stage_rc: no files to stage" + return 1 + fi + + if [[ ${DRY_RUN:-1} -eq 1 ]]; then + print_command "Dry-run, WOULD stage to ${dev_url}/${rc_tag}: ${files[*]}" + return 0 + fi + + local checkout_dir + checkout_dir=$(mktemp -d -t parquet-release-svn-XXXXXX) + # shellcheck disable=SC2064 # we want $checkout_dir expanded now, not at trap time + trap "rm -rf '${checkout_dir}'" RETURN + + exec_process_with_retries 5 60 "${checkout_dir}" \ + svn co --depth=empty \ + --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + "${dev_url}" "${checkout_dir}" + + mkdir -p "${checkout_dir}/${rc_tag}" + cp "${files[@]}" "${checkout_dir}/${rc_tag}/" + + ( cd "${checkout_dir}" && exec_process svn add "${rc_tag}" ) + ( cd "${checkout_dir}" && exec_process svn ci \ + --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + -m "${commit_msg}" ) +} + +# svn_promote_rc_to_release +# Moves dist/dev/parquet// to dist/release/parquet//. +function svn_promote_rc_to_release { + local version="$1" + local rc_num="$2" + + local rc_tag="${TAG_PREFIX}${version}-rc${rc_num}" + local final_tag="${TAG_PREFIX}${version}" + local dev_url="${APACHE_DIST_URL}${APACHE_DIST_DEV_PATH}/${rc_tag}" + local release_url="${APACHE_DIST_URL}${APACHE_DIST_RELEASE_PATH}/${final_tag}" + + exec_process svn mv \ + --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + "${dev_url}" "${release_url}" \ + -m "Release Apache Parquet ${version}" +} + +# svn_remove_rc +# Removes dist/dev/parquet// (used by cancel-rc.sh). +function svn_remove_rc { + local version="$1" + local rc_num="$2" + + local rc_tag="${TAG_PREFIX}${version}-rc${rc_num}" + local dev_url="${APACHE_DIST_URL}${APACHE_DIST_DEV_PATH}/${rc_tag}" + + if [[ ${DRY_RUN:-1} -ne 1 ]]; then + if ! svn ls --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" \ + --non-interactive "${dev_url}" >/dev/null 2>&1; then + print_warning "SVN directory not found: ${dev_url} (already deleted?)" + return 0 + fi + fi + + exec_process svn rm \ + --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + "${dev_url}" \ + -m "Cancel Apache Parquet ${version} RC${rc_num}" +} + +# svn_list_old_releases +# Echoes (one per line, on stdout) the names of release directories +# under dist/release/parquet/ that should be removed when promoting +# as the new latest release. Returns non-zero only +# on real svn-listing failure. +function svn_list_old_releases { + local version_to_keep="$1" + local release_base_url="${APACHE_DIST_URL}${APACHE_DIST_RELEASE_PATH}" + + if [[ ${DRY_RUN:-1} -eq 1 ]]; then + print_command "Dry-run, WOULD list ${release_base_url}" >&2 + return 0 + fi + + local listing + if ! listing=$(svn list \ + --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + "${release_base_url}" 2>&1); then + print_error "Failed to list SVN releases at ${release_base_url}: ${listing}" + return 1 + fi + + echo "${listing}" | grep -E "^${TAG_PREFIX}[0-9]" | sed 's|/$||' \ + | grep -v "^${TAG_PREFIX}${version_to_keep}\$" || true +} + +# svn_remove_release +# svn rm a release directory under dist/release/parquet/. +function svn_remove_release { + local release_dir="$1" + local new_version="$2" + local release_base_url="${APACHE_DIST_URL}${APACHE_DIST_RELEASE_PATH}" + + exec_process svn rm \ + --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ + "${release_base_url}/${release_dir}" \ + -m "Remove old release ${release_dir} (superseded by ${new_version})" +} diff --git a/release/tests/svn.bats b/release/tests/svn.bats new file mode 100644 index 0000000000..dae6d7a961 --- /dev/null +++ b/release/tests/svn.bats @@ -0,0 +1,187 @@ +#!/usr/bin/env bats +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +setup() { + load test_helper/common + source "${LIBS_DIR}/_svn.sh" + export SVN_USERNAME="testuser" + export SVN_PASSWORD="testpass" +} + +# ---- svn_stage_rc ---- + +@test "svn_stage_rc: dry-run does not call svn" { + DRY_RUN=1 + local f="${BATS_TEST_TMPDIR}/parquet-1.18.0.tar.gz" + echo "fake tarball" > "$f" + run svn_stage_rc "1.18.0" "0" "$f" + [ "$status" -eq 0 ] + [[ "$output" == *"Dry-run"* ]] + [[ "$output" == *"apache-parquet-1.18.0-rc0"* ]] +} + +@test "svn_stage_rc: rejects empty file list" { + DRY_RUN=1 + run svn_stage_rc "1.18.0" "0" + [ "$status" -ne 0 ] + [[ "$output" == *"no files to stage"* ]] +} + +@test "svn_stage_rc: dry-run does not write to repo root tmp/" { + DRY_RUN=1 + cd "${BATS_TEST_TMPDIR}" + local f="${BATS_TEST_TMPDIR}/x.tar.gz" + echo x > "$f" + run svn_stage_rc "1.18.0" "0" "$f" + [ "$status" -eq 0 ] + [ ! -d "${BATS_TEST_TMPDIR}/tmp" ] +} + +# ---- svn_promote_rc_to_release ---- + +@test "svn_promote_rc_to_release: dry-run shows mv command" { + DRY_RUN=1 + run svn_promote_rc_to_release "1.18.0" "2" + [ "$status" -eq 0 ] + [[ "$output" == *"svn mv"* ]] + [[ "$output" == *"apache-parquet-1.18.0-rc2"* ]] + [[ "$output" == *"apache-parquet-1.18.0"* ]] + [[ "$output" == *"/dev/parquet"* ]] + [[ "$output" == *"/release/parquet"* ]] +} + +@test "svn_promote_rc_to_release: real mode invokes svn mv" { + DRY_RUN=0 + svn() { echo "SVN: $*"; return 0; } + export -f svn + run svn_promote_rc_to_release "1.18.0" "0" + [ "$status" -eq 0 ] + [[ "$output" == *"SVN: mv"* ]] +} + +# ---- svn_remove_rc ---- + +@test "svn_remove_rc: dry-run shows rm command" { + DRY_RUN=1 + run svn_remove_rc "1.18.0" "1" + [ "$status" -eq 0 ] + [[ "$output" == *"svn rm"* ]] + [[ "$output" == *"apache-parquet-1.18.0-rc1"* ]] + [[ "$output" == *"/dev/parquet"* ]] +} + +@test "svn_remove_rc: real mode skips rm if path does not exist" { + DRY_RUN=0 + svn() { + if [[ "$1" == "ls" ]]; then return 1; fi + echo "UNEXPECTED svn $*"; return 99 + } + export -f svn + run svn_remove_rc "1.18.0" "0" + [ "$status" -eq 0 ] + [[ "$output" == *"already deleted"* ]] || [[ "$output" == *"not found"* ]] +} + +@test "svn_remove_rc: real mode invokes svn rm when path exists" { + DRY_RUN=0 + svn() { + if [[ "$1" == "ls" ]]; then return 0; fi + echo "SVN: $*"; return 0 + } + export -f svn + run svn_remove_rc "1.18.0" "0" + [ "$status" -eq 0 ] + [[ "$output" == *"SVN: rm"* ]] +} + +# ---- svn_list_old_releases ---- + +@test "svn_list_old_releases: dry-run returns empty without calling svn" { + DRY_RUN=1 + run svn_list_old_releases "1.18.0" + [ "$status" -eq 0 ] + # No actual entries, just the dry-run banner on stderr + [ -z "$(echo "$output" | grep -v Dry-run | grep -v WOULD)" ] +} + +@test "svn_list_old_releases: filters to apache-parquet-* dirs and excludes the kept version" { + DRY_RUN=0 + svn() { + if [[ "$1" == "list" ]]; then + printf 'apache-parquet-1.16.0/\napache-parquet-1.17.0/\napache-parquet-1.18.0/\nKEYS\nREADME\n' + return 0 + fi + return 99 + } + export -f svn + run svn_list_old_releases "1.18.0" + [ "$status" -eq 0 ] + [[ "$output" == *"apache-parquet-1.16.0"* ]] + [[ "$output" == *"apache-parquet-1.17.0"* ]] + [[ "$output" != *"apache-parquet-1.18.0"* ]] + [[ "$output" != *"KEYS"* ]] + [[ "$output" != *"README"* ]] +} + +@test "svn_list_old_releases: returns empty when only the kept version exists" { + DRY_RUN=0 + svn() { + if [[ "$1" == "list" ]]; then printf 'apache-parquet-1.18.0/\n'; return 0; fi + return 99 + } + export -f svn + run svn_list_old_releases "1.18.0" + [ "$status" -eq 0 ] + [[ "$output" != *"apache-parquet"* ]] +} + +@test "svn_list_old_releases: fails when svn list errors" { + DRY_RUN=0 + svn() { + if [[ "$1" == "list" ]]; then echo "svn: E170013"; return 1; fi + return 99 + } + export -f svn + run svn_list_old_releases "1.18.0" + [ "$status" -eq 1 ] + [[ "$output" == *"Failed to list"* ]] +} + +# ---- svn_remove_release ---- + +@test "svn_remove_release: dry-run shows rm command" { + DRY_RUN=1 + run svn_remove_release "apache-parquet-1.16.0" "1.18.0" + [ "$status" -eq 0 ] + [[ "$output" == *"svn rm"* ]] + [[ "$output" == *"apache-parquet-1.16.0"* ]] + [[ "$output" == *"superseded by 1.18.0"* ]] +} + +@test "svn_remove_release: real mode invokes svn rm with commit message" { + DRY_RUN=0 + svn() { echo "SVN: $*"; return 0; } + export -f svn + run svn_remove_release "apache-parquet-1.16.0" "1.18.0" + [ "$status" -eq 0 ] + [[ "$output" == *"SVN: rm"* ]] + [[ "$output" == *"apache-parquet-1.16.0"* ]] + [[ "$output" == *"superseded by 1.18.0"* ]] +} diff --git a/release/tests/test_helper/common.bash b/release/tests/test_helper/common.bash index 70385e1374..4f4623a63b 100644 --- a/release/tests/test_helper/common.bash +++ b/release/tests/test_helper/common.bash @@ -22,7 +22,7 @@ LIBS_DIR="${BATS_TEST_DIRNAME}/../libs" # Reset include guards so libraries can be re-sourced in each test unset _CONSTANTS_LOADED _LOG_LOADED _EXEC_LOADED _VERSION_LOADED -unset _GITHUB_LOADED _NEXUS_LOADED _MAVEN_LOADED +unset _GITHUB_LOADED _NEXUS_LOADED _MAVEN_LOADED _SVN_LOADED # Reset global variables that library functions set _reset_version_vars() { From 22fb3e7bb1ca852bb0d8bb035109e2c129982267 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Fri, 8 May 2026 16:49:46 -0500 Subject: [PATCH 14/14] Scope dist/release cleanup to older patches of the same minor Previously svn_list_old_releases returned every apache-parquet-X.Y.Z directory other than the version being released, which would have swept the entire dist/release/parquet/ tree on first run and would clobber a newer minor on a back-patch release (e.g. shipping 1.9.4 after 1.10.0 is out). Restrict the policy to "older patches of the same minor branch": releasing 1.9.2 removes 1.9.0 and 1.9.1 but never touches 1.8.x, 1.10.x, or a higher 1.9.x patch. The new implementation validates the input with VERSION_REGEX, then uses awk with literal-prefix matching (index($0, prefix) == 1) and numeric tail comparison so '.' is never interpreted as a regex metacharacter and double-digit patches sort correctly. The publish-release.sh step summary now reads "Patch Release Cleanup (parquet-X.Y.x)" so the operator can see the policy at a glance. Adds seven new bats cases covering same-minor cleanup, leaving higher patches alone, releasing .0 (no-op), ignoring -rc and sister-project siblings, double-digit patches, and invalid input. Full suite at 119 tests, all green. --- release/bin/publish-release.sh | 16 +++-- release/libs/_svn.sh | 35 ++++++++-- release/tests/svn.bats | 116 +++++++++++++++++++++++++++++++-- 3 files changed, 151 insertions(+), 16 deletions(-) diff --git a/release/bin/publish-release.sh b/release/bin/publish-release.sh index c34a9db25c..32607c8602 100755 --- a/release/bin/publish-release.sh +++ b/release/bin/publish-release.sh @@ -216,10 +216,14 @@ svn_promote_rc_to_release "${version}" "${rc_num}" step_summary "Promoted \`${rc_tag}\` -> \`${final_tag}\` on dist.apache.org" # --------------------------------------------------------------------------- -# Step 2: Clean up old releases from dist/release +# Step 2: Clean up superseded patch releases on the same minor branch # --------------------------------------------------------------------------- +# Policy: only patch releases of the *same* minor (parquet-${major}.${minor}.x) +# that are older than ${version} are removed from dist/release/parquet/. +# Releases on other minor branches are left in place and must be retired +# manually. See svn_list_old_releases for the full rationale. step_summary "" -step_summary "### Old Release Cleanup" +step_summary "### Patch Release Cleanup (parquet-${major}.${minor}.x)" if [[ ${DRY_RUN:-1} -ne 1 ]]; then if ! old_versions=$(svn_list_old_releases "${version}"); then @@ -227,18 +231,18 @@ if [[ ${DRY_RUN:-1} -ne 1 ]]; then fi if [[ -n "${old_versions}" ]]; then - step_summary "Removing old releases:" + step_summary "Removing superseded patches on parquet-${major}.${minor}.x:" while IFS= read -r old_dir; do [[ -z "${old_dir}" ]] && continue svn_remove_release "${old_dir}" "${version}" step_summary "- Removed \`${old_dir}\`" done <<< "${old_versions}" else - step_summary "No old releases to clean up" + step_summary "No superseded ${major}.${minor}.x patches to clean up" fi else - print_command "Dry-run, WOULD clean up old releases from dist/release" - step_summary "Would clean up old releases (dry-run)" + print_command "Dry-run, WOULD remove superseded ${major}.${minor}.x patches from dist/release" + step_summary "Would remove superseded ${major}.${minor}.x patches (dry-run)" fi # --------------------------------------------------------------------------- diff --git a/release/libs/_svn.sh b/release/libs/_svn.sh index b9b8cc6509..9faad854bf 100644 --- a/release/libs/_svn.sh +++ b/release/libs/_svn.sh @@ -124,8 +124,17 @@ function svn_remove_rc { # svn_list_old_releases # Echoes (one per line, on stdout) the names of release directories # under dist/release/parquet/ that should be removed when promoting -# as the new latest release. Returns non-zero only -# on real svn-listing failure. +# as the new latest release. +# +# Policy: only older patch releases of the *same* minor branch are +# considered eligible for cleanup. Releasing 1.9.2 cleans up 1.9.0 and +# 1.9.1; it never touches releases on a different X.Y branch (e.g. +# 1.8.x, 1.10.x) and never touches a higher patch on the same branch +# (e.g. 1.9.3 if it somehow exists). This matches "only one patch per +# supported minor on dist.apache.org/release"; older minor branches are +# left in place and must be retired manually. +# +# Returns non-zero only on real svn-listing failure or invalid input. function svn_list_old_releases { local version_to_keep="$1" local release_base_url="${APACHE_DIST_URL}${APACHE_DIST_RELEASE_PATH}" @@ -135,6 +144,14 @@ function svn_list_old_releases { return 0 fi + if [[ ! "${version_to_keep}" =~ ^${VERSION_REGEX}$ ]]; then + print_error "svn_list_old_releases: invalid version '${version_to_keep}'" + return 1 + fi + local keep_major="${BASH_REMATCH[1]}" + local keep_minor="${BASH_REMATCH[2]}" + local keep_patch="${BASH_REMATCH[3]}" + local listing if ! listing=$(svn list \ --username "${SVN_USERNAME}" --password "${SVN_PASSWORD}" --non-interactive \ @@ -143,8 +160,18 @@ function svn_list_old_releases { return 1 fi - echo "${listing}" | grep -E "^${TAG_PREFIX}[0-9]" | sed 's|/$||' \ - | grep -v "^${TAG_PREFIX}${version_to_keep}\$" || true + # Match only `apache-parquet-${keep_major}.${keep_minor}./` entries + # with a numeric patch strictly less than ${keep_patch}. The literal-prefix + # check via awk's `index` avoids any chance of `.` being interpreted as a + # regex metacharacter and matching a sibling tree. + local minor_prefix="${TAG_PREFIX}${keep_major}.${keep_minor}." + echo "${listing}" \ + | sed 's|/$||' \ + | awk -v prefix="${minor_prefix}" -v keep_patch="${keep_patch}" ' + index($0, prefix) == 1 { + tail = substr($0, length(prefix) + 1) + if (tail ~ /^[0-9]+$/ && (tail + 0) < (keep_patch + 0)) print $0 + }' } # svn_remove_release diff --git a/release/tests/svn.bats b/release/tests/svn.bats index dae6d7a961..1647010a49 100644 --- a/release/tests/svn.bats +++ b/release/tests/svn.bats @@ -112,6 +112,11 @@ setup() { } # ---- svn_list_old_releases ---- +# +# Policy under test: cleanup is scoped to *older patch releases on the same +# minor branch*. Releasing 1.9.2 cleans up 1.9.0 and 1.9.1; it must never +# touch other minors (1.8.x, 1.10.x), the kept version itself, or any +# higher patch on the same minor. @test "svn_list_old_releases: dry-run returns empty without calling svn" { DRY_RUN=1 @@ -121,25 +126,124 @@ setup() { [ -z "$(echo "$output" | grep -v Dry-run | grep -v WOULD)" ] } -@test "svn_list_old_releases: filters to apache-parquet-* dirs and excludes the kept version" { +@test "svn_list_old_releases: returns older patches on the same minor only" { DRY_RUN=0 svn() { if [[ "$1" == "list" ]]; then - printf 'apache-parquet-1.16.0/\napache-parquet-1.17.0/\napache-parquet-1.18.0/\nKEYS\nREADME\n' + printf '%s\n' \ + 'apache-parquet-1.8.3/' \ + 'apache-parquet-1.9.0/' \ + 'apache-parquet-1.9.1/' \ + 'apache-parquet-1.9.2/' \ + 'apache-parquet-1.10.0/' \ + 'KEYS' 'README' return 0 fi return 99 } export -f svn - run svn_list_old_releases "1.18.0" + run svn_list_old_releases "1.9.2" [ "$status" -eq 0 ] - [[ "$output" == *"apache-parquet-1.16.0"* ]] - [[ "$output" == *"apache-parquet-1.17.0"* ]] - [[ "$output" != *"apache-parquet-1.18.0"* ]] + [[ "$output" == *"apache-parquet-1.9.0"* ]] + [[ "$output" == *"apache-parquet-1.9.1"* ]] + # Same-minor current and other-minor entries must not be returned. + [[ "$output" != *"apache-parquet-1.9.2"* ]] + [[ "$output" != *"apache-parquet-1.8.3"* ]] + [[ "$output" != *"apache-parquet-1.10.0"* ]] [[ "$output" != *"KEYS"* ]] [[ "$output" != *"README"* ]] } +@test "svn_list_old_releases: never touches a higher patch on the same minor" { + DRY_RUN=0 + svn() { + if [[ "$1" == "list" ]]; then + printf '%s\n' \ + 'apache-parquet-1.9.1/' \ + 'apache-parquet-1.9.2/' \ + 'apache-parquet-1.9.3/' + return 0 + fi + return 99 + } + export -f svn + run svn_list_old_releases "1.9.2" + [ "$status" -eq 0 ] + [[ "$output" == *"apache-parquet-1.9.1"* ]] + [[ "$output" != *"apache-parquet-1.9.2"* ]] + [[ "$output" != *"apache-parquet-1.9.3"* ]] +} + +@test "svn_list_old_releases: returns empty when releasing the .0 of a minor" { + DRY_RUN=0 + svn() { + if [[ "$1" == "list" ]]; then + printf '%s\n' \ + 'apache-parquet-1.8.5/' \ + 'apache-parquet-1.9.0/' \ + 'apache-parquet-1.10.0/' + return 0 + fi + return 99 + } + export -f svn + run svn_list_old_releases "1.9.0" + [ "$status" -eq 0 ] + [ -z "$(echo "$output" | grep -v '^$')" ] +} + +@test "svn_list_old_releases: ignores rc and otherwise-malformed siblings" { + DRY_RUN=0 + svn() { + if [[ "$1" == "list" ]]; then + printf '%s\n' \ + 'apache-parquet-1.9.0/' \ + 'apache-parquet-1.9.1-rc0/' \ + 'apache-parquet-1.9.x/' \ + 'apache-parquet-format-2.10.0/' \ + 'apache-parquet-cpp-1.5.0/' + return 0 + fi + return 99 + } + export -f svn + run svn_list_old_releases "1.9.2" + [ "$status" -eq 0 ] + [[ "$output" == *"apache-parquet-1.9.0"* ]] + # Only the bare X.Y.Z/ sibling on the same minor counts. + [[ "$output" != *"rc0"* ]] + [[ "$output" != *"1.9.x"* ]] + [[ "$output" != *"format"* ]] + [[ "$output" != *"cpp"* ]] +} + +@test "svn_list_old_releases: handles double-digit patch numbers numerically" { + DRY_RUN=0 + svn() { + if [[ "$1" == "list" ]]; then + printf '%s\n' \ + 'apache-parquet-1.9.2/' \ + 'apache-parquet-1.9.9/' \ + 'apache-parquet-1.9.10/' + return 0 + fi + return 99 + } + export -f svn + run svn_list_old_releases "1.9.10" + [ "$status" -eq 0 ] + [[ "$output" == *"apache-parquet-1.9.2"* ]] + [[ "$output" == *"apache-parquet-1.9.9"* ]] + [[ "$output" != *"apache-parquet-1.9.10"* ]] +} + +@test "svn_list_old_releases: rejects an invalid version-to-keep" { + DRY_RUN=0 + run svn_list_old_releases "1.9" + [ "$status" -ne 0 ] + [[ "$output" == *"invalid version"* ]] +} + @test "svn_list_old_releases: returns empty when only the kept version exists" { DRY_RUN=0 svn() {