diff --git a/.github/workflows/release-notes-check.yml b/.github/workflows/release-notes-check.yml new file mode 100644 index 0000000..732232f --- /dev/null +++ b/.github/workflows/release-notes-check.yml @@ -0,0 +1,37 @@ +name: Release Notes Check + +# Guards the release toolchain: a floating dependency once dropped the +# Features/Bug Fixes sections from the generated notes. This runs the pinned +# semantic-release in dry-run mode against the real history and, when a release +# is due, fails if the notes carry no sections. Scoped to the release +# config/scripts so ordinary PRs are unaffected. +on: + pull_request: + paths: + - .releaserc.mjs + - ci/release/** + - .github/workflows/release-notes-check.yml + push: + branches: [ main ] + paths: + - .releaserc.mjs + - ci/release/** + - .github/workflows/release-notes-check.yml + +permissions: + contents: read + +jobs: + release-notes-check: + name: Release Notes Check + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-node@v6 + with: + node-version: "24" + - name: Assert generated release notes contain the expected sections + run: ./ci/release/dry_run.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4c43ac..8318801 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,12 +2,11 @@ name: Release and Publish on: push: - branches: [ main ] tags: [ 'v[0-9]+.[0-9]+.[0-9]+' ] permissions: id-token: write - contents: write + contents: read jobs: build: @@ -17,6 +16,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v7 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v6 with: @@ -33,18 +34,41 @@ jobs: name: dist path: dist/ retention-days: 1 - release: - name: Release to GitHub + github-release: + name: Attach assets to GitHub release runs-on: ubuntu-latest needs: build - if: startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write steps: - name: Download artifact uses: actions/download-artifact@v8 with: name: dist path: dist/ - - name: Publish to GitHub release page + # semantic-release creates the release for this tag in the same run that + # pushed the tag, normally seconds before this build finishes. Wait for it + # so the upload below attaches to that existing release (with its generated + # notes) instead of racing ahead and creating a bare release -- which would + # in turn make semantic-release's own release step fail with a 409. + - name: Wait for the semantic-release GitHub release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: | + for attempt in $(seq 1 30); do + if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo "Release $TAG found." + exit 0 + fi + echo "Release $TAG not present yet (attempt $attempt/30); waiting 10s..." + sleep 10 + done + echo "::error::Release $TAG did not appear within 5 minutes; not creating a bare release." >&2 + exit 1 + # Omitting `body` leaves the generated notes untouched (the action retains + # existing release info on update); only the wheel/sdist are attached. + - name: Attach artifacts to the GitHub release uses: softprops/action-gh-release@v3 with: files: | @@ -53,7 +77,7 @@ jobs: publish: name: Publish to PyPI runs-on: ubuntu-latest - needs: release + needs: [build, github-release] environment: pypi steps: - name: Download artifact diff --git a/.github/workflows/semantic-release.yml b/.github/workflows/semantic-release.yml new file mode 100644 index 0000000..306e3dd --- /dev/null +++ b/.github/workflows/semantic-release.yml @@ -0,0 +1,45 @@ +name: Semantic Release + +on: + schedule: + # 2 AM on Sunday + - cron: "0 2 * * 0" + workflow_dispatch: + +concurrency: + group: release + cancel-in-progress: false + +jobs: + semantic-release: + if: github.repository == 'substrait-io/substrait-python' + runs-on: ubuntu-latest + steps: + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ secrets.RELEASER_ID }} + private-key: ${{ secrets.RELEASER_KEY }} + - name: Checkout code + uses: actions/checkout@v6 + with: + token: ${{ steps.app-token.outputs.token }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version: "24" + - name: Get bot user ID + id: bot-user-id + run: | + echo "user-id=$(gh api "/users/${{ steps.app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT" + env: + GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} + - name: Run semantic-release + run: ./ci/release/run.sh + env: + GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} + GIT_AUTHOR_NAME: "${{ steps.app-token.outputs.app-slug }}[bot]" + GIT_COMMITTER_NAME: "${{ steps.app-token.outputs.app-slug }}[bot]" + GIT_AUTHOR_EMAIL: "${{ steps.bot-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com" + GIT_COMMITTER_EMAIL: "${{ steps.bot-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com" diff --git a/.releaserc.mjs b/.releaserc.mjs new file mode 100644 index 0000000..16f0d0b --- /dev/null +++ b/.releaserc.mjs @@ -0,0 +1,141 @@ +// semantic-release configuration. +// +// This is an ESM (.mjs) config rather than .releaserc.json for two reasons: the +// release-notes-generator needs a `writerOpts.transform` function to strip git +// trailers (e.g. `Signed-off-by:`) that the conventional-commits parser folds +// into BREAKING CHANGE notes (JSON cannot carry functions), and the set of +// plugins depends on RELEASE_DRY_RUN. +// +// The config defaults to a dry run as a fail-safe: the side-effecting plugins +// (@semantic-release/github, @semantic-release/git) are only included when +// RELEASE_DRY_RUN=false. ci/release/run.sh opts in to a real release that way; +// every other invocation -- the credential-free notes check in +// ci/release/dry_run.sh, or a forgotten/typo'd env var -- stays harmless. +// +// The env var is needed on top of --dry-run because --dry-run alone is not +// enough: semantic-release still runs every plugin's verifyConditions step in +// dry-run mode (it skips only prepare, publish, addChannel, success and fail). +// @semantic-release/github's verifyConditions fails without a GITHUB_TOKEN, so +// the side-effecting plugins must be omitted entirely, not merely guarded by +// --dry-run, to allow a credential-free dry run. + +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; + +// Load the conventionalcommits preset. The release scripts run semantic-release +// via `npx -p ...`, which installs the preset as a sibling of semantic-release +// in a temporary node_modules. A bare `import` here would resolve relative to +// this config file's directory (the repo, which has no node_modules) and fail, +// so fall back to resolving the preset relative to the running semantic-release +// binary (process.argv[1]). The plain import path still covers local dev where +// the preset is installed alongside the project. +const loadPreset = async () => { + try { + return (await import("conventional-changelog-conventionalcommits")).default; + } catch { + const require = createRequire(pathToFileURL(process.argv[1])); + const resolved = pathToFileURL( + require.resolve("conventional-changelog-conventionalcommits"), + ).href; + return (await import(resolved)).default; + } +}; +const conventionalcommits = await loadPreset(); + +// Git trailers that should never appear in the changelog or release notes. +const TRAILER_KEYS = [ + "Signed-off-by", + "Co-authored-by", + "Co-developed-by", + "Reviewed-by", + "Acked-by", + "Tested-by", + "Reported-by", + "Suggested-by", + "Helped-by", + "Cc", +]; +const TRAILER = new RegExp(`^(?:${TRAILER_KEYS.join("|")}):\\s`, "i"); + +// The conventional-commits parser ends a BREAKING CHANGE note only at a +// recognized reference (closes #..., fixes #...) or another note keyword, not +// at a git trailer -- so a trailing `Signed-off-by:` gets absorbed into the +// note text. Strip such trailing trailer lines. +const stripTrailers = (text) => { + if (!text) return text; + const lines = text.split("\n"); + while (lines.length) { + const last = lines[lines.length - 1].trim(); + if (last === "" || TRAILER.test(last)) { + lines.pop(); + } else { + break; + } + } + return lines.join("\n"); +}; + +const preset = await conventionalcommits(); +const presetTransform = preset.writer.transform; + +// Dry run unless explicitly disabled (ci/release/run.sh sets RELEASE_DRY_RUN=false). +const dryRun = process.env.RELEASE_DRY_RUN !== "false"; + +export default { + branches: ["main"], + preset: "conventionalcommits", + dryRun, + plugins: [ + [ + "@semantic-release/commit-analyzer", + { + // Pre-1.0, matching substrait-java: a breaking change is a minor bump. + releaseRules: [{ breaking: true, release: "minor" }], + }, + ], + [ + "@semantic-release/release-notes-generator", + { + // Only `transform` is overridden; the generator merges this over the + // preset's writer options, so templates/grouping/sorting are kept. + writerOpts: { + transform(commit, context) { + const out = presetTransform(commit, context); + if (out && Array.isArray(out.notes)) { + out.notes = out.notes.map((note) => ({ + ...note, + text: stripTrailers(note.text), + })); + } + return out; + }, + }, + }, + ], + [ + "@semantic-release/changelog", + { + changelogTitle: "Release Notes\n---", + changelogFile: "CHANGELOG.md", + }, + ], + // GitHub releases and the release commit only happen in a real release. + ...(dryRun + ? [] + : [ + [ + "@semantic-release/github", + { + successComment: false, + }, + ], + [ + "@semantic-release/git", + { + assets: ["CHANGELOG.md"], + message: "chore(release): ${nextRelease.version}", + }, + ], + ]), + ], +}; diff --git a/RELEASING.md b/RELEASING.md index 300868c..5f7a4cf 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,14 +1,61 @@ # Releasing substrait-python -Given that you are a Substrait committer or PMC member and have the appropriate permissions, releasing a new version of substrait-python is done by simply creating new Github Release through the UI: +Releases of substrait-python are **fully automated** using +[semantic-release](https://semantic-release.gitbook.io/) and follow the same model +as the other Substrait projects (e.g. substrait-java). -1. Go to https://github.com/substrait-io/substrait-python/releases -2. Click `Draft a new release` -3. Enter the version to be released in the `Tag` field prefixed with a lower case `v`, e.g. `v0.99.0`. Then click `Create new tag` which tells Github to create the tag on publication of the release. -4. Click `Generate release notes` which will automatically populate the release title and release notes fields. -5. If you are happy with the release notes and you are ready for an immediate release simply click `Publish release` otherwise save the release as a draft for later. -6. Monitor the Github Actions release build for the newly created release and tag. +The [Semantic Release](.github/workflows/semantic-release.yml) workflow runs on a +weekly schedule (2 AM UTC on Sundays). It inspects the [Conventional +Commits](https://www.conventionalcommits.org/en/v1.0.0/) merged since the last +release, computes the next version, updates `CHANGELOG.md`, creates the `vX.Y.Z` +git tag, and publishes a GitHub Release with auto-generated notes. + +Creating the tag triggers the [Release and Publish](.github/workflows/release.yml) +workflow, which builds the package (the version is derived from the tag via +`setuptools_scm`) and publishes it to [PyPI](https://pypi.org/project/substrait/) +using Trusted Publishing. + +As a result, there is nothing to do by hand for a normal release — just make sure +your PR titles/commit messages follow the Conventional Commits specification (this +is enforced by the [PR Title Check](.github/workflows/pr_title.yml) workflow). + +## Triggering an off-cycle release + +If you need a release before the next scheduled run (and you are a Substrait +committer or PMC member with the appropriate permissions): + +1. Go to https://github.com/substrait-io/substrait-python/actions/workflows/semantic-release.yml +2. Click `Run workflow` and select the `main` branch. +3. Monitor the workflow run, then the triggered `Release and Publish` run, to + confirm the new version reaches PyPI. + +If there are no release-worthy commits since the last release (only `chore`, `docs`, +`ci`, etc.), semantic-release will report that no release is necessary and do +nothing. ## Versioning -substrait-python follows semantic versioning as described for the Substrait specification here: https://substrait.io/spec/versioning/. \ No newline at end of file +Version bumps are derived from commit types: + +| Commit type | Version bump | +| -------------------------------------------- | ------------ | +| `fix:` | patch | +| `feat:` | minor | +| breaking change (`feat!:`, `BREAKING CHANGE`)| minor | + +substrait-python follows semantic versioning as described for the Substrait +specification here: https://substrait.io/spec/versioning/. Because the project is +pre-1.0, breaking changes produce a **minor** bump rather than a major one (matching +substrait-java). + +## Release toolchain + +The semantic-release toolchain versions are pinned in +[`ci/release/run.sh`](ci/release/run.sh) and the release plugin set lives in +[`.releaserc.mjs`](.releaserc.mjs). Pinning is deliberate: a floating +`conventional-changelog-conventionalcommits` once dropped the `Features` / +`Bug Fixes` sections from the generated notes. When bumping any of these +versions, the [Release Notes Check](.github/workflows/release-notes-check.yml) +workflow runs [`ci/release/dry_run.sh`](ci/release/dry_run.sh) — a credential-free +dry run against the real history (in a throwaway worktree) that fails if a release +is due but its notes come out with no sections. diff --git a/ci/release/dry_run.sh b/ci/release/dry_run.sh new file mode 100755 index 0000000..cbb09a3 --- /dev/null +++ b/ci/release/dry_run.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# +# Credential-free release-notes dry run. +# +# Runs the *pinned* semantic-release toolchain in dry-run mode against the real +# repository history (in a throwaway worktree so nothing is mutated) and prints +# the notes it would generate. When a release is actually due, it asserts the +# notes contain sections rather than just a version heading -- guarding the +# regression seen on PR #171, where a floating conventional-changelog preset +# emitted only the heading and silently dropped the Features/Bug Fixes sections. +# +# No GitHub token is needed: RELEASE_DRY_RUN is left at its default so +# .releaserc.mjs omits the @semantic-release/github and @semantic-release/git +# plugins, whose verifyConditions would otherwise demand credentials. +# +# Keep the pinned versions here in sync with ci/release/run.sh. + +set -euo pipefail + +# semantic-release uses env-ci to detect the branch. Under GitHub Actions on a +# pull_request it resolves GITHUB_REF=refs/pull/N/merge to a PR build and refuses +# to run; clear the Actions markers so env-ci falls back to the worktree's own +# branch below (created fresh from HEAD, so this works from a detached checkout). +unset GITHUB_ACTIONS GITHUB_EVENT_NAME GITHUB_REF GITHUB_HEAD_REF GITHUB_BASE_REF GITHUB_SHA + +curdir="$PWD" +worktree="$(mktemp -d)" +branch="$(basename "$worktree")" + +git worktree add -q "$worktree" + +cleanup() { + cd "$curdir" || exit 1 + git worktree remove --force "$worktree" >/dev/null 2>&1 || true + git worktree prune + git branch -D "$branch" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +cd "$worktree" + +echo "Running semantic-release dry run against real history (branch ${branch})..." +if notes="$( + npx --yes \ + -p "semantic-release@25.0.5" \ + -p "@semantic-release/commit-analyzer@13.0.1" \ + -p "@semantic-release/release-notes-generator@14.1.1" \ + -p "@semantic-release/changelog@6.0.3" \ + -p "@semantic-release/github@12.0.8" \ + -p "@semantic-release/git@10.0.1" \ + -p "conventional-changelog-conventionalcommits@9.3.1" \ + semantic-release \ + --ci false \ + --dry-run \ + --branches "$branch" \ + --repository-url "file://$worktree" 2>&1 +)"; then + sr_exit=0 +else + sr_exit=$? +fi + +echo "$notes" + +if [[ "$sr_exit" -ne 0 ]]; then + echo "ERROR: semantic-release dry run failed (exit ${sr_exit})." >&2 + exit 1 +fi + +if ! grep -qE 'The next release version is' <<<"$notes"; then + echo "NOTE: no release is due from the current history, so there are no notes to assert." + echo "OK: semantic-release dry run completed without errors." + exit 0 +fi + +# A release is due, so at least one feat/fix/breaking commit exists and the notes +# must carry a matching section. The regression produced only the "## " +# heading (h2) with no "###
" (h3) beneath it. +if ! grep -qE '^### ' <<<"$notes"; then + echo "ERROR: a release is due but the generated notes contain no sections -- only the heading." >&2 + echo "This is the PR #171 regression; check the pinned toolchain in ci/release/run.sh and .releaserc.mjs." >&2 + exit 1 +fi + +echo "OK: a release is due and the generated notes contain the expected sections." diff --git a/ci/release/run.sh b/ci/release/run.sh new file mode 100755 index 0000000..af862a9 --- /dev/null +++ b/ci/release/run.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +set -euo pipefail + +# Versions are pinned deliberately: a floating conventional-changelog preset +# silently dropped the Features/Bug Fixes sections from the generated notes. +# ci/release/dry_run.sh guards against a regression when these are bumped. +# RELEASE_DRY_RUN=false opts .releaserc.mjs in to a real release (it defaults +# to a dry run as a fail-safe). +RELEASE_DRY_RUN=false npx --yes \ + -p "semantic-release@25.0.5" \ + -p "@semantic-release/commit-analyzer@13.0.1" \ + -p "@semantic-release/release-notes-generator@14.1.1" \ + -p "@semantic-release/changelog@6.0.3" \ + -p "@semantic-release/github@12.0.8" \ + -p "@semantic-release/git@10.0.1" \ + -p "conventional-changelog-conventionalcommits@9.3.1" \ + semantic-release --ci