Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 69 additions & 18 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ commands:
command: |
. venv/bin/activate
pip install -r <<parameters.requirements>>
- run:
name: Apply grace period to installed packages
command: |
. venv/bin/activate
pip install --quiet requests packaging
python .circleci/pin_safe_versions.py <<parameters.requirements>>

run-tests-with-coverage-report:
parameters:
Expand Down Expand Up @@ -81,6 +87,22 @@ commands:
paths:
- coverage_results

capture-installed-versions:
parameters:
label:
type: string
steps:
- run:
name: Capture installed package versions
when: on_success
command: |
. venv/bin/activate
pip freeze > /tmp/installed_<<parameters.label>>.txt
- persist_to_workspace:
root: /tmp
paths:
- installed_<<parameters.label>>.txt

store-pytest-results:
steps:
- store_test_results:
Expand Down Expand Up @@ -158,57 +180,65 @@ jobs:
- pip-install-deps
- pip-install-tests-deps
- run-tests-with-coverage-report
- capture-installed-versions:
label: "py<<parameters.py-version>>"
- store-pytest-results
- store-coverage-report

py39cassandra:
py39gevent:
docker:
- image: public.ecr.aws/docker/library/python:3.9
- image: public.ecr.aws/docker/library/cassandra:3.11.16-jammy
environment:
MAX_HEAP_SIZE: 2048m
HEAP_NEWSIZE: 512m
working_directory: ~/repo
steps:
- checkout
- check-if-tests-needed
- pip-install-deps
- pip-install-tests-deps:
requirements: "tests/requirements-cassandra.txt"
requirements: "tests/requirements-gevent-starlette.txt"
- run-tests-with-coverage-report:
cassandra: "true"
tests: "tests/clients/test_cassandra-driver.py"
gevent: "true"
tests: "tests/frameworks/test_gevent.py"
- capture-installed-versions:
label: "gevent"
- store-pytest-results
- store-coverage-report

py39gevent:
py312aws:
docker:
- image: public.ecr.aws/docker/library/python:3.9
- image: public.ecr.aws/docker/library/python:3.12
working_directory: ~/repo
steps:
- checkout
- check-if-tests-needed
- pip-install-deps
- pip-install-tests-deps:
requirements: "tests/requirements-gevent-starlette.txt"
requirements: "tests/requirements-aws.txt"
- run-tests-with-coverage-report:
gevent: "true"
tests: "tests/frameworks/test_gevent.py"
tests: "tests_aws"
- capture-installed-versions:
label: "aws"
- store-pytest-results
- store-coverage-report

py312aws:
py312cassandra:
docker:
- image: public.ecr.aws/docker/library/python:3.12
- image: public.ecr.aws/docker/library/cassandra:3.11.16-jammy
environment:
MAX_HEAP_SIZE: 2048m
HEAP_NEWSIZE: 512m
working_directory: ~/repo
steps:
- checkout
- check-if-tests-needed
- pip-install-deps
- pip-install-tests-deps:
requirements: "tests/requirements-aws.txt"
requirements: "tests/requirements-cassandra.txt"
- run-tests-with-coverage-report:
tests: "tests_aws"
cassandra: "true"
tests: "tests/clients/test_cassandra-driver.py"
- capture-installed-versions:
label: "cassandra"
- store-pytest-results
- store-coverage-report

Expand Down Expand Up @@ -253,6 +283,8 @@ jobs:
- run-tests-with-coverage-report:
kafka: "true"
tests: "tests/clients/kafka/test*.py"
- capture-installed-versions:
label: "kafka"
- store-pytest-results
- store-coverage-report

Expand Down Expand Up @@ -285,6 +317,22 @@ jobs:
- check-if-tests-needed
- run_sonarqube

update-currency-versions:
docker:
- image: public.ecr.aws/docker/library/alpine:latest
steps:
- attach_workspace:
at: /tmp/workspace
- run:
name: Collect pip freeze files
command: |
mkdir -p /tmp/pip-freeze
cp /tmp/workspace/installed_*.txt /tmp/pip-freeze/
ls -la /tmp/pip-freeze/
- store_artifacts:
path: /tmp/pip-freeze
destination: pip-freeze

workflows:
tests:
max_auto_reruns: 2
Expand All @@ -293,9 +341,9 @@ workflows:
matrix:
parameters:
py-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
- py39cassandra
- py39gevent
- py312aws
- py312cassandra
- py313kafka
- autowrapt:
matrix:
Expand All @@ -304,8 +352,11 @@ workflows:
- final_job:
requires:
- python3x
- py39cassandra
- py39gevent
- py312aws
- py312cassandra
- py313kafka
- autowrapt
- update-currency-versions:
requires:
- final_job
146 changes: 146 additions & 0 deletions .circleci/pin_safe_versions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
#!/usr/bin/env python3
# (c) Copyright IBM Corp. 2026

"""
Downgrades any installed packages that were released within the 5-day grace
period to their latest safe version. Run after pip install so that CI tests
only exercise versions that have cleared the supply-chain safety window.

Usage:
python scripts/pin_safe_versions.py [requirements_file]

If a requirements file is given, only the packages listed there are checked.
Otherwise every installed package is checked (slow).
"""
from typing import Any, Union


import re
import subprocess
import sys
from datetime import datetime, timedelta

import requests
from packaging.version import Version

GRACE_PERIOD_DAYS = 5


def _get_pypi_releases(package_name: str) -> list[Any]:
try:
r = requests.get(f"https://pypi.org/pypi/{package_name}/json", timeout=10)
r.raise_for_status()
data = r.json()
except Exception:
return []

result = []
for ver, files in data["releases"].items():
if not files or re.search(r"(a|b|rc|dev)\d*$", ver, re.I):
continue
try:
Version(ver)
except Exception:
continue
upload_time = files[-1].get("upload_time_iso_8601", "")
match = re.search(r"([\d-]+)T", upload_time)
if not match:
continue
date = datetime.strptime(match[1], "%Y-%m-%d").date()
result.append((ver, date))
result.sort(key=lambda x: (x[1], Version(x[0])), reverse=True)
return result


def _get_safe_version(releases: list[Any]) -> Union[tuple[Any, Any], tuple[None, None]]:
today = datetime.today().date()
grace_cutoff = today - timedelta(days=GRACE_PERIOD_DAYS)
for i, (ver, date) in enumerate(releases):
grace_end = date + timedelta(days=GRACE_PERIOD_DAYS)
superseded = any(nd < grace_end for _, nd in releases[:i])
if not superseded and date <= grace_cutoff:
return ver, date
return None, None


def _installed_packages() -> dict[Any, Any]:
result = subprocess.run(["pip", "freeze"], capture_output=True, text=True, check=True)
packages = {}
for line in result.stdout.strip().splitlines():
if "==" in line:
pkg, ver = line.split("==", 1)
packages[pkg.lower()] = ver.strip()
return packages


def _parse_req_file(path: str) -> set[str]:
names = set()
try:
with open(path) as f:
for line in f:
line = line.strip()
if not line or line.startswith("#"):
continue
if line.startswith("-r "):
# Recurse into included requirement files (same directory)
import os
included = os.path.join(os.path.dirname(path), line[3:].strip())
names |= _parse_req_file(included)
continue
if line.startswith("-"):
continue
name = re.split(r"[><=!;[\s]", line)[0].strip().lower()
if name:
names.add(name)
except FileNotFoundError:
print(f"Warning: requirements file '{path}' not found.")
return names


def main() -> None:
packages_to_check = None
if len(sys.argv) > 1:
packages_to_check = _parse_req_file(sys.argv[1])
print(f"Checking {len(packages_to_check)} packages from {sys.argv[1]}")

installed = _installed_packages()
today = datetime.today().date()
grace_cutoff = today - timedelta(days=GRACE_PERIOD_DAYS)

to_pin = []
for pkg, installed_ver in installed.items():
if packages_to_check is not None and pkg not in packages_to_check:
continue

releases = _get_pypi_releases(pkg)
if not releases:
continue

installed_date = next((d for v, d in releases if v == installed_ver), None)
if installed_date is None or installed_date <= grace_cutoff:
continue

safe_ver, safe_date = _get_safe_version(releases)
if safe_ver is None:
print(
f"[grace-period] {pkg}=={installed_ver} (released {installed_date}) "
f"is within grace period but no safe version exists — skipping"
)
continue

print(
f"[grace-period] {pkg}: {installed_ver} (released {installed_date}) "
f"→ pinning to {safe_ver} (released {safe_date})"
)
to_pin.append(f"{pkg}=={safe_ver}")

if to_pin:
print(f"\nPinning {len(to_pin)} package(s) to grace-period-safe versions...")
subprocess.run(["pip", "install"] + to_pin, check=True)
print("Grace period enforcement complete.")
else:
print("All checked packages comply with the grace period.")


if __name__ == "__main__":
main()
Loading