-
Notifications
You must be signed in to change notification settings - Fork 44
Add PEP 723 inline script metadata support #452
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,6 +17,7 @@ | |
| """A collection of utilities for fades.""" | ||
|
|
||
| import json | ||
| import shutil | ||
| import logging | ||
| import os | ||
| import subprocess | ||
|
|
@@ -28,12 +29,17 @@ | |
| from urllib.error import HTTPError | ||
|
|
||
| from packaging.requirements import Requirement | ||
| from packaging.specifiers import InvalidSpecifier, SpecifierSet | ||
| from packaging.version import Version | ||
|
|
||
| from fades import FadesError, _version | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
| # range of CPython 3 minor versions probed when auto-selecting an interpreter to | ||
| # satisfy a PEP 723 'requires-python' specifier | ||
| PYTHON_MINOR_RANGE = range(6, 30) | ||
|
|
||
| # command to retrieve the version from an external Python | ||
| SHOW_VERSION_CMD = """ | ||
| import sys, json | ||
|
|
@@ -162,6 +168,93 @@ def get_interpreter_version(requested_interpreter): | |
| return (requested_interpreter, is_current) | ||
|
|
||
|
|
||
| def _get_interpreter_full_version(interpreter=None): | ||
| """Return the (major, minor, micro) version tuple of an interpreter.""" | ||
| if interpreter is None: | ||
| return tuple(sys.version_info[:3]) | ||
| args = [interpreter, '-c', SHOW_VERSION_CMD] | ||
| try: | ||
| raw = logged_exec(args) | ||
| # parse inside the try: a noisy interpreter (e.g. a shim printing to stderr, which | ||
| # logged_exec merges into stdout) can make raw[0] not be the expected JSON; turning | ||
| # any such failure into a FadesError lets _find_interpreter skip that candidate | ||
| info = json.loads(raw[0]) | ||
| return (info['major'], info['minor'], info['micro']) | ||
| except Exception as error: | ||
| logger.error("Error getting requested interpreter version: %s", error) | ||
| raise FadesError("Could not get interpreter version") | ||
|
|
||
|
|
||
| def _find_interpreter(specifier): | ||
| """Search PATH for a python interpreter whose version satisfies the specifier.""" | ||
| candidate_names = ["python3.{}".format(minor) for minor in PYTHON_MINOR_RANGE] | ||
| candidate_names += ["python3", "python"] | ||
|
|
||
| found = {} # path -> Version, to avoid probing the same interpreter twice | ||
| for name in candidate_names: | ||
| path = shutil.which(name) | ||
| if path is None or path in found: | ||
| continue | ||
| try: | ||
| major, minor, micro = _get_interpreter_full_version(path) | ||
| except FadesError: | ||
| continue | ||
| found[path] = Version("{}.{}.{}".format(major, minor, micro)) | ||
|
|
||
| candidates = sorted( | ||
| (version, path) for path, version in found.items() | ||
| if specifier.contains(version, prereleases=True)) | ||
| if not candidates: | ||
| return None | ||
| # pick the highest satisfying version | ||
| return candidates[-1][1] | ||
|
|
||
|
|
||
| def get_interpreter_for_requirement(requires_python, requested_python): | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These names are very confusing. I know What about being explicit and call them |
||
| """Honor a PEP 723 'requires-python' specifier, returning the interpreter to use. | ||
|
|
||
| The returned value is the python executable/path to use, which may be | ||
| 'requested_python' unchanged or an auto-discovered one. Raises FadesError if no | ||
| available interpreter satisfies the specifier. | ||
| """ | ||
| if not requires_python: | ||
| return requested_python | ||
|
|
||
| try: | ||
| specifier = SpecifierSet(requires_python) | ||
| except (InvalidSpecifier, TypeError) as error: | ||
| # TypeError happens when requires-python is not a string (e.g. a TOML number) | ||
| logger.error("Invalid PEP 723 requires-python %r: %s", requires_python, error) | ||
| raise FadesError("Invalid PEP 723 requires-python") | ||
| logger.debug("Honoring PEP 723 requires-python %r", requires_python) | ||
|
|
||
| # check the currently selected interpreter (explicit -p or fades' own) | ||
| major, minor, micro = _get_interpreter_full_version(requested_python) | ||
| current_version = Version("{}.{}.{}".format(major, minor, micro)) | ||
| if specifier.contains(current_version, prereleases=True): | ||
| return requested_python | ||
|
|
||
| if requested_python is not None: | ||
| # the user explicitly chose an interpreter that conflicts with the script; don't | ||
| # silently override that choice, fail so they can resolve it | ||
| msg = ("The chosen Python interpreter (version {}) does not satisfy the script's " | ||
| "requires-python ({!r})".format(current_version, requires_python)) | ||
| logger.error(msg) | ||
| raise FadesError(msg) | ||
|
|
||
| # nothing was explicitly requested and fades' own python doesn't satisfy the spec: | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is a little confusing... if nothing was explicitly requested, what is "the spec"? |
||
| # try to discover a suitable interpreter in PATH | ||
| discovered = _find_interpreter(specifier) | ||
| if discovered is None: | ||
| msg = ("No available Python interpreter satisfies the script's requires-python " | ||
| "({!r})".format(requires_python)) | ||
| logger.error(msg) | ||
| raise FadesError(msg) | ||
| logger.info("Using Python interpreter %r to satisfy requires-python %r", | ||
| discovered, requires_python) | ||
| return discovered | ||
|
|
||
|
|
||
| def get_latest_version_number(project_name): | ||
| """Return latest version of a package.""" | ||
| try: | ||
|
|
@@ -217,9 +310,12 @@ def check_pypi_updates(dependencies): | |
|
|
||
| def _pypi_head_package(dependency): | ||
| """Hit pypi with a http HEAD to check if pkg_name exists.""" | ||
| if dependency.specifier: | ||
| spec = list(dependency.specifier)[0] | ||
| version = spec.version | ||
| # Only an exact pin (== or ===) maps to a version-specific URL; range specifiers | ||
| # like '<3' or '>=2' must be checked by package name, otherwise we'd query a | ||
| # non-existent "version" and wrongly conclude the package is missing. | ||
| exact_specs = [spec for spec in dependency.specifier if spec.operator in ("==", "===")] | ||
| if exact_specs: | ||
| version = exact_specs[0].version | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This may be a bug we have... but this code is not the solution at all. The bug I think we have: "if you declare a version X not using ==, this will check for X anyway" The bug this is introducing: "if you declare a version X not using ==, this will just check if the package exists (not caring the version)". You should open a GH issue (if not there already) and let's handle separately, |
||
| url = BASE_PYPI_URL_WITH_VERSION.format(name=dependency.name, version=version) | ||
| else: | ||
| url = BASE_PYPI_URL.format(name=dependency.name) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -132,11 +132,14 @@ def consolidate_dependencies(needs_ipython, child_program, requirement_files, ma | |
| logger.debug("Dependencies from source file: %s", srcfile_deps) | ||
| docstring_deps = parsing.parse_docstring(child_program) | ||
| logger.debug("Dependencies from docstrings: %s", docstring_deps) | ||
| pep723_deps, _ = parsing.parse_pep723(child_program) | ||
| logger.debug("Dependencies from PEP 723 metadata: %s", pep723_deps) | ||
| else: | ||
| srcfile_deps = {} | ||
| docstring_deps = {} | ||
| pep723_deps = {} | ||
|
|
||
| all_dependencies = [ipython_dep, srcfile_deps, docstring_deps] | ||
| all_dependencies = [ipython_dep, srcfile_deps, docstring_deps, pep723_deps] | ||
|
|
||
| if requirement_files is not None: | ||
| for rf_path in requirement_files: | ||
|
|
@@ -388,8 +391,13 @@ def go(): | |
| if args.check_updates: | ||
| helpers.check_pypi_updates(indicated_deps) | ||
|
|
||
| # honor a PEP 723 'requires-python' (this may pick a different interpreter than the one | ||
| # explicitly requested with --python or fades' own) | ||
| _, requires_python = parsing.parse_pep723(analyzable_child_program) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we avoid the duplicate parsing? |
||
| python_interpreter = helpers.get_interpreter_for_requirement(requires_python, args.python) | ||
|
|
||
| # get the interpreter version requested for the child_program | ||
| interpreter, is_current = helpers.get_interpreter_version(args.python) | ||
| interpreter, is_current = helpers.get_interpreter_version(python_interpreter) | ||
|
|
||
| # options | ||
| pip_options = args.pip_options # pip_options mustn't store. | ||
|
|
@@ -423,7 +431,8 @@ def go(): | |
|
|
||
| # Create a new venv | ||
| venv_data, installed = envbuilder.create_venv( | ||
| indicated_deps, args.python, is_current, options, pip_options, args.avoid_pip_upgrade) | ||
| indicated_deps, python_interpreter, is_current, options, pip_options, | ||
| args.avoid_pip_upgrade) | ||
| # store this new venv in the cache | ||
| venvscache.store(installed, venv_data, interpreter, options) | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,14 +21,27 @@ | |
| from pathlib import Path | ||
| from typing import Generator | ||
|
|
||
| from packaging.requirements import Requirement | ||
| try: | ||
| import tomllib | ||
| except ModuleNotFoundError: # Python < 3.11 | ||
| try: | ||
| import tomli as tomllib | ||
| except ModuleNotFoundError: | ||
| tomllib = None # PEP 723 support degrades gracefully; warned at use site | ||
|
|
||
| from packaging.requirements import InvalidRequirement, Requirement | ||
| from packaging.version import Version | ||
|
|
||
| from fades import REPO_PYPI, REPO_VCS | ||
| from fades import FadesError, REPO_PYPI, REPO_VCS | ||
| from fades.pkgnamesdb import MODULE_TO_PACKAGE | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
| # Canonical regular expression to find a PEP 723 metadata block (see | ||
| # https://peps.python.org/pep-0723/#reference-implementation). | ||
| PEP723_REGEX = re.compile( | ||
| r'(?m)^# /// (?P<type>[a-zA-Z0-9-]+)$\s(?P<content>(^#(| .*)$\s)+)^# ///$') | ||
|
|
||
|
|
||
| class _VCSSpecifier: | ||
| """A simple specifier that works with VCSDependency.""" | ||
|
|
@@ -304,3 +317,64 @@ def parse_docstring(filepath: Path): | |
| return {} | ||
| with open(filepath, 'rt', encoding='utf8') as fh: | ||
| return _parse_docstring(fh) | ||
|
|
||
|
|
||
| def _parse_pep723(content): | ||
| """Parse a PEP 723 inline metadata block. | ||
|
|
||
| Return a tuple ``(deps, requires_python)`` where ``deps`` is the usual repo->deps | ||
| mapping and ``requires_python`` is the raw 'requires-python' specifier (or None). | ||
| """ | ||
| matches = [m for m in PEP723_REGEX.finditer(content) if m.group('type') == 'script'] | ||
| if not matches: | ||
| return {}, None | ||
| if len(matches) > 1: | ||
| # The PEP mandates that tools error when several blocks of the same type exist. | ||
| logger.error("Found %d PEP 723 'script' blocks, but only one is allowed", len(matches)) | ||
| raise FadesError("Multiple PEP 723 'script' blocks found") | ||
| logger.debug("Found a PEP 723 'script' metadata block") | ||
|
|
||
| if tomllib is None: | ||
| logger.warning( | ||
| "Found a PEP 723 metadata block but no TOML parser is available; " | ||
| "install the 'tomli' package to enable PEP 723 support in Python <3.11") | ||
| return {}, None | ||
|
|
||
| # Rebuild the TOML content stripping the comment prefix of each line, as per the PEP. | ||
| toml_content = ''.join( | ||
| line[2:] if line.startswith('# ') else line[1:] | ||
| for line in matches[0].group('content').splitlines(keepends=True)) | ||
| try: | ||
| metadata = tomllib.loads(toml_content) | ||
| except tomllib.TOMLDecodeError as error: | ||
| logger.error("Invalid TOML in the PEP 723 metadata block: %s", error) | ||
| raise FadesError("Invalid TOML in the PEP 723 metadata block") | ||
| logger.debug("Parsed PEP 723 metadata: %s", metadata) | ||
|
|
||
| deps = {} | ||
| dependencies = metadata.get('dependencies', []) | ||
| if not isinstance(dependencies, list): | ||
| logger.error( | ||
| "PEP 723 'dependencies' must be a list, got %s", type(dependencies).__name__) | ||
| raise FadesError("PEP 723 'dependencies' must be a list") | ||
| if dependencies: | ||
| # PEP 723 dependencies are standard PEP 508 strings (including 'name @ url' direct | ||
| # references that pip understands), so they all go to the PyPI repo. | ||
| try: | ||
| deps[REPO_PYPI] = [Requirement(dep) for dep in dependencies] | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Dependencies from other backends should not go in |
||
| except InvalidRequirement as error: | ||
| logger.error("Invalid dependency in the PEP 723 metadata block: %s", error) | ||
| raise FadesError("Invalid dependency in the PEP 723 metadata block") | ||
|
|
||
| return deps, metadata.get('requires-python') | ||
|
|
||
|
|
||
| def parse_pep723(filepath): | ||
| """Parse a source file's PEP 723 metadata block. | ||
|
|
||
| Return a tuple ``(deps, requires_python)``; see ``_parse_pep723``. | ||
| """ | ||
| if filepath is None: | ||
| return {}, None | ||
| with open(filepath, 'rt', encoding='utf8') as fh: | ||
| return _parse_pep723(fh.read()) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,3 +6,4 @@ pytest | |
| pyuca | ||
| pyxdg | ||
| rst2html5 | ||
| tomli | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| # Copyright 2014-2026 Facundo Batista, Nicolás Demarchi | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. New files (here and all below) should start with copyright in current year. And they're missing the rest of the header. |
||
|
|
||
| # /// script | ||
| # requires-python = ">=3.11" | ||
| # dependencies = [ | ||
| # "requests<3", | ||
| # ] | ||
| # /// | ||
|
|
||
| import requests # fades | ||
| import rich # fades | ||
|
|
||
| rich.print(requests.get("https://example.com")) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| # Copyright 2014-2026 Facundo Batista, Nicolás Demarchi | ||
|
|
||
| # /// script | ||
| # dependencies = [ | ||
| # "requests<3", | ||
| # "rich", | ||
| # ] | ||
| # /// | ||
|
|
||
| import requests | ||
| from rich.pretty import pprint | ||
|
|
||
| pprint(requests.get("https://example.com")) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| # Copyright 2014-2026 Facundo Batista, Nicolás Demarchi | ||
|
|
||
| # /// script | ||
| # requires-python = ">=3.11" | ||
| # dependencies = [ | ||
| # "requests<3", | ||
| # ] | ||
| # /// | ||
|
|
||
| import requests | ||
|
|
||
| requests.get("https://example.com") |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It feels a little clumsy to run an external subprocess 25 times just in case to find something.
What about exploring the PATH to see what's out there? See this code, for example, in my machine it produces:
We can work from there...