From 0dc0cb2090ba12dcd7d1c15aa6fd992e72fbbaeb Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Mon, 22 Jun 2026 02:53:49 +0300 Subject: [PATCH 01/23] Add OAuth `cld login` with PKCE loopback flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `cld login` / `cld logout` backed by an OAuth 2.0 Authorization Code + PKCE loopback flow (RFC 8252). A login is persisted as a named `cloudinary://` entry in config.json so it flows through the SDK parser and existing config machinery; tokens refresh transparently when a saved login is selected. Highlights: - auth/: protocol helpers (flow), session codec + JWT decode (session), single-shot loopback server (loopback_server), and the login façade. - Region drives both the API host and the OAuth host; `--region` on the login subcommand, falling back to CLOUDINARY_REGION. - config_resolver: explicit-config guard so a sole OAuth login never hijacks an explicitly chosen `-c`/`-C` account; strips OAuth bookkeeping keys before they reach the API/upload kwargs. - Hardening: request timeouts on token calls, broadened refresh except, cloud_name guard, loopback path check, key-based is_oauth_url, and an expires_in fallback. - logout is OAuth-only and, with no name, lists saved logins for a validated numbered selection. Co-Authored-By: Claude Opus 4.8 (1M context) --- cloudinary_cli/auth/__init__.py | 118 +++++++++++ cloudinary_cli/auth/flow.py | 60 ++++++ cloudinary_cli/auth/loopback_server.py | 72 +++++++ cloudinary_cli/auth/session.py | 113 +++++++++++ cloudinary_cli/cli_group.py | 16 +- cloudinary_cli/core/__init__.py | 3 + cloudinary_cli/core/auth.py | 70 +++++++ cloudinary_cli/core/config.py | 5 +- cloudinary_cli/defaults.py | 52 +++++ cloudinary_cli/modules/clone.py | 4 +- cloudinary_cli/utils/config_resolver.py | 61 ++++++ cloudinary_cli/utils/config_utils.py | 55 +++-- test/test_auth_flow.py | 57 ++++++ test/test_auth_loopback.py | 66 ++++++ test/test_auth_region.py | 34 ++++ test/test_auth_session.py | 156 +++++++++++++++ test/test_cli.py | 2 + test/test_cli_config_oauth.py | 255 ++++++++++++++++++++++++ test/test_modules/test_cli_clone.py | 33 +++ 19 files changed, 1196 insertions(+), 36 deletions(-) create mode 100644 cloudinary_cli/auth/__init__.py create mode 100644 cloudinary_cli/auth/flow.py create mode 100644 cloudinary_cli/auth/loopback_server.py create mode 100644 cloudinary_cli/auth/session.py create mode 100644 cloudinary_cli/core/auth.py create mode 100644 cloudinary_cli/utils/config_resolver.py create mode 100644 test/test_auth_flow.py create mode 100644 test/test_auth_loopback.py create mode 100644 test/test_auth_region.py create mode 100644 test/test_auth_session.py create mode 100644 test/test_cli_config_oauth.py diff --git a/cloudinary_cli/auth/__init__.py b/cloudinary_cli/auth/__init__.py new file mode 100644 index 0000000..b736412 --- /dev/null +++ b/cloudinary_cli/auth/__init__.py @@ -0,0 +1,118 @@ +"""OAuth login façade: runs the PKCE loopback flow, persists each login as a named +`cloudinary://` entry in `config.json`, and refreshes tokens when a saved login is selected.""" +import secrets +import webbrowser + +import requests + +from cloudinary_cli.auth import flow +from cloudinary_cli.auth.loopback_server import start_callback_server, wait_for_callback +from cloudinary_cli.auth.session import ( + Session, + to_cloudinary_url, + from_cloudinary_url, + is_oauth_url, +) +from cloudinary_cli.defaults import logger, normalize_region, DEFAULT_REGION, CLOUDINARY_REGION +from cloudinary_cli.utils.config_utils import load_config, update_config, remove_config_keys +from cloudinary_cli.utils.utils import log_exception + + +def login(region=None, name=None): + """ + Run the interactive browser login and persist the resulting session as a named config entry. + + Returns the saved config name, or None on failure. + """ + region = normalize_region(region or CLOUDINARY_REGION) + session = _run_browser_flow(region) + if not session.cloud_name: + raise RuntimeError("Login token did not include a cloud name; cannot save this login.") + config_name = name or _derive_config_name(session.cloud_name, region) + update_config({config_name: to_cloudinary_url(session)}) + return config_name + + +def logout(name): + """Remove a saved OAuth login by name. Returns "removed", "not_found", or "not_oauth".""" + saved = load_config() + if name not in saved: + return "not_found" + if not is_oauth_url(saved[name]): + return "not_oauth" + remove_config_keys(name) + return "removed" + + +def refresh_url_if_stale(name, url): + """ + Given a saved config value, refresh it if it is a stale OAuth login (rewriting the stored + URL on token rotation). Non-OAuth and still-fresh URLs are returned unchanged. + """ + if not is_oauth_url(url): + return url + + session = from_cloudinary_url(url) + if session.is_fresh(): + return url + if not session.refresh_token: + return url + + try: + token_response = flow.refresh(session.refresh_token, session.region) + except requests.RequestException as e: + log_exception(e, debug_message="OAuth token refresh failed") + return url + + # Hydra rotates refresh tokens; keep the old one only if a new one was not returned. + token_response.setdefault("refresh_token", session.refresh_token) + refreshed_url = to_cloudinary_url(session.updated_from(token_response)) + update_config({name: refreshed_url}) + return refreshed_url + + +def find_sole_oauth_login(): + """Return (name, url) of the only saved OAuth login, or None if there are zero or many.""" + oauth_logins = [(name, url) for name, url in load_config().items() if is_oauth_url(url)] + return oauth_logins[0] if len(oauth_logins) == 1 else None + + +def list_oauth_login_names(): + """Return the names of all saved OAuth logins.""" + return [name for name, url in load_config().items() if is_oauth_url(url)] + + +def _run_browser_flow(region): + verifier, challenge = flow.generate_pkce_pair() + state = secrets.token_urlsafe(16) + httpd, redirect_uri = start_callback_server() + + authorize_url = flow.build_authorize_url(challenge, state, redirect_uri, region) + logger.info("Opening browser to log in to Cloudinary...") + if not webbrowser.open(authorize_url): + logger.info(f"Could not open a browser. Visit this URL to log in:\n{authorize_url}") + else: + logger.info(f"If it doesn't open automatically, visit:\n{authorize_url}") + + auth_code, returned_state = wait_for_callback(httpd) + if returned_state != state: + raise RuntimeError("State mismatch - possible CSRF, aborting.") + + token_response = flow.exchange_code(auth_code, verifier, redirect_uri, region) + return Session.from_token_response(token_response, region=region) + + +def _derive_config_name(cloud_name, region): + """ + Build the saved name: cloud_name + region geo suffix (when not default) + auth-type suffix + only when the base name collides with a DIFFERENT auth type (re-login overwrites in place). + """ + base = cloud_name + if region != DEFAULT_REGION: + base = f"{base}-{region[len('api-'):]}" # api-eu -> "-eu" + + config = load_config() + existing = config.get(base) + if existing is None or is_oauth_url(existing): + return base # free, or same (oauth) type -> overwrite in place + return f"{base}-oauth" # taken by an api-key config -> suffix the new oauth entry diff --git a/cloudinary_cli/auth/flow.py b/cloudinary_cli/auth/flow.py new file mode 100644 index 0000000..8b7811c --- /dev/null +++ b/cloudinary_cli/auth/flow.py @@ -0,0 +1,60 @@ +"""OAuth 2.0 Authorization Code + PKCE protocol helpers (RFC 8252): build the authorize URL, +exchange a code, refresh a token. Pure protocol, no file I/O or global state.""" +import base64 +import hashlib +import secrets +import urllib.parse + +import requests + +from cloudinary_cli.defaults import ( + oauth_authorize_url_for_region, + oauth_token_url_for_region, + OAUTH_CLIENT_ID, + OAUTH_SCOPES, + OAUTH_HTTP_TIMEOUT_SECONDS, +) + + +def generate_pkce_pair(): + """Return (code_verifier, code_challenge) for the S256 PKCE method.""" + verifier = base64.urlsafe_b64encode(secrets.token_bytes(32)).rstrip(b"=").decode("ascii") + digest = hashlib.sha256(verifier.encode("ascii")).digest() + challenge = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + return verifier, challenge + + +def build_authorize_url(challenge, state, redirect_uri, region): + query = urllib.parse.urlencode({ + "client_id": OAUTH_CLIENT_ID, + "response_type": "code", + "scope": OAUTH_SCOPES, + "redirect_uri": redirect_uri, + "state": state, + "code_challenge": challenge, + "code_challenge_method": "S256", + }) + return f"{oauth_authorize_url_for_region(region)}?{query}" + + +def exchange_code(auth_code, verifier, redirect_uri, region): + """Exchange the authorization code for tokens. Public PKCE client - no client_secret.""" + resp = requests.post(oauth_token_url_for_region(region), data={ + "grant_type": "authorization_code", + "code": auth_code, + "redirect_uri": redirect_uri, + "client_id": OAUTH_CLIENT_ID, + "code_verifier": verifier, + }, timeout=OAUTH_HTTP_TIMEOUT_SECONDS) + resp.raise_for_status() + return resp.json() + + +def refresh(refresh_token, region): + resp = requests.post(oauth_token_url_for_region(region), data={ + "grant_type": "refresh_token", + "refresh_token": refresh_token, + "client_id": OAUTH_CLIENT_ID, + }, timeout=OAUTH_HTTP_TIMEOUT_SECONDS) + resp.raise_for_status() + return resp.json() diff --git a/cloudinary_cli/auth/loopback_server.py b/cloudinary_cli/auth/loopback_server.py new file mode 100644 index 0000000..1b3eda0 --- /dev/null +++ b/cloudinary_cli/auth/loopback_server.py @@ -0,0 +1,72 @@ +"""Single-shot loopback HTTP server that captures the OAuth redirect: binds a localhost port, +serves until the `?code=&state=` (or `?error=`) redirect arrives or it times out.""" +import time +import urllib.parse +from http.server import BaseHTTPRequestHandler, HTTPServer + +from cloudinary_cli.defaults import ( + OAUTH_REDIRECT_HOST, + OAUTH_REDIRECT_PORT, + OAUTH_CALLBACK_PATH, + OAUTH_CALLBACK_TIMEOUT_SECONDS, +) + + +class _CallbackHandler(BaseHTTPRequestHandler): + """Captures the ?code=&state= redirect from the authorization server.""" + + def do_GET(self): # noqa: N802 (http.server API) + parsed = urllib.parse.urlparse(self.path) + params = urllib.parse.parse_qs(parsed.query) + + # Ignore stray requests (e.g. /favicon.ico, wrong path) so they don't consume the wait. + if parsed.path != OAUTH_CALLBACK_PATH or ("code" not in params and "error" not in params): + self.send_response(404) + self.end_headers() + return + + self.server.auth_code = params.get("code", [None])[0] + self.server.auth_state = params.get("state", [None])[0] + self.server.auth_error = params.get("error", [None])[0] + + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.end_headers() + if self.server.auth_error: + body = f"

Login failed

{self.server.auth_error}

" + else: + body = "

Login successful

You can close this tab and return to the terminal.

" + self.wfile.write(f"{body}".encode("utf-8")) + + def log_message(self, *args): + pass # silence the default stderr request logging + + +def start_callback_server(): + """Bind the loopback server and return (httpd, redirect_uri).""" + httpd = HTTPServer((OAUTH_REDIRECT_HOST, OAUTH_REDIRECT_PORT), _CallbackHandler) + httpd.auth_code = httpd.auth_state = httpd.auth_error = None + httpd.timeout = OAUTH_CALLBACK_TIMEOUT_SECONDS + redirect_uri = f"http://{OAUTH_REDIRECT_HOST}:{OAUTH_REDIRECT_PORT}{OAUTH_CALLBACK_PATH}" + return httpd, redirect_uri + + +def wait_for_callback(httpd): + """ + Serve requests until the redirect arrives (ignoring favicon/etc.) or the timeout elapses. + Returns (auth_code, auth_state); raises on error or timeout. + """ + deadline = time.monotonic() + OAUTH_CALLBACK_TIMEOUT_SECONDS + try: + while httpd.auth_code is None and httpd.auth_error is None: + if time.monotonic() > deadline: + break + httpd.handle_request() + finally: + httpd.server_close() + + if httpd.auth_error: + raise RuntimeError(f"Authorization failed: {httpd.auth_error}") + if not httpd.auth_code: + raise RuntimeError("Timed out waiting for the authorization redirect.") + return httpd.auth_code, httpd.auth_state diff --git a/cloudinary_cli/auth/session.py b/cloudinary_cli/auth/session.py new file mode 100644 index 0000000..b126ecb --- /dev/null +++ b/cloudinary_cli/auth/session.py @@ -0,0 +1,113 @@ +"""The OAuth session and its `cloudinary://` URL codec. A login is persisted as a `cloudinary://` +URL so it flows through the SDK parser and existing config machinery unchanged; the `Session` +dataclass is the in-memory form, `to_cloudinary_url`/`from_cloudinary_url` the persisted one.""" +import base64 +import json +import time +import urllib.parse +from dataclasses import dataclass + +from cloudinary_cli.defaults import ( + logger, + OAUTH_EXPIRY_SKEW_SECONDS, + OAUTH_FALLBACK_EXPIRES_IN_SECONDS, + api_host_for_region, +) + +# Query-string keys that carry the OAuth session inside a cloudinary:// URL. +_OAUTH_MARKER = "oauth_token" + +_OAUTH_INTERNAL_KEYS = frozenset({"refresh_token", "expires_at", "region", "issuer"}) + + +def strip_oauth_internal_keys(config_dict): + return {k: v for k, v in config_dict.items() if k not in _OAUTH_INTERNAL_KEYS} + + +@dataclass +class Session: + cloud_name: str + access_token: str + refresh_token: str = None + expires_at: int = 0 + region: str = "api" + issuer: str = None + + def is_fresh(self, skew=OAUTH_EXPIRY_SKEW_SECONDS): + return int(self.expires_at or 0) - skew > int(time.time()) + + @classmethod + def from_token_response(cls, token_response, cloud_name=None, region="api", issuer=None): + access_token = token_response["access_token"] + expires_in = int(token_response.get("expires_in") or 0) or OAUTH_FALLBACK_EXPIRES_IN_SECONDS + return cls( + cloud_name=cloud_name or decode_cloud_name(access_token), + access_token=access_token, + refresh_token=token_response.get("refresh_token"), + expires_at=int(time.time()) + expires_in, + region=region, + issuer=decode_issuer(access_token), + ) + + def updated_from(self, token_response): + """Return a new Session with refreshed tokens, preserving cloud_name/region.""" + return Session.from_token_response( + token_response, cloud_name=self.cloud_name, region=self.region) + + +def to_cloudinary_url(session): + """Encode a Session as a key-less cloudinary:// URL (Bearer auth, region-derived host).""" + params = { + "oauth_token": session.access_token, + "refresh_token": session.refresh_token or "", + "expires_at": session.expires_at, + "region": session.region, + "issuer": session.issuer or "", + "upload_prefix": api_host_for_region(session.region), + } + return f"cloudinary://{session.cloud_name}?{urllib.parse.urlencode(params)}" + + +def from_cloudinary_url(url): + """Parse an OAuth cloudinary:// URL back into a Session.""" + parsed = urllib.parse.urlparse(url) + q = {k: v[0] for k, v in urllib.parse.parse_qs(parsed.query).items()} + return Session( + cloud_name=parsed.hostname, + access_token=q.get("oauth_token"), + refresh_token=q.get("refresh_token") or None, + expires_at=int(q.get("expires_at", 0) or 0), + region=q.get("region", "api"), + issuer=q.get("issuer") or None, + ) + + +def is_oauth_url(url): + if not isinstance(url, str): + return False + query = urllib.parse.urlparse(url).query + return _OAUTH_MARKER in urllib.parse.parse_qs(query) + + +def _decode_jwt_payload(access_token): + payload_b64 = access_token.split(".")[1] + payload_b64 += "=" * (-len(payload_b64) % 4) # pad to a multiple of 4 + return json.loads(base64.urlsafe_b64decode(payload_b64)) + + +def decode_cloud_name(access_token): + """Best-effort extraction of cloud_name from the JWT's `ext` claim.""" + try: + payload = _decode_jwt_payload(access_token) + return (payload.get("ext") or {}).get("cloud_name") or payload.get("cloud_name") + except Exception as e: + logger.debug(f"Could not decode cloud_name from token: {e}") + return None + + +def decode_issuer(access_token): + try: + return _decode_jwt_payload(access_token).get("iss") + except Exception as e: + logger.debug(f"Could not decode issuer from token: {e}") + return None diff --git a/cloudinary_cli/cli_group.py b/cloudinary_cli/cli_group.py index 9cd4147..a1b5301 100644 --- a/cloudinary_cli/cli_group.py +++ b/cloudinary_cli/cli_group.py @@ -7,8 +7,7 @@ import cloudinary from cloudinary_cli.defaults import logger -from cloudinary_cli.utils.config_utils import load_config, refresh_cloudinary_config, \ - is_valid_cloudinary_config +from cloudinary_cli.utils.config_resolver import resolve_cli_config from cloudinary_cli.version import __version__ as cli_version CONTEXT_SETTINGS = dict(max_content_width=shutil.get_terminal_size()[0], terminal_width=shutil.get_terminal_size()[0]) @@ -29,19 +28,8 @@ @click_log.simple_verbosity_option(logger) @click.pass_context def cli(ctx, config, config_saved): - if config: - refresh_cloudinary_config(config) - elif config_saved: - config = load_config() - if config_saved not in config: - raise Exception(f"Config {config_saved} does not exist") + resolve_cli_config(config, config_saved) - refresh_cloudinary_config(config[config_saved]) - - if not is_valid_cloudinary_config(): - logger.warning("No Cloudinary configuration found.") - - # If no subcommand was invoked, show help and exit with code 0 if ctx.invoked_subcommand is None: click.echo(ctx.get_help()) ctx.exit(0) diff --git a/cloudinary_cli/core/__init__.py b/cloudinary_cli/core/__init__.py index 7646e1c..c9afce8 100644 --- a/cloudinary_cli/core/__init__.py +++ b/cloudinary_cli/core/__init__.py @@ -1,6 +1,7 @@ import click from cloudinary_cli.core.admin import admin +from cloudinary_cli.core.auth import login, logout from cloudinary_cli.core.config import config from cloudinary_cli.core.search import search, search_folders from cloudinary_cli.core.uploader import uploader @@ -12,6 +13,8 @@ commands = [ config, + login, + logout, search, search_folders, admin, diff --git a/cloudinary_cli/core/auth.py b/cloudinary_cli/core/auth.py new file mode 100644 index 0000000..6b66734 --- /dev/null +++ b/cloudinary_cli/core/auth.py @@ -0,0 +1,70 @@ +from click import command, argument, option, echo + +from cloudinary_cli.auth import login as run_login, logout as run_logout, list_oauth_login_names +from cloudinary_cli.defaults import logger +from cloudinary_cli.utils.utils import log_exception + + +@command("login", help="Log in to Cloudinary via OAuth (opens a browser). The session is saved " + "as a named configuration you can select with `-C`.") +@argument("name", required=False) +@option("--region", + help="Cloudinary region to log in to (e.g. eu, ap, or api-eu). Defaults to the " + "global region (api).") +def login(name, region): + try: + config_name = run_login(region=region, name=name) + except Exception as e: + log_exception(e, "Login failed") + return False + + logger.info(f"Logged in. Saved as '{config_name}'.") + logger.info(f"Example usage: cld -C {config_name} ") + return True + + +@command("logout", help="Log out by removing a saved OAuth configuration. " + "Run without a name to choose from the saved logins.") +@argument("name", required=False) +def logout(name): + if not name: + action, name = _select_oauth_login() + if action == "invalid": + return False + if action != "selected": + return True + + status = run_logout(name) + if status == "removed": + logger.info(f"Logged out of '{name}'.") + elif status == "not_oauth": + logger.error(f"'{name}' is not an OAuth login; refusing to remove it. " + f"Use `config -rm {name}` to delete a saved configuration.") + return False + else: + logger.info(f"No saved OAuth configuration named '{name}'.") + return True + + +def _select_oauth_login(): + """ + Prompt the user to pick a saved OAuth login by number. + + Returns ("selected", name), ("cancelled", None), ("none", None), or ("invalid", None). + """ + names = list_oauth_login_names() + if not names: + logger.info("No saved OAuth logins to log out of.") + return "none", None + + echo("Saved OAuth logins:") + for i, name in enumerate(names, start=1): + echo(f" {i}) {name}") + + choice = input(f"Select a login to log out of [1-{len(names)}] (or Enter to cancel): ").strip() + if not choice: + return "cancelled", None + if not (choice.isdigit() and 1 <= int(choice) <= len(names)): + logger.error(f"Invalid selection '{choice}'. Expected a number between 1 and {len(names)}.") + return "invalid", None + return "selected", names[int(choice) - 1] diff --git a/cloudinary_cli/core/config.py b/cloudinary_cli/core/config.py index c558e17..b90dcee 100644 --- a/cloudinary_cli/core/config.py +++ b/cloudinary_cli/core/config.py @@ -3,7 +3,8 @@ from cloudinary_cli.defaults import logger from cloudinary_cli.utils.config_utils import load_config, verify_cloudinary_url, update_config, remove_config_keys, \ - show_cloudinary_config + show_cloudinary_config, is_valid_cloudinary_config +from cloudinary_cli.utils.utils import ConfigurationError @command("config", help="Display the current configuration, and manage additional configurations.") @@ -47,4 +48,6 @@ def config(new, ls, show, rm, from_url): return show_cloudinary_config(config_obj) else: + if not is_valid_cloudinary_config(): + raise ConfigurationError("No Cloudinary configuration found.") return show_cloudinary_config(cloudinary.config()) diff --git a/cloudinary_cli/defaults.py b/cloudinary_cli/defaults.py index eb41905..565a885 100644 --- a/cloudinary_cli/defaults.py +++ b/cloudinary_cli/defaults.py @@ -24,6 +24,58 @@ CLOUDINARY_CLI_CONFIG_FILE = abspath(path_join(CLOUDINARY_HOME, 'config.json')) +# OAuth (ORY Hydra) configuration for `cld login`. The region string derives both the API and +# OAuth hosts; an unknown region simply fails to resolve. +DEFAULT_REGION = 'api' + + +def normalize_region(region): + # Bare geo codes ('eu') become 'api-'; 'api' and 'api-*' pass through. + region = (region or DEFAULT_REGION).strip() + return region if region.startswith('api') else f'api-{region}' + + +def _oauth_host_for(region): + # Short suffixes (geo codes) use the central authz server; longer ones route to oauth-. + _, _, suffix = region.partition('-') + return 'oauth.cloudinary.com' if len(suffix) <= 2 else f'oauth-{suffix}.cloudinary.com' + + +def api_host_for_region(region): + return f'https://{normalize_region(region)}.cloudinary.com' + + +def oauth_base_url_for_region(region): + return f'https://{_oauth_host_for(normalize_region(region))}' + + +def oauth_authorize_url_for_region(region): + return f'{oauth_base_url_for_region(region)}/oauth2/auth' + + +def oauth_token_url_for_region(region): + return f'{oauth_base_url_for_region(region)}/oauth2/token' + + +CLOUDINARY_REGION = normalize_region(os.environ.get('CLOUDINARY_REGION')) + +# Public PKCE client (no secret). +OAUTH_CLIENT_ID = 'cld_cli' +OAUTH_SCOPES = 'openid offline_access asset_management upload' + +# Hydra requires an exact redirect match, so the port is fixed and must match the registered client. +OAUTH_DEFAULT_REDIRECT_HOST = '127.0.0.1' +OAUTH_REDIRECT_HOST = os.environ.get('CLOUDINARY_OAUTH_REDIRECT_HOST', OAUTH_DEFAULT_REDIRECT_HOST) +OAUTH_DEFAULT_REDIRECT_PORT = 49421 +OAUTH_REDIRECT_PORT = int(os.environ.get('CLOUDINARY_OAUTH_REDIRECT_PORT', OAUTH_DEFAULT_REDIRECT_PORT)) +OAUTH_CALLBACK_PATH = '/callback' + +OAUTH_CALLBACK_TIMEOUT_SECONDS = 300 +OAUTH_EXPIRY_SKEW_SECONDS = 30 +OAUTH_HTTP_TIMEOUT_SECONDS = 30 +# Fallback when the token response omits expires_in, so it can't pin expires_at to "now". +OAUTH_FALLBACK_EXPIRES_IN_SECONDS = 3600 + TEMPLATE_FOLDER_NAME = 'templates' CLOUDINARY_CLI_ROOT = dirname(__file__) TEMPLATE_FOLDER = path_join(CLOUDINARY_CLI_ROOT, TEMPLATE_FOLDER_NAME) diff --git a/cloudinary_cli/modules/clone.py b/cloudinary_cli/modules/clone.py index 44bc856..c93b8d1 100644 --- a/cloudinary_cli/modules/clone.py +++ b/cloudinary_cli/modules/clone.py @@ -4,7 +4,7 @@ from cloudinary.auth_token import _digest from cloudinary_cli.utils.utils import run_tasks_concurrently from cloudinary_cli.utils.api_utils import upload_file -from cloudinary_cli.utils.config_utils import get_cloudinary_config, config_to_dict +from cloudinary_cli.utils.config_resolver import get_cloudinary_config, config_to_api_kwargs from cloudinary_cli.defaults import logger from cloudinary_cli.core.search import execute_single_request, handle_auto_pagination import time @@ -115,7 +115,7 @@ def _prepare_upload_list(source_assets, target_config, overwrite, async_, notification_url, auth_token, url_expiry, normalize_list_params(fields)) - updated_options.update(config_to_dict(target_config)) + updated_options.update(config_to_api_kwargs(target_config)) upload_list.append((asset_url, {**updated_options})) return upload_list diff --git a/cloudinary_cli/utils/config_resolver.py b/cloudinary_cli/utils/config_resolver.py new file mode 100644 index 0000000..024046f --- /dev/null +++ b/cloudinary_cli/utils/config_resolver.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +import cloudinary + +from cloudinary_cli.auth import refresh_url_if_stale, find_sole_oauth_login +from cloudinary_cli.auth.session import strip_oauth_internal_keys +from cloudinary_cli.defaults import logger +from cloudinary_cli.utils.config_utils import ( + load_config, + config_to_dict, + ping_cloudinary, + refresh_cloudinary_config, + is_valid_cloudinary_config, + is_env_configured, +) + + +def resolve_cli_config(config=None, config_saved=None): + explicit_config = bool(config or config_saved) or is_env_configured() + + if config: + refresh_cloudinary_config(config) + elif config_saved: + saved = load_config() + if config_saved not in saved: + raise Exception(f"Config {config_saved} does not exist") + refresh_cloudinary_config(refresh_url_if_stale(config_saved, saved[config_saved])) + + if not explicit_config and not is_valid_cloudinary_config(): + sole_login = find_sole_oauth_login() + if sole_login: + name, url = sole_login + refresh_cloudinary_config(refresh_url_if_stale(name, url)) + + if not is_valid_cloudinary_config(): + logger.warning("No Cloudinary configuration found.") + return False + + return True + + +def get_cloudinary_config(target): + target_config = cloudinary.Config() + if target.startswith("cloudinary://"): + parsed_url = target_config._parse_cloudinary_url(target) + elif target in load_config(): + url = refresh_url_if_stale(target, load_config().get(target)) + parsed_url = target_config._parse_cloudinary_url(url) + else: + return False + + target_config._setup_from_parsed_url(parsed_url) + + if not ping_cloudinary(**config_to_api_kwargs(target_config)): + logger.error(f"Invalid Cloudinary config: {target}") + return False + + return target_config + + +def config_to_api_kwargs(config): + return strip_oauth_internal_keys(config_to_dict(config)) diff --git a/cloudinary_cli/utils/config_utils.py b/cloudinary_cli/utils/config_utils.py index 7b5732a..b15a9bc 100644 --- a/cloudinary_cli/utils/config_utils.py +++ b/cloudinary_cli/utils/config_utils.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import os +import re import cloudinary from click import echo @@ -17,6 +18,7 @@ def load_config(): def save_config(config): _verify_file_path(CLOUDINARY_CLI_CONFIG_FILE) write_json_to_file(config, CLOUDINARY_CLI_CONFIG_FILE) + _restrict_permissions(CLOUDINARY_CLI_CONFIG_FILE) def update_config(new_config): @@ -38,8 +40,8 @@ def remove_config_keys(*keys): def refresh_cloudinary_config(cloudinary_url): - os.environ.update({'CLOUDINARY_URL': cloudinary_url}) cloudinary.reset_config() + cloudinary.config()._load_from_url(cloudinary_url) def verify_cloudinary_url(cloudinary_url): @@ -47,32 +49,33 @@ def verify_cloudinary_url(cloudinary_url): return ping_cloudinary() -def get_cloudinary_config(target): - target_config = cloudinary.Config() - if target.startswith("cloudinary://"): - parsed_url = target_config._parse_cloudinary_url(target) - elif target in load_config(): - parsed_url = target_config._parse_cloudinary_url(load_config().get(target)) - else: - return False +def config_to_dict(config): + return {k: v for k, v in config.__dict__.items() if not k.startswith("_")} - target_config._setup_from_parsed_url(parsed_url) - if not ping_cloudinary(**config_to_dict(target_config)): - logger.error(f"Invalid Cloudinary config: {target}") - return False +_SECRET_KEYS = {"api_secret", "oauth_token", "refresh_token"} +_URL_SECRET_KEYS = {"account_url"} - return target_config -def config_to_dict(config): - return {k: v for k, v in config.__dict__.items() if not k.startswith("_")} +def _mask_secret(value): + value = str(value) + return "*" * (len(value) - 4) + value[-4:] if len(value) > 4 else "*" * len(value) + + +def _mask_url_secret(url): + # Mask the password between `:` and `@` in scheme://user:secret@host. + return re.sub(r'(://[^:/?#]+:)([^@]+)(@)', + lambda m: m.group(1) + _mask_secret(m.group(2)) + m.group(3), str(url)) + def show_cloudinary_config(cloudinary_config): obfuscated_config = config_to_dict(cloudinary_config) - if "api_secret" in obfuscated_config: - api_secret = obfuscated_config["api_secret"] - obfuscated_config["api_secret"] = "*" * (len(api_secret) - 4) + f"{api_secret[-4:]}" + for key, value in obfuscated_config.items(): + if value and key in _SECRET_KEYS: + obfuscated_config[key] = _mask_secret(value) + elif value and key in _URL_SECRET_KEYS: + obfuscated_config[key] = _mask_url_secret(value) # omit default signature algorithm if obfuscated_config.get("signature_algorithm", None) == cloudinary.utils.SIGNATURE_SHA1: @@ -105,9 +108,15 @@ def migrate_old_config(): def is_valid_cloudinary_config(): + if cloudinary.config().cloud_name and cloudinary.config().oauth_token: + return True return None not in [cloudinary.config().cloud_name, cloudinary.config().api_key, cloudinary.config().api_secret] +def is_env_configured(): + return bool(cloudinary.Config().cloud_name) + + def initialize(): migrate_old_config() @@ -122,3 +131,11 @@ def ping_cloudinary(**options): def _verify_file_path(file): os.makedirs(os.path.dirname(file), exist_ok=True) + + +def _restrict_permissions(file): + # The config file holds secrets (api_secret, account_url, OAuth tokens), so keep it 0600. + try: + os.chmod(file, 0o600) + except OSError as e: + logger.debug(f"Could not restrict permissions on {file}: {e}") diff --git a/test/test_auth_flow.py b/test/test_auth_flow.py new file mode 100644 index 0000000..d96b48c --- /dev/null +++ b/test/test_auth_flow.py @@ -0,0 +1,57 @@ +import base64 +import hashlib +import unittest +from unittest.mock import patch, MagicMock +from urllib.parse import urlparse, parse_qs + +from cloudinary_cli.auth import flow + + +class TestAuthFlow(unittest.TestCase): + def test_pkce_pair_s256_no_padding(self): + verifier, challenge = flow.generate_pkce_pair() + self.assertNotIn("=", verifier) + self.assertNotIn("=", challenge) + # challenge must be the S256 of the verifier + expected = base64.urlsafe_b64encode( + hashlib.sha256(verifier.encode("ascii")).digest()).rstrip(b"=").decode("ascii") + self.assertEqual(expected, challenge) + + def test_build_authorize_url(self): + url = flow.build_authorize_url("the_challenge", "the_state", "http://127.0.0.1:49421/callback", "api") + q = parse_qs(urlparse(url).query) + self.assertTrue(url.startswith("https://oauth.cloudinary.com/oauth2/auth?")) + self.assertEqual("code", q["response_type"][0]) + self.assertEqual("S256", q["code_challenge_method"][0]) + self.assertEqual("the_challenge", q["code_challenge"][0]) + self.assertEqual("the_state", q["state"][0]) + self.assertEqual("http://127.0.0.1:49421/callback", q["redirect_uri"][0]) + self.assertIn("client_id", q) + + def test_build_authorize_url_region_drives_host(self): + url = flow.build_authorize_url("c", "s", "http://127.0.0.1:49421/callback", "test") + self.assertTrue(url.startswith("https://oauth-test.cloudinary.com/oauth2/auth?")) + + def test_exchange_code_posts_pkce_no_secret(self): + resp = MagicMock() + resp.json.return_value = {"access_token": "tok"} + with patch("cloudinary_cli.auth.flow.requests.post", return_value=resp) as post: + flow.exchange_code("the_code", "the_verifier", "http://127.0.0.1:49421/callback", "test") + self.assertEqual("https://oauth-test.cloudinary.com/oauth2/token", post.call_args.args[0]) + data = post.call_args.kwargs["data"] + self.assertEqual("authorization_code", data["grant_type"]) + self.assertEqual("the_code", data["code"]) + self.assertEqual("the_verifier", data["code_verifier"]) + self.assertNotIn("client_secret", data) + self.assertIn("timeout", post.call_args.kwargs) + + def test_refresh_posts_refresh_token(self): + resp = MagicMock() + resp.json.return_value = {"access_token": "tok2"} + with patch("cloudinary_cli.auth.flow.requests.post", return_value=resp) as post: + flow.refresh("rt_abc", "api-eu") + self.assertEqual("https://oauth.cloudinary.com/oauth2/token", post.call_args.args[0]) + data = post.call_args.kwargs["data"] + self.assertEqual("refresh_token", data["grant_type"]) + self.assertEqual("rt_abc", data["refresh_token"]) + self.assertIn("timeout", post.call_args.kwargs) diff --git a/test/test_auth_loopback.py b/test/test_auth_loopback.py new file mode 100644 index 0000000..5d8a139 --- /dev/null +++ b/test/test_auth_loopback.py @@ -0,0 +1,66 @@ +import threading +import unittest +import urllib.request +from http.server import HTTPServer + +from cloudinary_cli.auth.loopback_server import _CallbackHandler, wait_for_callback + + +class TestLoopbackServer(unittest.TestCase): + def setUp(self): + # Bind an OS-assigned port on loopback so tests don't collide with the real default. + self.httpd = HTTPServer(("127.0.0.1", 0), _CallbackHandler) + self.httpd.auth_code = self.httpd.auth_state = self.httpd.auth_error = None + self.httpd.timeout = 5 + self.port = self.httpd.server_address[1] + + def tearDown(self): + try: + self.httpd.server_close() + except Exception: + pass + + def _get(self, path): + try: + urllib.request.urlopen(f"http://127.0.0.1:{self.port}{path}", timeout=5).read() + except Exception: + pass + + def test_captures_code_and_state(self): + waiter = threading.Thread(target=lambda: setattr(self, "result", wait_for_callback(self.httpd))) + waiter.start() + self._get("/callback?code=the_code&state=the_state") + waiter.join(timeout=5) + self.assertEqual(("the_code", "the_state"), self.result) + + def test_ignores_favicon_then_captures(self): + waiter = threading.Thread(target=lambda: setattr(self, "result", wait_for_callback(self.httpd))) + waiter.start() + self._get("/favicon.ico") # must NOT end the wait + self._get("/callback?code=c2&state=s2") + waiter.join(timeout=5) + self.assertEqual(("c2", "s2"), self.result) + + def test_ignores_code_on_wrong_path_then_captures(self): + waiter = threading.Thread(target=lambda: setattr(self, "result", wait_for_callback(self.httpd))) + waiter.start() + self._get("/anything?code=stray&state=s") # wrong path must NOT end the wait + self._get("/callback?code=real&state=s3") + waiter.join(timeout=5) + self.assertEqual(("real", "s3"), self.result) + + def test_error_raises(self): + error = {} + + def run(): + try: + wait_for_callback(self.httpd) + except Exception as e: + error["e"] = e + + waiter = threading.Thread(target=run) + waiter.start() + self._get("/callback?error=access_denied") + waiter.join(timeout=5) + self.assertIsInstance(error.get("e"), RuntimeError) + self.assertIn("access_denied", str(error["e"])) diff --git a/test/test_auth_region.py b/test/test_auth_region.py new file mode 100644 index 0000000..86d6bd3 --- /dev/null +++ b/test/test_auth_region.py @@ -0,0 +1,34 @@ +import unittest + +from cloudinary_cli.defaults import normalize_region, _oauth_host_for, api_host_for_region + + +class TestAuthRegion(unittest.TestCase): + def test_normalize_region(self): + self.assertEqual('api', normalize_region(None)) + self.assertEqual('api', normalize_region('')) + self.assertEqual('api', normalize_region('api')) + self.assertEqual('api-eu', normalize_region('eu')) + self.assertEqual('api-ap', normalize_region('ap')) + self.assertEqual('api-eu', normalize_region('api-eu')) + self.assertEqual('api-eu', normalize_region(' api-eu ')) + self.assertEqual('api-test', normalize_region('test')) + self.assertEqual('api-test', normalize_region('api-test')) + + def test_api_host_for_region(self): + self.assertEqual('https://api.cloudinary.com', api_host_for_region('api')) + self.assertEqual('https://api-eu.cloudinary.com', api_host_for_region('api-eu')) + # short codes are normalized first + self.assertEqual('https://api-ap.cloudinary.com', api_host_for_region('ap')) + self.assertEqual('https://api-test.cloudinary.com', api_host_for_region('test')) + self.assertEqual('https://api-test.cloudinary.com', api_host_for_region('api-test')) + + def test_oauth_host_central_for_geo_regions(self): + # <= 2-char suffixes (and bare 'api') use the central authz server + self.assertEqual('oauth.cloudinary.com', _oauth_host_for('api')) + self.assertEqual('oauth.cloudinary.com', _oauth_host_for('api-eu')) + self.assertEqual('oauth.cloudinary.com', _oauth_host_for('api-ap')) + + def test_oauth_host_dedicated_for_long_suffix(self): + # longer suffixes route to their own oauth- host + self.assertEqual('oauth-test.cloudinary.com', _oauth_host_for('api-test')) diff --git a/test/test_auth_session.py b/test/test_auth_session.py new file mode 100644 index 0000000..7e03bdb --- /dev/null +++ b/test/test_auth_session.py @@ -0,0 +1,156 @@ +import time +import unittest +from unittest.mock import patch + +import cloudinary + +from cloudinary_cli.auth import login, refresh_url_if_stale, _derive_config_name +from cloudinary_cli.auth.session import ( + Session, + to_cloudinary_url, + from_cloudinary_url, + is_oauth_url, + strip_oauth_internal_keys, +) + + +def _session(**overrides): + base = dict(cloud_name="eu-cloud", access_token="eyJ.aaa.bbb", refresh_token="rt_123", + expires_at=int(time.time()) + 300, region="api-eu", + issuer="https://oauth.cloudinary.com/") + base.update(overrides) + return Session(**base) + + +class TestSessionCodec(unittest.TestCase): + def test_round_trip(self): + s = _session() + parsed = from_cloudinary_url(to_cloudinary_url(s)) + self.assertEqual(s.cloud_name, parsed.cloud_name) + self.assertEqual(s.access_token, parsed.access_token) + self.assertEqual(s.refresh_token, parsed.refresh_token) + self.assertEqual(s.region, parsed.region) + self.assertEqual(s.issuer, parsed.issuer) + self.assertEqual(s.expires_at, parsed.expires_at) + self.assertIsInstance(parsed.expires_at, int) + + def test_parses_through_sdk_as_bearer(self): + url = to_cloudinary_url(_session()) + config = cloudinary.Config() + config._setup_from_parsed_url(config._parse_cloudinary_url(url)) + self.assertEqual("eu-cloud", config.cloud_name) + self.assertEqual("eyJ.aaa.bbb", config.oauth_token) + self.assertIsNone(config.api_key) + self.assertIsNone(config.api_secret) + self.assertEqual("https://api-eu.cloudinary.com", config.upload_prefix) + + def test_is_oauth_url(self): + self.assertTrue(is_oauth_url(to_cloudinary_url(_session()))) + self.assertFalse(is_oauth_url("cloudinary://key:secret@cloud")) + self.assertFalse(is_oauth_url(None)) + # substring 'oauth_token' outside the query key must not match + self.assertFalse(is_oauth_url("cloudinary://key:secret@oauth_token.example.com")) + self.assertFalse(is_oauth_url("cloudinary://key:secret@cloud?cname=oauth_token.io")) + + def test_is_fresh(self): + self.assertTrue(_session().is_fresh()) + self.assertFalse(_session(expires_at=int(time.time()) - 10).is_fresh()) + + def test_missing_expires_in_falls_back_to_fresh(self): + s = Session.from_token_response({"access_token": "eyJ.aaa.bbb"}, cloud_name="c") + self.assertGreater(s.expires_at, int(time.time())) + self.assertTrue(s.is_fresh()) + + def test_zero_expires_in_falls_back_to_fresh(self): + s = Session.from_token_response( + {"access_token": "eyJ.aaa.bbb", "expires_in": 0}, cloud_name="c") + self.assertTrue(s.is_fresh()) + + +class TestStripOAuthInternalKeys(unittest.TestCase): + def test_drops_bookkeeping_keeps_auth_and_host(self): + url = to_cloudinary_url(_session()) + config = cloudinary.Config() + config._setup_from_parsed_url(config._parse_cloudinary_url(url)) + full = {k: v for k, v in config.__dict__.items() if not k.startswith("_")} + self.assertEqual({"refresh_token", "expires_at", "region", "issuer"}, full.keys() & + {"refresh_token", "expires_at", "region", "issuer"}) + + sanitized = strip_oauth_internal_keys(full) + for leaked in ("refresh_token", "expires_at", "region", "issuer"): + self.assertNotIn(leaked, sanitized) + self.assertEqual("eyJ.aaa.bbb", sanitized["oauth_token"]) + self.assertEqual("https://api-eu.cloudinary.com", sanitized["upload_prefix"]) + self.assertEqual("eu-cloud", sanitized["cloud_name"]) + + def test_noop_on_api_key_config(self): + full = {"cloud_name": "c", "api_key": "k", "api_secret": "s"} + self.assertEqual(full, strip_oauth_internal_keys(full)) + + +class TestRefreshUrlIfStale(unittest.TestCase): + def test_non_oauth_passthrough(self): + url = "cloudinary://key:secret@cloud" + self.assertEqual(url, refresh_url_if_stale("c", url)) + + def test_fresh_unchanged(self): + url = to_cloudinary_url(_session()) + with patch("cloudinary_cli.auth.flow.refresh") as refresh: + self.assertEqual(url, refresh_url_if_stale("eu-cloud", url)) + refresh.assert_not_called() + + def test_stale_refreshes_and_rewrites(self): + stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) + token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.update_config") as update_config: + new_url = refresh_url_if_stale("eu-cloud", stale_url) + self.assertIn("oauth_token=eyJ.new.tok", new_url) + self.assertIn("refresh_token=rt_new", new_url) + update_config.assert_called_once() + + def test_no_refresh_token_returns_unchanged(self): + url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10, refresh_token=None)) + self.assertEqual(url, refresh_url_if_stale("eu-cloud", url)) + + def test_refresh_timeout_returns_stale_url(self): + import requests + stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) + with patch("cloudinary_cli.auth.flow.refresh", side_effect=requests.Timeout()), \ + patch("cloudinary_cli.auth.update_config") as update_config: + self.assertEqual(stale_url, refresh_url_if_stale("eu-cloud", stale_url)) + update_config.assert_not_called() + + +class TestLoginGuards(unittest.TestCase): + def test_missing_cloud_name_raises_and_saves_nothing(self): + session = _session(cloud_name=None) + with patch("cloudinary_cli.auth._run_browser_flow", return_value=session), \ + patch("cloudinary_cli.auth.update_config") as update_config: + with self.assertRaises(RuntimeError): + login(region="api-eu") + update_config.assert_not_called() + + +class TestDeriveConfigName(unittest.TestCase): + def _derive(self, cloud, region, config): + with patch("cloudinary_cli.auth.load_config", return_value=config): + return _derive_config_name(cloud, region) + + def test_default_region_bare(self): + self.assertEqual("my_cloud", self._derive("my_cloud", "api", {})) + + def test_region_suffix(self): + self.assertEqual("my_cloud-eu", self._derive("my_cloud", "api-eu", {})) + + def test_relogin_overwrites_same_type(self): + existing = {"my_cloud-eu": "cloudinary://my_cloud-eu?oauth_token=x"} + self.assertEqual("my_cloud-eu", self._derive("my_cloud", "api-eu", existing)) + + def test_cross_type_collision_gets_oauth_suffix(self): + existing = {"my_cloud": "cloudinary://key:secret@my_cloud"} + self.assertEqual("my_cloud-oauth", self._derive("my_cloud", "api", existing)) + + def test_cross_type_collision_with_region(self): + existing = {"my_cloud-eu": "cloudinary://key:secret@my_cloud"} + self.assertEqual("my_cloud-eu-oauth", self._derive("my_cloud", "api-eu", existing)) diff --git a/test/test_cli.py b/test/test_cli.py index ca8251d..87e1208 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -11,6 +11,8 @@ class TestCLI(unittest.TestCase): COMMANDS = [ 'admin', 'config', + 'login', + 'logout', 'make', 'migrate', 'provisioning', diff --git a/test/test_cli_config_oauth.py b/test/test_cli_config_oauth.py new file mode 100644 index 0000000..6e9da03 --- /dev/null +++ b/test/test_cli_config_oauth.py @@ -0,0 +1,255 @@ +import os +import time +import unittest +from unittest.mock import patch + +import cloudinary +from click.testing import CliRunner + +from cloudinary_cli.auth.session import Session, to_cloudinary_url +from cloudinary_cli.cli import cli +from cloudinary_cli.utils.config_resolver import config_to_api_kwargs, get_cloudinary_config +from cloudinary_cli.utils.config_utils import config_to_dict, show_cloudinary_config + + +def _oauth_url(cloud="eu-cloud", region="api-eu"): + return to_cloudinary_url(Session( + cloud_name=cloud, access_token="eyJ.secret_access.tok", refresh_token="rt_secret_value", + expires_at=int(time.time()) + 300, region=region, issuer="https://oauth.cloudinary.com/")) + + +class _RestoresSdkConfig(unittest.TestCase): + def setUp(self): + self._env_snapshot = dict(os.environ) + self.addCleanup(self._restore_sdk_config) + + def _restore_sdk_config(self): + os.environ.clear() + os.environ.update(self._env_snapshot) + cloudinary.reset_config() + + +class TestLogoutScope(unittest.TestCase): + """logout must only remove OAuth logins, never plain saved configs.""" + + def test_removes_oauth_login(self): + from cloudinary_cli.auth import logout + saved = {"eu-cloud": _oauth_url()} + with patch("cloudinary_cli.auth.load_config", return_value=saved), \ + patch("cloudinary_cli.auth.remove_config_keys") as remove: + self.assertEqual("removed", logout("eu-cloud")) + remove.assert_called_once_with("eu-cloud") + + def test_refuses_non_oauth_config(self): + from cloudinary_cli.auth import logout + saved = {"mykey": "cloudinary://key:secret@cloud"} + with patch("cloudinary_cli.auth.load_config", return_value=saved), \ + patch("cloudinary_cli.auth.remove_config_keys") as remove: + self.assertEqual("not_oauth", logout("mykey")) + remove.assert_not_called() + + def test_missing_name(self): + from cloudinary_cli.auth import logout + with patch("cloudinary_cli.auth.load_config", return_value={}), \ + patch("cloudinary_cli.auth.remove_config_keys") as remove: + self.assertEqual("not_found", logout("nope")) + remove.assert_not_called() + + +class TestLogoutInteractiveSelect(unittest.TestCase): + """`cld logout` with no name lists OAuth logins and removes the chosen one.""" + + runner = CliRunner() + + def test_lists_only_oauth_and_removes_selected(self): + saved = {"mykey": "cloudinary://key:secret@cloud", + "cloud-a": _oauth_url("cloud-a"), "cloud-b": _oauth_url("cloud-b")} + with patch("cloudinary_cli.auth.load_config", return_value=saved), \ + patch("cloudinary_cli.auth.remove_config_keys") as remove: + result = self.runner.invoke(cli, ["logout"], input="2\n") + self.assertIn("cloud-a", result.output) + self.assertIn("cloud-b", result.output) + self.assertNotIn("mykey", result.output) # non-oauth not offered + remove.assert_called_once_with("cloud-b") + + def test_no_oauth_logins(self): + with patch("cloudinary_cli.auth.load_config", + return_value={"mykey": "cloudinary://key:secret@cloud"}), \ + patch("cloudinary_cli.auth.remove_config_keys") as remove: + result = self.runner.invoke(cli, ["logout"], input="\n") + self.assertIn("No saved OAuth logins", result.output) + remove.assert_not_called() + + def test_cancel_on_empty_input(self): + with patch("cloudinary_cli.auth.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ + patch("cloudinary_cli.auth.remove_config_keys") as remove: + result = self.runner.invoke(cli, ["logout"], input="\n") + remove.assert_not_called() + self.assertEqual(0, result.exit_code) + + def test_invalid_non_numeric_errors(self): + with patch("cloudinary_cli.auth.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ + patch("cloudinary_cli.auth.remove_config_keys") as remove: + result = self.runner.invoke(cli, ["logout"], input="sdfdsf\n", standalone_mode=False) + self.assertIn("Invalid selection", result.output) + self.assertFalse(result.return_value) # main() maps falsy -> exit 1 + remove.assert_not_called() + + def test_out_of_range_errors(self): + with patch("cloudinary_cli.auth.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ + patch("cloudinary_cli.auth.remove_config_keys") as remove: + result = self.runner.invoke(cli, ["logout"], input="5\n", standalone_mode=False) + self.assertIn("Invalid selection", result.output) + self.assertFalse(result.return_value) + remove.assert_not_called() + + +class TestConfigSecretMasking(unittest.TestCase): + """show_cloudinary_config must never print a secret in the clear.""" + + def test_masks_api_secret(self): + config = cloudinary.Config() + config.update(cloud_name="c", api_key="k", api_secret="abcdefghIJKLMNOP") + with patch("cloudinary_cli.utils.config_utils.echo") as echo: + show_cloudinary_config(config) + out = echo.call_args[0][0] + self.assertNotIn("abcdefghIJKLMNOP", out) + self.assertIn("MNOP", out) # last 4 kept + + def test_masks_account_url_password(self): + config = cloudinary.Config() + config.update(account_url="account://acc_key:SUPERSECRETPASSWORD@account_id") + with patch("cloudinary_cli.utils.config_utils.echo") as echo: + show_cloudinary_config(config) + out = echo.call_args[0][0] + self.assertNotIn("SUPERSECRETPASSWORD", out) + self.assertIn("acc_key", out) # identifier kept + self.assertIn("account_id", out) # host kept + + def test_masks_oauth_and_refresh_tokens(self): + config = cloudinary.Config() + config.update(cloud_name="c", oauth_token="eyJ.secret_access.tok", + refresh_token="rt_secret_value") + with patch("cloudinary_cli.utils.config_utils.echo") as echo: + show_cloudinary_config(config) + out = echo.call_args[0][0] + self.assertNotIn("eyJ.secret_access.tok", out) + self.assertNotIn("rt_secret_value", out) + + +class TestOAuthConfigCoexistence(_RestoresSdkConfig): + runner = CliRunner() + + CONFIG = { + "prod-account": "cloudinary://key:secret@prod_cloud", + "eu-cloud": _oauth_url(), + } + + def test_ls_shows_both(self): + with patch("cloudinary_cli.core.config.load_config", return_value=dict(self.CONFIG)): + result = self.runner.invoke(cli, ['config', '--ls']) + self.assertEqual(0, result.exit_code) + self.assertIn("prod-account", result.output) + self.assertIn("eu-cloud", result.output) + + def test_show_oauth_masks_token(self): + with patch("cloudinary_cli.core.config.load_config", return_value=dict(self.CONFIG)): + result = self.runner.invoke(cli, ['config', '--show', 'eu-cloud']) + self.assertEqual(0, result.exit_code) + self.assertIn("eu-cloud", result.output) + self.assertNotIn("eyJ.secret_access.tok", result.output) + self.assertNotIn("rt_secret_value", result.output) + + def test_select_oauth_login_configures_sdk(self): + with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(self.CONFIG)): + result = self.runner.invoke(cli, ['-C', 'eu-cloud', 'url', 'sample']) + self.assertEqual(0, result.exit_code, result.output) + self.assertIn("eu-cloud", result.output) + + +class TestSoleOAuthLoginFallbackGate(_RestoresSdkConfig): + runner = CliRunner() + + def _invoke(self, args, sole_login, saved=None, env=None): + env = dict(env or {}) + with patch("cloudinary_cli.utils.config_resolver.find_sole_oauth_login", return_value=sole_login), \ + patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved or {})), \ + patch.dict("os.environ", env, clear=False): + for key in ("CLOUDINARY_URL", "CLOUDINARY_CLOUD_NAME", "CLOUDINARY_API_KEY", + "CLOUDINARY_API_SECRET"): + if key not in env: + os.environ.pop(key, None) + cloudinary.reset_config() + return self.runner.invoke(cli, args) + + def test_fires_when_no_explicit_config(self): + sole = ("eu-cloud", _oauth_url()) + result = self._invoke(['url', 'sample'], sole_login=sole) + self.assertEqual(0, result.exit_code, result.output) + self.assertEqual("eu-cloud", cloudinary.config().cloud_name) + + def test_does_not_hijack_explicit_invalid_minus_C(self): + saved = {"myaccount": "cloudinary://key@chosen_cloud"} # incomplete: no secret -> invalid + sole = ("eu-cloud", _oauth_url()) + result = self._invoke(['-C', 'myaccount', 'url', 'sample'], sole_login=sole, saved=saved) + self.assertEqual("chosen_cloud", cloudinary.config().cloud_name) + self.assertIsNone(cloudinary.config().oauth_token) + + def test_does_not_hijack_explicit_cloudinary_url(self): + sole = ("eu-cloud", _oauth_url()) + self._invoke(['url', 'sample'], sole_login=sole, + env={"CLOUDINARY_URL": "cloudinary://key@env_cloud"}) + self.assertEqual("env_cloud", cloudinary.config().cloud_name) + self.assertIsNone(cloudinary.config().oauth_token) + + + +class TestConfigToApiKwargs(unittest.TestCase): + def _oauth_config(self): + config = cloudinary.Config() + config._setup_from_parsed_url(config._parse_cloudinary_url(_oauth_url())) + return config + + def test_drops_oauth_bookkeeping(self): + config = self._oauth_config() + kwargs = config_to_api_kwargs(config) + for leaked in ("refresh_token", "expires_at", "region", "issuer"): + self.assertNotIn(leaked, kwargs) + self.assertEqual("eyJ.secret_access.tok", kwargs["oauth_token"]) + self.assertEqual("eu-cloud", kwargs["cloud_name"]) + + def test_config_to_dict_still_faithful(self): + full = config_to_dict(self._oauth_config()) + self.assertIn("refresh_token", full) + self.assertIn("region", full) + + +class TestGetCloudinaryConfigOAuth(_RestoresSdkConfig): + def _stale_url(self): + return to_cloudinary_url(Session( + cloud_name="eu-cloud", access_token="eyJ.old.tok", refresh_token="rt_old", + expires_at=int(time.time()) - 10, region="api-eu", + issuer="https://oauth.cloudinary.com/")) + + def test_refreshes_stale_target_before_use(self): + config = {"eu-cloud": self._stale_url()} + token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=config), \ + patch("cloudinary_cli.auth.load_config", return_value=config), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.update_config"), \ + patch("cloudinary_cli.utils.config_resolver.ping_cloudinary", return_value=True): + target_config = get_cloudinary_config("eu-cloud") + self.assertTrue(target_config) + self.assertEqual("eyJ.new.tok", target_config.oauth_token) + + def test_ping_receives_sanitized_config(self): + config = {"eu-cloud": _oauth_url()} + with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=config), \ + patch("cloudinary_cli.auth.load_config", return_value=config), \ + patch("cloudinary_cli.utils.config_resolver.ping_cloudinary", return_value=True) as ping: + get_cloudinary_config("eu-cloud") + ping_kwargs = ping.call_args.kwargs + for leaked in ("refresh_token", "expires_at", "region", "issuer"): + self.assertNotIn(leaked, ping_kwargs) + self.assertEqual("eyJ.secret_access.tok", ping_kwargs["oauth_token"]) diff --git a/test/test_modules/test_cli_clone.py b/test/test_modules/test_cli_clone.py index a4843b1..7abe383 100644 --- a/test/test_modules/test_cli_clone.py +++ b/test/test_modules/test_cli_clone.py @@ -313,5 +313,38 @@ def test_process_metadata_restricted_raw_asset_with_auth_token(self, mock_cloudi self.assertEqual(url, ('https://res.cloudinary.com/demo/raw/upload/s--XyZaBcDeF--/sample_document', {})) +class TestCloneOAuthTarget(unittest.TestCase): + def _oauth_target_config(self): + import cloudinary + from cloudinary_cli.auth.session import Session, to_cloudinary_url + import time + session = Session( + cloud_name="target_cloud", access_token="eyJ.access.tok", + refresh_token="rt", expires_at=int(time.time()) + 3600, + region="api-eu", issuer="https://oauth.cloudinary.com/") + config = cloudinary.Config() + config._setup_from_parsed_url(config._parse_cloudinary_url(to_cloudinary_url(session))) + return config + + def test_upload_list_drops_oauth_bookkeeping(self): + source_assets = { + 'resources': [{ + 'public_id': 'sample', 'type': 'upload', 'resource_type': 'image', + 'format': 'jpg', + 'secure_url': 'https://res.cloudinary.com/demo/image/upload/v1/sample.jpg', + }] + } + upload_list = clone_module._prepare_upload_list( + source_assets, self._oauth_target_config(), overwrite=False, + async_=False, notification_url=None, auth_token=None, + url_expiry=3600, fields=()) + + _, options = upload_list[0] + for leaked in ("refresh_token", "expires_at", "region", "issuer"): + self.assertNotIn(leaked, options) + self.assertEqual("eyJ.access.tok", options["oauth_token"]) + self.assertEqual("target_cloud", options["cloud_name"]) + + if __name__ == '__main__': unittest.main() From 688708399ccd2f4f60c07fdb326558d2e915bf6a Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Wed, 24 Jun 2026 14:56:33 +0300 Subject: [PATCH 02/23] Make config persistence atomic and OAuth refresh concurrency-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add atomic_write primitive (temp file + os.replace) so an interrupted or interleaved write can never truncate/corrupt the config or sync meta file. write_json_to_file gains an opt-in atomic= flag (default False); enabled for save_config and the .cld-sync meta write. atomic_write normalizes the temp file's mkstemp 0600 mode to the umask default so non-config output files keep their usual permissions. Add a reentrant cross-process FileLock around config read-modify-write (update_config / remove_config_keys), and use it in refresh_url_if_stale to re-read and re-check token freshness inside the lock. This prevents two concurrent processes from both refreshing — and thus burning — Hydra's single-use rotated refresh token; a peer's fresh token is adopted instead. Fix OAUTH_CLIENT_ID to the registered Hydra client. Add filelock dependency. Add OAUTH_ATOMIC_CONFIG_REVIEW.md for the next reviewer. Tests: atomic write (incl. read-only dir/target, umask), both json modes, config 0600 + concurrency, peer-refresh adoption vs re-refresh. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 1 + cloudinary_cli/auth/__init__.py | 38 +++++---- cloudinary_cli/defaults.py | 2 +- cloudinary_cli/modules/sync.py | 2 +- cloudinary_cli/utils/config_utils.py | 35 +++++--- cloudinary_cli/utils/file_utils.py | 37 +++++++++ cloudinary_cli/utils/json_utils.py | 16 +++- requirements.txt | 1 + test/test_auth_session.py | 24 ++++++ test/test_config_concurrency.py | 46 +++++++++++ test/test_config_permissions.py | 45 ++++++++++ test/test_file_utils.py | 118 ++++++++++++++++++++++++++- test/test_json_utils.py | 101 +++++++++++++++++++++++ 13 files changed, 433 insertions(+), 33 deletions(-) create mode 100644 test/test_config_concurrency.py create mode 100644 test/test_config_permissions.py create mode 100644 test/test_json_utils.py diff --git a/.gitignore b/.gitignore index 796c08f..413b071 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ venv .cld-sync .cld-settings +.cld-config .venv diff --git a/cloudinary_cli/auth/__init__.py b/cloudinary_cli/auth/__init__.py index b736412..d0fb64b 100644 --- a/cloudinary_cli/auth/__init__.py +++ b/cloudinary_cli/auth/__init__.py @@ -14,7 +14,7 @@ is_oauth_url, ) from cloudinary_cli.defaults import logger, normalize_region, DEFAULT_REGION, CLOUDINARY_REGION -from cloudinary_cli.utils.config_utils import load_config, update_config, remove_config_keys +from cloudinary_cli.utils.config_utils import load_config, update_config, remove_config_keys, config_lock from cloudinary_cli.utils.utils import log_exception @@ -48,27 +48,35 @@ def refresh_url_if_stale(name, url): """ Given a saved config value, refresh it if it is a stale OAuth login (rewriting the stored URL on token rotation). Non-OAuth and still-fresh URLs are returned unchanged. + + The refresh consumes a single-use refresh token, so the whole read-refresh-write runs under + a cross-process lock with the freshness re-checked inside it: a peer that refreshed while we + waited leaves a fresh token we adopt instead of refreshing (and burning) it again. """ if not is_oauth_url(url): return url session = from_cloudinary_url(url) - if session.is_fresh(): - return url - if not session.refresh_token: - return url - - try: - token_response = flow.refresh(session.refresh_token, session.region) - except requests.RequestException as e: - log_exception(e, debug_message="OAuth token refresh failed") + if session.is_fresh() or not session.refresh_token: return url - # Hydra rotates refresh tokens; keep the old one only if a new one was not returned. - token_response.setdefault("refresh_token", session.refresh_token) - refreshed_url = to_cloudinary_url(session.updated_from(token_response)) - update_config({name: refreshed_url}) - return refreshed_url + with config_lock(): + url = load_config().get(name, url) # re-read: a peer may have refreshed while we waited + session = from_cloudinary_url(url) + if session.is_fresh() or not session.refresh_token: + return url + + try: + token_response = flow.refresh(session.refresh_token, session.region) + except requests.RequestException as e: + log_exception(e, debug_message="OAuth token refresh failed") + return url + + # Hydra rotates refresh tokens; keep the old one only if a new one was not returned. + token_response.setdefault("refresh_token", session.refresh_token) + refreshed_url = to_cloudinary_url(session.updated_from(token_response)) + update_config({name: refreshed_url}) + return refreshed_url def find_sole_oauth_login(): diff --git a/cloudinary_cli/defaults.py b/cloudinary_cli/defaults.py index 565a885..a01fb12 100644 --- a/cloudinary_cli/defaults.py +++ b/cloudinary_cli/defaults.py @@ -60,7 +60,7 @@ def oauth_token_url_for_region(region): CLOUDINARY_REGION = normalize_region(os.environ.get('CLOUDINARY_REGION')) # Public PKCE client (no secret). -OAUTH_CLIENT_ID = 'cld_cli' +OAUTH_CLIENT_ID = 'a920ea9c-531b-4613-9783-1d4f4cc10655' OAUTH_SCOPES = 'openid offline_access asset_management upload' # Hydra requires an exact redirect match, so the port is fixed and must match the registered client. diff --git a/cloudinary_cli/modules/sync.py b/cloudinary_cli/modules/sync.py index 8e82c82..483ac4c 100644 --- a/cloudinary_cli/modules/sync.py +++ b/cloudinary_cli/modules/sync.py @@ -336,7 +336,7 @@ def _save_sync_meta_file(self, upload_results): current_diverse_files.update(diverse_filenames) try: logger.debug(f"Updating '{self.sync_meta_file}' file") - write_json_to_file(current_diverse_files, self.sync_meta_file) + write_json_to_file(current_diverse_files, self.sync_meta_file, atomic=True) logger.debug(f"Updated '{self.sync_meta_file}' file") except Exception as e: # Meta file is not critical for the sync itself, in case we cannot write it, we just log a warning diff --git a/cloudinary_cli/utils/config_utils.py b/cloudinary_cli/utils/config_utils.py index b15a9bc..01607ca 100644 --- a/cloudinary_cli/utils/config_utils.py +++ b/cloudinary_cli/utils/config_utils.py @@ -5,11 +5,22 @@ import cloudinary from click import echo from cloudinary import api +from filelock import FileLock from cloudinary_cli.defaults import CLOUDINARY_CLI_CONFIG_FILE, OLD_CLOUDINARY_CLI_CONFIG_FILE, logger from cloudinary_cli.utils.json_utils import write_json_to_file, read_json_from_file from cloudinary_cli.utils.utils import log_exception +# Cross-process lock guarding read-modify-write of the config file. Reentrant within a process, +# so callers may hold it across a multi-step update (e.g. token refresh) without deadlocking. +_config_lock = FileLock(CLOUDINARY_CLI_CONFIG_FILE + ".lock") + + +def config_lock(): + # The lock file lives in the config dir, which may not exist yet on a fresh install. + _verify_file_path(CLOUDINARY_CLI_CONFIG_FILE) + return _config_lock + def load_config(): return read_json_from_file(CLOUDINARY_CLI_CONFIG_FILE, does_not_exist_ok=True) @@ -17,24 +28,26 @@ def load_config(): def save_config(config): _verify_file_path(CLOUDINARY_CLI_CONFIG_FILE) - write_json_to_file(config, CLOUDINARY_CLI_CONFIG_FILE) + write_json_to_file(config, CLOUDINARY_CLI_CONFIG_FILE, atomic=True) _restrict_permissions(CLOUDINARY_CLI_CONFIG_FILE) def update_config(new_config): - curr_config = load_config() - curr_config.update(new_config) - save_config(curr_config) + with config_lock(): + curr_config = load_config() + curr_config.update(new_config) + save_config(curr_config) def remove_config_keys(*keys): - curr_config = load_config() - not_found = [] - for key in keys: - if not curr_config.pop(key, None): - not_found.append(key) - - save_config(curr_config) + with config_lock(): + curr_config = load_config() + not_found = [] + for key in keys: + if not curr_config.pop(key, None): + not_found.append(key) + + save_config(curr_config) return not_found diff --git a/cloudinary_cli/utils/file_utils.py b/cloudinary_cli/utils/file_utils.py index 561d686..8b7dc9a 100644 --- a/cloudinary_cli/utils/file_utils.py +++ b/cloudinary_cli/utils/file_utils.py @@ -1,5 +1,6 @@ import os import stat +import tempfile from os import walk, path, listdir, rmdir, sep from os.path import split, relpath, abspath from pathlib import PurePath @@ -39,6 +40,42 @@ } +def atomic_write(filename, write_fn): + """ + Writes via a temp file in the same directory, then atomically replaces the target, so a + concurrent reader never sees a half-written file and an interleaved write can't truncate it. + + :param filename: The destination file path. + :param write_fn: Callable receiving the open temp file object; performs the actual write. + """ + directory = path.dirname(filename) or "." + fd, tmp_path = tempfile.mkstemp(dir=directory, prefix=".tmp-") + try: + with os.fdopen(fd, 'w') as file: + write_fn(file) + _apply_umask_permissions(tmp_path) + os.replace(tmp_path, filename) + except BaseException: + try: + os.remove(tmp_path) + except OSError: + pass + raise + + +def _apply_umask_permissions(file): + # mkstemp creates the temp file as 0600, and os.replace preserves that mode onto the + # destination. Normalize to the process umask default so output files keep the same + # permissions a plain open() would have produced; callers needing 0600 (e.g. the config + # file) tighten it explicitly afterwards. + current_umask = os.umask(0) + os.umask(current_umask) + try: + os.chmod(file, 0o666 & ~current_umask) + except OSError as e: + logger.debug(f"Could not normalize permissions on {file}: {e}") + + def walk_dir(root_dir, include_hidden=False): all_files = {} for root, dirs, files in walk(root_dir): diff --git a/cloudinary_cli/utils/json_utils.py b/cloudinary_cli/utils/json_utils.py index 7f90869..58ab1f0 100644 --- a/cloudinary_cli/utils/json_utils.py +++ b/cloudinary_cli/utils/json_utils.py @@ -4,6 +4,8 @@ import click from pygments import highlight, lexers, formatters +from cloudinary_cli.utils.file_utils import atomic_write + def read_json_from_file(filename, does_not_exist_ok=False): if does_not_exist_ok and (not path.exists(filename) or path.getsize(filename) < 1): @@ -13,15 +15,21 @@ def read_json_from_file(filename, does_not_exist_ok=False): return json.loads(file.read() or "{}") -def write_json_to_file(json_obj, filename, indent=2, sort_keys=False): - with open(filename, 'w') as file: +def write_json_to_file(json_obj, filename, indent=2, sort_keys=False, atomic=False): + def dump(file): json.dump(json_obj, file, indent=indent, sort_keys=sort_keys) + if atomic: + atomic_write(filename, dump) + else: + with open(filename, 'w') as file: + dump(file) + -def update_json_file(json_obj, filename, indent=2, sort_keys=False): +def update_json_file(json_obj, filename, indent=2, sort_keys=False, atomic=False): curr_obj = read_json_from_file(filename, True) curr_obj.update(json_obj) - write_json_to_file(curr_obj, filename, indent, sort_keys) + write_json_to_file(curr_obj, filename, indent, sort_keys, atomic) def print_json(res): diff --git a/requirements.txt b/requirements.txt index a0ab04f..7576a44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ pygments jinja2 click click-log +filelock requests docstring-parser urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/test/test_auth_session.py b/test/test_auth_session.py index 7e03bdb..babea18 100644 --- a/test/test_auth_session.py +++ b/test/test_auth_session.py @@ -121,6 +121,30 @@ def test_refresh_timeout_returns_stale_url(self): self.assertEqual(stale_url, refresh_url_if_stale("eu-cloud", stale_url)) update_config.assert_not_called() + def test_adopts_peer_refresh_without_calling_refresh(self): + # Peer already rewrote the saved URL to a fresh token while we waited for the lock. + stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) + peer_fresh_url = to_cloudinary_url(_session( + access_token="eyJ.peer.tok", expires_at=int(time.time()) + 300)) + with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": peer_fresh_url}), \ + patch("cloudinary_cli.auth.flow.refresh") as refresh, \ + patch("cloudinary_cli.auth.update_config") as update_config: + result = refresh_url_if_stale("eu-cloud", stale_url) + self.assertEqual(peer_fresh_url, result) + refresh.assert_not_called() # we did not burn the (already-rotated) refresh token + update_config.assert_not_called() + + def test_refreshes_when_peer_value_still_stale(self): + stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) + token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response) as refresh, \ + patch("cloudinary_cli.auth.update_config") as update_config: + result = refresh_url_if_stale("eu-cloud", stale_url) + self.assertIn("oauth_token=eyJ.new.tok", result) + refresh.assert_called_once() + update_config.assert_called_once() + class TestLoginGuards(unittest.TestCase): def test_missing_cloud_name_raises_and_saves_nothing(self): diff --git a/test/test_config_concurrency.py b/test/test_config_concurrency.py new file mode 100644 index 0000000..12bce31 --- /dev/null +++ b/test/test_config_concurrency.py @@ -0,0 +1,46 @@ +import json +import os +import subprocess +import sys +import tempfile +import unittest + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Each worker refreshes config under the cross-process lock; with the lock the read-modify-write +# is serialized, so disjoint keys from concurrent writers must all survive (no last-writer-wins). +_WORKER = """ +import os, sys, time +sys.path.insert(0, {repo!r}) +import cloudinary, cloudinary.api # noqa +from cloudinary_cli.utils.config_utils import update_config +key = sys.argv[1] +update_config({{key: "cloudinary://k:s@" + key}}) +""" + + +class TestConfigConcurrency(unittest.TestCase): + def test_concurrent_writers_lose_no_keys(self): + tmp = tempfile.mkdtemp() + env = dict(os.environ, CLOUDINARY_HOME=tmp) + worker = _WORKER.format(repo=REPO_ROOT) + + n = 12 + procs = [ + subprocess.Popen([sys.executable, "-c", worker, f"cloud{i}"], + env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + for i in range(n) + ] + for p in procs: + _, err = p.communicate(timeout=60) + self.assertEqual(0, p.returncode, err.decode()) + + with open(os.path.join(tmp, "config.json")) as f: + config = json.load(f) # must be valid JSON (never half-written) + + for i in range(n): + self.assertIn(f"cloud{i}", config) # every concurrent write survived + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_config_permissions.py b/test/test_config_permissions.py new file mode 100644 index 0000000..f1df01a --- /dev/null +++ b/test/test_config_permissions.py @@ -0,0 +1,45 @@ +import json +import os +import stat +import subprocess +import sys +import tempfile +import unittest + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# save_config resolves CLOUDINARY_CLI_CONFIG_FILE from CLOUDINARY_HOME at import time, so the +# write must happen in a subprocess with CLOUDINARY_HOME pointed at a temp dir. +_WRITER = """ +import sys +sys.path.insert(0, {repo!r}) +import cloudinary, cloudinary.api # noqa +from cloudinary_cli.utils.config_utils import save_config +save_config({{"cloud": "cloudinary://key:secret@cloud?oauth_token=tok&refresh_token=r"}}) +""" + + +@unittest.skipIf(sys.platform == "win32", "POSIX file modes not applicable on Windows") +class TestConfigPermissions(unittest.TestCase): + def test_saved_config_is_owner_only(self): + tmp = tempfile.mkdtemp() + env = dict(os.environ, CLOUDINARY_HOME=tmp) + + proc = subprocess.run( + [sys.executable, "-c", _WRITER.format(repo=REPO_ROOT)], + env=env, capture_output=True, + ) + self.assertEqual(0, proc.returncode, proc.stderr.decode()) + + config_file = os.path.join(tmp, "config.json") + # The file holds api_secret + OAuth tokens, so it must not be group/world readable. + mode = stat.S_IMODE(os.stat(config_file).st_mode) + self.assertEqual(0o600, mode, f"expected 0600, got {oct(mode)}") + + # Sanity: it's still valid JSON carrying the secret-bearing value. + with open(config_file) as f: + self.assertIn("oauth_token", json.load(f)["cloud"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_file_utils.py b/test/test_file_utils.py index 7dd8f60..212b37d 100644 --- a/test/test_file_utils.py +++ b/test/test_file_utils.py @@ -1,7 +1,16 @@ +import os +import stat +import sys +import tempfile import unittest from pathlib import Path -from cloudinary_cli.utils.file_utils import get_destination_folder, walk_dir, normalize_file_extension +from cloudinary_cli.utils.file_utils import ( + get_destination_folder, + walk_dir, + normalize_file_extension, + atomic_write, +) from test.helper_test import RESOURCES_DIR @@ -33,3 +42,110 @@ def test_normalize_file_extension(self): "SAMPLE.JPEG": "SAMPLE.jpg", }.items(): self.assertEqual(expected, normalize_file_extension(value)) + + +class AtomicWriteTest(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.path = os.path.join(self.dir, "out.txt") + + def _leftover(self): + return [f for f in os.listdir(self.dir) if f != os.path.basename(self.path)] + + def test_writes_content(self): + atomic_write(self.path, lambda f: f.write("hello")) + with open(self.path) as f: + self.assertEqual("hello", f.read()) + + def test_overwrite_replaces_contents(self): + atomic_write(self.path, lambda f: f.write("old")) + atomic_write(self.path, lambda f: f.write("new")) + with open(self.path) as f: + self.assertEqual("new", f.read()) + + def test_leaves_no_temp_files(self): + atomic_write(self.path, lambda f: f.write("x")) + self.assertEqual([], self._leftover()) + + def test_failed_write_removes_temp_and_keeps_original(self): + atomic_write(self.path, lambda f: f.write("keep")) + + def boom(f): + f.write("partial") + raise ValueError("write failed") + + with self.assertRaises(ValueError): + atomic_write(self.path, boom) + + with open(self.path) as f: + self.assertEqual("keep", f.read()) + self.assertEqual([], self._leftover()) + + def test_missing_target_is_not_created_on_failure(self): + with self.assertRaises(ValueError): + atomic_write(self.path, lambda f: (_ for _ in ()).throw(ValueError())) + self.assertFalse(os.path.exists(self.path)) + self.assertEqual([], os.listdir(self.dir)) + + def test_normalizes_to_umask_mode(self): + # mkstemp creates the temp as 0600; atomic_write must relax it to the umask default + # so output files are not silently owner-only. + old_umask = os.umask(0o022) + try: + atomic_write(self.path, lambda f: f.write("x")) + finally: + os.umask(old_umask) + mode = stat.S_IMODE(os.stat(self.path).st_mode) + self.assertEqual(0o644, mode) + + def test_respects_restrictive_umask(self): + old_umask = os.umask(0o077) + try: + atomic_write(self.path, lambda f: f.write("x")) + finally: + os.umask(old_umask) + mode = stat.S_IMODE(os.stat(self.path).st_mode) + self.assertEqual(0o600, mode) + + def test_writes_to_filename_in_cwd_without_dir(self): + # path.dirname("") is "" -> must fall back to "." rather than failing. + old_cwd = os.getcwd() + os.chdir(self.dir) + try: + atomic_write("bare.txt", lambda f: f.write("x")) + with open("bare.txt") as f: + self.assertEqual("x", f.read()) + finally: + os.chdir(old_cwd) + + @unittest.skipIf(sys.platform == "win32", "POSIX directory modes not applicable on Windows") + @unittest.skipIf(hasattr(os, "geteuid") and os.geteuid() == 0, "root bypasses permission bits") + def test_readonly_directory_raises_and_leaves_nothing(self): + # mkstemp needs to create the temp inside the directory, so a read-only directory must + # fail loudly rather than silently writing nothing, and must not leave a temp file behind. + os.chmod(self.dir, 0o500) + try: + with self.assertRaises(OSError): + atomic_write(self.path, lambda f: f.write("x")) + self.assertFalse(os.path.exists(self.path)) + self.assertEqual([], os.listdir(self.dir)) + finally: + os.chmod(self.dir, 0o700) + + @unittest.skipIf(sys.platform == "win32", "POSIX file modes not applicable on Windows") + @unittest.skipIf(hasattr(os, "geteuid") and os.geteuid() == 0, "root bypasses permission bits") + def test_overwrites_readonly_target_in_writable_dir(self): + # os.replace only needs write permission on the directory, not the target, so atomic_write + # can replace a read-only file (where a plain open(file, 'w') would fail) and normalizes + # the result to the umask default. + old_umask = os.umask(0o022) + try: + atomic_write(self.path, lambda f: f.write("old")) + os.chmod(self.path, 0o400) + atomic_write(self.path, lambda f: f.write("new")) + finally: + os.umask(old_umask) + with open(self.path) as f: + self.assertEqual("new", f.read()) + self.assertEqual(0o644, stat.S_IMODE(os.stat(self.path).st_mode)) + self.assertEqual([], self._leftover()) diff --git a/test/test_json_utils.py b/test/test_json_utils.py new file mode 100644 index 0000000..03687c5 --- /dev/null +++ b/test/test_json_utils.py @@ -0,0 +1,101 @@ +import os +import stat +import sys +import tempfile +import unittest + +from cloudinary_cli.utils.json_utils import ( + write_json_to_file, + read_json_from_file, + update_json_file, +) + + +class WriteJsonToFileTest(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.path = os.path.join(self.dir, "config.json") + + def _leftover(self): + return [f for f in os.listdir(self.dir) if f != "config.json"] + + @unittest.skipIf(sys.platform == "win32", "POSIX directory modes not applicable on Windows") + @unittest.skipIf(hasattr(os, "geteuid") and os.geteuid() == 0, "root bypasses permission bits") + def test_readonly_directory_raises_in_both_modes(self): + os.chmod(self.dir, 0o500) + try: + with self.assertRaises(OSError): + write_json_to_file({"a": 1}, self.path, atomic=True) + with self.assertRaises(OSError): + write_json_to_file({"a": 1}, self.path, atomic=False) + self.assertFalse(os.path.exists(self.path)) + self.assertEqual([], os.listdir(self.dir)) + finally: + os.chmod(self.dir, 0o700) + + def test_writes_valid_json(self): + write_json_to_file({"a": 1, "b": "two"}, self.path) + self.assertEqual({"a": 1, "b": "two"}, read_json_from_file(self.path)) + + def test_overwrite_replaces_contents(self): + write_json_to_file({"old": True}, self.path) + write_json_to_file({"new": True}, self.path) + self.assertEqual({"new": True}, read_json_from_file(self.path)) + + def test_respects_indent_and_sort_keys(self): + write_json_to_file({"b": 1, "a": 2}, self.path, indent=2, sort_keys=True) + with open(self.path) as f: + content = f.read() + self.assertEqual('{\n "a": 2,\n "b": 1\n}', content) + + def test_atomic_writes_valid_json(self): + write_json_to_file({"a": 1}, self.path, atomic=True) + self.assertEqual({"a": 1}, read_json_from_file(self.path)) + + def test_atomic_leaves_no_temp_files(self): + write_json_to_file({"a": 1}, self.path, atomic=True) + self.assertEqual([], self._leftover()) + + def test_atomic_failed_write_removes_temp_and_keeps_original(self): + write_json_to_file({"keep": True}, self.path, atomic=True) + # An unserializable object makes json.dump raise mid-write. + with self.assertRaises(TypeError): + write_json_to_file({"bad": object()}, self.path, atomic=True) + self.assertEqual({"keep": True}, read_json_from_file(self.path)) + self.assertEqual([], self._leftover()) + + def test_non_atomic_is_default(self): + # The non-atomic path writes in place: open('w') truncates the target up front, so a + # mid-write failure leaves a corrupted file. This documents why atomic=True exists and + # must be opted into for files that matter (config, sync meta). + write_json_to_file({"keep": True}, self.path) + with self.assertRaises(TypeError): + write_json_to_file({"bad": object()}, self.path) + with self.assertRaises(ValueError): # JSONDecodeError on the partial write + read_json_from_file(self.path) + + +class UpdateJsonFileTest(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.path = os.path.join(self.dir, "data.json") + + def test_creates_file_when_missing(self): + update_json_file({"a": 1}, self.path) + self.assertEqual({"a": 1}, read_json_from_file(self.path)) + + def test_merges_into_existing(self): + write_json_to_file({"a": 1, "b": 2}, self.path) + update_json_file({"b": 20, "c": 3}, self.path) + self.assertEqual({"a": 1, "b": 20, "c": 3}, read_json_from_file(self.path)) + + def test_atomic_flag_merges_and_leaves_no_temp(self): + write_json_to_file({"a": 1}, self.path) + update_json_file({"b": 2}, self.path, atomic=True) + self.assertEqual({"a": 1, "b": 2}, read_json_from_file(self.path)) + leftover = [f for f in os.listdir(self.dir) if f != "data.json"] + self.assertEqual([], leftover) + + +if __name__ == "__main__": + unittest.main() From 43fb67a19b41ce9ab767bdb3900976f12d3d69d0 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 01:47:47 +0300 Subject: [PATCH 03/23] Add selectable default config, config inventory views, and OAuth token refresh Replace the implicit "sole OAuth login auto-applies" rule with an explicit, stored default configuration, and split config handling into an offline format-check (group level) and a lazy network refresh at point-of-use. - Stored default in config.json under reserved key `__default__`; settable via `cld login --set-default`, `cld config -d/--set-default/--unset-default`, and auto-set when a login is the only config with no env/default. - Resolution precedence: -c > -C > stored default > environment > unconfigured. The default outranks env; env is the fallback when no default is set. - Resolver does selection + load only (no network). Stale OAuth tokens refresh lazily at the API chokepoints (call_api + search .execute()), closing the eager-refresh hang (Finding 1). - `cld config -ls` table + `--json`; `cld config`/`-s` gain a header and `--json` with masked secrets, structured expires_at, decomposed account_url, and an active/default/source view. Synthetic (environment)/(command-line) rows. - `cld config --refresh [name] / --refresh-all [--force]` to refresh OAuth tokens; failure hint preserves the config's region in the re-login command. - Fixed-width secret masking (no asterisk walls / length leak); empty fields hidden. - Extract config inventory presentation into utils/config_listing.py; move cloud_name_from_url / config_type into config_utils. Co-Authored-By: Claude Opus 4.8 (1M context) --- cloudinary_cli/auth/__init__.py | 91 +++- cloudinary_cli/core/auth.py | 7 +- cloudinary_cli/core/config.py | 156 ++++++- cloudinary_cli/core/search.py | 2 + cloudinary_cli/defaults.py | 4 + cloudinary_cli/utils/api_utils.py | 4 + cloudinary_cli/utils/config_listing.py | 119 +++++ cloudinary_cli/utils/config_resolver.py | 95 +++- cloudinary_cli/utils/config_utils.py | 181 +++++++- test/test_auth_session.py | 84 +++- test/test_cli_config_oauth.py | 569 +++++++++++++++++++++++- test/test_default_config.py | 57 +++ 12 files changed, 1304 insertions(+), 65 deletions(-) create mode 100644 cloudinary_cli/utils/config_listing.py create mode 100644 test/test_default_config.py diff --git a/cloudinary_cli/auth/__init__.py b/cloudinary_cli/auth/__init__.py index d0fb64b..bc59fd6 100644 --- a/cloudinary_cli/auth/__init__.py +++ b/cloudinary_cli/auth/__init__.py @@ -14,25 +14,57 @@ is_oauth_url, ) from cloudinary_cli.defaults import logger, normalize_region, DEFAULT_REGION, CLOUDINARY_REGION -from cloudinary_cli.utils.config_utils import load_config, update_config, remove_config_keys, config_lock +from cloudinary_cli.utils.config_utils import ( + load_config, + update_config, + remove_config_keys, + config_lock, + user_config_names, + get_default_config_name, + set_default_config, + is_reserved_config_name, + is_env_configured, +) from cloudinary_cli.utils.utils import log_exception -def login(region=None, name=None): +def login(region=None, name=None, set_default=False): """ Run the interactive browser login and persist the resulting session as a named config entry. Returns the saved config name, or None on failure. """ + if name and is_reserved_config_name(name): + raise RuntimeError(f"'{name}' is a reserved configuration name.") region = normalize_region(region or CLOUDINARY_REGION) session = _run_browser_flow(region) if not session.cloud_name: raise RuntimeError("Login token did not include a cloud name; cannot save this login.") config_name = name or _derive_config_name(session.cloud_name, region) update_config({config_name: to_cloudinary_url(session)}) + + if set_default or _should_auto_default(config_name): + set_default_config(config_name) return config_name +def _should_auto_default(name): + """ + True when the just-saved login should become the default without an explicit flag: it is the + only saved config, the environment configures nothing, and no default is already stored. + + A stored default outranks the environment, so auto-defaulting is suppressed when an env config + is present: a single `cld login` must not silently override a user's CLOUDINARY_URL. They can + still opt in with `--set-default`. + """ + cfg = load_config() + return ( + user_config_names(cfg) == [name] + and not is_env_configured() + and not get_default_config_name() + ) + + def logout(name): """Remove a saved OAuth login by name. Returns "removed", "not_found", or "not_oauth".""" saved = load_config() @@ -44,10 +76,11 @@ def logout(name): return "removed" -def refresh_url_if_stale(name, url): +def refresh_url_if_stale(name, url, force=False): """ Given a saved config value, refresh it if it is a stale OAuth login (rewriting the stored - URL on token rotation). Non-OAuth and still-fresh URLs are returned unchanged. + URL on token rotation). Non-OAuth and still-fresh URLs are returned unchanged. With force=True + a still-fresh token is refreshed too (used by the explicit `config --refresh --force`). The refresh consumes a single-use refresh token, so the whole read-refresh-write runs under a cross-process lock with the freshness re-checked inside it: a peer that refreshed while we @@ -57,13 +90,13 @@ def refresh_url_if_stale(name, url): return url session = from_cloudinary_url(url) - if session.is_fresh() or not session.refresh_token: + if (session.is_fresh() and not force) or not session.refresh_token: return url with config_lock(): url = load_config().get(name, url) # re-read: a peer may have refreshed while we waited session = from_cloudinary_url(url) - if session.is_fresh() or not session.refresh_token: + if (session.is_fresh() and not force) or not session.refresh_token: return url try: @@ -79,15 +112,51 @@ def refresh_url_if_stale(name, url): return refreshed_url -def find_sole_oauth_login(): - """Return (name, url) of the only saved OAuth login, or None if there are zero or many.""" - oauth_logins = [(name, url) for name, url in load_config().items() if is_oauth_url(url)] - return oauth_logins[0] if len(oauth_logins) == 1 else None +def refresh_config(name, force=False): + """ + Refresh a single saved OAuth config by name and report the outcome. Returns one of: + "not_found", "not_oauth", "fresh" (skipped, still valid), "refreshed", or "failed" + ("failed" = stale/forced but no refresh token, or the network refresh did not rotate it). + """ + cfg = load_config() + if name not in user_config_names(cfg): + return "not_found" + url = cfg[name] + if not is_oauth_url(url): + return "not_oauth" + + session = from_cloudinary_url(url) + if session.is_fresh() and not force: + return "fresh" + if not session.refresh_token: + return "failed" + + new_url = refresh_url_if_stale(name, url, force=force) + return "refreshed" if new_url != url else "failed" + + +def refresh_configs(force=False): + """Refresh every saved OAuth config. Returns {name: outcome} (see refresh_config).""" + return {name: refresh_config(name, force=force) for name in list_oauth_login_names()} + + +def relogin_command(name): + """ + Build the `cld login` command to re-authenticate a saved OAuth config, preserving its region + (a non-default region must be passed explicitly so the right OAuth host is used). + """ + cmd = f"cld login {name}" + url = load_config().get(name) + region = from_cloudinary_url(url).region if url and is_oauth_url(url) else None + if region and region != DEFAULT_REGION: + cmd += f" --region {region}" + return cmd def list_oauth_login_names(): """Return the names of all saved OAuth logins.""" - return [name for name, url in load_config().items() if is_oauth_url(url)] + cfg = load_config() + return [name for name in user_config_names(cfg) if is_oauth_url(cfg[name])] def _run_browser_flow(region): diff --git a/cloudinary_cli/core/auth.py b/cloudinary_cli/core/auth.py index 6b66734..af1d6f5 100644 --- a/cloudinary_cli/core/auth.py +++ b/cloudinary_cli/core/auth.py @@ -11,9 +11,12 @@ @option("--region", help="Cloudinary region to log in to (e.g. eu, ap, or api-eu). Defaults to the " "global region (api).") -def login(name, region): +@option("--set-default", "set_default", is_flag=True, + help="Set this login as the default configuration used when no -c/-C and no environment " + "config is given.") +def login(name, region, set_default): try: - config_name = run_login(region=region, name=name) + config_name = run_login(region=region, name=name, set_default=set_default) except Exception as e: log_exception(e, "Login failed") return False diff --git a/cloudinary_cli/core/config.py b/cloudinary_cli/core/config.py index b90dcee..b54abf0 100644 --- a/cloudinary_cli/core/config.py +++ b/cloudinary_cli/core/config.py @@ -1,26 +1,79 @@ import cloudinary -from click import command, option, echo, BadParameter +from click import command, option, echo, BadParameter, UsageError -from cloudinary_cli.defaults import logger -from cloudinary_cli.utils.config_utils import load_config, verify_cloudinary_url, update_config, remove_config_keys, \ - show_cloudinary_config, is_valid_cloudinary_config +from cloudinary_cli.defaults import logger, DEFAULT_CONFIG_KEY +from cloudinary_cli.utils.config_utils import ( + load_config, + verify_cloudinary_url, + update_config, + remove_config_keys, + show_cloudinary_config, + is_valid_cloudinary_config, + user_config_names, + get_default_config_name, + set_default_config, + clear_default_config, + is_reserved_config_name, + config_type, +) from cloudinary_cli.utils.utils import ConfigurationError +from cloudinary_cli.utils.json_utils import print_json +from cloudinary_cli.utils.config_resolver import active_config_name, active_config_is_url +from cloudinary_cli.auth import refresh_config, refresh_configs, relogin_command +from cloudinary_cli.utils.config_listing import ( + list_configs, + render_config_table, + config_meta, + active_config_meta, + SYNTHETIC_NAMES, +) @command("config", help="Display the current configuration, and manage additional configurations.") @option("-n", "--new", help="""\b Create and name a configuration from a Cloudinary account environment variable. e.g. cld config -n """, nargs=2) @option("-ls", "--ls", help="List all saved configurations.", is_flag=True) +@option("-j", "--json", "as_json", + help="Output as JSON (with -ls, -s, or the bare config view).", is_flag=True) @option("-s", "--show", help="Show details of a specified configuration.", nargs=1) @option("-rm", "--rm", help="Delete a specified configuration.", nargs=1) @option("-url", "--from_url", help="Create a configuration from a Cloudinary account environment variable. " "The configuration name is the cloud name.", nargs=1) -def config(new, ls, show, rm, from_url): +@option("-d", "--default", "default", nargs=1, + help="Set the named saved configuration as the default.") +@option("--set-default", "set_default", is_flag=True, + help="Set the configuration created by this command (-n / --from_url) as the default.") +@option("-ud", "--unset-default", "unset_default", is_flag=True, + help="Clear the stored default configuration.") +@option("-r", "--refresh", "refresh", nargs=1, + help="Refresh the OAuth token of a saved configuration (use the active config if no name).", + is_flag=False, flag_value="") +@option("-ra", "--refresh-all", "refresh_all", is_flag=True, + help="Refresh every saved OAuth configuration whose token is stale.") +@option("-f", "--force", "force", is_flag=True, + help="With --refresh/--refresh-all, refresh even tokens that are still fresh.") +def config(new, ls, as_json, show, rm, from_url, default, set_default, unset_default, + refresh, refresh_all, force): + if set_default and not (new or from_url): + raise UsageError("--set-default requires -n or --from_url; " + "to default an existing config use -d .") + + if force and refresh is None and not refresh_all: + raise UsageError("--force only applies to --refresh or --refresh-all.") + + if refresh_all: + return _refresh_all(force) + if refresh is not None: + return _refresh_one(refresh, force) + if new or from_url: config_name, cloudinary_url = new or [None, from_url] + if config_name and is_reserved_config_name(config_name): + raise BadParameter(f"'{config_name}' is a reserved configuration name.") + if not verify_cloudinary_url(cloudinary_url): return False @@ -30,24 +83,111 @@ def config(new, ls, show, rm, from_url): logger.info("Config '{}' saved!".format(config_name)) logger.info("Example usage: cld -C {} ".format(config_name)) + + if set_default: + set_default_config(config_name) + logger.info(f"Default set to '{config_name}'.") + elif default: + if default not in user_config_names(load_config()): + raise BadParameter(f"Configuration {default} does not exist, " + f"use -ls to list available configurations.") + set_default_config(default) + logger.info(f"Default set to '{default}'.") + elif unset_default: + clear_default_config() + logger.info("Default configuration cleared.") elif rm: if remove_config_keys(rm): logger.warning(f"Configuration '{rm}' not found.") else: + if get_default_config_name() == rm: + clear_default_config() logger.info(f"Configuration '{rm}' deleted.") elif ls: - echo("\n".join(load_config().keys())) + rows = list_configs() + if as_json: + print_json(rows) + else: + echo(render_config_table(rows)) elif show: curr_config = load_config() - if show not in curr_config: + if show not in user_config_names(curr_config): raise BadParameter(f"Configuration {show} does not exist, use -ls to list available configurations.") config_obj = cloudinary.Config() # noinspection PyProtectedMember - config_obj._setup_from_parsed_url(config_obj._parse_cloudinary_url(load_config()[show])) + config_obj._setup_from_parsed_url(config_obj._parse_cloudinary_url(curr_config[show])) + + if as_json: + return print_json(config_meta(show, curr_config, config_obj)) + _show_config_header(show, curr_config) return show_cloudinary_config(config_obj) else: if not is_valid_cloudinary_config(): raise ConfigurationError("No Cloudinary configuration found.") + if as_json: + return print_json(active_config_meta(cloudinary.config())) + _show_active_header() return show_cloudinary_config(cloudinary.config()) + + +_REFRESH_MESSAGES = { + "not_oauth": ("info", "'{name}' is an api-key config; nothing to refresh."), + "fresh": ("info", "'{name}' token is still fresh; nothing to refresh (use --force to refresh anyway)."), + "refreshed": ("info", "Refreshed '{name}'."), + "failed": ("error", "Could not refresh '{name}'. Please re-login with `{relogin}`."), +} + + +def _report_refresh(name, outcome): + """Log the outcome of a single refresh. Returns True on success (or a benign no-op).""" + level, template = _REFRESH_MESSAGES[outcome] + # The re-login hint must carry the config's region so the right OAuth host is used. + relogin = relogin_command(name) if outcome == "failed" else None + getattr(logger, level)(template.format(name=name, relogin=relogin)) + return outcome != "failed" + + +def _refresh_one(name, force): + name = name or active_config_name() + if not name: + raise UsageError("No active saved configuration to refresh; pass a name: " + "cld config --refresh .") + outcome = refresh_config(name, force=force) + if outcome == "not_found": + raise BadParameter(f"Configuration {name} does not exist, use -ls to list available configurations.") + return _report_refresh(name, outcome) + + +def _refresh_all(force): + results = refresh_configs(force=force) + if not results: + logger.info("No saved OAuth configurations to refresh.") + return True + ok = True + for name, outcome in results.items(): + ok = _report_refresh(name, outcome) and ok + return ok + + +def _show_config_header(name, cfg): + flags = [] + if cfg.get(DEFAULT_CONFIG_KEY) == name: + flags.append("default") + if active_config_name() == name: + flags.append("active") + suffix = f" [{', '.join(flags)}]" if flags else "" + echo(f"name: {name} ({config_type(cfg[name])}){suffix}\n") + + +def _show_active_header(): + """Header for bare `cld config`: identify the active config (saved name, -c URL, or env).""" + name = active_config_name() + if name is not None: + _show_config_header(name, load_config()) + return + active = cloudinary.config() + type_label = "oauth" if active.oauth_token else "api_key" + label = SYNTHETIC_NAMES["url"] if active_config_is_url() else SYNTHETIC_NAMES["env"] + echo(f"name: {label} ({type_label}) [active]\n") diff --git a/cloudinary_cli/core/search.py b/cloudinary_cli/core/search.py index 55cb6b3..9296af2 100644 --- a/cloudinary_cli/core/search.py +++ b/cloudinary_cli/core/search.py @@ -7,6 +7,7 @@ from cloudinary_cli.utils.utils import write_json_list_to_csv, confirm_action, whitelist_keys, \ normalize_list_params from cloudinary_cli.utils.search_utils import parse_aggregate +from cloudinary_cli.utils.config_resolver import ensure_active_config_fresh DEFAULT_MAX_RESULTS = 500 @@ -134,6 +135,7 @@ def _perform_search(query, with_field, fields, sort_by, aggregate, max_results, def execute_single_request(expression, fields_to_keep, result_field='resources'): + ensure_active_config_fresh() res = expression.execute() if fields_to_keep: diff --git a/cloudinary_cli/defaults.py b/cloudinary_cli/defaults.py index a01fb12..5458053 100644 --- a/cloudinary_cli/defaults.py +++ b/cloudinary_cli/defaults.py @@ -24,6 +24,10 @@ CLOUDINARY_CLI_CONFIG_FILE = abspath(path_join(CLOUDINARY_HOME, 'config.json')) +# Reserved key inside config.json that names the default saved configuration. Double-underscore +# names are rejected as user config names, so this can't collide with a saved config. +DEFAULT_CONFIG_KEY = "__default__" + # OAuth (ORY Hydra) configuration for `cld login`. The region string derives both the API and # OAuth hosts; an unknown region simply fails to resolve. DEFAULT_REGION = 'api' diff --git a/cloudinary_cli/utils/api_utils.py b/cloudinary_cli/utils/api_utils.py index 3cd0b89..8f936d6 100644 --- a/cloudinary_cli/utils/api_utils.py +++ b/cloudinary_cli/utils/api_utils.py @@ -8,6 +8,7 @@ from cloudinary_cli.defaults import logger from cloudinary_cli.utils.config_utils import is_valid_cloudinary_config +from cloudinary_cli.utils.config_resolver import ensure_active_config_fresh from cloudinary_cli.utils.file_utils import (normalize_file_extension, posix_rel_path, get_destination_folder, populate_duplicate_name) from cloudinary_cli.utils.json_utils import print_json, write_json_to_file @@ -65,6 +66,7 @@ def query_cld_folder(folder, folder_mode, status=None): logger.debug(f"Search expression: {search.to_json()}") + ensure_active_config_fresh() next_cursor = True while next_cursor: res = search.execute() @@ -104,6 +106,7 @@ def cld_folder_exists(folder): if not folder: return True # root folder + ensure_active_config_fresh() res = SearchFolders().expression(f"path=\"{folder}\"").execute() return res.get("total_count", 0) > 0 @@ -289,6 +292,7 @@ def get_folder_mode(): def call_api(func, args, kwargs): + ensure_active_config_fresh() try: return func(*args, **kwargs) except Exception as e: diff --git a/cloudinary_cli/utils/config_listing.py b/cloudinary_cli/utils/config_listing.py new file mode 100644 index 0000000..f8caf7b --- /dev/null +++ b/cloudinary_cli/utils/config_listing.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Presentation of the saved-config inventory: the rows behind `config -ls`, the table renderer, +and the per-config metadata used for `config`/`config -s` (text headers and JSON).""" +import cloudinary + +from cloudinary_cli.defaults import DEFAULT_CONFIG_KEY +from cloudinary_cli.utils.config_utils import ( + load_config, + user_config_names, + cloud_name_from_url, + config_type, + cloudinary_config_details, + is_env_configured, +) +from cloudinary_cli.utils.config_resolver import ( + active_config_name, + active_config_is_env, + active_config_is_url, +) + +# Display names for the synthetic (non-saved) configs. Parenthesized so they read as a source +# label, not a saved config name, in both the table and JSON. +SYNTHETIC_NAMES = {"env": "(environment)", "url": "(command-line)"} + +_TABLE_COLUMNS = [("name", "NAME"), ("cloud_name", "CLOUD"), ("type", "TYPE"), + ("default", "DEFAULT"), ("active", "ACTIVE")] + + +def list_configs(): + cfg = load_config() + # "default" is the persistent user choice (-d); "active" is the config this very invocation + # resolved to (honoring -c/-C/default/env precedence), as recorded by the resolver. + default = cfg.get(DEFAULT_CONFIG_KEY) + active_name = active_config_name() + + rows = [] + if active_config_is_url(): + rows.append(_url_row()) # an inline -c URL: not a saved config, but it is what's active now + if is_env_configured(): + rows.append(_env_row(env_active=active_config_is_env())) + rows += [ + { + "name": name, + "cloud_name": cloud_name_from_url(cfg[name]), + "type": config_type(cfg[name]), + "source": "saved", + "default": name == default, + "active": name == active_name, + } + for name in user_config_names(cfg) + ] + return rows + + +def config_meta(name, cfg, config_obj): + """JSON view of a named saved config: header metadata plus the masked detail fields.""" + return { + "name": name, + "source": "saved", + "type": config_type(cfg[name]), + "default": cfg.get(DEFAULT_CONFIG_KEY) == name, + "active": active_config_name() == name, + **cloudinary_config_details(config_obj), + } + + +def active_config_meta(config_obj): + """JSON view of the active config for bare `cld config` (saved name, -c URL, or env).""" + name = active_config_name() + if name is not None: + return config_meta(name, load_config(), config_obj) + source = "url" if active_config_is_url() else "env" + return { + "name": SYNTHETIC_NAMES[source], + "source": source, + "type": "oauth" if config_obj.oauth_token else "api_key", + "default": False, + "active": True, + **cloudinary_config_details(config_obj), + } + + +def render_config_table(rows): + headers = [title for _, title in _TABLE_COLUMNS] + cells = [[_cell(row, key) for key, _ in _TABLE_COLUMNS] for row in rows] + widths = [max(len(headers[i]), *(len(r[i]) for r in cells)) if cells else len(headers[i]) + for i in range(len(headers))] + line = lambda values: " ".join(v.ljust(widths[i]) for i, v in enumerate(values)).rstrip() + return "\n".join([line(headers)] + [line(r) for r in cells]) + + +def _url_row(): + active = cloudinary.config() # the CLI global, which the resolver loaded from the -c URL + return { + "name": SYNTHETIC_NAMES["url"], + "cloud_name": active.cloud_name or "", + "type": "oauth" if active.oauth_token else "api_key", + "source": "url", + "default": False, # an inline URL is never the stored default + "active": True, # it outranks everything else for this invocation + } + + +def _env_row(env_active): + env_config = cloudinary.Config() # constructed fresh from the environment, not the CLI global + return { + "name": SYNTHETIC_NAMES["env"], + "cloud_name": env_config.cloud_name or "", + "type": "oauth" if env_config.oauth_token else "api_key", + "source": "env", + "default": False, # the environment is never the *stored* default + "active": env_active, # active only when no stored default outranks it + } + + +def _cell(row, key): + if key in ("default", "active"): + return "*" if row[key] else "" + return str(row.get(key) or "") diff --git a/cloudinary_cli/utils/config_resolver.py b/cloudinary_cli/utils/config_resolver.py index 024046f..378559f 100644 --- a/cloudinary_cli/utils/config_resolver.py +++ b/cloudinary_cli/utils/config_resolver.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import cloudinary -from cloudinary_cli.auth import refresh_url_if_stale, find_sole_oauth_login +from cloudinary_cli.auth import refresh_url_if_stale from cloudinary_cli.auth.session import strip_oauth_internal_keys -from cloudinary_cli.defaults import logger +from cloudinary_cli.defaults import logger, DEFAULT_CONFIG_KEY from cloudinary_cli.utils.config_utils import ( load_config, config_to_dict, @@ -11,33 +11,100 @@ refresh_cloudinary_config, is_valid_cloudinary_config, is_env_configured, + user_config_names, ) +_UNCONFIGURED_MESSAGE = ( + "No Cloudinary configuration found.\n" + " - Log in with OAuth: cld login\n" + " - Add an API-key config: cld config -n " + "cloudinary://:@ --set-default\n" + " - Set an existing config\n" + " as the default: cld config -d " +) + +# What the last resolve_cli_config (Phase A) selected, by precedence. One of: +# "url" -> an inline -c CLOUDINARY_URL +# "env" -> the environment fallback +# None -> nothing configured +# plus _active_name, the saved-config name when a -C/default saved entry was selected (else None). +# Read by ensure_active_config_fresh (Phase B) to know which saved login may need a lazy refresh, +# and by `config -ls` to mark the row that is actually active for this invocation. +_active_name = None +_active_source = None + def resolve_cli_config(config=None, config_saved=None): - explicit_config = bool(config or config_saved) or is_env_configured() + """Select a config by precedence and load it into the SDK global. No network I/O.""" + global _active_name, _active_source + _active_name = None + _active_source = None if config: + _active_source = "url" refresh_cloudinary_config(config) - elif config_saved: - saved = load_config() - if config_saved not in saved: + return _format_ok() + + cfg = load_config() + + if config_saved: + if config_saved not in user_config_names(cfg): raise Exception(f"Config {config_saved} does not exist") - refresh_cloudinary_config(refresh_url_if_stale(config_saved, saved[config_saved])) + _active_name = config_saved + _active_source = "saved" + refresh_cloudinary_config(cfg[config_saved]) + return _format_ok() + + default = cfg.get(DEFAULT_CONFIG_KEY) + if default and default in cfg: + _active_name = default + _active_source = "saved" + refresh_cloudinary_config(cfg[default]) + return _format_ok() + + # No stored default: the SDK global already holds the environment config (if any), so + # _format_ok validates it; otherwise it warns that nothing is configured. + if is_env_configured(): + _active_source = "env" + return _format_ok() + - if not explicit_config and not is_valid_cloudinary_config(): - sole_login = find_sole_oauth_login() - if sole_login: - name, url = sole_login - refresh_cloudinary_config(refresh_url_if_stale(name, url)) +def active_config_name(): + """The saved-config name selected by the last resolution, or None for -c/env/unconfigured.""" + return _active_name + +def active_config_is_env(): + """True when the last resolution fell through to the environment fallback.""" + return _active_source == "env" + + +def active_config_is_url(): + """True when the last resolution loaded an inline -c CLOUDINARY_URL.""" + return _active_source == "url" + + +def _format_ok(): + """Format-only check: is a usable-SHAPED config loaded? Does NOT contact the network.""" if not is_valid_cloudinary_config(): - logger.warning("No Cloudinary configuration found.") + logger.warning(_UNCONFIGURED_MESSAGE) return False - return True +def ensure_active_config_fresh(): + """Refresh the active OAuth token if stale, just before an API call. No-op otherwise.""" + name = _active_name + if name is None: + return + url = load_config().get(name) + if url is None: + return + fresh = refresh_url_if_stale(name, url) + if fresh != url: + refresh_cloudinary_config(fresh) + + def get_cloudinary_config(target): target_config = cloudinary.Config() if target.startswith("cloudinary://"): diff --git a/cloudinary_cli/utils/config_utils.py b/cloudinary_cli/utils/config_utils.py index 01607ca..7e00140 100644 --- a/cloudinary_cli/utils/config_utils.py +++ b/cloudinary_cli/utils/config_utils.py @@ -1,13 +1,20 @@ #!/usr/bin/env python3 import os import re +import time +from datetime import datetime, timezone import cloudinary from click import echo from cloudinary import api from filelock import FileLock -from cloudinary_cli.defaults import CLOUDINARY_CLI_CONFIG_FILE, OLD_CLOUDINARY_CLI_CONFIG_FILE, logger +from cloudinary_cli.defaults import ( + CLOUDINARY_CLI_CONFIG_FILE, + OLD_CLOUDINARY_CLI_CONFIG_FILE, + DEFAULT_CONFIG_KEY, + logger, +) from cloudinary_cli.utils.json_utils import write_json_to_file, read_json_from_file from cloudinary_cli.utils.utils import log_exception @@ -52,6 +59,30 @@ def remove_config_keys(*keys): return not_found +def get_default_config_name(): + """Return the stored default config name, or None if none is set.""" + return load_config().get(DEFAULT_CONFIG_KEY) + + +def set_default_config(name): + update_config({DEFAULT_CONFIG_KEY: name}) + + +def clear_default_config(): + remove_config_keys(DEFAULT_CONFIG_KEY) + + +def user_config_names(cfg=None): + """Saved config names with the reserved default key filtered out.""" + cfg = cfg if cfg is not None else load_config() + return [k for k in cfg if k != DEFAULT_CONFIG_KEY] + + +def is_reserved_config_name(name): + """Names wrapped in double underscores are reserved for internal keys (e.g. the default).""" + return name.startswith("__") and name.endswith("__") + + def refresh_cloudinary_config(cloudinary_url): cloudinary.reset_config() cloudinary.config()._load_from_url(cloudinary_url) @@ -66,13 +97,33 @@ def config_to_dict(config): return {k: v for k, v in config.__dict__.items() if not k.startswith("_")} +def cloud_name_from_url(url): + """Parse a saved cloudinary:// URL and return its cloud name, or "" if it cannot be parsed.""" + config_obj = cloudinary.Config() + try: + # noinspection PyProtectedMember + config_obj._setup_from_parsed_url(config_obj._parse_cloudinary_url(url)) + except Exception: + return "" + return config_obj.cloud_name or "" + + +def config_type(url): + """Classify a saved config URL as "oauth" or "api_key".""" + from cloudinary_cli.auth.session import is_oauth_url + return "oauth" if is_oauth_url(url) else "api_key" + + _SECRET_KEYS = {"api_secret", "oauth_token", "refresh_token"} _URL_SECRET_KEYS = {"account_url"} +# Fixed mask width so a long secret (e.g. an OAuth JWT) does not print a wall of asterisks and the +# real length is not leaked. The last 4 chars are kept as a fingerprint to identify the value. +_MASK_PREFIX = "****" def _mask_secret(value): value = str(value) - return "*" * (len(value) - 4) + value[-4:] if len(value) > 4 else "*" * len(value) + return _MASK_PREFIX + value[-4:] if len(value) > 4 else "*" * len(value) def _mask_url_secret(url): @@ -81,24 +132,132 @@ def _mask_url_secret(url): lambda m: m.group(1) + _mask_secret(m.group(2)) + m.group(3), str(url)) +# account://:@ +_ACCOUNT_URL_RE = re.compile(r'^account://([^:/?#]+):([^@]+)@(.+)$') + + +def _format_account_url(url): + """Render the provisioning account URL as a labeled, secret-masked block (or None if unparsable).""" + match = _ACCOUNT_URL_RE.match(str(url)) + if not match: + return None + api_key, api_secret, account_id = match.groups() + fields = { + "account_id": account_id, + "provisioning_api_key": api_key, + "provisioning_api_secret": _mask_secret(api_secret), + } + width = len(max(fields, key=len)) + 1 + template = "{0:" + str(width) + "} {1}" + return "\n".join(template.format(f"{k}:", v) for k, v in fields.items()) + + +def _format_expires_at(value): + # OAuth token expiry: show the raw epoch plus a human-readable UTC time and live/expired state. + try: + epoch = int(value) + except (TypeError, ValueError): + return value + human = datetime.fromtimestamp(epoch, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + state = "expired" if epoch <= int(time.time()) else "valid" + return f"{epoch} ({human}, {state})" + + def show_cloudinary_config(cloudinary_config): obfuscated_config = config_to_dict(cloudinary_config) - for key, value in obfuscated_config.items(): - if value and key in _SECRET_KEYS: - obfuscated_config[key] = _mask_secret(value) - elif value and key in _URL_SECRET_KEYS: - obfuscated_config[key] = _mask_url_secret(value) - # omit default signature algorithm if obfuscated_config.get("signature_algorithm", None) == cloudinary.utils.SIGNATURE_SHA1: obfuscated_config.pop("signature_algorithm") - if not obfuscated_config: + # The account URL is long and structurally distinct, so it is shown in its own section below. + account_url = obfuscated_config.pop("account_url", None) + + obfuscated_config = { + key: _display_value(key, value) + for key, value in obfuscated_config.items() + if value not in (None, "") # drop empty/None fields (e.g. api_key on an OAuth config) + } + + if not obfuscated_config and not account_url: return False - template = "{0:" + str(len(max(obfuscated_config, key=len)) + 1) + "} {1}" # Gets the maximal length of the keys. - echo('\n'.join([template.format(f"{k}:", v) for k, v in obfuscated_config.items()])) + if obfuscated_config: + width = len(max(obfuscated_config, key=len)) + 1 + template = "{0:" + str(width) + "} {1}" + echo('\n'.join([template.format(f"{k}:", v) for k, v in obfuscated_config.items()])) + + if account_url: + structured = _format_account_url(account_url) + if structured is not None: + echo(f"\nAccount (provisioning) API:\n{structured}") + else: + echo(f"\naccount_url: {_mask_url_secret(account_url)}") + + +def cloudinary_config_details(cloudinary_config): + """ + JSON-friendly, secret-masked view of a Cloudinary config: the same fields shown by + show_cloudinary_config, with secrets masked, empties dropped, expires_at expanded into a + structured object, and account_url decomposed into a nested `account` object. + """ + raw = config_to_dict(cloudinary_config) + + if raw.get("signature_algorithm", None) == cloudinary.utils.SIGNATURE_SHA1: + raw.pop("signature_algorithm") + + account_url = raw.pop("account_url", None) + + details = {} + for key, value in raw.items(): + if value in (None, ""): + continue + if key in _SECRET_KEYS: + details[key] = _mask_secret(value) + elif key == "expires_at": + details[key] = _expires_at_details(value) + else: + details[key] = value + + account = _account_url_details(account_url) if account_url else None + if account is not None: + details["account"] = account + elif account_url: + details["account_url"] = _mask_url_secret(account_url) + + return details + + +def _expires_at_details(value): + try: + epoch = int(value) + except (TypeError, ValueError): + return value + return { + "epoch": epoch, + "utc": datetime.fromtimestamp(epoch, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC"), + "expired": epoch <= int(time.time()), + } + + +def _account_url_details(url): + match = _ACCOUNT_URL_RE.match(str(url)) + if not match: + return None + api_key, api_secret, account_id = match.groups() + return { + "account_id": account_id, + "provisioning_api_key": api_key, + "provisioning_api_secret": _mask_secret(api_secret), + } + + +def _display_value(key, value): + if key in _SECRET_KEYS: + return _mask_secret(value) + if key == "expires_at": + return _format_expires_at(value) + return value def migrate_old_config(): diff --git a/test/test_auth_session.py b/test/test_auth_session.py index babea18..97edcb2 100644 --- a/test/test_auth_session.py +++ b/test/test_auth_session.py @@ -4,7 +4,13 @@ import cloudinary -from cloudinary_cli.auth import login, refresh_url_if_stale, _derive_config_name +from cloudinary_cli.auth import ( + login, + refresh_url_if_stale, + refresh_config, + refresh_configs, + _derive_config_name, +) from cloudinary_cli.auth.session import ( Session, to_cloudinary_url, @@ -99,6 +105,16 @@ def test_fresh_unchanged(self): self.assertEqual(url, refresh_url_if_stale("eu-cloud", url)) refresh.assert_not_called() + def test_force_refreshes_fresh_token(self): + url = to_cloudinary_url(_session()) # fresh + token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": url}), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response) as refresh, \ + patch("cloudinary_cli.auth.update_config"): + new_url = refresh_url_if_stale("eu-cloud", url, force=True) + refresh.assert_called_once() + self.assertIn("oauth_token=eyJ.new.tok", new_url) + def test_stale_refreshes_and_rewrites(self): stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} @@ -146,6 +162,72 @@ def test_refreshes_when_peer_value_still_stale(self): update_config.assert_called_once() +class TestRefreshConfig(unittest.TestCase): + def _cfg(self, **extra): + cfg = { + "stale": to_cloudinary_url(_session(cloud_name="stale", expires_at=int(time.time()) - 10)), + "fresh": to_cloudinary_url(_session(cloud_name="fresh")), + "key": "cloudinary://k:s@kc", + } + cfg.update(extra) + return cfg + + def test_not_found(self): + with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()): + self.assertEqual("not_found", refresh_config("ghost")) + + def test_not_oauth(self): + with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()): + self.assertEqual("not_oauth", refresh_config("key")) + + def test_fresh_skipped(self): + with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()), \ + patch("cloudinary_cli.auth.flow.refresh") as refresh: + self.assertEqual("fresh", refresh_config("fresh")) + refresh.assert_not_called() + + def test_stale_refreshed(self): + token_response = {"access_token": "new", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.update_config"): + self.assertEqual("refreshed", refresh_config("stale")) + + def test_force_refreshes_fresh(self): + token_response = {"access_token": "new", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response) as refresh, \ + patch("cloudinary_cli.auth.update_config"): + self.assertEqual("refreshed", refresh_config("fresh", force=True)) + refresh.assert_called_once() + + def test_failed_when_no_refresh_token(self): + cfg = self._cfg(stale=to_cloudinary_url(_session( + cloud_name="stale", expires_at=int(time.time()) - 10, refresh_token=None))) + with patch("cloudinary_cli.auth.load_config", return_value=cfg): + self.assertEqual("failed", refresh_config("stale")) + + def test_relogin_command_includes_non_default_region(self): + from cloudinary_cli.auth import relogin_command + cfg = { + "global": to_cloudinary_url(_session(cloud_name="global", region="api")), + "stg": to_cloudinary_url(_session(cloud_name="stg", region="api-staging")), + "key": "cloudinary://k:s@kc", + } + with patch("cloudinary_cli.auth.load_config", return_value=cfg): + self.assertEqual("cld login global", relogin_command("global")) + self.assertEqual("cld login stg --region api-staging", relogin_command("stg")) + self.assertEqual("cld login key", relogin_command("key")) # non-oauth: no region + + def test_refresh_configs_sweeps_oauth_only(self): + token_response = {"access_token": "new", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.update_config"): + results = refresh_configs() + self.assertEqual({"stale": "refreshed", "fresh": "fresh"}, results) # "key" not swept + + class TestLoginGuards(unittest.TestCase): def test_missing_cloud_name_raises_and_saves_nothing(self): session = _session(cloud_name=None) diff --git a/test/test_cli_config_oauth.py b/test/test_cli_config_oauth.py index 6e9da03..4cb3a37 100644 --- a/test/test_cli_config_oauth.py +++ b/test/test_cli_config_oauth.py @@ -1,3 +1,4 @@ +import json import os import time import unittest @@ -104,6 +105,69 @@ def test_out_of_range_errors(self): remove.assert_not_called() +class TestLoginSetDefault(unittest.TestCase): + """`login` sets the default explicitly with --set-default and auto-defaults a sole login.""" + + def _patches(self, saved): + session = Session(cloud_name="eu-cloud", access_token="a", refresh_token="r", + expires_at=int(time.time()) + 300, region="api-eu", + issuer="https://oauth.cloudinary.com/") + return patch.multiple( + "cloudinary_cli.auth", + _run_browser_flow=lambda region: session, + load_config=lambda: dict(saved), + update_config=lambda *a, **k: None, + is_env_configured=lambda: False, + ) + + def test_set_default_flag_marks_default(self): + from cloudinary_cli import auth + with self._patches({"eu-cloud": _oauth_url(), "other": _oauth_url("other")}), \ + patch("cloudinary_cli.auth.set_default_config") as set_default, \ + patch("cloudinary_cli.auth.get_default_config_name", return_value=None): + auth.login(region="eu", name="eu-cloud", set_default=True) + set_default.assert_called_once_with("eu-cloud") + + def test_auto_default_when_sole_config_no_env_no_default(self): + from cloudinary_cli import auth + with self._patches({"eu-cloud": _oauth_url()}), \ + patch("cloudinary_cli.auth.set_default_config") as set_default, \ + patch("cloudinary_cli.auth.get_default_config_name", return_value=None): + auth.login(region="eu", name="eu-cloud") + set_default.assert_called_once_with("eu-cloud") + + def test_no_auto_default_when_other_configs_exist(self): + from cloudinary_cli import auth + with self._patches({"eu-cloud": _oauth_url(), "other": _oauth_url("other")}), \ + patch("cloudinary_cli.auth.set_default_config") as set_default, \ + patch("cloudinary_cli.auth.get_default_config_name", return_value=None): + auth.login(region="eu", name="eu-cloud") + set_default.assert_not_called() + + def test_no_auto_default_when_env_configured(self): + from cloudinary_cli import auth + with self._patches({"eu-cloud": _oauth_url()}), \ + patch("cloudinary_cli.auth.is_env_configured", return_value=True), \ + patch("cloudinary_cli.auth.set_default_config") as set_default, \ + patch("cloudinary_cli.auth.get_default_config_name", return_value=None): + auth.login(region="eu", name="eu-cloud") + set_default.assert_not_called() + + def test_no_auto_default_when_default_already_stored(self): + from cloudinary_cli import auth + with self._patches({"eu-cloud": _oauth_url()}), \ + patch("cloudinary_cli.auth.set_default_config") as set_default, \ + patch("cloudinary_cli.auth.get_default_config_name", return_value="something"): + auth.login(region="eu", name="eu-cloud") + set_default.assert_not_called() + + def test_reserved_name_rejected(self): + from cloudinary_cli import auth + with patch("cloudinary_cli.auth._run_browser_flow"): + with self.assertRaises(RuntimeError): + auth.login(region="eu", name="__default__") + + class TestConfigSecretMasking(unittest.TestCase): """show_cloudinary_config must never print a secret in the clear.""" @@ -136,6 +200,72 @@ def test_masks_oauth_and_refresh_tokens(self): self.assertNotIn("eyJ.secret_access.tok", out) self.assertNotIn("rt_secret_value", out) + def test_mask_is_fixed_width_for_long_secret(self): + config = cloudinary.Config() + config.update(cloud_name="c", oauth_token="eyJ" + "A" * 2000 + "N2dQ") + with patch("cloudinary_cli.utils.config_utils.echo") as echo: + show_cloudinary_config(config) + out = echo.call_args[0][0] + self.assertIn("****N2dQ", out) # fixed prefix + last 4 + self.assertNotIn("*" * 8, out) # never a wall of asterisks + + def test_hides_empty_fields(self): + config = cloudinary.Config() + config.update(cloud_name="c", oauth_token="eyJ.tok", api_key=None, api_secret=None) + with patch("cloudinary_cli.utils.config_utils.echo") as echo: + show_cloudinary_config(config) + out = echo.call_args[0][0] + self.assertNotIn("api_key", out) + self.assertNotIn("api_secret", out) + self.assertNotIn("None", out) + self.assertIn("cloud_name", out) + + def test_expires_at_human_readable_and_state(self): + future = cloudinary.Config() + future.update(cloud_name="c", oauth_token="eyJ.tok", expires_at=int(time.time()) + 3600) + with patch("cloudinary_cli.utils.config_utils.echo") as echo: + show_cloudinary_config(future) + out = echo.call_args[0][0] + self.assertIn("UTC", out) + self.assertIn("valid", out) + + past = cloudinary.Config() + past.update(cloud_name="c", oauth_token="eyJ.tok", expires_at=1782310673) + with patch("cloudinary_cli.utils.config_utils.echo") as echo: + show_cloudinary_config(past) + out = echo.call_args[0][0] + self.assertIn("1782310673", out) # raw epoch kept + self.assertIn("2026-06-24", out) # human-readable date + self.assertIn("expired", out) + + def test_account_url_shown_as_structured_section(self): + config = cloudinary.Config() + config.update(cloud_name="c", api_key="k", api_secret="abcdefghIJKLMNOP", + account_url="account://acc_key:SUPERSECRETPASSWORD@account_id") + with patch("cloudinary_cli.utils.config_utils.echo") as echo: + show_cloudinary_config(config) + # two echo calls: the main block, then the account (provisioning) section + self.assertEqual(2, echo.call_count) + main = echo.call_args_list[0][0][0] + account = echo.call_args_list[1][0][0] + self.assertNotIn("account://", main) + # the account URL is decomposed into labeled fields, secret masked + self.assertIn("account_id:", account) + self.assertIn("provisioning_api_key:", account) + self.assertIn("provisioning_api_secret:", account) + self.assertIn("acc_key", account) + self.assertIn("account_id", account) + self.assertNotIn("SUPERSECRETPASSWORD", account) + self.assertNotIn("account://", account) # no raw URL string + + def test_malformed_account_url_falls_back_to_raw_line(self): + config = cloudinary.Config() + config.update(cloud_name="c", account_url="account://garbage") + with patch("cloudinary_cli.utils.config_utils.echo") as echo: + show_cloudinary_config(config) + account = echo.call_args_list[-1][0][0] + self.assertIn("account_url: account://garbage", account) + class TestOAuthConfigCoexistence(_RestoresSdkConfig): runner = CliRunner() @@ -146,7 +276,7 @@ class TestOAuthConfigCoexistence(_RestoresSdkConfig): } def test_ls_shows_both(self): - with patch("cloudinary_cli.core.config.load_config", return_value=dict(self.CONFIG)): + with patch("cloudinary_cli.utils.config_listing.load_config", return_value=dict(self.CONFIG)): result = self.runner.invoke(cli, ['config', '--ls']) self.assertEqual(0, result.exit_code) self.assertIn("prod-account", result.output) @@ -166,14 +296,115 @@ def test_select_oauth_login_configures_sdk(self): self.assertEqual(0, result.exit_code, result.output) self.assertIn("eu-cloud", result.output) + def test_show_header_includes_name_type_and_flags(self): + cfg = {"__default__": "eu-cloud", "prod-account": "cloudinary://key:secret@prod_cloud", + "eu-cloud": _oauth_url()} + with patch("cloudinary_cli.core.config.load_config", return_value=dict(cfg)), \ + patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(cfg)), \ + patch.dict("os.environ", {}, clear=False): + for key in ("CLOUDINARY_URL", "CLOUDINARY_CLOUD_NAME", "CLOUDINARY_API_KEY", + "CLOUDINARY_API_SECRET"): + os.environ.pop(key, None) + cloudinary.reset_config() + default_active = self.runner.invoke(cli, ['config', '-s', 'eu-cloud']) + plain = self.runner.invoke(cli, ['config', '-s', 'prod-account']) + self.assertIn("name: eu-cloud (oauth) [default, active]", default_active.output) + self.assertIn("name: prod-account (api_key)\n", plain.output) # no flag bracket + + def test_bare_config_header_matches_active(self): + cfg = {"__default__": "eu-cloud", "eu-cloud": _oauth_url()} + with patch("cloudinary_cli.core.config.load_config", return_value=dict(cfg)), \ + patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(cfg)), \ + patch.dict("os.environ", {}, clear=False): + for key in ("CLOUDINARY_URL", "CLOUDINARY_CLOUD_NAME", "CLOUDINARY_API_KEY", + "CLOUDINARY_API_SECRET"): + os.environ.pop(key, None) + cloudinary.reset_config() + bare = self.runner.invoke(cli, ['config']) + shown = self.runner.invoke(cli, ['config', '-s', 'eu-cloud']) + # bare `config` identifies the active config the same way `-s ` does + self.assertIn("name: eu-cloud (oauth) [default, active]", bare.output) + self.assertIn("name: eu-cloud (oauth) [default, active]", shown.output) + + def test_bare_config_header_for_command_line_url(self): + cfg = {"__default__": "eu-cloud", "eu-cloud": _oauth_url()} + with patch("cloudinary_cli.core.config.load_config", return_value=dict(cfg)), \ + patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(cfg)), \ + patch.dict("os.environ", {}, clear=False): + for key in ("CLOUDINARY_URL", "CLOUDINARY_CLOUD_NAME", "CLOUDINARY_API_KEY", + "CLOUDINARY_API_SECRET"): + os.environ.pop(key, None) + cloudinary.reset_config() + result = self.runner.invoke(cli, ['-c', 'cloudinary://a:b@cmdcloud', 'config']) + self.assertIn("name: (command-line) (api_key) [active]", result.output) + + def _show_json(self, args, cfg): + with patch("cloudinary_cli.core.config.load_config", return_value=dict(cfg)), \ + patch("cloudinary_cli.utils.config_listing.load_config", return_value=dict(cfg)), \ + patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(cfg)), \ + patch.dict("os.environ", {}, clear=False): + for key in ("CLOUDINARY_URL", "CLOUDINARY_CLOUD_NAME", "CLOUDINARY_API_KEY", + "CLOUDINARY_API_SECRET"): + os.environ.pop(key, None) + cloudinary.reset_config() + result = self.runner.invoke(cli, args) + self.assertEqual(0, result.exit_code, result.output) + return json.loads(result.output[result.output.index("{"):]) + + def test_show_json_includes_meta_and_masks_secrets(self): + cfg = {"__default__": "eu-cloud", "eu-cloud": _oauth_url()} + data = self._show_json(['config', '-s', 'eu-cloud', '--json'], cfg) + self.assertEqual("eu-cloud", data["name"]) + self.assertEqual("saved", data["source"]) + self.assertEqual("oauth", data["type"]) + self.assertTrue(data["default"]) + self.assertTrue(data["active"]) + self.assertEqual("eu-cloud", data["cloud_name"]) + # secrets masked, never in the clear + self.assertNotIn("eyJ.secret_access.tok", json.dumps(data)) + self.assertNotIn("rt_secret_value", json.dumps(data)) + # expires_at expanded into a structured object + self.assertIn("epoch", data["expires_at"]) + self.assertIn("expired", data["expires_at"]) + + def test_bare_config_json_matches_show_json(self): + cfg = {"__default__": "eu-cloud", "eu-cloud": _oauth_url()} + bare = self._show_json(['config', '--json'], cfg) + shown = self._show_json(['config', '-s', 'eu-cloud', '--json'], cfg) + self.assertEqual(shown, bare) + + def test_bare_config_json_env_carries_source(self): + # a synthetic active source is disambiguated by `source`, matching the -ls -j rows + with patch("cloudinary_cli.core.config.load_config", return_value={}), \ + patch("cloudinary_cli.utils.config_resolver.load_config", return_value={}), \ + patch.dict("os.environ", {"CLOUDINARY_URL": "cloudinary://ek:es@env_cloud"}, clear=False): + cloudinary.reset_config() + result = self.runner.invoke(cli, ['config', '--json']) + data = json.loads(result.output[result.output.index("{"):]) + self.assertEqual("(environment)", data["name"]) + self.assertEqual("env", data["source"]) + + def test_config_details_decomposes_account_url(self): + from cloudinary_cli.utils.config_utils import cloudinary_config_details + config = cloudinary.Config() + config.update(cloud_name="c", account_url="account://pk:SUPERSECRETxyz@acc_id") + details = cloudinary_config_details(config) + self.assertEqual("acc_id", details["account"]["account_id"]) + self.assertEqual("pk", details["account"]["provisioning_api_key"]) + self.assertNotIn("SUPERSECRETxyz", json.dumps(details)) + self.assertTrue(details["account"]["provisioning_api_secret"].endswith("txyz") + or details["account"]["provisioning_api_secret"].startswith("****")) + self.assertNotIn("account_url", details) # decomposed, not raw + + +class TestDefaultConfigResolution(_RestoresSdkConfig): + """Resolution precedence: -c > -C > stored default > environment > unconfigured.""" -class TestSoleOAuthLoginFallbackGate(_RestoresSdkConfig): runner = CliRunner() - def _invoke(self, args, sole_login, saved=None, env=None): + def _invoke(self, args, saved, env=None): env = dict(env or {}) - with patch("cloudinary_cli.utils.config_resolver.find_sole_oauth_login", return_value=sole_login), \ - patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved or {})), \ + with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved)), \ patch.dict("os.environ", env, clear=False): for key in ("CLOUDINARY_URL", "CLOUDINARY_CLOUD_NAME", "CLOUDINARY_API_KEY", "CLOUDINARY_API_SECRET"): @@ -182,26 +413,328 @@ def _invoke(self, args, sole_login, saved=None, env=None): cloudinary.reset_config() return self.runner.invoke(cli, args) - def test_fires_when_no_explicit_config(self): - sole = ("eu-cloud", _oauth_url()) - result = self._invoke(['url', 'sample'], sole_login=sole) + def test_stored_default_applies_when_no_explicit_config(self): + saved = {"__default__": "eu-cloud", "eu-cloud": _oauth_url()} + result = self._invoke(['url', 'sample'], saved=saved) self.assertEqual(0, result.exit_code, result.output) self.assertEqual("eu-cloud", cloudinary.config().cloud_name) - def test_does_not_hijack_explicit_invalid_minus_C(self): - saved = {"myaccount": "cloudinary://key@chosen_cloud"} # incomplete: no secret -> invalid - sole = ("eu-cloud", _oauth_url()) - result = self._invoke(['-C', 'myaccount', 'url', 'sample'], sole_login=sole, saved=saved) - self.assertEqual("chosen_cloud", cloudinary.config().cloud_name) - self.assertIsNone(cloudinary.config().oauth_token) + def test_no_implicit_sole_login_without_default(self): + # A single saved login with no stored default no longer auto-applies. + saved = {"eu-cloud": _oauth_url()} + result = self._invoke(['url', 'sample'], saved=saved) + self.assertIn("No Cloudinary configuration found.", result.output) + self.assertIsNone(cloudinary.config().cloud_name) + + def test_stored_default_beats_env(self): + saved = {"__default__": "eu-cloud", "eu-cloud": _oauth_url()} + self._invoke(['url', 'sample'], saved=saved, + env={"CLOUDINARY_URL": "cloudinary://key:secret@env_cloud"}) + self.assertEqual("eu-cloud", cloudinary.config().cloud_name) - def test_does_not_hijack_explicit_cloudinary_url(self): - sole = ("eu-cloud", _oauth_url()) - self._invoke(['url', 'sample'], sole_login=sole, - env={"CLOUDINARY_URL": "cloudinary://key@env_cloud"}) + def test_env_applies_when_no_stored_default(self): + saved = {"eu-cloud": _oauth_url()} + self._invoke(['url', 'sample'], saved=saved, + env={"CLOUDINARY_URL": "cloudinary://key:secret@env_cloud"}) self.assertEqual("env_cloud", cloudinary.config().cloud_name) self.assertIsNone(cloudinary.config().oauth_token) + def test_explicit_minus_C_overrides_default(self): + saved = {"__default__": "eu-cloud", "eu-cloud": _oauth_url(), + "other": "cloudinary://key:secret@other_cloud"} + result = self._invoke(['-C', 'other', 'url', 'sample'], saved=saved) + self.assertEqual(0, result.exit_code, result.output) + self.assertEqual("other_cloud", cloudinary.config().cloud_name) + self.assertIsNone(cloudinary.config().oauth_token) + + def test_default_pointing_at_deleted_config_is_ignored(self): + saved = {"__default__": "gone"} + result = self._invoke(['url', 'sample'], saved=saved) + self.assertIn("No Cloudinary configuration found.", result.output) + + +class TestResolverNoNetworkIO(_RestoresSdkConfig): + """Finding 1 regression: resolution never refreshes a stale OAuth token (no network I/O).""" + + runner = CliRunner() + + def _stale_url(self): + return to_cloudinary_url(Session( + cloud_name="eu-cloud", access_token="eyJ.old.tok", refresh_token="rt_old", + expires_at=int(time.time()) - 10, region="api-eu", + issuer="https://oauth.cloudinary.com/")) + + def test_resolve_does_not_call_flow_refresh(self): + saved = {"__default__": "eu-cloud", "eu-cloud": self._stale_url()} + with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh") as refresh, \ + patch.dict("os.environ", {}, clear=False): + for key in ("CLOUDINARY_URL", "CLOUDINARY_CLOUD_NAME", "CLOUDINARY_API_KEY", + "CLOUDINARY_API_SECRET"): + os.environ.pop(key, None) + cloudinary.reset_config() + from cloudinary_cli.utils.config_resolver import resolve_cli_config + resolve_cli_config() + refresh.assert_not_called() + # The stale token is loaded as-is, awaiting lazy refresh at point-of-use. + self.assertEqual("eyJ.old.tok", cloudinary.config().oauth_token) + + def test_help_does_not_reach_phase_b(self): + with patch("cloudinary_cli.auth.flow.refresh") as refresh: + self.runner.invoke(cli, ['--help']) + refresh.assert_not_called() + + +class TestEnsureActiveConfigFresh(_RestoresSdkConfig): + """Phase B: the lazy freshen shim refreshes a stale active OAuth token, no-op otherwise.""" + + def _stale_url(self): + return to_cloudinary_url(Session( + cloud_name="eu-cloud", access_token="eyJ.old.tok", refresh_token="rt_old", + expires_at=int(time.time()) - 10, region="api-eu", + issuer="https://oauth.cloudinary.com/")) + + def test_refreshes_stale_active_login(self): + import cloudinary_cli.utils.config_resolver as resolver + saved = {"eu-cloud": self._stale_url()} + token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.update_config"): + resolver.resolve_cli_config(config_saved="eu-cloud") + resolver.ensure_active_config_fresh() + self.assertEqual("eyJ.new.tok", cloudinary.config().oauth_token) + + def test_noop_for_inline_url(self): + import cloudinary_cli.utils.config_resolver as resolver + with patch("cloudinary_cli.auth.flow.refresh") as refresh: + resolver.resolve_cli_config(config="cloudinary://key:secret@cloud") + resolver.ensure_active_config_fresh() + refresh.assert_not_called() + + def test_noop_for_api_key_config(self): + import cloudinary_cli.utils.config_resolver as resolver + saved = {"mykey": "cloudinary://key:secret@cloud"} + with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh") as refresh: + resolver.resolve_cli_config(config_saved="mykey") + resolver.ensure_active_config_fresh() + refresh.assert_not_called() + + +class TestConfigDefaultCommands(_RestoresSdkConfig): + """`cld config` default management: -d, --set-default, --unset-default, -ls marker, -rm cleanup.""" + + runner = CliRunner() + + def test_d_marks_existing(self): + saved = {"prod": "cloudinary://k:s@prod", "eu-cloud": _oauth_url()} + with patch("cloudinary_cli.core.config.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.core.config.set_default_config") as set_default: + result = self.runner.invoke(cli, ['config', '-d', 'prod']) + self.assertEqual(0, result.exit_code, result.output) + set_default.assert_called_once_with("prod") + self.assertIn("Default set to 'prod'", result.output) + + def test_d_nonexistent_errors(self): + with patch("cloudinary_cli.core.config.load_config", return_value={"prod": "cloudinary://k:s@prod"}): + result = self.runner.invoke(cli, ['config', '-d', 'nope']) + self.assertEqual(2, result.exit_code) + self.assertIn("does not exist", result.output) + + def test_set_default_without_create_flag_errors(self): + result = self.runner.invoke(cli, ['config', '--set-default']) + self.assertEqual(2, result.exit_code) + self.assertIn("requires -n or --from_url", result.output) + + def test_new_with_set_default(self): + with patch("cloudinary_cli.core.config.verify_cloudinary_url", return_value=True), \ + patch("cloudinary_cli.core.config.update_config"), \ + patch("cloudinary_cli.core.config.set_default_config") as set_default: + result = self.runner.invoke( + cli, ['config', '-n', 'prod', 'cloudinary://k:s@prod', '--set-default']) + self.assertEqual(0, result.exit_code, result.output) + set_default.assert_called_once_with("prod") + self.assertIn("Default set to 'prod'", result.output) + + def test_set_default_on_failing_url_neither_saves_nor_defaults(self): + with patch("cloudinary_cli.core.config.verify_cloudinary_url", return_value=False), \ + patch("cloudinary_cli.core.config.update_config") as update, \ + patch("cloudinary_cli.core.config.set_default_config") as set_default: + self.runner.invoke( + cli, ['config', '-n', 'prod', 'cloudinary://bad', '--set-default']) + update.assert_not_called() + set_default.assert_not_called() + + def test_unset_default(self): + with patch("cloudinary_cli.core.config.load_config", return_value={}), \ + patch("cloudinary_cli.core.config.clear_default_config") as clear: + result = self.runner.invoke(cli, ['config', '--unset-default']) + self.assertEqual(0, result.exit_code, result.output) + clear.assert_called_once() + self.assertIn("cleared", result.output) + + def _run_ls(self, args, saved, env=None): + # Both the resolver (Phase A, which records the active config) and the config command read + # the same saved dict; env is controlled via os.environ so is_env_configured() is genuine. + env = dict(env or {}) + with patch("cloudinary_cli.core.config.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.utils.config_listing.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved)), \ + patch.dict("os.environ", env, clear=False): + for key in ("CLOUDINARY_URL", "CLOUDINARY_CLOUD_NAME", "CLOUDINARY_API_KEY", + "CLOUDINARY_API_SECRET"): + if key not in env: + os.environ.pop(key, None) + cloudinary.reset_config() + return self.runner.invoke(cli, args) + + def _ls_json(self, saved, env=None): + result = self._run_ls(['config', '-ls', '--json'], saved, env) + self.assertEqual(0, result.exit_code, result.output) + return {r["name"]: r for r in json.loads(result.output[result.output.index("["):])} + + def test_ls_table_marks_default(self): + saved = {"__default__": "eu-cloud", "prod": "cloudinary://k:s@prod", "eu-cloud": _oauth_url()} + result = self._run_ls(['config', '-ls'], saved) + self.assertEqual(0, result.exit_code, result.output) + for header in ("NAME", "CLOUD", "TYPE", "DEFAULT", "ACTIVE"): + self.assertIn(header, result.output) + self.assertIn("prod", result.output) + self.assertIn("oauth", result.output) + self.assertIn("api_key", result.output) + # with no env, the stored default is both default and active (two markers) + rows = {line.split()[0]: line for line in result.output.splitlines() if line.startswith(("prod", "eu-cloud"))} + self.assertEqual(2, rows["eu-cloud"].count("*")) + self.assertEqual(0, rows["prod"].count("*")) + self.assertNotIn("__default__", result.output) + + def test_ls_json(self): + saved = {"__default__": "eu-cloud", "prod": "cloudinary://k:s@prodcloud", "eu-cloud": _oauth_url()} + by_name = self._ls_json(saved) + self.assertNotIn("__default__", by_name) + self.assertEqual("oauth", by_name["eu-cloud"]["type"]) + self.assertEqual("saved", by_name["eu-cloud"]["source"]) + self.assertTrue(by_name["eu-cloud"]["default"]) + self.assertTrue(by_name["eu-cloud"]["active"]) # no env -> stored default is active + self.assertEqual("api_key", by_name["prod"]["type"]) + self.assertEqual("prodcloud", by_name["prod"]["cloud_name"]) + self.assertFalse(by_name["prod"]["default"]) + self.assertFalse(by_name["prod"]["active"]) + + def test_ls_minus_C_marks_selected_active(self): + saved = {"__default__": "eu-cloud", "eu-cloud": _oauth_url(), + "test": "cloudinary://k:s@test_cloud"} + # an explicit -C selects the active config for this invocation, overriding the default + result = self._run_ls(['-C', 'test', 'config', '-ls', '--json'], saved) + self.assertEqual(0, result.exit_code, result.output) + by_name = {r["name"]: r for r in json.loads(result.output[result.output.index("["):])} + self.assertTrue(by_name["test"]["active"]) + self.assertFalse(by_name["eu-cloud"]["active"]) # not active, but still the stored default + self.assertTrue(by_name["eu-cloud"]["default"]) + + def test_ls_inline_url_shown_as_active_command_line_row(self): + saved = {"__default__": "eu-cloud", "eu-cloud": _oauth_url()} + # an inline -c URL is not a saved config, but it is what's active for this invocation + result = self._run_ls(['-c', 'cloudinary://a:b@inline_cloud', 'config', '-ls', '--json'], saved) + self.assertEqual(0, result.exit_code, result.output) + by_name = {r["name"]: r for r in json.loads(result.output[result.output.index("["):])} + cmd = by_name["(command-line)"] # synthetic source: parenthesized in both table and JSON + self.assertEqual("url", cmd["source"]) + self.assertEqual("inline_cloud", cmd["cloud_name"]) + self.assertTrue(cmd["active"]) + self.assertFalse(cmd["default"]) + # the stored default is still recorded, but it is not active while -c wins + self.assertTrue(by_name["eu-cloud"]["default"]) + self.assertFalse(by_name["eu-cloud"]["active"]) + + _ENV = {"CLOUDINARY_URL": "cloudinary://k:s@env_cloud"} + + def test_ls_env_row_active_when_no_default(self): + by_name = self._ls_json({"eu-cloud": _oauth_url()}, env=self._ENV) + env = by_name["(environment)"] + self.assertEqual("env", env["source"]) + self.assertEqual("env_cloud", env["cloud_name"]) + self.assertFalse(env["default"]) # the environment is never the *stored* default + self.assertTrue(env["active"]) # active because nothing outranks it + self.assertFalse(by_name["eu-cloud"]["active"]) + + def test_ls_stored_default_outranks_env_row(self): + by_name = self._ls_json({"__default__": "eu-cloud", "eu-cloud": _oauth_url()}, env=self._ENV) + env = by_name["(environment)"] + self.assertFalse(env["active"]) # stored default outranks the environment + # the stored default is both recorded and active + self.assertTrue(by_name["eu-cloud"]["default"]) + self.assertTrue(by_name["eu-cloud"]["active"]) + + def test_synthetic_row_name_parenthesized_in_table_and_json(self): + # synthetic (environment / command-line) configs read as parenthesized in both views + result = self._run_ls(['config', '-ls'], {"eu-cloud": _oauth_url()}, env=dict(self._ENV)) + self.assertIn("(environment)", result.output) + by_name = self._ls_json({"eu-cloud": _oauth_url()}, env=dict(self._ENV)) + self.assertIn("(environment)", by_name) + + def test_rm_of_default_clears_it(self): + with patch("cloudinary_cli.core.config.remove_config_keys", return_value=[]), \ + patch("cloudinary_cli.core.config.get_default_config_name", return_value="prod"), \ + patch("cloudinary_cli.core.config.clear_default_config") as clear: + result = self.runner.invoke(cli, ['config', '-rm', 'prod']) + self.assertEqual(0, result.exit_code, result.output) + clear.assert_called_once() + + def test_reserved_name_rejected_on_new(self): + result = self.runner.invoke( + cli, ['config', '-n', '__default__', 'cloudinary://k:s@c']) + self.assertEqual(2, result.exit_code) + self.assertIn("reserved", result.output) + + def test_refresh_named_delegates_to_refresh_config(self): + with patch("cloudinary_cli.core.config.refresh_config", return_value="refreshed") as rc: + result = self.runner.invoke(cli, ['config', '--refresh', 'eu-cloud']) + self.assertEqual(0, result.exit_code, result.output) + rc.assert_called_once_with("eu-cloud", force=False) + self.assertIn("Refreshed 'eu-cloud'", result.output) + + def test_refresh_force_passes_flag(self): + with patch("cloudinary_cli.core.config.refresh_config", return_value="refreshed") as rc: + self.runner.invoke(cli, ['config', '--refresh', 'eu-cloud', '--force']) + rc.assert_called_once_with("eu-cloud", force=True) + + def test_refresh_no_name_uses_active_config(self): + with patch("cloudinary_cli.core.config.active_config_name", return_value="active-one"), \ + patch("cloudinary_cli.core.config.refresh_config", return_value="refreshed") as rc: + self.runner.invoke(cli, ['config', '--refresh']) + rc.assert_called_once_with("active-one", force=False) + + def test_refresh_unknown_name_errors(self): + with patch("cloudinary_cli.core.config.refresh_config", return_value="not_found"): + result = self.runner.invoke(cli, ['config', '--refresh', 'ghost']) + self.assertEqual(2, result.exit_code) + self.assertIn("does not exist", result.output) + + def test_refresh_failed_reports_relogin_with_region(self): + with patch("cloudinary_cli.core.config.refresh_config", return_value="failed"), \ + patch("cloudinary_cli.core.config.relogin_command", + return_value="cld login eu-cloud --region api-eu") as relogin: + result = self.runner.invoke(cli, ['config', '--refresh', 'eu-cloud'], standalone_mode=False) + self.assertFalse(result.return_value) # main() maps falsy -> exit 1 + relogin.assert_called_once_with("eu-cloud") + self.assertIn("cld login eu-cloud --region api-eu", result.output) + + def test_refresh_all_reports_each(self): + with patch("cloudinary_cli.core.config.refresh_configs", + return_value={"a": "refreshed", "b": "fresh"}) as rc: + result = self.runner.invoke(cli, ['config', '--refresh-all']) + self.assertEqual(0, result.exit_code, result.output) + rc.assert_called_once_with(force=False) + self.assertIn("Refreshed 'a'", result.output) + self.assertIn("'b' token is still fresh", result.output) + + def test_force_without_refresh_errors(self): + result = self.runner.invoke(cli, ['config', '--force']) + self.assertEqual(2, result.exit_code) + self.assertIn("--force only applies", result.output) class TestConfigToApiKwargs(unittest.TestCase): diff --git a/test/test_default_config.py b/test/test_default_config.py new file mode 100644 index 0000000..5cafc89 --- /dev/null +++ b/test/test_default_config.py @@ -0,0 +1,57 @@ +import unittest +from contextlib import contextmanager +from unittest.mock import patch + +import cloudinary_cli.utils.config_utils as config_utils + + +@contextmanager +def _in_memory_config(initial=None): + """Back load_config/save_config with an in-memory dict (no real config.json or lock).""" + store = {"cfg": dict(initial or {})} + + def _load(): + return dict(store["cfg"]) + + def _save(cfg): + store["cfg"] = dict(cfg) + + @contextmanager + def _noop_lock(): + yield + + with patch("cloudinary_cli.utils.config_utils.load_config", side_effect=_load), \ + patch("cloudinary_cli.utils.config_utils.save_config", side_effect=_save), \ + patch("cloudinary_cli.utils.config_utils.config_lock", _noop_lock): + yield store + + +class TestDefaultConfigStorage(unittest.TestCase): + def test_get_set_clear_round_trip(self): + with _in_memory_config({"prod": "cloudinary://k:s@prod"}): + self.assertIsNone(config_utils.get_default_config_name()) + config_utils.set_default_config("prod") + self.assertEqual("prod", config_utils.get_default_config_name()) + config_utils.clear_default_config() + self.assertIsNone(config_utils.get_default_config_name()) + + def test_user_config_names_filters_reserved_key(self): + with _in_memory_config({"prod": "cloudinary://k:s@prod"}): + config_utils.set_default_config("prod") + self.assertEqual(["prod"], config_utils.user_config_names()) + + def test_default_key_present_in_raw_dict_only(self): + with _in_memory_config({"a": "cloudinary://k:s@a", "b": "cloudinary://k:s@b"}): + config_utils.set_default_config("b") + self.assertIn("__default__", config_utils.load_config()) + self.assertNotIn("__default__", config_utils.user_config_names()) + + def test_is_reserved_config_name(self): + self.assertTrue(config_utils.is_reserved_config_name("__default__")) + self.assertTrue(config_utils.is_reserved_config_name("__foo__")) + self.assertFalse(config_utils.is_reserved_config_name("prod")) + self.assertFalse(config_utils.is_reserved_config_name("__prod")) + + +if __name__ == "__main__": + unittest.main() From f7f57dc2ef0c6c6a949d70751e001817aff7406d Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 03:22:27 +0300 Subject: [PATCH 04/23] Self-refreshing OAuth token; remove scattered refresh hooks Replace ensure_active_config_fresh() and its four hand-maintained API chokepoints with a single self-refreshing oauth_token on the active config object. Access tokens are valid ~5 minutes, so refreshes are frequent; the SDK reads cloudinary.config().oauth_token per request, which is the one seam that needs a fresh token. - New cloudinary_cli/auth/oauth_config.py: OAuthConfig(cloudinary.Config). oauth_token is a class-level property that refreshes-if-stale on read (reusing refresh_url_if_stale's lock + double-check + atomic persist) and short-circuits with no I/O once the in-object session is fresh. It never calls reset_config() in the getter (avoids the global-swap thread race). has_oauth reports token presence without refreshing. - Every active config the CLI installs is now an OAuthConfig (saved, env-fallback, inline -c) via install_oauth_config / install_env_config, so has_oauth is universal and offline paths stay offline. - Delete ensure_active_config_fresh and its call sites in core/search.py and utils/api_utils.py; refresh_cloudinary_config delegates to the install seam. - load_config() gains an (mtime_ns, size) cache (copy-on-return, invalidated on save) to cut redundant reads on the remaining hot paths. - Type/validity/-ls classifiers read has_oauth (presence), never the refreshing property, so config -ls / -s / the group-level check do no network even on a stale token. Test isolation fixes (pre-existing bugs surfaced by the refactor): - test_auth_session: patch load_config so refresh tests never read/write the real ~/.cloudinary-cli/config.json (this had poisoned a real entry). - test_cli_config show_default_no_config: clear CLOUDINARY_* so it actually exercises the unconfigured path. - test_cli_config_oauth TestConfigSecretMasking: strip CLOUDINARY_* in setUp so a dev env's account_url does not leak into masking assertions (fixed 4 failures on account-enabled machines). - Rewrite TestEnsureActiveConfigFresh -> TestSelfRefreshingOAuthToken for the new model, with a presence-does-not-refresh regression guard. - Add CONFIG_PRESENT/REQUIRES_CONFIG skip predicate so the 13 mocked-HTTP tests that still need a resolvable config skip cleanly on a bare machine. See OAUTH_LAZY_TOKEN_HANDOFF.md for the full writeup and the remaining follow-ups (thread-local refresh lock, refresh-on-401 retry, transactional multi-step config ops). Builds on OAUTH_DEFAULT_CONFIG_IMPLEMENTATION.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md | 221 ++++++++++++++++++++ OAUTH_LAZY_TOKEN_HANDOFF.md | 137 ++++++++++++ OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md | 196 +++++++++++++++++ OAUTH_REFRESH_CONCURRENCY_REVIEW.md | 266 ++++++++++++++++++++++++ cloudinary_cli/auth/oauth_config.py | 93 +++++++++ cloudinary_cli/core/config.py | 3 +- cloudinary_cli/core/search.py | 2 - cloudinary_cli/utils/api_utils.py | 4 - cloudinary_cli/utils/config_listing.py | 15 +- cloudinary_cli/utils/config_resolver.py | 33 ++- cloudinary_cli/utils/config_utils.py | 46 +++- test/helper_test.py | 6 + test/test_auth_session.py | 6 +- test/test_cli_api.py | 11 +- test/test_cli_config.py | 14 +- test/test_cli_config_oauth.py | 42 +++- test/test_cli_search_api.py | 6 +- test/test_cli_url.py | 4 + test/test_cli_utils.py | 2 + 19 files changed, 1053 insertions(+), 54 deletions(-) create mode 100644 OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md create mode 100644 OAUTH_LAZY_TOKEN_HANDOFF.md create mode 100644 OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md create mode 100644 OAUTH_REFRESH_CONCURRENCY_REVIEW.md create mode 100644 cloudinary_cli/auth/oauth_config.py diff --git a/OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md b/OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md new file mode 100644 index 0000000..8d3e855 --- /dev/null +++ b/OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md @@ -0,0 +1,221 @@ +# Self-refreshing `oauth_token` — caveats explained with code, + an mtime-cached `load_config` + +> Companion to `OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md`. Two questions: +> 1. The two "make-or-break" caveats, shown concretely in code. +> 2. Can we skip reloading `config.json` when its modified-time is unchanged, to kill the per-call overhead? + +--- + +## Caveat A — reading `oauth_token` for *truthiness* must NOT trigger a refresh + +### The problem, concretely + +If `oauth_token` becomes a property that refreshes-on-read, then **any** access triggers it — +including the places that read it only to *classify* a config, none of which want network I/O. +Here are the exact current sites (all offline paths): + +```python +# cloudinary_cli/utils/config_utils.py:283 — runs at the GROUP level, on every command +def is_valid_cloudinary_config(): + if cloudinary.config().cloud_name and cloudinary.config().oauth_token: # <-- truthiness read + return True + return None not in [cloudinary.config().cloud_name, + cloudinary.config().api_key, cloudinary.config().api_secret] + +# cloudinary_cli/utils/config_listing.py:76, 97, 109 — `config -ls`, fully offline +"type": "oauth" if config_obj.oauth_token else "api_key", +"type": "oauth" if active.oauth_token else "api_key", +"type": "oauth" if env_config.oauth_token else "api_key", + +# cloudinary_cli/core/config.py:191 — `config` header, offline +type_label = "oauth" if active.oauth_token else "api_key" +``` + +A property can't distinguish "are you OAuth?" from "give me a token to send." So a naive property +turns `cld config -ls` — which should be 100% offline — into something that refreshes every stale +saved token just to print a table. **That is the exact Finding-1 hang we removed**, reintroduced. + +### The fix: split *presence* from *value* + +Keep a non-refreshing way to ask "is this OAuth / is a token present," and let only the **value** +read on the request path refresh. Two private fields back the public property: + +```python +# cloudinary_cli/auth/oauth_config.py (new) +import cloudinary +from cloudinary_cli.auth import refresh_url_if_stale +from cloudinary_cli.auth.session import from_cloudinary_url + +class OAuthConfig(cloudinary.Config): + """A Config whose oauth_token refreshes itself on read for the request path, while presence/ + type checks read the raw stored token and never touch the network.""" + + def bind_saved(self, name, url): + # association the resolver used to keep in a module global — now lives on the object + self._saved_name = name + self._raw_oauth_token = from_cloudinary_url(url).access_token + self._oauth_url = url + + # --- presence: cheap, no network. Use THIS in type/validity checks. --- + @property + def has_oauth(self): + return bool(getattr(self, "_raw_oauth_token", None)) + + # --- value: refresh-if-stale, used by the SDK on the request path. --- + @property + def oauth_token(self): + name = getattr(self, "_saved_name", None) + if name is None: + # env / -c / api-key config: serve the static value, never refresh + return getattr(self, "_raw_oauth_token", None) + fresh_url = refresh_url_if_stale(name, self._oauth_url) # existing lock+double-check+persist + if fresh_url != self._oauth_url: + self._oauth_url = fresh_url + self._raw_oauth_token = from_cloudinary_url(fresh_url).access_token + return self._raw_oauth_token +``` + +Then repoint the offline checks at `has_oauth` (presence), not `oauth_token` (value): + +```python +# is_valid_cloudinary_config +cfg = cloudinary.config() +if cfg.cloud_name and getattr(cfg, "has_oauth", False): # no refresh + return True + +# config_listing / core.config type labels +"type": "oauth" if getattr(config_obj, "has_oauth", False) else "api_key" +``` + +> Note the property must be a **data descriptor on the class** (a `property` is). The SDK's +> `__getattr__` (`return self.__dict__.get(i)`) only fires for *missing* attributes; the token is in +> `__dict__`, so `__getattr__` never sees it — but a class-level `property` *overrides* `__dict__` on +> read. That's why this works where a `__getattr__` hook wouldn't. + +### Subtlety: `config_to_dict` enumerates `__dict__` + +```python +# config_utils.py:97 +def config_to_dict(config): + return {k: v for k, v in config.__dict__.items() if not k.startswith("_")} +``` + +A `property` lives on the class, not in `__dict__`, so `config_to_dict` would **lose** `oauth_token` +from the masked/JSON views. Fix: have the masking layer read the raw token explicitly, e.g. add it +back from `_raw_oauth_token`, or store the raw token under the public key `oauth_token` in `__dict__` +*and* let the property shadow it on read (a property shadows the instance dict on attribute access, +but `__dict__["oauth_token"]` is still there for `config_to_dict` to enumerate). The cleanest: keep +`oauth_token` in `__dict__` for serialization, and have the property's getter read/refresh from it. + +--- + +## Caveat B — refreshing inside the getter must not `reset_config()` or deadlock + +### The problem, concretely + +Today refresh goes through `refresh_cloudinary_config`: + +```python +# config_utils.py +def refresh_cloudinary_config(cloudinary_url): + cloudinary.reset_config() # <-- clears the global singleton + cloudinary.config()._load_from_url(cloudinary_url) +``` + +If the **getter** for `oauth_token` called this, then reading `config().oauth_token` would, mid-read, +`reset_config()` — destroying the very object whose property is executing, and replacing our +`OAuthConfig` with a plain `Config` (property gone). It also opens the reset-then-reload window where +a concurrent worker thread reads a half-cleared global. And `refresh_url_if_stale` → `update_config` +takes the reentrant `config_lock`; doing a global swap underneath that is the fragile part. + +### The fix: refresh in place, never swap the global from inside the getter + +The getter (above) only mutates its **own** `_oauth_url` / `_raw_oauth_token` and lets +`refresh_url_if_stale` handle the **persist** (atomic write under the lock — already correct, +single-flight across threads and processes). No `reset_config()`, no global swap, no half-cleared +window. The lock is reentrant, so the getter running inside an in-progress operation that already +holds it won't deadlock. + +The one place that legitimately swaps the global is the **resolver** (Phase A), once per process — +that's where `OAuthConfig(...).bind_saved(name, url)` gets installed as `cloudinary._config`. Audit +every `reset_config()` call so a saved OAuth login always ends up installed as `OAuthConfig`, never a +plain `Config` (else the property silently disappears — the #1 risk from the design doc). + +--- + +## The mtime cache — yes, and it's the cleaner overhead fix + +Your instinct is good: most `load_config()` calls re-read and re-parse an unchanged file. Cache the +parsed dict keyed on the file's `(mtime, size)`; reload only when it changes. Current code: + +```python +# config_utils.py:32 +def load_config(): + return read_json_from_file(CLOUDINARY_CLI_CONFIG_FILE, does_not_exist_ok=True) +``` + +Cached version: + +```python +import os + +_config_cache = None +_config_cache_stat = None # (st_mtime_ns, st_size) + +def _config_stat(): + try: + st = os.stat(CLOUDINARY_CLI_CONFIG_FILE) + return (st.st_mtime_ns, st.st_size) + except FileNotFoundError: + return None + +def load_config(): + global _config_cache, _config_cache_stat + stat = _config_stat() + if stat is not None and stat == _config_cache_stat and _config_cache is not None: + return _config_cache # unchanged file: skip read + JSON parse + cfg = read_json_from_file(CLOUDINARY_CLI_CONFIG_FILE, does_not_exist_ok=True) + _config_cache, _config_cache_stat = cfg, stat + return cfg +``` + +And invalidate on our own writes so a writer sees its own update immediately: + +```python +def save_config(config): + global _config_cache, _config_cache_stat + _verify_file_path(CLOUDINARY_CLI_CONFIG_FILE) + write_json_to_file(config, CLOUDINARY_CLI_CONFIG_FILE, atomic=True) + _restrict_permissions(CLOUDINARY_CLI_CONFIG_FILE) + _config_cache = None # force re-stat/reload next load_config() + _config_cache_stat = None +``` + +### Caveats on the cache (important) + +1. **mtime granularity.** Use `st_mtime_ns` (nanosecond) + `st_size`, not 1-second `st_mtime`. A + sub-second refresh-then-read on a coarse FS could otherwise miss a change. The `os.replace` in + `atomic_write` updates mtime, so cross-process changes are detected. +2. **Return a copy if callers mutate.** Several callers do `cfg = load_config(); cfg.update(...)`. + If they mutate the returned dict in place they'd corrupt the shared cache. Either return + `dict(cfg)` (cheap; safest) or audit that mutators always go through `update_config` + (which builds on `load_config` then `save_config`). Returning a shallow copy is the safe default. +3. **It does NOT remove the need to read under the lock for refresh.** The + read-modify-write in `refresh_url_if_stale`/`update_config` must still `load_config()` *inside* + the lock to re-check freshness — the cache is fine there too (it re-stats; if a peer just wrote, + mtime changed, it reloads). The cache only saves the *redundant* reads, not the correctness read. +4. **Still mostly moot if we adopt the property.** With the self-refreshing `oauth_token`, the + per-call `load_config()` in `ensure_active_config_fresh` disappears entirely. The mtime cache is + still worth having (other hot reads: `is_valid_cloudinary_config` at group level, refresh loops), + but it's a complementary optimization, not the primary fix. + +### Recommendation on overhead + +- **Primary:** the property approach removes the per-API-call `load_config()` on the hot path + outright (no more `ensure_active_config_fresh`). +- **Secondary:** add the mtime+size cache to `load_config()` (with copy-on-return) so the remaining + reads — group-level validity check, `-ls`, refresh sweeps — stop re-parsing an unchanged file. + +Together they take the steady-state per-request config overhead from "disk read + JSON parse + +URL parse, every call, every thread" down to "one `os.stat`, and a token refresh only when the +5-minute token has actually expired." diff --git a/OAUTH_LAZY_TOKEN_HANDOFF.md b/OAUTH_LAZY_TOKEN_HANDOFF.md new file mode 100644 index 0000000..d86362a --- /dev/null +++ b/OAUTH_LAZY_TOKEN_HANDOFF.md @@ -0,0 +1,137 @@ +# Self-refreshing OAuth token — implementation & handoff + +> **For the next agent.** This is the implementation that replaces the scattered +> `ensure_active_config_fresh()` lazy-refresh hooks with a single self-refreshing `oauth_token` on +> the active config object. It builds on the change documented in +> `OAUTH_DEFAULT_CONFIG_IMPLEMENTATION.md` (commit `43fb67a`, branch `oauth-login`) and the design +> rationale in `OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md` + `OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md`. +> Concurrency analysis that motivated the redesign is in `OAUTH_REFRESH_CONCURRENCY_REVIEW.md`. +> +> Read `OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md` first for the "why"; this doc is the "what landed". + +--- + +## 1. What changed and why + +### The problem (see `OAUTH_REFRESH_CONCURRENCY_REVIEW.md`) +The previous design refreshed a stale OAuth token via `ensure_active_config_fresh()` called at four +hand-maintained API chokepoints (`call_api`, `execute_single_request`, `query_cld_folder`, +`cld_folder_exists`). Problems: per-call `load_config()` on the hot path; a hand-maintained +chokepoint list that silently misses new API paths (→ stale-token 401 mid-bulk-run); and a +`reset_config()`-based global swap that races with worker threads under `sync`/`upload-dir`. + +### The fix +Access tokens live ~5 minutes, so refreshes are frequent and expected. Instead of probing before +every call, the **active config's `oauth_token` refreshes itself when the SDK reads it** at request +time. The SDK reads `cloudinary.config().oauth_token` per request (`call_api.py:63`, +`uploader.py:877`) — one universal seam. + +- **New `cloudinary_cli/auth/oauth_config.py`** — `OAuthConfig(cloudinary.Config)`: + - `oauth_token` is a **property** (class-level data descriptor; overrides `__dict__` on read, + which a `__getattr__` hook could not — the SDK stores the token in `__dict__`). + - On read: if the in-object parsed `_session` is fresh (or has no refresh token), return the + stored token with **no I/O**. Only when stale does it `load_config()` + call + `refresh_url_if_stale` (the existing lock + double-check + atomic persist, reused verbatim), + update the in-object token, and return it. Subsequent reads short-circuit on the now-fresh + `_session` — **no per-call disk read or lock once fresh.** + - It does **not** `reset_config()` inside the getter (avoids the global-swap thread race and + self-destruction of the executing object). It mutates only its own fields. + - `has_oauth` property — token *presence* without refreshing. Used by all type/validity checks. + - `from_env()` / `from_url()` factories + `install_oauth_config()` / `install_env_config()`: + **every** active config the CLI installs is now an `OAuthConfig` (saved, env-fallback, and + inline `-c`), so `has_oauth` is universal and the resolver's env branch installs a static + (never-refreshing) OAuthConfig. +- **Deleted** `ensure_active_config_fresh()` and all four call sites (`core/search.py`, + `utils/api_utils.py` ×3). +- **`config_utils.refresh_cloudinary_config(url, saved_name=None)`** now delegates to + `install_oauth_config` (single install seam). +- **mtime cache in `load_config()`** — caches the parsed dict keyed on `(st_mtime_ns, st_size)`, + returns a **copy** (callers mutate in place), invalidated in `save_config`. Cuts the remaining + redundant reads (group-level validity check, `-ls`, refresh sweeps). +- **Classifiers consolidated** — `config_listing.config_type_label(obj)` is now + `"oauth" if obj.has_oauth else "api_key"` (no `__dict__` peeking, no `getattr` fallback); + `is_valid_cloudinary_config` reads `has_oauth` (lazily, without evaluating the refreshing + property — see Caveat A). + +### Caveats handled (detail in `OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md`) +- **A — truthiness reads must not refresh.** Type/validity/`-ls` read `has_oauth` (presence), never + the refreshing `oauth_token`. NOTE: do **not** write `getattr(cfg, "has_oauth", bool(cfg.oauth_token))` + — the default arg is evaluated eagerly and *would* trigger a refresh. Use a `hasattr` guard. +- **B — no `reset_config()` in the getter.** Refresh in place; the only global swap is at install + time, once per process. + +--- + +## 2. Test fixes in this commit (all pre-existing isolation bugs, surfaced by the refactor) + +1. **`test_auth_session.py`** — `test_stale_refreshes_and_rewrites` / `test_refresh_timeout_returns_stale_url` + did not patch `load_config`, so they read/wrote the developer's **real** `~/.cloudinary-cli/config.json`. + This had previously **poisoned a real config entry** (`eu-cloud`) with MagicMock garbage. Both now + patch `load_config`. (The poisoned entry was removed from the dev machine.) +2. **`test_cli_config.py::test_cli_config_show_default_no_config`** — asserts the "nothing + configured" path but passed on base only via cross-`invoke` global pollution. Now clears + `CLOUDINARY_*` env (via `patch.dict(..., clear=True)` over a filtered env) so it genuinely tests + the unconfigured path. The new resolver behavior (env re-read on resolve) is *more* correct; + this test just needed a clean env to assert the negative case. +3. **`test_cli_config_oauth.py::TestConfigSecretMasking`** — built `cloudinary.Config()` which + auto-loads the dev env; when `CLOUDINARY_CLOUD_NAME`+`CLOUDINARY_ACCOUNT_URL` are set (common + PyCharm setup), a real `account_url` leaked in, adding a 2nd `echo` call so the assertions + (reading only the last call) missed the masked fields → **4 failures on the developer's machine**. + Fixed by extending `_RestoresSdkConfig` to strip `CLOUDINARY_*` in `setUp` and inheriting it. +4. **`TestEnsureActiveConfigFresh` → `TestSelfRefreshingOAuthToken`** — rewritten for the new model: + presence check (`has_oauth`) does no network; reading `oauth_token` refreshes once; env/`-c`/api-key + never refresh. Plus `test_presence_check_does_not_refresh` as the Caveat-A regression guard. +5. **Skip gating for offline/no-account runs** — `helper_test.CONFIG_PRESENT` / `REQUIRES_CONFIG`, + applied per-method via `@unittest.skipUnless` to the 13 tests that mock HTTP but still need a + resolvable config (`test_cli_url`, `test_cli_utils`, `test_cli_search_api`, `test_cli_api`). They + now **skip** cleanly on a bare machine instead of failing. + +### Test status +- **No config (bare machine):** 191 passed, 21 skipped, 0 failed. +- **Real test cloud (via `tools/allocate_test_cloud.sh`):** 211 passed, 1 skipped (`test_provisioning`, + needs `account_id`), 0 failed. +- **Account-enabled dev env:** the 4 masking failures are fixed; remaining `test_cli_config` failures + there are only because the simulated creds were fake (401) — they pass against a real account. + +--- + +## 3. What is NOT done — for the next agent + +These were identified in the reviews but are **out of scope of this commit**: + +1. **Thread-safety of the refresh under `sync`/`upload-dir` (the original concern in + `OAUTH_REFRESH_CONCURRENCY_REVIEW.md` §2).** The cross-process single-flight (lock + double-check) + is correct and preserved. But the in-process getter, when stale, can have N worker threads enter + `refresh_url_if_stale` together; they serialize on the reentrant `config_lock` and all but one + adopt the peer-refreshed token (correct, but a synchronized stall). Consider a process-local + `threading.Lock` around the getter's stale branch so only one thread per process attempts it. +2. **Refresh-on-401 retry (`OAUTH_REFRESH_CONCURRENCY_REVIEW.md` Fix C).** The lazy property closes + the "stale at request build time" gap, but a token that expires *mid-flight* (between read and + server receipt) still 401s with no retry. A reactive retry at `call_api` would be the robust + complement. +3. **Transactional multi-step config ops (Fix D).** `login` + auto-default and refresh-and-default + still read-decide-write across separate `config_lock` scopes (TOCTOU). Wrap in one lock scope. +4. **"config removed mid-run" message (Fix E).** The getter returns the stale token if the saved + config vanished underneath it; surface a clear "re-login" error instead of a later raw 401. +5. **Chokepoint-completeness is now moot** (the property covers all read paths), but a test that + asserts the active config is always an `OAuthConfig` post-resolve would lock that invariant in. +6. **`reset_config()` audit.** Verified the resolver/install paths, but any *future* direct + `cloudinary.reset_config()` call would replace the OAuthConfig with a plain Config and silently + disable self-refresh. Keep all config installation routed through + `oauth_config.install_oauth_config` / `install_env_config`. + +--- + +## 4. Key files + +| File | Role | +|---|---| +| `cloudinary_cli/auth/oauth_config.py` | **New.** `OAuthConfig`, `has_oauth`, install seams. | +| `cloudinary_cli/utils/config_resolver.py` | Installs OAuthConfig per branch; `ensure_active_config_fresh` deleted. | +| `cloudinary_cli/utils/config_utils.py` | `refresh_cloudinary_config` delegates to install; mtime cache; `is_valid_cloudinary_config` via has_oauth. | +| `cloudinary_cli/utils/config_listing.py` | `config_type_label(obj)` → `has_oauth`. | +| `cloudinary_cli/utils/api_utils.py`, `core/search.py` | Chokepoint calls removed. | +| `test/helper_test.py` | `CONFIG_PRESENT` / `REQUIRES_CONFIG` skip predicate. | + +Run tests: `.venv/bin/python -m pytest test/ -q --ignore=test/test_modules` +(excludes `test/test_modules`, which makes a real Admin API call at import). diff --git a/OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md b/OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md new file mode 100644 index 0000000..fa55e0a --- /dev/null +++ b/OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md @@ -0,0 +1,196 @@ +# Design: self-refreshing `oauth_token` — collapse the scattered refresh hooks + +> Proposal under evaluation: make the SDK config's `oauth_token` resolve through a callable/property +> that refreshes a stale token on read, so we can delete `ensure_active_config_fresh` and every +> chokepoint call (`call_api`, `execute_single_request`, `query_cld_folder`, `cld_folder_exists`). +> Context: access tokens are valid ~5 minutes, so refreshes are frequent and expected. +> +> Verdict: **the idea is sound and the SDK seam exists, but a plain `__getattr__` hook will NOT +> work** because of how the SDK stores the token. A `property`/descriptor approach works, with two +> caveats that must be handled. Details below, traced against the installed SDK. + +--- + +## 1. Why this is the right instinct + +Every place that needs auth reads the token the same way, at call time: + +```python +# cloudinary/api_client/call_api.py:63 +oauth_token = options.pop("oauth_token", cloudinary.config().oauth_token) +# cloudinary/uploader.py:877 +oauth_token = options.get("oauth_token", cloudinary.config().oauth_token) +``` + +Both read **`cloudinary.config().oauth_token`** at the moment of the request. That is a single, +universal seam. If reading that attribute returns a *fresh* token, then: + +- `ensure_active_config_fresh()` and all four chokepoint calls can be **deleted**. +- Search's direct `.execute()` paths, sync's threaded `call_api`, provisioning, and any future API + entry point all get correct tokens for free — no hand-maintained chokepoint list. +- The "stale token mid-bulk-run → 401" gap closes, because the token is re-evaluated per request. + +This directly answers the 5-minute-validity reality: refresh becomes demand-driven at the exact +point of use, once per request, with no proactive probing. + +--- + +## 2. The blocker: the SDK does NOT route `oauth_token` through `__getattr__` + +`cloudinary.Config` (BaseConfig): + +```python +def __getattr__(self, i): + return self.__dict__.get(i) +``` + +`__getattr__` fires **only when the attribute is absent from the instance `__dict__`.** But the +token is written *into* `__dict__`: + +```python +def _setup_from_parsed_url(self, parsed_url): + ... + self.__dict__[k] = v[0] # oauth_token lands here +``` + +So `config().oauth_token` is a normal dict hit and `__getattr__` never runs. **A `__getattr__`-based +hook is dead on arrival.** This is the trap to avoid. + +### What does work + +A **data descriptor** (a `property`) defined on the *class* takes precedence over the instance +`__dict__`, so it intercepts the read even when `oauth_token` is also in `__dict__`. Options, in +order of preference: + +1. **Subclass + property (cleanest, CLI-local, no SDK fork).** Define a `Config` subclass with an + `oauth_token` property whose getter refreshes-if-stale and returns the live access token; the + raw stored URL/refresh-token live in private attributes. Install it as `cloudinary._config` (the + object `cloudinary.config()` returns). The SDK's `config().oauth_token` then hits our property. + - Requires confirming the SDK lets us swap the config singleton (it stores a module-level + `_config`; `reset_config()` rebuilds it — see §4 risk). + +2. **Store the token outside `__dict__` so the existing `__getattr__` fires.** Pop `oauth_token` + out of `__dict__` and serve it from `__getattr__`. Fragile: anything that writes + `config().oauth_token = x` (or `update(oauth_token=...)`) puts it back in `__dict__` and silently + disables the hook. Not recommended. + +3. **Monkeypatch `BaseConfig.oauth_token` as a property at CLI import.** Works (class-level data + descriptor), but mutates SDK global state for the whole process — affects every Config instance, + including the env-derived `cloudinary.Config()` we build in `config_listing`. Has to no-op for + non-OAuth / non-CLI-managed configs. Workable but the broadest blast radius. + +**Recommendation: option 1** (subclass + property installed as the active config). + +--- + +## 3. Two semantic caveats that MUST be handled + +### Caveat A — `oauth_token` is read for *truthiness*, not just for the value + +The CLI itself does this in several places: + +```python +"type": "oauth" if config_obj.oauth_token else "api_key" # config_listing.py x3, core/config.py +if cloudinary.config().cloud_name and cloudinary.config().oauth_token: # is_valid_cloudinary_config +``` + +If `oauth_token` becomes a refresh-on-read property, then **a type check or a validity check would +trigger a network refresh** — exactly the kind of accidental I/O on an offline path (`config -ls`, +`config -s`, `is_valid_cloudinary_config` at the group level) that this whole effort was trying to +remove (the original Finding 1 hang). This is the subtle regression risk. + +Mitigations: +- The property getter must **refresh only when the caller actually needs a live token for a + request** — but a property can't tell "truthiness check" from "real use." So: keep a separate, + non-refreshing attribute for *presence* (e.g. an `is_oauth` flag or a raw `_oauth_token_raw`) and + point the type/validity checks at *that*, leaving the refreshing `oauth_token` only on the + request path. i.e. the property refreshes; the CLI's own introspection reads the raw field. +- Or gate the getter: refresh only if `expires_at` is set AND we're not in a "describe" context. + Context flags are ugly; prefer the separate-presence-field approach. + +### Caveat B — reentrancy and thread safety on read + +A 5-minute token under a multi-threaded sync means many threads may read `oauth_token` near +expiry simultaneously. The getter must reuse the **existing** lock + double-check from +`refresh_url_if_stale` (which is already correct and single-flight across threads and processes — +preserve it verbatim). The getter also must not deadlock: it runs *inside* SDK request code, and +`refresh_url_if_stale` → `update_config` takes the reentrant `config_lock` and then calls +`refresh_cloudinary_config` (which does `reset_config()` + reload of the **global**). Rebuilding the +global config object *from within a getter on that same global* is the dangerous part: + +- Do **not** `reset_config()` inside the getter. Instead, refresh, persist, and update only the + private token fields on the *current* object (no global swap). The getter returns the new access + token; no `cloudinary.reset_config()` on the hot path. This also fixes the latent + reset-then-reload thread race documented in the concurrency review. + +--- + +## 4. Other risks / unknowns to confirm before building + +1. **`reset_config()` rebuilds the singleton.** The SDK's `reset_config()` constructs a fresh + `Config()`. If we installed a subclass instance, any code path that calls `reset_config()` (we do, + in `refresh_cloudinary_config`, and the SDK may internally) would replace our self-refreshing + object with a plain one, silently disabling the property. Audit every `reset_config()` call and + route config installation through one helper that always installs the subclass. + +2. **Env / `-c` configs.** Env-derived and inline-`-c` OAuth URLs currently never refresh (no saved + entry, no refresh token persisted). The property must no-op for configs with no refresh token + (return the static token) — same as `refresh_url_if_stale`'s existing early-out. + +3. **Persistence on rotation.** When the getter refreshes, it must still write the rotated token back + to `config.json` under the lock (so the next *process* benefits and the single-use token isn't + re-burned). That write must use the existing atomic `update_config`. The getter therefore still + needs to know *which saved name* it maps to — i.e. it needs the `_active_name` association, just + carried on the config object instead of a module global. (This is strictly better: the binding + lives with the object, not in resolver module state.) + +4. **`config_to_dict` / masking.** `config_to_dict` iterates `__dict__`; a property is on the class, + not in `__dict__`, so the masking/listing code that enumerates `__dict__` would **miss** + `oauth_token` unless the raw value is still stored as an instance attr. Keep the raw token in + `__dict__` (under a private name) so masking still sees it; expose the live one via the property. + +5. **SDK upgrades.** This couples us to two SDK internals: `config().oauth_token` being read per + request (stable, it's the documented OAuth path) and the descriptor-vs-`__dict__` precedence + (Python language guarantee, safe). The fragile coupling is `reset_config()` behavior (#1). + +--- + +## 5. What gets deleted if this lands + +- `config_resolver.ensure_active_config_fresh` (whole function). +- The 4 call sites: `api_utils.call_api`, `api_utils.query_cld_folder`, `api_utils.cld_folder_exists`, + `core/search.execute_single_request`. +- The module-global `_active_name` *as a refresh input* (still useful for `config -ls` "active" + marker — keep it, or move the active-name onto the config object too). +- The per-call `load_config()` on the hot path. + +What stays (and should be reused verbatim inside the getter): +- `refresh_url_if_stale`'s lock + double-check + atomic persist (the genuinely-correct core). +- The resolver's Phase-A selection/precedence and the offline format check. + +--- + +## 6. Recommended shape + +1. Add a CLI `OAuthConfig(cloudinary.Config)` subclass: + - stores `_raw_url` / `_saved_name` (the saved-config association), + - `oauth_token` is a **property**: if a saved OAuth token, refresh-if-stale (reusing + `refresh_url_if_stale`'s lock/double-check/persist), update the in-object token in place (no + `reset_config`), return the live access token; otherwise return the static stored value, + - keeps the raw token in `__dict__` under a private key for masking/introspection. +2. Route all config installation (resolver, `refresh_cloudinary_config`) through one helper that + installs this subclass and never leaves a plain `Config` as the active global for a saved OAuth + login. Audit `reset_config()`. +3. Point **presence/type/validity** checks (`is_valid_cloudinary_config`, the three + `"oauth" if ... else "api_key"` sites) at the **raw** field / an `is_oauth` flag — NOT the + refreshing property — so offline `config`/`-ls`/`-s` never touch the network (preserves the + Finding-1 fix). +4. Delete `ensure_active_config_fresh` and its four call sites. +5. Tests: (a) reading `oauth_token` on a stale saved config refreshes once and persists; (b) reading + it on a fresh / api-key / env config does no network; (c) `config -ls` / `-s` / + `is_valid_cloudinary_config` do **zero** network even with a stale token (regression guard for + Caveat A); (d) concurrent threaded reads single-flight (one refresh, others adopt). + +This collapses the scattered hooks into one well-placed seam, fixes the mid-run-401 and the +reset-then-reload race, and keeps the offline paths offline — provided Caveat A (truthiness reads) +and the `reset_config` audit (#1/#4) are handled. Those two are the make-or-break items. diff --git a/OAUTH_REFRESH_CONCURRENCY_REVIEW.md b/OAUTH_REFRESH_CONCURRENCY_REVIEW.md new file mode 100644 index 0000000..983b3c3 --- /dev/null +++ b/OAUTH_REFRESH_CONCURRENCY_REVIEW.md @@ -0,0 +1,266 @@ +# OAuth lazy-refresh concurrency review — `ensure_active_config_fresh` + +> Scope: the lazy token-refresh design landed in `43fb67a` (`oauth-login` branch). Focuses on +> behavior under (1) high in-process + cross-process parallelism (sync / upload-dir with up to ~30 +> `cld` instances) and (2) one long-running instance while a second instance mutates config +> (changing the default, dropping configs). Every claim below was traced against the code, not the +> design doc. + +--- + +## 1. How refresh is wired today + +`resolve_cli_config` (group callback, Phase A) does **no network**. It records the selected saved +config name in the module global `_active_name`. Then, at each API chokepoint, Phase B runs: + +```python +# config_resolver.ensure_active_config_fresh() +name = _active_name +if name is None: return +url = load_config().get(name) # disk read +if url is None: return +fresh = refresh_url_if_stale(name, url) # may take config_lock + network +if fresh != url: + refresh_cloudinary_config(fresh) # mutates the SDK *process global* +``` + +`refresh_url_if_stale` (auth/__init__.py): + +```python +if not is_oauth_url(url): return url +session = from_cloudinary_url(url) +if (session.is_fresh() and not force) or not session.refresh_token: + return url # FAST PATH: no lock, no I/O beyond the load_config above +with config_lock(): # FileLock on config.json.lock (cross-process, reentrant) + url = load_config().get(name, url) # re-read under lock + session = from_cloudinary_url(url) + if (session.is_fresh() and not force) or not session.refresh_token: + return url # peer already refreshed -> adopt, don't burn + token_response = flow.refresh(...) # NETWORK, single-use refresh token + update_config({name: refreshed_url})# atomic write (tmp + os.replace), reentrant lock + return refreshed_url +``` + +Chokepoints calling `ensure_active_config_fresh`: + +| Site | Frequency | +|---|---| +| `api_utils.call_api` | **once per API call** — and `upload_file` → `call_api`, run under `run_tasks_concurrently` (N worker threads) | +| `core/search.py::execute_single_request` | once per search request | +| `api_utils.query_cld_folder` | once per folder query (before the cursor loop — good) | +| `api_utils.cld_folder_exists` | once per existence check | + +Freshness: `is_fresh()` = `expires_at - 30s > now` (`OAUTH_EXPIRY_SKEW_SECONDS = 30`). + +--- + +## 2. The user's concern #1 — "30 instances all refresh at once" + +### 2.1 What actually happens — it is mostly safe, but the design is wasteful and fragile + +**The good news (correctness):** the refresh itself is *not* a correctness problem under +concurrency. The lock + double-check makes the actual refresh single-flight: + +- Only one process at a time holds `config_lock()`. The first to enter refreshes, rotates the + single-use token, and `os.replace`s the file atomically. Every other process, on acquiring the + lock, re-reads and sees a now-fresh token → adopts it → **does not** burn a second refresh. +- So we will **not** stampede the OAuth server with 30 concurrent refreshes for the same config, + and we will **not** burn 30 refresh tokens. That part of the design is sound. + +**The bad news (this is the poor design you flagged):** + +1. **Per-call fast-path cost, multiplied by every asset and every thread.** `ensure_active_config_fresh` + runs on **every** `call_api`. For a fresh token the cost is `load_config()` (a disk read + + JSON parse) **plus** `from_cloudinary_url` parse, on every single upload/admin call. In a sync + of 50k assets across N threads that is 50k×(disk read + parse) of `config.json`, on the hot + path, purely to discover "still fresh, nothing to do." It is not a hang, but it is real, + repeated, avoidable I/O and lock-adjacent work in the tightest loop the CLI has. + +2. **In-process thread contention on a reentrant lock is invisible but real.** `run_tasks_concurrently` + spins up worker threads; each calls `call_api` → `ensure_active_config_fresh`. On the fast path + they don't take `config_lock`, so threads mostly don't serialize there. **But** the moment the + token actually goes stale mid-run (long sync crossing the `expires_at` boundary), *every* worker + thread simultaneously fails `is_fresh()`, and they pile up on `config_lock()`. One refreshes; the + rest serialize behind it, re-read, adopt. Correct, but a synchronized stall of all workers at the + expiry boundary — and `refresh_cloudinary_config` mutates the **process-global** SDK config (see + §2.2), which those same threads are concurrently reading. + +3. **`refresh_cloudinary_config` mutates global SDK state while other threads make API calls.** + This is the sharpest in-process hazard. `refresh_cloudinary_config` does + `cloudinary.reset_config()` then `_load_from_url(...)`. There is a window where the global config + has been reset but not yet reloaded. A peer worker thread issuing `uploader.upload` during that + window can read a half-cleared global config. The cross-*process* path is safe (separate address + spaces, atomic file); the cross-*thread* path within one process shares one mutable + `cloudinary.config()` singleton with no lock around the reset+reload. This is a latent data race, + not currently covered by any test. + +4. **Cross-process thundering herd at the expiry boundary.** 30 processes started near the token's + expiry will each independently hit the stale branch, then queue on the *file* lock one-by-one. + The first refreshes (network); the other 29 each acquire the lock, re-read, see fresh, release. + That is 29 serialized lock acquisitions + 29 `load_config` re-reads gated on a single + cross-process `FileLock` — a serialization point that all 30 bulk jobs funnel through at the same + instant. No token burn, but a real latency cliff and a single point of contention. + +### 2.2 Summary of #1 + +- **Token burn / OAuth stampede:** safe. Single-flight via lock + double-check works across + processes and threads. +- **Performance:** poor. Per-call `load_config` on the hot path; synchronized stall of all + workers/processes at the expiry boundary; cross-process serialization on one file lock. +- **In-process thread safety of the SDK global:** **unsafe (latent race)** — `reset_config()` + + reload is not atomic w.r.t. concurrent reader threads. + +--- + +## 3. The user's concern #2 — long-running instance vs. a second instance mutating config + +Scenario: instance A is mid-`sync` (minutes long). Instance B runs `cld config -d other`, +`cld config -rm A's-config`, `cld config -ud`, etc. + +### 3.1 Default change (`-d` / `-ud`) — safe, ignored by A + +Instance A resolved its config **once** at startup and cached the selection in its own process +global `_active_name`. A re-reads `config.json` in `ensure_active_config_fresh`, but only to fetch +**A's own** `_active_name` entry's URL — it never re-reads `__default__`. So B changing or clearing +the default has **zero effect** on a running A. Correct and desirable (A shouldn't switch accounts +mid-sync). ✔ + +### 3.2 Dropping the config A is using (`cld config -rm `) — degrades, mostly safe + +- B's `-rm` does `remove_config_keys` under `config_lock()` + atomic write. A's in-flight API calls + are unaffected (A already loaded the URL into its SDK global at resolve time). +- The exposure is **only** inside `ensure_active_config_fresh`: `url = load_config().get(name)`. If + B removed the entry, this returns `None` → A **returns early and does not refresh**. If A's token + was about to expire, A then proceeds with a stale token and the next real API call fails with a + 401 — mid-sync, not a clean error. So: no crash, no corruption, but a removed-out-from-under-you + config can turn a refresh into a silent skip → late 401. +- If A is the default and B removes A's config, `core/config.py` clears `__default__` too — fine, + doesn't affect running A. + +### 3.3 Token rotation interleaving (A refreshes while B refreshes/edits) — safe + +Both go through `config_lock()` + atomic `os.replace`. No torn reads (atomic write guarantees a +reader sees either the old or the new whole file). The reentrant lock means A's +`refresh_url_if_stale` → `update_config` nests safely. ✔ + +### 3.4 The one real cross-process correctness gap — lost update on unrelated keys + +`update_config` is read-modify-write under the lock: + +```python +with config_lock(): + curr = load_config(); curr.update(new_config); save_config(curr) +``` + +Because every writer takes the lock, two writers don't lose each other's updates. **However**, +`set_default_config`, `clear_default_config`, `refresh` and `-rm` are *separate* lock acquisitions, +not one transaction. A multi-step CLI operation that does read-decide-write across two lock scopes +(e.g. `_should_auto_default` reads outside the lock, then `set_default_config` writes inside a new +lock) can interleave with a peer between the two scopes. Concretely: `login` does +`update_config({name: url})` (lock #1), then `_should_auto_default` does `load_config()` (no lock), +then `set_default_config` (lock #2). A peer running `cld config -d X` between #1 and #2 can have its +default silently overwritten by the auto-default, or vice-versa. Low probability, but it is a +genuine TOCTOU across separate lock scopes. ✔ data integrity (no torn file) / ✘ atomicity of the +logical operation. + +### 3.5 Summary of #2 + +| B's action while A runs | A's outcome | Safe? | +|---|---|---| +| `-d` / `-ud` change default | ignored (A cached its selection) | ✔ | +| `-rm` A's config | refresh becomes a no-op → possible late 401 if token expires | ⚠ degraded | +| refresh / rotate token | atomic, single-flight, adopted | ✔ | +| concurrent multi-step config edit | file never torn; logical op can interleave (TOCTOU) | ⚠ | + +--- + +## 4. Why `ensure_active_config_fresh` is the wrong shape + +1. **Wrong altitude.** Freshness is a property of *the session we resolved*, but the check is + re-derived from disk on every API call via a module global. It couples `api_utils` and + `core/search` to resolver internals (`_active_name`) and to `load_config`. +2. **Hot-path I/O.** A `load_config()` (disk + JSON parse) per API call, inside the busiest loops, + to almost always conclude "fresh." +3. **Process-global mutation under concurrency.** `refresh_cloudinary_config` resets the shared SDK + singleton with no guard against concurrent reader threads (§2.2.3). +4. **Chokepoint set is hand-maintained.** Four call sites enumerated by hand; any new API entry + point silently runs on a stale token. No test asserts completeness. +5. **Module-global `_active_name` is not thread-scoped.** Fine for one resolve per process, but it + makes the whole mechanism implicitly single-config-per-process and invisible to readers. + +--- + +## 5. Suggested fixes (in priority order) + +### Fix A — refresh ONCE, eagerly, after resolution; drop the per-call hook (recommended) + +The original eager refresh hung offline commands. The real fix for *that* was to **only refresh +when about to do network work**, not to refresh on *every* call. Move the refresh to a single point: +right after `resolve_cli_config` succeeds **and** the command is known to be a network command +(or lazily, but **once**, guarded by a process-level "already ensured" flag). + +```python +_ensured = False +def ensure_active_config_fresh(): + global _ensured + if _ensured or _active_name is None: + return + _ensured = True # one refresh attempt per process, not per call + url = load_config().get(_active_name) + ... +``` + +- Eliminates per-call `load_config` and per-call parse. +- Eliminates the synchronized all-threads stall at the expiry boundary (only the first call into + any chokepoint pays the cost; the flag short-circuits the rest). +- A token expiring *mid-very-long-sync* is then handled by Fix C, not by re-checking every call. + +### Fix B — make `refresh_cloudinary_config` atomic w.r.t. reader threads + +Guard the `reset_config()` + `_load_from_url()` pair with a process-local `threading.Lock`, and have +the per-thread API path either hold it for the read or only ever swap in a fully-built config. The +cleanest version builds the new `cloudinary.Config` object off to the side and assigns it in one +reference swap rather than reset-then-mutate, so a reader thread never sees a half-cleared global. + +### Fix C — refresh-on-401 retry, not refresh-on-every-call + +The robust pattern for long-running multi-threaded/multi-process jobs is **reactive**: attempt the +API call; on a 401/auth error, take the lock, refresh once (double-checked), reload, and retry the +call exactly once. This: + +- removes all proactive per-call freshness work, +- naturally single-flights across threads and processes (same lock + double-check already in place), +- correctly handles the token expiring mid-run and the "B removed my config" case (the retry can + surface a clear "re-login" error instead of a raw 401). + +Wrap it at `call_api` (which already catches exceptions) and at the two direct `.execute()` sites. + +### Fix D — make logical config operations transactional + +Provide a single `with config_lock(): read; mutate; write` helper for multi-step operations +(`login` + auto-default, refresh-and-default) so read-decide-write happens under **one** lock scope, +closing the §3.4 TOCTOU. At minimum, move `_should_auto_default`'s read and `set_default_config`'s +write into the same lock acquisition. + +### Fix E — handle "config removed mid-run" explicitly + +In the refresh path, distinguish "config gone" (`url is None` after the resolver had a name) from +"nothing to refresh," and surface a clear message (`config '' was removed; please re-login`) +rather than silently skipping and letting a later 401 fall out. + +### Fix F — assert chokepoint completeness + +A test that monkeypatches the refresh entry point to a counter and exercises each top-level network +command, asserting it fired exactly once (pairs with Fix A's once-per-process flag). + +--- + +## 6. Recommendation + +Adopt **Fix A + Fix C** as the core redesign: resolve once, refresh once eagerly *or* reactively on +401, and stop probing the token on every call. Add **Fix B** to close the SDK-global thread race +(required before we trust multi-threaded sync under token rotation). **Fix D/E/F** are smaller +hardening steps. The current code is *correct on token burn* (the lock + double-check is genuinely +good and should be preserved verbatim inside whichever path survives), but the per-call hook and the +unguarded global mutation make it the wrong shape for the 30-instance / N-thread bulk workloads this +CLI is built for. diff --git a/cloudinary_cli/auth/oauth_config.py b/cloudinary_cli/auth/oauth_config.py new file mode 100644 index 0000000..8456c2b --- /dev/null +++ b/cloudinary_cli/auth/oauth_config.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +import cloudinary + +from cloudinary_cli.auth.session import is_oauth_url, from_cloudinary_url + + +class OAuthConfig(cloudinary.Config): + """ + A Cloudinary config whose `oauth_token` refreshes itself on read, at the moment the SDK builds + a request. Presence/type checks read `has_oauth` instead and never touch the network, so offline + paths (`config -ls`, `config -s`, the group-level validity check) stay offline. + + The raw access token is kept in `__dict__["oauth_token"]` so serialization (config_to_dict, + masking) still sees it; the class-level property shadows it on attribute *read* to + refresh-if-stale. A parsed Session (`_session`) carries expiry/refresh-token so a still-fresh + token short-circuits with no disk read and no lock — only a stale token reads config + refreshes. + """ + + def bind_saved(self, name, url): + # name: the saved-config name this maps to (None for env / inline -c -> never refreshes). + # url: the full cloudinary:// URL, kept parsed so we know expiry without re-reading disk. + self._saved_name = name + self._session = from_cloudinary_url(url) if (name and url and is_oauth_url(url)) else None + + @classmethod + def from_env(cls): + """An OAuthConfig populated from the environment (CLOUDINARY_URL/CLOUDINARY_*). Static: it is + not bound to a saved name, so reading its oauth_token never refreshes.""" + cfg = cls() # the base Config constructor loads the environment + cfg._saved_name = None + cfg._session = None + return cfg + + @classmethod + def from_url(cls, url): + """An OAuthConfig populated from a cloudinary:// URL, not bound to a saved name (static).""" + cfg = cls() + cfg._load_from_url(url) + cfg._saved_name = None + cfg._session = None + return cfg + + @property + def has_oauth(self): + """True if this config carries an OAuth token. Cheap, never refreshes.""" + return bool(self.__dict__.get("oauth_token")) + + @property + def oauth_token(self): + session = getattr(self, "_session", None) + if session is None: + return self.__dict__.get("oauth_token") # env / -c / api-key: static, no refresh + if session.is_fresh() or not session.refresh_token: + return self.__dict__.get("oauth_token") # still valid (or unrefreshable): no I/O + + # Stale: read the saved URL (a peer may already have refreshed it) and refresh under lock. + from cloudinary_cli.auth import refresh_url_if_stale + from cloudinary_cli.utils.config_utils import load_config + + url = load_config().get(self._saved_name) + if not url: + return self.__dict__.get("oauth_token") # config removed underneath us; serve what we have + fresh_url = refresh_url_if_stale(self._saved_name, url) + self._session = from_cloudinary_url(fresh_url) + self.__dict__["oauth_token"] = self._session.access_token + return self.__dict__["oauth_token"] + + @oauth_token.setter + def oauth_token(self, value): + self.__dict__["oauth_token"] = value + + +def install_oauth_config(cloudinary_url, saved_name=None): + """ + Load `cloudinary_url` and install it as the active SDK config. The installed object is always an + OAuthConfig (so every active config exposes `has_oauth`); it self-refreshes only when bound to a + saved OAuth `saved_name`, and is static for api-key / inline `-c` URLs. The single seam that + swaps the global config object. + """ + cloudinary.reset_config() + cfg = OAuthConfig() + cfg._load_from_url(cloudinary_url) + cfg.bind_saved(saved_name, cloudinary_url) + cloudinary._config = cfg + return cfg + + +def install_env_config(): + """Install the environment config as a (static) OAuthConfig, so the active global is always an + OAuthConfig and exposes has_oauth without a refresh. Used for the env fallback branch.""" + cfg = OAuthConfig.from_env() + cloudinary._config = cfg + return cfg diff --git a/cloudinary_cli/core/config.py b/cloudinary_cli/core/config.py index b54abf0..0025ecc 100644 --- a/cloudinary_cli/core/config.py +++ b/cloudinary_cli/core/config.py @@ -25,6 +25,7 @@ render_config_table, config_meta, active_config_meta, + config_type_label, SYNTHETIC_NAMES, ) @@ -188,6 +189,6 @@ def _show_active_header(): _show_config_header(name, load_config()) return active = cloudinary.config() - type_label = "oauth" if active.oauth_token else "api_key" + type_label = config_type_label(active) label = SYNTHETIC_NAMES["url"] if active_config_is_url() else SYNTHETIC_NAMES["env"] echo(f"name: {label} ({type_label}) [active]\n") diff --git a/cloudinary_cli/core/search.py b/cloudinary_cli/core/search.py index 9296af2..55cb6b3 100644 --- a/cloudinary_cli/core/search.py +++ b/cloudinary_cli/core/search.py @@ -7,7 +7,6 @@ from cloudinary_cli.utils.utils import write_json_list_to_csv, confirm_action, whitelist_keys, \ normalize_list_params from cloudinary_cli.utils.search_utils import parse_aggregate -from cloudinary_cli.utils.config_resolver import ensure_active_config_fresh DEFAULT_MAX_RESULTS = 500 @@ -135,7 +134,6 @@ def _perform_search(query, with_field, fields, sort_by, aggregate, max_results, def execute_single_request(expression, fields_to_keep, result_field='resources'): - ensure_active_config_fresh() res = expression.execute() if fields_to_keep: diff --git a/cloudinary_cli/utils/api_utils.py b/cloudinary_cli/utils/api_utils.py index 8f936d6..3cd0b89 100644 --- a/cloudinary_cli/utils/api_utils.py +++ b/cloudinary_cli/utils/api_utils.py @@ -8,7 +8,6 @@ from cloudinary_cli.defaults import logger from cloudinary_cli.utils.config_utils import is_valid_cloudinary_config -from cloudinary_cli.utils.config_resolver import ensure_active_config_fresh from cloudinary_cli.utils.file_utils import (normalize_file_extension, posix_rel_path, get_destination_folder, populate_duplicate_name) from cloudinary_cli.utils.json_utils import print_json, write_json_to_file @@ -66,7 +65,6 @@ def query_cld_folder(folder, folder_mode, status=None): logger.debug(f"Search expression: {search.to_json()}") - ensure_active_config_fresh() next_cursor = True while next_cursor: res = search.execute() @@ -106,7 +104,6 @@ def cld_folder_exists(folder): if not folder: return True # root folder - ensure_active_config_fresh() res = SearchFolders().expression(f"path=\"{folder}\"").execute() return res.get("total_count", 0) > 0 @@ -292,7 +289,6 @@ def get_folder_mode(): def call_api(func, args, kwargs): - ensure_active_config_fresh() try: return func(*args, **kwargs) except Exception as e: diff --git a/cloudinary_cli/utils/config_listing.py b/cloudinary_cli/utils/config_listing.py index f8caf7b..1c138e7 100644 --- a/cloudinary_cli/utils/config_listing.py +++ b/cloudinary_cli/utils/config_listing.py @@ -3,6 +3,7 @@ and the per-config metadata used for `config`/`config -s` (text headers and JSON).""" import cloudinary +from cloudinary_cli.auth.oauth_config import OAuthConfig from cloudinary_cli.defaults import DEFAULT_CONFIG_KEY from cloudinary_cli.utils.config_utils import ( load_config, @@ -22,6 +23,12 @@ # label, not a saved config name, in both the table and JSON. SYNTHETIC_NAMES = {"env": "(environment)", "url": "(command-line)"} + +def config_type_label(config_obj): + """oauth/api_key for a config OBJECT. Every active config the CLI installs is an OAuthConfig, so + presence is read via has_oauth (refresh-free). (config_utils.config_type classifies a URL str.)""" + return "oauth" if config_obj.has_oauth else "api_key" + _TABLE_COLUMNS = [("name", "NAME"), ("cloud_name", "CLOUD"), ("type", "TYPE"), ("default", "DEFAULT"), ("active", "ACTIVE")] @@ -73,7 +80,7 @@ def active_config_meta(config_obj): return { "name": SYNTHETIC_NAMES[source], "source": source, - "type": "oauth" if config_obj.oauth_token else "api_key", + "type": config_type_label(config_obj), "default": False, "active": True, **cloudinary_config_details(config_obj), @@ -94,7 +101,7 @@ def _url_row(): return { "name": SYNTHETIC_NAMES["url"], "cloud_name": active.cloud_name or "", - "type": "oauth" if active.oauth_token else "api_key", + "type": config_type_label(active), "source": "url", "default": False, # an inline URL is never the stored default "active": True, # it outranks everything else for this invocation @@ -102,11 +109,11 @@ def _url_row(): def _env_row(env_active): - env_config = cloudinary.Config() # constructed fresh from the environment, not the CLI global + env_config = OAuthConfig.from_env() # constructed fresh from the environment, not the CLI global return { "name": SYNTHETIC_NAMES["env"], "cloud_name": env_config.cloud_name or "", - "type": "oauth" if env_config.oauth_token else "api_key", + "type": config_type_label(env_config), "source": "env", "default": False, # the environment is never the *stored* default "active": env_active, # active only when no stored default outranks it diff --git a/cloudinary_cli/utils/config_resolver.py b/cloudinary_cli/utils/config_resolver.py index 378559f..252d89d 100644 --- a/cloudinary_cli/utils/config_resolver.py +++ b/cloudinary_cli/utils/config_resolver.py @@ -23,13 +23,14 @@ " as the default: cld config -d " ) -# What the last resolve_cli_config (Phase A) selected, by precedence. One of: +# What the last resolve_cli_config selected, by precedence. One of: # "url" -> an inline -c CLOUDINARY_URL # "env" -> the environment fallback # None -> nothing configured -# plus _active_name, the saved-config name when a -C/default saved entry was selected (else None). -# Read by ensure_active_config_fresh (Phase B) to know which saved login may need a lazy refresh, -# and by `config -ls` to mark the row that is actually active for this invocation. +# plus _active_name, the saved-config name when a -C/default saved entry was selected (else None), +# read by `config -ls` to mark the row active for this invocation. Token freshness is no longer +# handled here: a saved OAuth config installs a self-refreshing OAuthConfig that refreshes lazily +# when the SDK reads its oauth_token at request time. _active_name = None _active_source = None @@ -52,20 +53,23 @@ def resolve_cli_config(config=None, config_saved=None): raise Exception(f"Config {config_saved} does not exist") _active_name = config_saved _active_source = "saved" - refresh_cloudinary_config(cfg[config_saved]) + refresh_cloudinary_config(cfg[config_saved], saved_name=config_saved) return _format_ok() default = cfg.get(DEFAULT_CONFIG_KEY) if default and default in cfg: _active_name = default _active_source = "saved" - refresh_cloudinary_config(cfg[default]) + refresh_cloudinary_config(cfg[default], saved_name=default) return _format_ok() - # No stored default: the SDK global already holds the environment config (if any), so - # _format_ok validates it; otherwise it warns that nothing is configured. + # No stored default: fall back to the environment. Install it as an OAuthConfig (static, no + # saved name -> never refreshes) so the active global is always an OAuthConfig and exposes + # has_oauth uniformly; if nothing is configured, _format_ok warns. if is_env_configured(): _active_source = "env" + from cloudinary_cli.auth.oauth_config import install_env_config + install_env_config() return _format_ok() @@ -92,19 +96,6 @@ def _format_ok(): return True -def ensure_active_config_fresh(): - """Refresh the active OAuth token if stale, just before an API call. No-op otherwise.""" - name = _active_name - if name is None: - return - url = load_config().get(name) - if url is None: - return - fresh = refresh_url_if_stale(name, url) - if fresh != url: - refresh_cloudinary_config(fresh) - - def get_cloudinary_config(target): target_config = cloudinary.Config() if target.startswith("cloudinary://"): diff --git a/cloudinary_cli/utils/config_utils.py b/cloudinary_cli/utils/config_utils.py index 7e00140..9cf4557 100644 --- a/cloudinary_cli/utils/config_utils.py +++ b/cloudinary_cli/utils/config_utils.py @@ -29,14 +29,42 @@ def config_lock(): return _config_lock +# Parsed-config cache keyed on the file's (mtime_ns, size). The config file is read on nearly every +# code path; caching skips the re-read + JSON parse when it has not changed on disk (including +# changes written by a peer process, which os.replace stamps with a new mtime). +_config_cache = None +_config_cache_stat = None + + +def _config_stat(): + try: + st = os.stat(CLOUDINARY_CLI_CONFIG_FILE) + return st.st_mtime_ns, st.st_size + except FileNotFoundError: + return None + + +def _invalidate_config_cache(): + global _config_cache, _config_cache_stat + _config_cache = None + _config_cache_stat = None + + def load_config(): - return read_json_from_file(CLOUDINARY_CLI_CONFIG_FILE, does_not_exist_ok=True) + global _config_cache, _config_cache_stat + stat = _config_stat() + if stat is not None and stat == _config_cache_stat and _config_cache is not None: + return dict(_config_cache) # copy: callers mutate the result in place (e.g. cfg.update(...)) + cfg = read_json_from_file(CLOUDINARY_CLI_CONFIG_FILE, does_not_exist_ok=True) + _config_cache, _config_cache_stat = cfg, stat + return dict(cfg) def save_config(config): _verify_file_path(CLOUDINARY_CLI_CONFIG_FILE) write_json_to_file(config, CLOUDINARY_CLI_CONFIG_FILE, atomic=True) _restrict_permissions(CLOUDINARY_CLI_CONFIG_FILE) + _invalidate_config_cache() # next load_config re-stats and reloads our own write def update_config(new_config): @@ -83,9 +111,11 @@ def is_reserved_config_name(name): return name.startswith("__") and name.endswith("__") -def refresh_cloudinary_config(cloudinary_url): - cloudinary.reset_config() - cloudinary.config()._load_from_url(cloudinary_url) +def refresh_cloudinary_config(cloudinary_url, saved_name=None): + """Install cloudinary_url as the active config. OAuth URLs install a self-refreshing config + bound to saved_name (so token rotations persist); other URLs use the plain SDK config.""" + from cloudinary_cli.auth.oauth_config import install_oauth_config + install_oauth_config(cloudinary_url, saved_name=saved_name) def verify_cloudinary_url(cloudinary_url): @@ -280,9 +310,13 @@ def migrate_old_config(): def is_valid_cloudinary_config(): - if cloudinary.config().cloud_name and cloudinary.config().oauth_token: + config = cloudinary.config() + # has_oauth reports token presence without triggering OAuthConfig's refresh-on-read. Fall back + # to a refresh-free __dict__ read for a plain SDK Config (e.g. before any config is installed). + has_oauth = config.has_oauth if hasattr(config, "has_oauth") else bool(config.__dict__.get("oauth_token")) + if config.cloud_name and has_oauth: return True - return None not in [cloudinary.config().cloud_name, cloudinary.config().api_key, cloudinary.config().api_secret] + return None not in [config.cloud_name, config.api_key, config.api_secret] def is_env_configured(): diff --git a/test/helper_test.py b/test/helper_test.py index 0a3fb06..ce2f118 100644 --- a/test/helper_test.py +++ b/test/helper_test.py @@ -5,12 +5,18 @@ from functools import wraps from pathlib import Path +import cloudinary import cloudinary.api from cloudinary import logger from cloudinary_cli.utils.api_utils import query_cld_folder from urllib3 import HTTPResponse, disable_warnings from urllib3._collections import HTTPHeaderDict +# Many CLI tests mock the HTTP layer but still need a resolvable config to run; without one the +# command exits "No Cloudinary configuration found". Gate those tests on a config being present. +CONFIG_PRESENT = bool(cloudinary.config().cloud_name) +REQUIRES_CONFIG = "Requires a Cloudinary configuration (set CLOUDINARY_URL or a saved config)" + SUFFIX = os.environ.get('TRAVIS_JOB_ID') or random.randint(10000, 99999) RESOURCES_DIR = Path.joinpath(Path(__file__).resolve().parent, "resources") diff --git a/test/test_auth_session.py b/test/test_auth_session.py index 97edcb2..27430e9 100644 --- a/test/test_auth_session.py +++ b/test/test_auth_session.py @@ -118,7 +118,8 @@ def test_force_refreshes_fresh_token(self): def test_stale_refreshes_and_rewrites(self): stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} - with patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ patch("cloudinary_cli.auth.update_config") as update_config: new_url = refresh_url_if_stale("eu-cloud", stale_url) self.assertIn("oauth_token=eyJ.new.tok", new_url) @@ -132,7 +133,8 @@ def test_no_refresh_token_returns_unchanged(self): def test_refresh_timeout_returns_stale_url(self): import requests stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) - with patch("cloudinary_cli.auth.flow.refresh", side_effect=requests.Timeout()), \ + with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ + patch("cloudinary_cli.auth.flow.refresh", side_effect=requests.Timeout()), \ patch("cloudinary_cli.auth.update_config") as update_config: self.assertEqual(stale_url, refresh_url_if_stale("eu-cloud", stale_url)) update_config.assert_not_called() diff --git a/test/test_cli_api.py b/test/test_cli_api.py index d18361e..1ec464c 100644 --- a/test/test_cli_api.py +++ b/test/test_cli_api.py @@ -6,7 +6,8 @@ from click.testing import CliRunner from cloudinary_cli.cli import cli -from test.helper_test import api_response_mock, uploader_response_mock, URLLIB3_REQUEST +from test.helper_test import api_response_mock, uploader_response_mock, URLLIB3_REQUEST, \ + CONFIG_PRESENT, REQUIRES_CONFIG API_MOCK_RESPONSE = api_response_mock() UPLOAD_MOCK_RESPONSE = uploader_response_mock() @@ -17,6 +18,7 @@ class TestCLIApi(unittest.TestCase): runner = CliRunner() + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) @patch(URLLIB3_REQUEST) def test_admin(self, mocker): mocker.return_value = API_MOCK_RESPONSE @@ -25,6 +27,7 @@ def test_admin(self, mocker): self.assertEqual(0, result.exit_code, result.output) self.assertIn('"foo": "bar"', result.output) + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) @patch(URLLIB3_REQUEST) def test_upload(self, mocker): mocker.return_value = UPLOAD_MOCK_RESPONSE @@ -56,6 +59,7 @@ def test_delete_all_resources_decline_skips_call(self, http_mock, confirm_mock): confirm_mock.assert_called_once() self.assertFalse(http_mock.called, "SDK should not be called when user declines") + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) @patch(CONFIRM_ACTION_PATCH, return_value=True) @patch(URLLIB3_REQUEST) def test_delete_all_resources_accept_calls_sdk(self, http_mock, confirm_mock): @@ -66,6 +70,7 @@ def test_delete_all_resources_accept_calls_sdk(self, http_mock, confirm_mock): confirm_mock.assert_called_once() self.assertTrue(http_mock.called, "SDK should be called when user accepts") + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) @patch(CONFIRM_ACTION_PATCH, return_value=False) @patch(URLLIB3_REQUEST) def test_delete_all_resources_force_skips_prompt(self, http_mock, confirm_mock): @@ -86,6 +91,7 @@ def test_delete_resources_by_tag_decline_skips_call(self, http_mock, confirm_moc confirm_mock.assert_called_once() self.assertFalse(http_mock.called, "SDK should not be called when user declines") + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) @patch(CONFIRM_ACTION_PATCH, return_value=False) @patch(URLLIB3_REQUEST) def test_delete_resources_explicit_ids_no_prompt(self, http_mock, confirm_mock): @@ -96,6 +102,7 @@ def test_delete_resources_explicit_ids_no_prompt(self, http_mock, confirm_mock): self.assertFalse(confirm_mock.called, "Explicit-ID delete must not prompt") self.assertTrue(http_mock.called, "SDK should be called for explicit-ID delete") + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) @patch(CONFIRM_ACTION_PATCH, return_value=False) @patch(URLLIB3_REQUEST) def test_uploader_add_tag_no_prompt(self, http_mock, confirm_mock): @@ -106,6 +113,7 @@ def test_uploader_add_tag_no_prompt(self, http_mock, confirm_mock): self.assertFalse(confirm_mock.called, "Non-destructive bulk methods must not prompt") self.assertTrue(http_mock.called, "SDK should be called for non-destructive bulk methods") + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) @patch(CONFIRM_ACTION_PATCH, return_value=False) @patch(URLLIB3_REQUEST) def test_admin_resources_read_no_prompt(self, http_mock, confirm_mock): @@ -116,6 +124,7 @@ def test_admin_resources_read_no_prompt(self, http_mock, confirm_mock): self.assertFalse(confirm_mock.called, "Read commands must not prompt") self.assertTrue(http_mock.called, "SDK should be called for read commands") + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) @patch(CONFIRM_ACTION_PATCH, return_value=False) @patch(URLLIB3_REQUEST) def test_admin_resources_read_with_force_no_prompt(self, http_mock, confirm_mock): diff --git a/test/test_cli_config.py b/test/test_cli_config.py index fee18bf..fac0964 100644 --- a/test/test_cli_config.py +++ b/test/test_cli_config.py @@ -1,4 +1,6 @@ +import os import unittest +from unittest.mock import patch import cloudinary from click.testing import CliRunner @@ -6,6 +8,10 @@ from cloudinary_cli.cli import cli +def _env_without_cloudinary_vars(): + return {k: v for k, v in os.environ.items() if not k.startswith("CLOUDINARY_")} + + def _get_real_cloudinary_url(): cfg = cloudinary.config() @@ -88,9 +94,13 @@ def test_cli_config_show(self): @unittest.skipUnless(cloudinary.config().api_secret, "Requires api_key/api_secret") def test_cli_config_show_default_no_config(self): - self.runner.invoke(cli, ['config', '--from_url', self.EMPTY_CLOUDINARY_URL]) + # This asserts the "nothing configured" path, so it must run with no environment config: + # otherwise the resolver legitimately falls back to CLOUDINARY_URL and the config is valid. + with patch.dict(os.environ, _env_without_cloudinary_vars(), clear=True): + cloudinary.reset_config() + self.runner.invoke(cli, ['config', '--from_url', self.EMPTY_CLOUDINARY_URL]) - result = self.runner.invoke(cli, ['config']) + result = self.runner.invoke(cli, ['config']) self.assertEqual(1, result.exit_code) diff --git a/test/test_cli_config_oauth.py b/test/test_cli_config_oauth.py index 4cb3a37..9e20a51 100644 --- a/test/test_cli_config_oauth.py +++ b/test/test_cli_config_oauth.py @@ -22,6 +22,11 @@ def _oauth_url(cloud="eu-cloud", region="api-eu"): class _RestoresSdkConfig(unittest.TestCase): def setUp(self): self._env_snapshot = dict(os.environ) + # Strip ambient CLOUDINARY_* so a bare cloudinary.Config() built in a test is not polluted by + # the developer's env (e.g. a real account_url leaking into masking/display assertions). + for key in [k for k in os.environ if k.startswith("CLOUDINARY_")]: + del os.environ[key] + cloudinary.reset_config() self.addCleanup(self._restore_sdk_config) def _restore_sdk_config(self): @@ -168,7 +173,7 @@ def test_reserved_name_rejected(self): auth.login(region="eu", name="__default__") -class TestConfigSecretMasking(unittest.TestCase): +class TestConfigSecretMasking(_RestoresSdkConfig): """show_cloudinary_config must never print a secret in the clear.""" def test_masks_api_secret(self): @@ -476,8 +481,10 @@ def test_resolve_does_not_call_flow_refresh(self): from cloudinary_cli.utils.config_resolver import resolve_cli_config resolve_cli_config() refresh.assert_not_called() - # The stale token is loaded as-is, awaiting lazy refresh at point-of-use. - self.assertEqual("eyJ.old.tok", cloudinary.config().oauth_token) + # The stale token is loaded as-is (presence check is refresh-free), awaiting a lazy refresh + # only when the SDK reads oauth_token at request time. + self.assertTrue(cloudinary.config().has_oauth) + self.assertEqual("eyJ.old.tok", cloudinary.config().__dict__.get("oauth_token")) def test_help_does_not_reach_phase_b(self): with patch("cloudinary_cli.auth.flow.refresh") as refresh: @@ -485,8 +492,9 @@ def test_help_does_not_reach_phase_b(self): refresh.assert_not_called() -class TestEnsureActiveConfigFresh(_RestoresSdkConfig): - """Phase B: the lazy freshen shim refreshes a stale active OAuth token, no-op otherwise.""" +class TestSelfRefreshingOAuthToken(_RestoresSdkConfig): + """The active OAuth config refreshes its token lazily when the SDK reads oauth_token at request + time; presence/type checks (has_oauth) never trigger a refresh.""" def _stale_url(self): return to_cloudinary_url(Session( @@ -494,32 +502,46 @@ def _stale_url(self): expires_at=int(time.time()) - 10, region="api-eu", issuer="https://oauth.cloudinary.com/")) - def test_refreshes_stale_active_login(self): + def test_reading_oauth_token_refreshes_stale_active_login(self): import cloudinary_cli.utils.config_resolver as resolver saved = {"eu-cloud": self._stale_url()} token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ patch("cloudinary_cli.auth.update_config"): resolver.resolve_cli_config(config_saved="eu-cloud") - resolver.ensure_active_config_fresh() - self.assertEqual("eyJ.new.tok", cloudinary.config().oauth_token) + # The read of oauth_token is what triggers the refresh (as the SDK does per request). + self.assertEqual("eyJ.new.tok", cloudinary.config().oauth_token) + + def test_presence_check_does_not_refresh(self): + """has_oauth (used by type/validity/-ls) must NOT touch the network on a stale token.""" + import cloudinary_cli.utils.config_resolver as resolver + saved = {"eu-cloud": self._stale_url()} + with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh") as refresh: + resolver.resolve_cli_config(config_saved="eu-cloud") + self.assertTrue(cloudinary.config().has_oauth) + refresh.assert_not_called() def test_noop_for_inline_url(self): import cloudinary_cli.utils.config_resolver as resolver with patch("cloudinary_cli.auth.flow.refresh") as refresh: resolver.resolve_cli_config(config="cloudinary://key:secret@cloud") - resolver.ensure_active_config_fresh() + _ = cloudinary.config().oauth_token refresh.assert_not_called() def test_noop_for_api_key_config(self): import cloudinary_cli.utils.config_resolver as resolver saved = {"mykey": "cloudinary://key:secret@cloud"} with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh") as refresh: resolver.resolve_cli_config(config_saved="mykey") - resolver.ensure_active_config_fresh() + _ = cloudinary.config().oauth_token refresh.assert_not_called() diff --git a/test/test_cli_search_api.py b/test/test_cli_search_api.py index c48e775..a1e282d 100644 --- a/test/test_cli_search_api.py +++ b/test/test_cli_search_api.py @@ -4,7 +4,8 @@ from click.testing import CliRunner from cloudinary_cli.cli import cli -from test.helper_test import api_response_mock, uploader_response_mock, URLLIB3_REQUEST +from test.helper_test import api_response_mock, uploader_response_mock, URLLIB3_REQUEST, \ + CONFIG_PRESENT, REQUIRES_CONFIG API_MOCK_RESPONSE = api_response_mock() UPLOAD_MOCK_RESPONSE = uploader_response_mock() @@ -13,6 +14,7 @@ class TestCLISearchApi(unittest.TestCase): runner = CliRunner() + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) @patch(URLLIB3_REQUEST) def test_search(self, mocker): mocker.return_value = API_MOCK_RESPONSE @@ -27,6 +29,7 @@ def test_search_fields(self): self.assertEqual(0, result.exit_code) self.assertIn('"fields": [\n "url",\n "tags",\n "context"\n ]', result.output) + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) def test_search_url(self): result = self.runner.invoke(cli, ['search', 'cat', '-c', 'NEXT_CURSOR', '--ttl', '1000', '--url']) @@ -37,6 +40,7 @@ def test_search_url(self): self.assertIn('1000', result.output) self.assertIn('NEXT_CURSOR', result.output) + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) @patch(URLLIB3_REQUEST) def test_search_folders(self, mocker): mocker.return_value = API_MOCK_RESPONSE diff --git a/test/test_cli_url.py b/test/test_cli_url.py index 08bff9f..8d6511a 100644 --- a/test/test_cli_url.py +++ b/test/test_cli_url.py @@ -3,6 +3,7 @@ from click.testing import CliRunner from cloudinary_cli.cli import cli +from test.helper_test import CONFIG_PRESENT, REQUIRES_CONFIG class TestCLIURL(unittest.TestCase): @@ -14,18 +15,21 @@ def test_url_no_public_id(self): self.assertEqual(2, result.exit_code) self.assertIn("Error: Missing argument 'PUBLIC_ID'", result.output) + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) def test_url(self): result = self.runner.invoke(cli, ['url', 'sample']) self.assertEqual(0, result.exit_code) self.assertIn('image/upload/sample', result.output) + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) def test_url_list(self): result = self.runner.invoke(cli, ['url', 'sample', '--type', 'list']) self.assertEqual(0, result.exit_code) self.assertIn('image/list/sample.json', result.output) + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) def test_url_authenticated(self): result = self.runner.invoke(cli, ['url', 'sample', '--type', 'authenticated']) diff --git a/test/test_cli_utils.py b/test/test_cli_utils.py index 77dcfac..5db4e5b 100644 --- a/test/test_cli_utils.py +++ b/test/test_cli_utils.py @@ -3,6 +3,7 @@ from click.testing import CliRunner from cloudinary_cli.cli import cli +from test.helper_test import CONFIG_PRESENT, REQUIRES_CONFIG class TestCLIUtils(unittest.TestCase): @@ -26,6 +27,7 @@ def test_list_utils(self): for util in self.UTILS: self.assertIn(util, result.output) + @unittest.skipUnless(CONFIG_PRESENT, REQUIRES_CONFIG) def test_utils_cloudinary_url(self): result = self.runner.invoke(cli, ['utils', 'cloudinary_url', 'sample']) From 18e2db4d557479852396ec143c10897e778d6788 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 03:49:10 +0300 Subject: [PATCH 05/23] Fix OAuth login defects: mutual -c/-C, busy port, config perms, refresh visibility A1: reject -c and -C together with a UsageError instead of silently ignoring -C. A2: surface a clear error when the loopback login port is in use, not a raw OSError. A4: write config.json 0600 via the atomic temp file so it is never momentarily world-readable mid-write (drops the post-replace chmod that left a window). A3a: on a failed background token refresh, warn once per config with a re-login hint instead of a silent debug line followed by a bare downstream 401. Also adds the previously-uncommitted tests for the load_config mtime cache and the SDK oauth_token refresh seam. Co-Authored-By: Claude Opus 4.8 (1M context) --- cloudinary_cli/auth/__init__.py | 13 +++ cloudinary_cli/auth/loopback_server.py | 9 +- cloudinary_cli/utils/config_resolver.py | 4 + cloudinary_cli/utils/config_utils.py | 14 +-- cloudinary_cli/utils/file_utils.py | 11 +- cloudinary_cli/utils/json_utils.py | 4 +- test/test_auth_loopback.py | 22 +++- test/test_auth_session.py | 31 +++++ test/test_cli_config_oauth.py | 8 ++ test/test_config_cache.py | 62 ++++++++++ test/test_file_utils.py | 31 +++++ test/test_oauth_token_seam.py | 144 ++++++++++++++++++++++++ 12 files changed, 337 insertions(+), 16 deletions(-) create mode 100644 test/test_config_cache.py create mode 100644 test/test_oauth_token_seam.py diff --git a/cloudinary_cli/auth/__init__.py b/cloudinary_cli/auth/__init__.py index bc59fd6..e320cfe 100644 --- a/cloudinary_cli/auth/__init__.py +++ b/cloudinary_cli/auth/__init__.py @@ -27,6 +27,11 @@ ) from cloudinary_cli.utils.utils import log_exception +# Names already warned about a failed background refresh, so a bulk run (many workers, each reading +# the token) logs the re-login hint once per config instead of once per asset. Mutated only under +# config_lock, so no extra synchronization is needed. +_refresh_warned = set() + def login(region=None, name=None, set_default=False): """ @@ -102,9 +107,17 @@ def refresh_url_if_stale(name, url, force=False): try: token_response = flow.refresh(session.refresh_token, session.region) except requests.RequestException as e: + # Serve the stale token (a bulk run survives a transient blip) but make the failure + # visible once per config, not a silent debug line followed by a bare downstream 401. log_exception(e, debug_message="OAuth token refresh failed") + if name not in _refresh_warned: + _refresh_warned.add(name) + logger.warning(f"Could not refresh the OAuth token for '{name}'; using the existing " + f"token, which may be expired. Re-login with `{relogin_command(name)}`.") return url + _refresh_warned.discard(name) # a later success re-arms the warning for this config + # Hydra rotates refresh tokens; keep the old one only if a new one was not returned. token_response.setdefault("refresh_token", session.refresh_token) refreshed_url = to_cloudinary_url(session.updated_from(token_response)) diff --git a/cloudinary_cli/auth/loopback_server.py b/cloudinary_cli/auth/loopback_server.py index 1b3eda0..873b5fc 100644 --- a/cloudinary_cli/auth/loopback_server.py +++ b/cloudinary_cli/auth/loopback_server.py @@ -44,7 +44,14 @@ def log_message(self, *args): def start_callback_server(): """Bind the loopback server and return (httpd, redirect_uri).""" - httpd = HTTPServer((OAUTH_REDIRECT_HOST, OAUTH_REDIRECT_PORT), _CallbackHandler) + try: + httpd = HTTPServer((OAUTH_REDIRECT_HOST, OAUTH_REDIRECT_PORT), _CallbackHandler) + except OSError as e: + raise RuntimeError( + f"Could not start the local login server on {OAUTH_REDIRECT_HOST}:{OAUTH_REDIRECT_PORT} " + f"({e.strerror or e}). Another login may be in progress, or the port is in use. " + f"Close it and retry." + ) from e httpd.auth_code = httpd.auth_state = httpd.auth_error = None httpd.timeout = OAUTH_CALLBACK_TIMEOUT_SECONDS redirect_uri = f"http://{OAUTH_REDIRECT_HOST}:{OAUTH_REDIRECT_PORT}{OAUTH_CALLBACK_PATH}" diff --git a/cloudinary_cli/utils/config_resolver.py b/cloudinary_cli/utils/config_resolver.py index 252d89d..24fc308 100644 --- a/cloudinary_cli/utils/config_resolver.py +++ b/cloudinary_cli/utils/config_resolver.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import cloudinary +from click import UsageError from cloudinary_cli.auth import refresh_url_if_stale from cloudinary_cli.auth.session import strip_oauth_internal_keys @@ -41,6 +42,9 @@ def resolve_cli_config(config=None, config_saved=None): _active_name = None _active_source = None + if config and config_saved: + raise UsageError("-c/--config and -C/--config_saved are mutually exclusive; pass only one.") + if config: _active_source = "url" refresh_cloudinary_config(config) diff --git a/cloudinary_cli/utils/config_utils.py b/cloudinary_cli/utils/config_utils.py index 9cf4557..b47b786 100644 --- a/cloudinary_cli/utils/config_utils.py +++ b/cloudinary_cli/utils/config_utils.py @@ -61,9 +61,11 @@ def load_config(): def save_config(config): + # 0600 from the start: the config file holds secrets (api_secret, account_url, OAuth tokens), + # and writing the temp file 0600 before the atomic replace means it is never momentarily + # world-readable (unlike a chmod applied after the replace). _verify_file_path(CLOUDINARY_CLI_CONFIG_FILE) - write_json_to_file(config, CLOUDINARY_CLI_CONFIG_FILE, atomic=True) - _restrict_permissions(CLOUDINARY_CLI_CONFIG_FILE) + write_json_to_file(config, CLOUDINARY_CLI_CONFIG_FILE, atomic=True, mode=0o600) _invalidate_config_cache() # next load_config re-stats and reloads our own write @@ -337,11 +339,3 @@ def ping_cloudinary(**options): def _verify_file_path(file): os.makedirs(os.path.dirname(file), exist_ok=True) - - -def _restrict_permissions(file): - # The config file holds secrets (api_secret, account_url, OAuth tokens), so keep it 0600. - try: - os.chmod(file, 0o600) - except OSError as e: - logger.debug(f"Could not restrict permissions on {file}: {e}") diff --git a/cloudinary_cli/utils/file_utils.py b/cloudinary_cli/utils/file_utils.py index 8b7dc9a..a13a62e 100644 --- a/cloudinary_cli/utils/file_utils.py +++ b/cloudinary_cli/utils/file_utils.py @@ -40,20 +40,27 @@ } -def atomic_write(filename, write_fn): +def atomic_write(filename, write_fn, mode=None): """ Writes via a temp file in the same directory, then atomically replaces the target, so a concurrent reader never sees a half-written file and an interleaved write can't truncate it. :param filename: The destination file path. :param write_fn: Callable receiving the open temp file object; performs the actual write. + :param mode: Final permission bits to set on the file. When given, the temp file is set to + this mode before the replace, so the destination is never momentarily wider + (mkstemp creates it 0600, so a secret file is never world-readable mid-write). + When omitted, normalize to the process umask default like a plain open(). """ directory = path.dirname(filename) or "." fd, tmp_path = tempfile.mkstemp(dir=directory, prefix=".tmp-") try: with os.fdopen(fd, 'w') as file: write_fn(file) - _apply_umask_permissions(tmp_path) + if mode is not None: + os.chmod(tmp_path, mode) + else: + _apply_umask_permissions(tmp_path) os.replace(tmp_path, filename) except BaseException: try: diff --git a/cloudinary_cli/utils/json_utils.py b/cloudinary_cli/utils/json_utils.py index 58ab1f0..31d86d6 100644 --- a/cloudinary_cli/utils/json_utils.py +++ b/cloudinary_cli/utils/json_utils.py @@ -15,12 +15,12 @@ def read_json_from_file(filename, does_not_exist_ok=False): return json.loads(file.read() or "{}") -def write_json_to_file(json_obj, filename, indent=2, sort_keys=False, atomic=False): +def write_json_to_file(json_obj, filename, indent=2, sort_keys=False, atomic=False, mode=None): def dump(file): json.dump(json_obj, file, indent=indent, sort_keys=sort_keys) if atomic: - atomic_write(filename, dump) + atomic_write(filename, dump, mode=mode) else: with open(filename, 'w') as file: dump(file) diff --git a/test/test_auth_loopback.py b/test/test_auth_loopback.py index 5d8a139..5a0ddb5 100644 --- a/test/test_auth_loopback.py +++ b/test/test_auth_loopback.py @@ -1,9 +1,12 @@ +import socket import threading import unittest import urllib.request from http.server import HTTPServer +from unittest.mock import patch -from cloudinary_cli.auth.loopback_server import _CallbackHandler, wait_for_callback +import cloudinary_cli.auth.loopback_server as loopback_server +from cloudinary_cli.auth.loopback_server import _CallbackHandler, start_callback_server, wait_for_callback class TestLoopbackServer(unittest.TestCase): @@ -64,3 +67,20 @@ def run(): waiter.join(timeout=5) self.assertIsInstance(error.get("e"), RuntimeError) self.assertIn("access_denied", str(error["e"])) + + +class TestStartCallbackServerPortBusy(unittest.TestCase): + """A2: a busy redirect port must surface a clear RuntimeError, not a raw OSError.""" + + def test_port_in_use_raises_friendly_error(self): + busy = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + busy.bind(("127.0.0.1", 0)) + busy.listen(1) + port = busy.getsockname()[1] + self.addCleanup(busy.close) + with patch.object(loopback_server, "OAUTH_REDIRECT_HOST", "127.0.0.1"), \ + patch.object(loopback_server, "OAUTH_REDIRECT_PORT", port): + with self.assertRaises(RuntimeError) as ctx: + start_callback_server() + self.assertIn("local login server", str(ctx.exception)) + self.assertIn("in use", str(ctx.exception)) diff --git a/test/test_auth_session.py b/test/test_auth_session.py index 27430e9..afca8e4 100644 --- a/test/test_auth_session.py +++ b/test/test_auth_session.py @@ -139,6 +139,37 @@ def test_refresh_timeout_returns_stale_url(self): self.assertEqual(stale_url, refresh_url_if_stale("eu-cloud", stale_url)) update_config.assert_not_called() + def test_refresh_failure_warns_once_per_config(self): + # A3a: a failed background refresh must surface a re-login hint (not just a debug line), but + # only once per config so a bulk run does not log it per asset. + import requests + import cloudinary_cli.auth as auth + auth._refresh_warned.discard("eu-cloud") + self.addCleanup(auth._refresh_warned.discard, "eu-cloud") + stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) + with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ + patch("cloudinary_cli.auth.flow.refresh", side_effect=requests.ConnectionError()), \ + patch("cloudinary_cli.auth.update_config"), \ + patch("cloudinary_cli.auth.logger.warning") as warn: + refresh_url_if_stale("eu-cloud", stale_url) + refresh_url_if_stale("eu-cloud", stale_url) # second stale read in the same run + warn.assert_called_once() + self.assertIn("cld login eu-cloud", warn.call_args[0][0]) + + def test_refresh_success_rearms_the_warning(self): + # After a successful refresh the warning is re-armed, so a later failure warns again. + import requests + import cloudinary_cli.auth as auth + auth._refresh_warned.add("eu-cloud") + self.addCleanup(auth._refresh_warned.discard, "eu-cloud") + stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) + token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.update_config"): + refresh_url_if_stale("eu-cloud", stale_url) + self.assertNotIn("eu-cloud", auth._refresh_warned) + def test_adopts_peer_refresh_without_calling_refresh(self): # Peer already rewrote the saved URL to a fresh token while we waited for the lock. stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) diff --git a/test/test_cli_config_oauth.py b/test/test_cli_config_oauth.py index 9e20a51..27b4484 100644 --- a/test/test_cli_config_oauth.py +++ b/test/test_cli_config_oauth.py @@ -457,6 +457,14 @@ def test_default_pointing_at_deleted_config_is_ignored(self): result = self._invoke(['url', 'sample'], saved=saved) self.assertIn("No Cloudinary configuration found.", result.output) + def test_inline_url_and_saved_together_errors(self): + # A1: -c and -C are mutually exclusive; passing both must error, not silently drop one. + saved = {"eu-cloud": _oauth_url()} + result = self._invoke( + ['-c', 'cloudinary://a:b@inline', '-C', 'eu-cloud', 'url', 'sample'], saved=saved) + self.assertEqual(2, result.exit_code) + self.assertIn("mutually exclusive", result.output) + class TestResolverNoNetworkIO(_RestoresSdkConfig): """Finding 1 regression: resolution never refreshes a stale OAuth token (no network I/O).""" diff --git a/test/test_config_cache.py b/test/test_config_cache.py new file mode 100644 index 0000000..b411875 --- /dev/null +++ b/test/test_config_cache.py @@ -0,0 +1,62 @@ +"""The parsed-config cache in load_config(): it skips the re-read+parse when the file is unchanged +on disk, but must return a fresh copy each call (callers mutate in place), must invalidate on our +own save, and must reload when a peer rewrites the file (os.replace stamps a new mtime).""" +import os +import tempfile +import unittest +from unittest.mock import patch + +import cloudinary_cli.utils.config_utils as cu + + +class TestLoadConfigCache(unittest.TestCase): + def setUp(self): + self._dir = tempfile.mkdtemp() + self._path = os.path.join(self._dir, "config.json") + self._patch = patch.object(cu, "CLOUDINARY_CLI_CONFIG_FILE", self._path) + self._patch.start() + cu._invalidate_config_cache() + self.addCleanup(self._patch.stop) + self.addCleanup(cu._invalidate_config_cache) + + def _write(self, text): + with open(self._path, "w") as f: + f.write(text) + + def test_returns_fresh_copy_so_caller_mutation_does_not_leak(self): + self._write('{"a": "cloudinary://k:s@a"}') + first = cu.load_config() + first["injected"] = "mutated" # callers do cfg.update(...) on the result + second = cu.load_config() + self.assertNotIn("injected", second) # the cache was not poisoned by the caller's mutation + + def test_cache_hit_skips_reparse_when_unchanged(self): + self._write('{"a": "cloudinary://k:s@a"}') + cu.load_config() # populates the cache + with patch.object(cu, "read_json_from_file") as read: + cu.load_config() + read.assert_not_called() # served from cache: no second read+parse + + def test_reloads_when_file_changes_on_disk(self): + self._write('{"a": "cloudinary://k:s@a"}') + self.assertIn("a", cu.load_config()) + # A peer rewrite changes mtime/size; os.utime forces a distinct mtime even on a fast disk. + self._write('{"a": "cloudinary://k:s@a", "b": "cloudinary://k:s@b"}') + os.utime(self._path, (1, 1)) + self.assertIn("b", cu.load_config()) + + def test_save_config_invalidates_cache(self): + self._write('{"a": "cloudinary://k:s@a"}') + cu.load_config() # warm the cache + cu.save_config({"a": "cloudinary://k:s@a", "c": "cloudinary://k:s@c"}) + self.assertIn("c", cu.load_config()) # invalidated -> reloaded our own write + + def test_missing_file_caches_empty_without_error(self): + self.assertEqual({}, cu.load_config()) # no file: empty dict, no exception + self._write('{"a": "cloudinary://k:s@a"}') + os.utime(self._path, (2, 2)) + self.assertIn("a", cu.load_config()) # appears once the file is created + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_file_utils.py b/test/test_file_utils.py index 212b37d..54e52cb 100644 --- a/test/test_file_utils.py +++ b/test/test_file_utils.py @@ -4,6 +4,7 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch from cloudinary_cli.utils.file_utils import ( get_destination_folder, @@ -107,6 +108,36 @@ def test_respects_restrictive_umask(self): mode = stat.S_IMODE(os.stat(self.path).st_mode) self.assertEqual(0o600, mode) + @unittest.skipIf(sys.platform == "win32", "POSIX permission bits") + def test_explicit_mode_overrides_umask(self): + # A4: with an explicit mode the result is that mode regardless of a permissive umask, so the + # config file is never widened to the umask default. + old_umask = os.umask(0o000) + try: + atomic_write(self.path, lambda f: f.write("x"), mode=0o600) + finally: + os.umask(old_umask) + self.assertEqual(0o600, stat.S_IMODE(os.stat(self.path).st_mode)) + + @unittest.skipIf(sys.platform == "win32", "POSIX permission bits") + def test_explicit_mode_temp_file_never_wider_during_write(self): + # The temp file must already carry the final mode before the replace, so there is no instant + # at which the destination is world-readable. Capture the temp file's mode at replace time. + seen = {} + real_replace = os.replace + + def capturing_replace(src, dst): + seen["mode"] = stat.S_IMODE(os.stat(src).st_mode) + return real_replace(src, dst) + + old_umask = os.umask(0o000) + try: + with patch("cloudinary_cli.utils.file_utils.os.replace", side_effect=capturing_replace): + atomic_write(self.path, lambda f: f.write("x"), mode=0o600) + finally: + os.umask(old_umask) + self.assertEqual(0o600, seen["mode"]) # 0600 on the temp file, before it becomes the target + def test_writes_to_filename_in_cwd_without_dir(self): # path.dirname("") is "" -> must fall back to "." rather than failing. old_cwd = os.getcwd() diff --git a/test/test_oauth_token_seam.py b/test/test_oauth_token_seam.py new file mode 100644 index 0000000..b3a1558 --- /dev/null +++ b/test/test_oauth_token_seam.py @@ -0,0 +1,144 @@ +"""The redesign's central premise: the active OAuth config refreshes its token through the single +SDK seam, `cloudinary.config().oauth_token`, read at request build time. These tests exercise that +seam the way the SDK does (call_api / uploader) rather than reading oauth_token directly, and pin +the post-resolve invariant that the active config is always an OAuthConfig.""" +import time +import unittest +from unittest.mock import patch + +import cloudinary + +from cloudinary_cli.auth.oauth_config import OAuthConfig, install_oauth_config, install_env_config +from cloudinary_cli.auth.session import Session, to_cloudinary_url + + +def _url(cloud="eu-cloud", token="eyJ.tok", refresh="rt", region="api-eu", expires_delta=300): + return to_cloudinary_url(Session( + cloud_name=cloud, access_token=token, refresh_token=refresh, + expires_at=int(time.time()) + expires_delta, region=region, + issuer="https://oauth.cloudinary.com/")) + + +class _RestoresSdkConfig(unittest.TestCase): + def setUp(self): + import os + self._env_snapshot = dict(os.environ) + for key in [k for k in os.environ if k.startswith("CLOUDINARY_")]: + del os.environ[key] + cloudinary.reset_config() + self.addCleanup(self._restore) + + def _restore(self): + import os + os.environ.clear() + os.environ.update(self._env_snapshot) + cloudinary.reset_config() + + +class TestSdkSeamTriggersRefresh(_RestoresSdkConfig): + """The SDK reads cloudinary.config().oauth_token to build the Authorization header; that read + (not any CLI-side probe) is what refreshes a stale token.""" + + def _saved_stale(self): + return {"eu-cloud": _url(token="eyJ.old.tok", refresh="rt_old", expires_delta=-10)} + + def test_call_api_authorize_path_refreshes_stale_token(self): + saved = self._saved_stale() + token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.update_config"): + install_oauth_config(saved["eu-cloud"], saved_name="eu-cloud") + # Reproduce verbatim the read cloudinary.api_client.call_api performs at request build + # time (call_api.py:63): options.pop("oauth_token", cloudinary.config().oauth_token). + options = {} + oauth_token = options.pop("oauth_token", cloudinary.config().oauth_token) + self.assertEqual("eyJ.new.tok", oauth_token) + + def test_uploader_header_path_refreshes_stale_token(self): + saved = self._saved_stale() + token_response = {"access_token": "eyJ.fresh.tok", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.update_config"): + install_oauth_config(saved["eu-cloud"], saved_name="eu-cloud") + # The uploader reads the same attribute to set the Bearer header (uploader.py:877): + # oauth_token = options.get("oauth_token", cloudinary.config().oauth_token). + import cloudinary.uploader # noqa: F401 (ensures the seam module is importable) + options = {} + token = options.get("oauth_token", cloudinary.config().oauth_token) + self.assertEqual("eyJ.fresh.tok", token) + + def test_seam_read_refreshes_only_once_then_serves_cached(self): + saved = self._saved_stale() + token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=token_response) as refresh, \ + patch("cloudinary_cli.auth.update_config"): + install_oauth_config(saved["eu-cloud"], saved_name="eu-cloud") + first = cloudinary.config().oauth_token + second = cloudinary.config().oauth_token + self.assertEqual("eyJ.new.tok", first) + self.assertEqual("eyJ.new.tok", second) + refresh.assert_called_once() # the now-fresh _session short-circuits the second read + + +class TestPostResolveInvariant(_RestoresSdkConfig): + """Not-done item #5 / Caveat B: every install seam leaves an OAuthConfig as the active global, so + has_oauth is universal and self-refresh is never silently disabled by a plain Config swap.""" + + runner = None + + def test_saved_oauth_install_is_oauthconfig(self): + install_oauth_config(_url(), saved_name="eu-cloud") + self.assertIsInstance(cloudinary.config(), OAuthConfig) + + def test_inline_url_install_is_oauthconfig(self): + install_oauth_config("cloudinary://key:secret@cloud", saved_name=None) + self.assertIsInstance(cloudinary.config(), OAuthConfig) + + def test_env_install_is_oauthconfig(self): + import os + os.environ["CLOUDINARY_URL"] = "cloudinary://k:s@env_cloud" + cloudinary.reset_config() + install_env_config() + self.assertIsInstance(cloudinary.config(), OAuthConfig) + + def test_resolver_leaves_oauthconfig_for_every_branch(self): + from click.testing import CliRunner + from cloudinary_cli.cli import cli + import os + saved = {"__default__": "eu-cloud", "eu-cloud": _url()} + with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved)): + for key in ("CLOUDINARY_URL", "CLOUDINARY_CLOUD_NAME", "CLOUDINARY_API_KEY", + "CLOUDINARY_API_SECRET"): + os.environ.pop(key, None) + cloudinary.reset_config() + # default branch + CliRunner().invoke(cli, ['url', 'sample']) + self.assertIsInstance(cloudinary.config(), OAuthConfig) + + +class TestEnvConfigStatic(_RestoresSdkConfig): + """An env-installed OAuthConfig is static: it has no saved name, so reading oauth_token never + refreshes even if the token is expired (it cannot rotate an env-supplied token).""" + + def test_env_oauth_token_never_refreshes_even_when_stale(self): + import os + os.environ["CLOUDINARY_URL"] = ( + "cloudinary://env_cloud?oauth_token=eyJ.env.tok&refresh_token=rt&" + f"expires_at={int(time.time()) - 10}®ion=api") + cloudinary.reset_config() + with patch("cloudinary_cli.auth.flow.refresh") as refresh: + cfg = install_env_config() + token = cfg.oauth_token + self.assertEqual("eyJ.env.tok", token) + refresh.assert_not_called() + self.assertIsNone(getattr(cfg, "_session")) # static: no parsed session to drive a refresh + + +if __name__ == "__main__": + unittest.main() From 00f27d7f229137b84f6c7805d30a732241426f90 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 04:03:05 +0300 Subject: [PATCH 06/23] Handle non-interactive stdin/stdout for OAuth, confirms, and JSON output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously a closed/non-interactive stdin raised a bare EOFError from input(), surfacing as a blank "Command execution failed" with exit 0 — so a destructive bulk op or a name-less `cld logout` silently no-op'd while reporting success. Separately, colored --json output was only suppressed on Windows, so it relied on click.echo's implicit ANSI stripping for piped output. Interactivity is now centralized in cloudinary_cli/utils/utils.py: - is_interactive(): the single sys.stdin.isatty() check. - prompt_user(): the single input() call; returns None on EOF, logging an optional hint so the caller's decision is never a silent no-op. - get_user_action / confirm_action delegate to prompt_user; on no input they apply the default and hint at --force (-F). Behavior: - `cld logout` with no name: on EOF, error with the `cld logout ` form and exit non-zero (the selection has no flag substitute). - `cld login`: when no browser opens and stdin is not a TTY, fail fast with a headless-usage hint (-c/-C) instead of blocking until the 300s callback timeout. - print_json: colorize only when stdout is an interactive TTY (drops the Windows-only guard), so piped/captured JSON (LLM agents, `| jq`, redirects) is never corrupted by ANSI escapes. Co-Authored-By: Claude Opus 4.8 (1M context) --- cloudinary_cli/auth/__init__.py | 15 +++++++-- cloudinary_cli/core/auth.py | 11 +++++-- cloudinary_cli/utils/json_utils.py | 6 ++-- cloudinary_cli/utils/utils.py | 33 +++++++++++++++++-- test/test_auth_session.py | 34 +++++++++++++++++++ test/test_cli_config_oauth.py | 12 +++++++ test/test_json_utils.py | 26 +++++++++++++++ test/test_utils.py | 52 +++++++++++++++++++++++++++++- 8 files changed, 179 insertions(+), 10 deletions(-) diff --git a/cloudinary_cli/auth/__init__.py b/cloudinary_cli/auth/__init__.py index e320cfe..7eb5bc2 100644 --- a/cloudinary_cli/auth/__init__.py +++ b/cloudinary_cli/auth/__init__.py @@ -25,7 +25,7 @@ is_reserved_config_name, is_env_configured, ) -from cloudinary_cli.utils.utils import log_exception +from cloudinary_cli.utils.utils import log_exception, is_interactive # Names already warned about a failed background refresh, so a bulk run (many workers, each reading # the token) logs the re-login hint once per config instead of once per asset. Mutated only under @@ -179,7 +179,18 @@ def _run_browser_flow(region): authorize_url = flow.build_authorize_url(challenge, state, redirect_uri, region) logger.info("Opening browser to log in to Cloudinary...") - if not webbrowser.open(authorize_url): + opened = webbrowser.open(authorize_url) + if not opened and not is_interactive(): + # No browser and no interactive terminal: nobody can complete the redirect, so fail fast + # instead of blocking until the callback times out. Headless runs use a pre-set config. + httpd.server_close() + raise RuntimeError( + "cld login needs an interactive browser session, but no browser could be opened and " + "this is not an interactive terminal. For headless/CI use, configure credentials with " + "an API-key URL instead: `cld -c cloudinary://:@ ` or save " + "one with `cld config -n ` and select it via `-C `." + ) + if not opened: logger.info(f"Could not open a browser. Visit this URL to log in:\n{authorize_url}") else: logger.info(f"If it doesn't open automatically, visit:\n{authorize_url}") diff --git a/cloudinary_cli/core/auth.py b/cloudinary_cli/core/auth.py index af1d6f5..e46ab81 100644 --- a/cloudinary_cli/core/auth.py +++ b/cloudinary_cli/core/auth.py @@ -2,7 +2,7 @@ from cloudinary_cli.auth import login as run_login, logout as run_logout, list_oauth_login_names from cloudinary_cli.defaults import logger -from cloudinary_cli.utils.utils import log_exception +from cloudinary_cli.utils.utils import log_exception, prompt_user @command("login", help="Log in to Cloudinary via OAuth (opens a browser). The session is saved " @@ -64,7 +64,14 @@ def _select_oauth_login(): for i, name in enumerate(names, start=1): echo(f" {i}) {name}") - choice = input(f"Select a login to log out of [1-{len(names)}] (or Enter to cancel): ").strip() + # The selection needs real input that no flag replaces, so on non-interactive stdin prompt_user + # returns None (after logging the hint) and we report it as an invalid (non-zero) outcome. + choice = prompt_user( + f"Select a login to log out of [1-{len(names)}] (or Enter to cancel): ", + noninteractive_hint="Pass the configuration name directly: `cld logout `.") + if choice is None: + return "invalid", None + choice = choice.strip() if not choice: return "cancelled", None if not (choice.isdigit() and 1 <= int(choice) <= len(names)): diff --git a/cloudinary_cli/utils/json_utils.py b/cloudinary_cli/utils/json_utils.py index 31d86d6..b976c34 100644 --- a/cloudinary_cli/utils/json_utils.py +++ b/cloudinary_cli/utils/json_utils.py @@ -1,5 +1,5 @@ import json -from platform import system +import sys from os import path import click from pygments import highlight, lexers, formatters @@ -35,7 +35,9 @@ def update_json_file(json_obj, filename, indent=2, sort_keys=False, atomic=False def print_json(res): res_str = json.dumps(res, indent=2) - if system() != "Windows": + # Colorize only for an interactive terminal. When stdout is piped/redirected/captured (e.g. an + # LLM agent or `| jq`), emit plain JSON so ANSI escapes never corrupt the parsed output. + if sys.stdout.isatty(): res_str = highlight(res_str.encode('UTF-8'), lexers.JsonLexer(), formatters.TerminalFormatter()).strip() click.echo(res_str) diff --git a/cloudinary_cli/utils/utils.py b/cloudinary_cli/utils/utils.py index e0c69a4..8d282a2 100644 --- a/cloudinary_cli/utils/utils.py +++ b/cloudinary_cli/utils/utils.py @@ -2,6 +2,7 @@ import builtins import json import os +import sys from collections import OrderedDict from csv import DictWriter from functools import reduce @@ -198,6 +199,26 @@ def run_tasks_concurrently(func, tasks, concurrent_workers): thread_pool.starmap(func, tasks) +def is_interactive(): + """True when we can prompt the user (stdin is an interactive terminal). The single home for the + interactivity check, so flow code never pokes sys.stdin directly.""" + return sys.stdin.isatty() + + +def prompt_user(message, noninteractive_hint=None): + """ + Read a line of user input. The single place that calls input(): returns None when no input can + be read (closed/non-interactive stdin), logging noninteractive_hint (if given) so the caller's + decision is never a silent no-op. + """ + try: + return input(message) + except EOFError: + if noninteractive_hint: + logger.warning(f"No input available (non-interactive terminal). {noninteractive_hint}") + return None + + def confirm_action(message="Continue? (y/N)"): """ Confirms whether the user wants to continue. @@ -208,10 +229,12 @@ def confirm_action(message="Continue? (y/N)"): :return: Boolean indicating whether user wants to continue. :rtype bool """ - return get_user_action(message, {"y": True, "default": False}) + return get_user_action( + message, {"y": True, "default": False}, + noninteractive_hint="Pass --force (-F) to skip this confirmation in non-interactive runs.") -def get_user_action(message, options): +def get_user_action(message, options, noninteractive_hint=None): """ Reads user input and returns value specified in options. @@ -222,10 +245,14 @@ def get_user_action(message, options): :type message: string :param options: Options mapping. :type options: dict + :param noninteractive_hint: Logged when no input can be read (closed/non-interactive stdin), to + point the user at the flag or piped input that replaces this prompt. The default option is + then applied. :return: Value according to the user selection. """ - r = input(message).lower() + r = prompt_user(message, noninteractive_hint) + r = r.lower() if r is not None else "" # no input -> apply the default option return options.get(r, options.get("default")) diff --git a/test/test_auth_session.py b/test/test_auth_session.py index afca8e4..1a9cfdb 100644 --- a/test/test_auth_session.py +++ b/test/test_auth_session.py @@ -1,5 +1,6 @@ import time import unittest +from unittest import mock from unittest.mock import patch import cloudinary @@ -271,6 +272,39 @@ def test_missing_cloud_name_raises_and_saves_nothing(self): update_config.assert_not_called() +class TestBrowserFlowNonInteractive(unittest.TestCase): + """No browser + no TTY: _run_browser_flow must fail fast with a headless-usage hint, never block + in wait_for_callback until the callback times out.""" + + def test_no_browser_no_tty_fails_fast_without_waiting(self): + from cloudinary_cli.auth import _run_browser_flow + fake_httpd = mock.Mock() + with patch("cloudinary_cli.auth.start_callback_server", + return_value=(fake_httpd, "http://127.0.0.1:49421/callback")), \ + patch("cloudinary_cli.auth.webbrowser.open", return_value=False), \ + patch("cloudinary_cli.auth.is_interactive", return_value=False), \ + patch("cloudinary_cli.auth.wait_for_callback") as wait: + with self.assertRaises(RuntimeError) as ctx: + _run_browser_flow("api-eu") + wait.assert_not_called() # fails fast: no 5-minute callback wait + fake_httpd.server_close.assert_called_once() # releases the bound port + self.assertIn("-c", str(ctx.exception)) # points at the headless API-key alternative + + def test_no_browser_but_tty_still_waits(self): + # A human at a TTY can paste the printed URL, so we must NOT fail fast here. + from cloudinary_cli.auth import _run_browser_flow + with patch("cloudinary_cli.auth.start_callback_server", + return_value=(mock.Mock(), "http://127.0.0.1:49421/callback")), \ + patch("cloudinary_cli.auth.webbrowser.open", return_value=False), \ + patch("cloudinary_cli.auth.is_interactive", return_value=True), \ + patch("cloudinary_cli.auth.wait_for_callback", return_value=("code", "st")) as wait, \ + patch("cloudinary_cli.auth.flow.exchange_code", return_value={"access_token": "x"}): + # state mismatch is irrelevant here; we only assert it reached the wait (did not fast-fail) + with patch("cloudinary_cli.auth.secrets.token_urlsafe", return_value="st"): + _run_browser_flow("api-eu") + wait.assert_called_once() + + class TestDeriveConfigName(unittest.TestCase): def _derive(self, cloud, region, config): with patch("cloudinary_cli.auth.load_config", return_value=config): diff --git a/test/test_cli_config_oauth.py b/test/test_cli_config_oauth.py index 27b4484..3d6e81c 100644 --- a/test/test_cli_config_oauth.py +++ b/test/test_cli_config_oauth.py @@ -109,6 +109,18 @@ def test_out_of_range_errors(self): self.assertFalse(result.return_value) remove.assert_not_called() + def test_noninteractive_stdin_errors_with_hint(self): + # Closed stdin (no input at all): the selection cannot be made, so error with the + # non-interactive form (`cld logout `) and exit non-zero, not a silent no-op. + import builtins + with patch("cloudinary_cli.auth.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ + patch("cloudinary_cli.auth.remove_config_keys") as remove, \ + patch.object(builtins, "input", side_effect=EOFError()): + result = self.runner.invoke(cli, ["logout"], standalone_mode=False) + self.assertIn("cld logout ", result.output) + self.assertFalse(result.return_value) # main() maps falsy -> exit 1 + remove.assert_not_called() + class TestLoginSetDefault(unittest.TestCase): """`login` sets the default explicitly with --set-default and auto-defaults a sole login.""" diff --git a/test/test_json_utils.py b/test/test_json_utils.py index 03687c5..8c00eec 100644 --- a/test/test_json_utils.py +++ b/test/test_json_utils.py @@ -1,13 +1,16 @@ +import json import os import stat import sys import tempfile import unittest +from unittest.mock import patch from cloudinary_cli.utils.json_utils import ( write_json_to_file, read_json_from_file, update_json_file, + print_json, ) @@ -97,5 +100,28 @@ def test_atomic_flag_merges_and_leaves_no_temp(self): self.assertEqual([], leftover) +class PrintJsonColorTest(unittest.TestCase): + """print_json must emit plain (parseable) JSON when stdout is not a terminal, so piped/captured + output (LLM agents, `| jq`, redirects) is never corrupted by ANSI color escapes.""" + + DATA = {"a": 1, "b": "two", "nested": {"c": True}} + + def _captured(self): + with patch("cloudinary_cli.utils.json_utils.click.echo") as echo: + print_json(self.DATA) + return echo.call_args[0][0] + + def test_non_tty_is_plain_parseable_json(self): + with patch("cloudinary_cli.utils.json_utils.sys.stdout.isatty", return_value=False): + out = self._captured() + self.assertNotIn("\x1b[", out) # no ANSI escapes + self.assertEqual(self.DATA, json.loads(out)) # round-trips cleanly + + def test_tty_is_colorized(self): + with patch("cloudinary_cli.utils.json_utils.sys.stdout.isatty", return_value=True): + out = self._captured() + self.assertIn("\x1b[", out) # colorized for an interactive terminal + + if __name__ == "__main__": unittest.main() diff --git a/test/test_utils.py b/test/test_utils.py index b77b3fe..d186d2d 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,7 +1,57 @@ +import builtins import unittest +from unittest.mock import patch from cloudinary_cli.utils.utils import parse_option_value, parse_args_kwargs, whitelist_keys, merge_responses, \ - normalize_list_params, chunker, group_params + normalize_list_params, chunker, group_params, confirm_action, get_user_action, prompt_user, is_interactive + + +class NonInteractiveInputTest(unittest.TestCase): + """A confirmation/selection prompt on closed (non-interactive) stdin must apply the default and + surface a hint, not raise EOFError up to a blank 'Command execution failed' with exit 0.""" + + def _eof(self, *args): + raise EOFError("EOF when reading a line") + + def test_confirm_action_defaults_to_no_on_eof(self): + with patch.object(builtins, "input", self._eof), \ + patch("cloudinary_cli.utils.utils.logger.warning") as warn: + self.assertFalse(confirm_action()) + warn.assert_called_once() + self.assertIn("--force", warn.call_args[0][0]) + + def test_get_user_action_returns_default_on_eof(self): + with patch.object(builtins, "input", self._eof): + self.assertEqual("fallback", + get_user_action("pick: ", {"y": True, "default": "fallback"})) + + def test_get_user_action_no_hint_when_not_provided(self): + with patch.object(builtins, "input", self._eof), \ + patch("cloudinary_cli.utils.utils.logger.warning") as warn: + self.assertIsNone(get_user_action("pick: ", {"y": True})) + warn.assert_not_called() + + def test_empty_line_still_uses_default(self): + # An empty line (piped) is distinct from EOF and already used the default; keep that intact. + with patch.object(builtins, "input", lambda *a: ""): + self.assertFalse(confirm_action()) + + def test_prompt_user_returns_line_when_available(self): + with patch.object(builtins, "input", lambda *a: " 2 "): + self.assertEqual(" 2 ", prompt_user("pick: ")) + + def test_prompt_user_returns_none_and_hints_on_eof(self): + with patch.object(builtins, "input", self._eof), \ + patch("cloudinary_cli.utils.utils.logger.warning") as warn: + self.assertIsNone(prompt_user("pick: ", noninteractive_hint="do X instead")) + warn.assert_called_once() + self.assertIn("do X instead", warn.call_args[0][0]) + + def test_is_interactive_reflects_stdin_isatty(self): + with patch("cloudinary_cli.utils.utils.sys.stdin.isatty", return_value=True): + self.assertTrue(is_interactive()) + with patch("cloudinary_cli.utils.utils.sys.stdin.isatty", return_value=False): + self.assertFalse(is_interactive()) class UtilsTest(unittest.TestCase): From 62186353e681532de2e1ff02bd6ef7ef9c03bd65 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 04:27:43 +0300 Subject: [PATCH 07/23] Run CI on macOS and Windows; make tests cross-platform - CI matrix: full Python (3.10-3.14) on Linux, plus one latest-Python (3.14) smoke job each on macos-latest and windows-latest. A single bash shell runs every step identically on all three (Git Bash ships on the Windows runner). - Guard the remaining POSIX-mode assertions (umask/file-mode) with skipIf(win32); os.chmod only honors the read-only flag on Windows, so exact-mode checks do not apply there (the code still runs, the hardening is just POSIX-only). - Rewrite the loopback port-busy test to mock HTTPServer raising OSError instead of relying on a real double-bind, which Windows does not reject like POSIX does. - Add print_json tests asserting non-TTY output stays plain on Windows too and that colorization is decided by isatty(), not the OS. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/cloudinary-cli-test.yml | 20 +++++++++++++++++--- test/test_auth_loopback.py | 23 ++++++++++------------- test/test_auth_session.py | 1 - test/test_file_utils.py | 2 ++ test/test_json_utils.py | 20 +++++++++++++++++++- 5 files changed, 48 insertions(+), 18 deletions(-) diff --git a/.github/workflows/cloudinary-cli-test.yml b/.github/workflows/cloudinary-cli-test.yml index 9d0664f..dfef328 100644 --- a/.github/workflows/cloudinary-cli-test.yml +++ b/.github/workflows/cloudinary-cli-test.yml @@ -9,11 +9,25 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + # Full Python matrix on Linux; macOS and Windows get a single latest-Python smoke job each + # (enough to catch platform-specific regressions without 3x the runners and test clouds). + os: [ubuntu-latest] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + include: + - os: macos-latest + python-version: "3.14" + - os: windows-latest + python-version: "3.14" + + # Git Bash ships on the GitHub Windows runners, so a single bash shell keeps every step + # identical across Linux, macOS, and Windows (no PowerShell variants to maintain). + defaults: + run: + shell: bash steps: - uses: actions/checkout@v4 @@ -35,7 +49,7 @@ jobs: - name: Get test cloud run: echo "CLOUDINARY_URL=$(bash tools/get_test_cloud.sh)" >> $GITHUB_ENV - name: Show test cloud - run: echo $CLOUDINARY_URL | cut -d'@' -f2 + run: echo "$CLOUDINARY_URL" | cut -d'@' -f2 - name: Test with pytest run: | pytest diff --git a/test/test_auth_loopback.py b/test/test_auth_loopback.py index 5a0ddb5..8985e0e 100644 --- a/test/test_auth_loopback.py +++ b/test/test_auth_loopback.py @@ -1,11 +1,9 @@ -import socket import threading import unittest import urllib.request from http.server import HTTPServer from unittest.mock import patch -import cloudinary_cli.auth.loopback_server as loopback_server from cloudinary_cli.auth.loopback_server import _CallbackHandler, start_callback_server, wait_for_callback @@ -70,17 +68,16 @@ def run(): class TestStartCallbackServerPortBusy(unittest.TestCase): - """A2: a busy redirect port must surface a clear RuntimeError, not a raw OSError.""" + """A2: a failed bind (e.g. busy redirect port) must surface a clear RuntimeError, not a raw + OSError. The bind is mocked to fail so the test is deterministic across OSes (Windows does not + raise on a double-bind the way POSIX does).""" - def test_port_in_use_raises_friendly_error(self): - busy = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - busy.bind(("127.0.0.1", 0)) - busy.listen(1) - port = busy.getsockname()[1] - self.addCleanup(busy.close) - with patch.object(loopback_server, "OAUTH_REDIRECT_HOST", "127.0.0.1"), \ - patch.object(loopback_server, "OAUTH_REDIRECT_PORT", port): + def test_bind_failure_raises_friendly_error(self): + with patch("cloudinary_cli.auth.loopback_server.HTTPServer", + side_effect=OSError(48, "Address already in use")): with self.assertRaises(RuntimeError) as ctx: start_callback_server() - self.assertIn("local login server", str(ctx.exception)) - self.assertIn("in use", str(ctx.exception)) + msg = str(ctx.exception) + self.assertIn("local login server", msg) + self.assertIn("in use", msg) + self.assertIsInstance(ctx.exception.__cause__, OSError) # chains the original diff --git a/test/test_auth_session.py b/test/test_auth_session.py index 1a9cfdb..ee9a484 100644 --- a/test/test_auth_session.py +++ b/test/test_auth_session.py @@ -159,7 +159,6 @@ def test_refresh_failure_warns_once_per_config(self): def test_refresh_success_rearms_the_warning(self): # After a successful refresh the warning is re-armed, so a later failure warns again. - import requests import cloudinary_cli.auth as auth auth._refresh_warned.add("eu-cloud") self.addCleanup(auth._refresh_warned.discard, "eu-cloud") diff --git a/test/test_file_utils.py b/test/test_file_utils.py index 54e52cb..d663005 100644 --- a/test/test_file_utils.py +++ b/test/test_file_utils.py @@ -88,6 +88,7 @@ def test_missing_target_is_not_created_on_failure(self): self.assertFalse(os.path.exists(self.path)) self.assertEqual([], os.listdir(self.dir)) + @unittest.skipIf(sys.platform == "win32", "POSIX file modes not applicable on Windows") def test_normalizes_to_umask_mode(self): # mkstemp creates the temp as 0600; atomic_write must relax it to the umask default # so output files are not silently owner-only. @@ -99,6 +100,7 @@ def test_normalizes_to_umask_mode(self): mode = stat.S_IMODE(os.stat(self.path).st_mode) self.assertEqual(0o644, mode) + @unittest.skipIf(sys.platform == "win32", "POSIX file modes not applicable on Windows") def test_respects_restrictive_umask(self): old_umask = os.umask(0o077) try: diff --git a/test/test_json_utils.py b/test/test_json_utils.py index 8c00eec..381a126 100644 --- a/test/test_json_utils.py +++ b/test/test_json_utils.py @@ -1,6 +1,5 @@ import json import os -import stat import sys import tempfile import unittest @@ -122,6 +121,25 @@ def test_tty_is_colorized(self): out = self._captured() self.assertIn("\x1b[", out) # colorized for an interactive terminal + def test_non_tty_is_plain_on_windows_too(self): + # The automation guarantee must hold on every OS: a non-terminal stdout yields plain JSON + # regardless of platform (the old code special-cased Windows; the branch is OS-independent now). + with patch("cloudinary_cli.utils.json_utils.sys.platform", "win32"), \ + patch("cloudinary_cli.utils.json_utils.sys.stdout.isatty", return_value=False): + out = self._captured() + self.assertNotIn("\x1b[", out) + self.assertEqual(self.DATA, json.loads(out)) + + def test_colorization_is_decided_by_isatty_not_os(self): + # Same isatty() value -> same colorization decision under any reported platform, so Windows + # interactive consoles get color (click.echo translates ANSI) and Windows pipes stay plain. + for plat in ("win32", "darwin", "linux"): + with patch("cloudinary_cli.utils.json_utils.sys.platform", plat): + with patch("cloudinary_cli.utils.json_utils.sys.stdout.isatty", return_value=True): + self.assertIn("\x1b[", self._captured(), f"expected color on tty ({plat})") + with patch("cloudinary_cli.utils.json_utils.sys.stdout.isatty", return_value=False): + self.assertNotIn("\x1b[", self._captured(), f"expected plain off tty ({plat})") + if __name__ == "__main__": unittest.main() From 1d7793276765ab88f6812c7ad464648d26be0e39 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 05:03:49 +0300 Subject: [PATCH 08/23] Fix Python 3.10 test failures: rename config command to avoid module shadowing The `config` command function shadowed the `cloudinary_cli.core.config` submodule as a package attribute, so on Python 3.10 mock.patch resolved "cloudinary_cli.core.config." to the click.Command (no such attribute) instead of the module. Python 3.11+ prefers the real submodule, so only the 3.10 CI job failed. Rename the function to config_command (CLI command name "config" is unchanged) so the submodule is never shadowed. Co-Authored-By: Claude Opus 4.8 (1M context) --- cloudinary_cli/core/__init__.py | 4 ++-- cloudinary_cli/core/config.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinary_cli/core/__init__.py b/cloudinary_cli/core/__init__.py index c9afce8..8e2edeb 100644 --- a/cloudinary_cli/core/__init__.py +++ b/cloudinary_cli/core/__init__.py @@ -2,7 +2,7 @@ from cloudinary_cli.core.admin import admin from cloudinary_cli.core.auth import login, logout -from cloudinary_cli.core.config import config +from cloudinary_cli.core.config import config_command from cloudinary_cli.core.search import search, search_folders from cloudinary_cli.core.uploader import uploader from cloudinary_cli.core.provisioning import provisioning @@ -12,7 +12,7 @@ setattr(click.Group, "resolve_command", resolve_command) commands = [ - config, + config_command, login, logout, search, diff --git a/cloudinary_cli/core/config.py b/cloudinary_cli/core/config.py index 0025ecc..258e118 100644 --- a/cloudinary_cli/core/config.py +++ b/cloudinary_cli/core/config.py @@ -55,8 +55,8 @@ help="Refresh every saved OAuth configuration whose token is stale.") @option("-f", "--force", "force", is_flag=True, help="With --refresh/--refresh-all, refresh even tokens that are still fresh.") -def config(new, ls, as_json, show, rm, from_url, default, set_default, unset_default, - refresh, refresh_all, force): +def config_command(new, ls, as_json, show, rm, from_url, default, set_default, unset_default, + refresh, refresh_all, force): if set_default and not (new or from_url): raise UsageError("--set-default requires -n or --from_url; " "to default an existing config use -d .") From 49aa9772cc12cf6c48814f3b121122edd2e51eed Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 05:05:22 +0300 Subject: [PATCH 09/23] Document OAuth login, default config, and token refresh in the README Add `cld login`/`logout` as the recommended setup path (no API secret on disk), and document choosing a default configuration, resolution precedence, and manual OAuth token refresh. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3769a80..aa54055 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,17 @@ Python 3.6 or later. You can install Python from [https://www.python.org/](http ## Setup and Installation 1. To install this package, run: `pip3 install cloudinary-cli` -2. To make all your `cld` commands point to your Cloudinary account, set up your CLOUDINARY\_URL environment variable. For example: +2. Point your `cld` commands at a Cloudinary account using **either** of the following: + + **Option A — Log in with OAuth (recommended).** Run: + + ``` + cld login + ``` + + This opens your browser to authorize the CLI, then saves the login as a configuration (named after the cloud) and sets it as the default. No API secret is stored on disk — the saved login holds a short-lived token that the CLI refreshes automatically. For an account in a non-default region, pass `--region`, for example `cld login --region eu`. + + **Option B — Set your CLOUDINARY\_URL environment variable.** For example: * On Mac or Linux:
`export CLOUDINARY_URL=cloudinary://123456789012345:abcdefghijklmnopqrstuvwxyzA@cloud_name` * On Windows (cmd.exe):
`set CLOUDINARY_URL=cloudinary://123456789012345:abcdefghijklmnopqrstuvwxyzA@cloud_name` * On Windows (PowerShell):
`$Env:CLOUDINARY_URL="cloudinary://123456789012345:abcdefghijklmnopqrstuvwxyzA@cloud_name"` @@ -47,6 +57,8 @@ Usage: cld [cli options] [command] [command options] [method] [method parameters ``` cld --help # Lists available commands. +cld login # Logs in to a Cloudinary account via OAuth in your browser. +cld logout # Removes a saved OAuth login. cld search --help # Shows usage for the Search API. cld admin # Lists Admin API methods. cld uploader # Lists Upload API methods. @@ -243,7 +255,7 @@ Whereas using the saved configuration "accountx": cld -C accountx admin usage ``` -_**Caution:** Creating a saved configuration may put your API secret at risk as it is stored in a local plain text file._ +_**Caution:** A saved API-key configuration stores your API secret in a local file. An OAuth login (see below) avoids this by storing a short-lived, auto-refreshed token instead._ You can create, delete and list saved configurations using the `config` command. @@ -252,3 +264,41 @@ cld config [options] ``` For details, see the [Cloudinary CLI documentation](https://cloudinary.com/documentation/cloudinary_cli#config). + +### Logging in with OAuth + +Instead of saving an API key and secret, you can log in to a Cloudinary account through your browser. The CLI saves the resulting session as a named configuration and refreshes its token automatically. + +``` +cld login # Log in and save the configuration (named after the cloud). +cld login --region eu # Log in to an account in a non-default region. +cld login my-account # Save the login under a specific name. +cld logout # Choose a saved OAuth login to remove. +cld logout my-account # Remove a specific saved OAuth login. +``` + +Once saved, an OAuth login is selected with `-C ` just like any other saved configuration. + +### Choosing a default configuration + +The default configuration is used when no `-c`/`-C` option is given and no `CLOUDINARY_URL` environment variable is set. The first OAuth login becomes the default automatically; you can change it at any time. + +``` +cld config -d # Set an existing saved configuration as the default. +cld config --unset-default # Clear the stored default. +cld config -ls # List saved configurations, marking the default and the active one. +``` + +When creating a configuration with `-n` or `--from_url`, add `--set-default` to make it the default in the same step. Resolution precedence is: `-c` (inline URL) > `-C` (saved name) > stored default > `CLOUDINARY_URL` environment variable. + +### Refreshing OAuth tokens + +OAuth tokens are refreshed automatically as needed, but you can refresh them manually. + +``` +cld config --refresh # Refresh a saved OAuth configuration's token. +cld config --refresh-all # Refresh every saved OAuth configuration whose token is stale. +cld config --refresh --force # Refresh even if the token is still fresh. +``` + +If a token can no longer be refreshed (for example, the login was revoked), the CLI reports the configuration and the `cld login` command to use to log in again. From 0598ba4d6c968984083e9cff5001e98ec2e7a960 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 05:07:22 +0300 Subject: [PATCH 10/23] Drop --region from README OAuth docs (advanced use case) Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index aa54055..750bca3 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Python 3.6 or later. You can install Python from [https://www.python.org/](http cld login ``` - This opens your browser to authorize the CLI, then saves the login as a configuration (named after the cloud) and sets it as the default. No API secret is stored on disk — the saved login holds a short-lived token that the CLI refreshes automatically. For an account in a non-default region, pass `--region`, for example `cld login --region eu`. + This opens your browser to authorize the CLI, then saves the login as a configuration (named after the cloud) and sets it as the default. No API secret is stored on disk — the saved login holds a short-lived token that the CLI refreshes automatically. **Option B — Set your CLOUDINARY\_URL environment variable.** For example: * On Mac or Linux:
`export CLOUDINARY_URL=cloudinary://123456789012345:abcdefghijklmnopqrstuvwxyzA@cloud_name` @@ -271,7 +271,6 @@ Instead of saving an API key and secret, you can log in to a Cloudinary account ``` cld login # Log in and save the configuration (named after the cloud). -cld login --region eu # Log in to an account in a non-default region. cld login my-account # Save the login under a specific name. cld logout # Choose a saved OAuth login to remove. cld logout my-account # Remove a specific saved OAuth login. From 7f9a2f62a03a59a8d305d233b77ed3c28a8f3e74 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 05:11:10 +0300 Subject: [PATCH 11/23] Untrack internal OAuth design/review docs These are working notes, not part of the published package; keep them local. Co-Authored-By: Claude Opus 4.8 (1M context) --- OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md | 221 --------------------- OAUTH_LAZY_TOKEN_HANDOFF.md | 137 ------------- OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md | 196 ------------------- OAUTH_REFRESH_CONCURRENCY_REVIEW.md | 266 -------------------------- 4 files changed, 820 deletions(-) delete mode 100644 OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md delete mode 100644 OAUTH_LAZY_TOKEN_HANDOFF.md delete mode 100644 OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md delete mode 100644 OAUTH_REFRESH_CONCURRENCY_REVIEW.md diff --git a/OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md b/OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md deleted file mode 100644 index 8d3e855..0000000 --- a/OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md +++ /dev/null @@ -1,221 +0,0 @@ -# Self-refreshing `oauth_token` — caveats explained with code, + an mtime-cached `load_config` - -> Companion to `OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md`. Two questions: -> 1. The two "make-or-break" caveats, shown concretely in code. -> 2. Can we skip reloading `config.json` when its modified-time is unchanged, to kill the per-call overhead? - ---- - -## Caveat A — reading `oauth_token` for *truthiness* must NOT trigger a refresh - -### The problem, concretely - -If `oauth_token` becomes a property that refreshes-on-read, then **any** access triggers it — -including the places that read it only to *classify* a config, none of which want network I/O. -Here are the exact current sites (all offline paths): - -```python -# cloudinary_cli/utils/config_utils.py:283 — runs at the GROUP level, on every command -def is_valid_cloudinary_config(): - if cloudinary.config().cloud_name and cloudinary.config().oauth_token: # <-- truthiness read - return True - return None not in [cloudinary.config().cloud_name, - cloudinary.config().api_key, cloudinary.config().api_secret] - -# cloudinary_cli/utils/config_listing.py:76, 97, 109 — `config -ls`, fully offline -"type": "oauth" if config_obj.oauth_token else "api_key", -"type": "oauth" if active.oauth_token else "api_key", -"type": "oauth" if env_config.oauth_token else "api_key", - -# cloudinary_cli/core/config.py:191 — `config` header, offline -type_label = "oauth" if active.oauth_token else "api_key" -``` - -A property can't distinguish "are you OAuth?" from "give me a token to send." So a naive property -turns `cld config -ls` — which should be 100% offline — into something that refreshes every stale -saved token just to print a table. **That is the exact Finding-1 hang we removed**, reintroduced. - -### The fix: split *presence* from *value* - -Keep a non-refreshing way to ask "is this OAuth / is a token present," and let only the **value** -read on the request path refresh. Two private fields back the public property: - -```python -# cloudinary_cli/auth/oauth_config.py (new) -import cloudinary -from cloudinary_cli.auth import refresh_url_if_stale -from cloudinary_cli.auth.session import from_cloudinary_url - -class OAuthConfig(cloudinary.Config): - """A Config whose oauth_token refreshes itself on read for the request path, while presence/ - type checks read the raw stored token and never touch the network.""" - - def bind_saved(self, name, url): - # association the resolver used to keep in a module global — now lives on the object - self._saved_name = name - self._raw_oauth_token = from_cloudinary_url(url).access_token - self._oauth_url = url - - # --- presence: cheap, no network. Use THIS in type/validity checks. --- - @property - def has_oauth(self): - return bool(getattr(self, "_raw_oauth_token", None)) - - # --- value: refresh-if-stale, used by the SDK on the request path. --- - @property - def oauth_token(self): - name = getattr(self, "_saved_name", None) - if name is None: - # env / -c / api-key config: serve the static value, never refresh - return getattr(self, "_raw_oauth_token", None) - fresh_url = refresh_url_if_stale(name, self._oauth_url) # existing lock+double-check+persist - if fresh_url != self._oauth_url: - self._oauth_url = fresh_url - self._raw_oauth_token = from_cloudinary_url(fresh_url).access_token - return self._raw_oauth_token -``` - -Then repoint the offline checks at `has_oauth` (presence), not `oauth_token` (value): - -```python -# is_valid_cloudinary_config -cfg = cloudinary.config() -if cfg.cloud_name and getattr(cfg, "has_oauth", False): # no refresh - return True - -# config_listing / core.config type labels -"type": "oauth" if getattr(config_obj, "has_oauth", False) else "api_key" -``` - -> Note the property must be a **data descriptor on the class** (a `property` is). The SDK's -> `__getattr__` (`return self.__dict__.get(i)`) only fires for *missing* attributes; the token is in -> `__dict__`, so `__getattr__` never sees it — but a class-level `property` *overrides* `__dict__` on -> read. That's why this works where a `__getattr__` hook wouldn't. - -### Subtlety: `config_to_dict` enumerates `__dict__` - -```python -# config_utils.py:97 -def config_to_dict(config): - return {k: v for k, v in config.__dict__.items() if not k.startswith("_")} -``` - -A `property` lives on the class, not in `__dict__`, so `config_to_dict` would **lose** `oauth_token` -from the masked/JSON views. Fix: have the masking layer read the raw token explicitly, e.g. add it -back from `_raw_oauth_token`, or store the raw token under the public key `oauth_token` in `__dict__` -*and* let the property shadow it on read (a property shadows the instance dict on attribute access, -but `__dict__["oauth_token"]` is still there for `config_to_dict` to enumerate). The cleanest: keep -`oauth_token` in `__dict__` for serialization, and have the property's getter read/refresh from it. - ---- - -## Caveat B — refreshing inside the getter must not `reset_config()` or deadlock - -### The problem, concretely - -Today refresh goes through `refresh_cloudinary_config`: - -```python -# config_utils.py -def refresh_cloudinary_config(cloudinary_url): - cloudinary.reset_config() # <-- clears the global singleton - cloudinary.config()._load_from_url(cloudinary_url) -``` - -If the **getter** for `oauth_token` called this, then reading `config().oauth_token` would, mid-read, -`reset_config()` — destroying the very object whose property is executing, and replacing our -`OAuthConfig` with a plain `Config` (property gone). It also opens the reset-then-reload window where -a concurrent worker thread reads a half-cleared global. And `refresh_url_if_stale` → `update_config` -takes the reentrant `config_lock`; doing a global swap underneath that is the fragile part. - -### The fix: refresh in place, never swap the global from inside the getter - -The getter (above) only mutates its **own** `_oauth_url` / `_raw_oauth_token` and lets -`refresh_url_if_stale` handle the **persist** (atomic write under the lock — already correct, -single-flight across threads and processes). No `reset_config()`, no global swap, no half-cleared -window. The lock is reentrant, so the getter running inside an in-progress operation that already -holds it won't deadlock. - -The one place that legitimately swaps the global is the **resolver** (Phase A), once per process — -that's where `OAuthConfig(...).bind_saved(name, url)` gets installed as `cloudinary._config`. Audit -every `reset_config()` call so a saved OAuth login always ends up installed as `OAuthConfig`, never a -plain `Config` (else the property silently disappears — the #1 risk from the design doc). - ---- - -## The mtime cache — yes, and it's the cleaner overhead fix - -Your instinct is good: most `load_config()` calls re-read and re-parse an unchanged file. Cache the -parsed dict keyed on the file's `(mtime, size)`; reload only when it changes. Current code: - -```python -# config_utils.py:32 -def load_config(): - return read_json_from_file(CLOUDINARY_CLI_CONFIG_FILE, does_not_exist_ok=True) -``` - -Cached version: - -```python -import os - -_config_cache = None -_config_cache_stat = None # (st_mtime_ns, st_size) - -def _config_stat(): - try: - st = os.stat(CLOUDINARY_CLI_CONFIG_FILE) - return (st.st_mtime_ns, st.st_size) - except FileNotFoundError: - return None - -def load_config(): - global _config_cache, _config_cache_stat - stat = _config_stat() - if stat is not None and stat == _config_cache_stat and _config_cache is not None: - return _config_cache # unchanged file: skip read + JSON parse - cfg = read_json_from_file(CLOUDINARY_CLI_CONFIG_FILE, does_not_exist_ok=True) - _config_cache, _config_cache_stat = cfg, stat - return cfg -``` - -And invalidate on our own writes so a writer sees its own update immediately: - -```python -def save_config(config): - global _config_cache, _config_cache_stat - _verify_file_path(CLOUDINARY_CLI_CONFIG_FILE) - write_json_to_file(config, CLOUDINARY_CLI_CONFIG_FILE, atomic=True) - _restrict_permissions(CLOUDINARY_CLI_CONFIG_FILE) - _config_cache = None # force re-stat/reload next load_config() - _config_cache_stat = None -``` - -### Caveats on the cache (important) - -1. **mtime granularity.** Use `st_mtime_ns` (nanosecond) + `st_size`, not 1-second `st_mtime`. A - sub-second refresh-then-read on a coarse FS could otherwise miss a change. The `os.replace` in - `atomic_write` updates mtime, so cross-process changes are detected. -2. **Return a copy if callers mutate.** Several callers do `cfg = load_config(); cfg.update(...)`. - If they mutate the returned dict in place they'd corrupt the shared cache. Either return - `dict(cfg)` (cheap; safest) or audit that mutators always go through `update_config` - (which builds on `load_config` then `save_config`). Returning a shallow copy is the safe default. -3. **It does NOT remove the need to read under the lock for refresh.** The - read-modify-write in `refresh_url_if_stale`/`update_config` must still `load_config()` *inside* - the lock to re-check freshness — the cache is fine there too (it re-stats; if a peer just wrote, - mtime changed, it reloads). The cache only saves the *redundant* reads, not the correctness read. -4. **Still mostly moot if we adopt the property.** With the self-refreshing `oauth_token`, the - per-call `load_config()` in `ensure_active_config_fresh` disappears entirely. The mtime cache is - still worth having (other hot reads: `is_valid_cloudinary_config` at group level, refresh loops), - but it's a complementary optimization, not the primary fix. - -### Recommendation on overhead - -- **Primary:** the property approach removes the per-API-call `load_config()` on the hot path - outright (no more `ensure_active_config_fresh`). -- **Secondary:** add the mtime+size cache to `load_config()` (with copy-on-return) so the remaining - reads — group-level validity check, `-ls`, refresh sweeps — stop re-parsing an unchanged file. - -Together they take the steady-state per-request config overhead from "disk read + JSON parse + -URL parse, every call, every thread" down to "one `os.stat`, and a token refresh only when the -5-minute token has actually expired." diff --git a/OAUTH_LAZY_TOKEN_HANDOFF.md b/OAUTH_LAZY_TOKEN_HANDOFF.md deleted file mode 100644 index d86362a..0000000 --- a/OAUTH_LAZY_TOKEN_HANDOFF.md +++ /dev/null @@ -1,137 +0,0 @@ -# Self-refreshing OAuth token — implementation & handoff - -> **For the next agent.** This is the implementation that replaces the scattered -> `ensure_active_config_fresh()` lazy-refresh hooks with a single self-refreshing `oauth_token` on -> the active config object. It builds on the change documented in -> `OAUTH_DEFAULT_CONFIG_IMPLEMENTATION.md` (commit `43fb67a`, branch `oauth-login`) and the design -> rationale in `OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md` + `OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md`. -> Concurrency analysis that motivated the redesign is in `OAUTH_REFRESH_CONCURRENCY_REVIEW.md`. -> -> Read `OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md` first for the "why"; this doc is the "what landed". - ---- - -## 1. What changed and why - -### The problem (see `OAUTH_REFRESH_CONCURRENCY_REVIEW.md`) -The previous design refreshed a stale OAuth token via `ensure_active_config_fresh()` called at four -hand-maintained API chokepoints (`call_api`, `execute_single_request`, `query_cld_folder`, -`cld_folder_exists`). Problems: per-call `load_config()` on the hot path; a hand-maintained -chokepoint list that silently misses new API paths (→ stale-token 401 mid-bulk-run); and a -`reset_config()`-based global swap that races with worker threads under `sync`/`upload-dir`. - -### The fix -Access tokens live ~5 minutes, so refreshes are frequent and expected. Instead of probing before -every call, the **active config's `oauth_token` refreshes itself when the SDK reads it** at request -time. The SDK reads `cloudinary.config().oauth_token` per request (`call_api.py:63`, -`uploader.py:877`) — one universal seam. - -- **New `cloudinary_cli/auth/oauth_config.py`** — `OAuthConfig(cloudinary.Config)`: - - `oauth_token` is a **property** (class-level data descriptor; overrides `__dict__` on read, - which a `__getattr__` hook could not — the SDK stores the token in `__dict__`). - - On read: if the in-object parsed `_session` is fresh (or has no refresh token), return the - stored token with **no I/O**. Only when stale does it `load_config()` + call - `refresh_url_if_stale` (the existing lock + double-check + atomic persist, reused verbatim), - update the in-object token, and return it. Subsequent reads short-circuit on the now-fresh - `_session` — **no per-call disk read or lock once fresh.** - - It does **not** `reset_config()` inside the getter (avoids the global-swap thread race and - self-destruction of the executing object). It mutates only its own fields. - - `has_oauth` property — token *presence* without refreshing. Used by all type/validity checks. - - `from_env()` / `from_url()` factories + `install_oauth_config()` / `install_env_config()`: - **every** active config the CLI installs is now an `OAuthConfig` (saved, env-fallback, and - inline `-c`), so `has_oauth` is universal and the resolver's env branch installs a static - (never-refreshing) OAuthConfig. -- **Deleted** `ensure_active_config_fresh()` and all four call sites (`core/search.py`, - `utils/api_utils.py` ×3). -- **`config_utils.refresh_cloudinary_config(url, saved_name=None)`** now delegates to - `install_oauth_config` (single install seam). -- **mtime cache in `load_config()`** — caches the parsed dict keyed on `(st_mtime_ns, st_size)`, - returns a **copy** (callers mutate in place), invalidated in `save_config`. Cuts the remaining - redundant reads (group-level validity check, `-ls`, refresh sweeps). -- **Classifiers consolidated** — `config_listing.config_type_label(obj)` is now - `"oauth" if obj.has_oauth else "api_key"` (no `__dict__` peeking, no `getattr` fallback); - `is_valid_cloudinary_config` reads `has_oauth` (lazily, without evaluating the refreshing - property — see Caveat A). - -### Caveats handled (detail in `OAUTH_LAZY_TOKEN_CAVEATS_AND_MTIME.md`) -- **A — truthiness reads must not refresh.** Type/validity/`-ls` read `has_oauth` (presence), never - the refreshing `oauth_token`. NOTE: do **not** write `getattr(cfg, "has_oauth", bool(cfg.oauth_token))` - — the default arg is evaluated eagerly and *would* trigger a refresh. Use a `hasattr` guard. -- **B — no `reset_config()` in the getter.** Refresh in place; the only global swap is at install - time, once per process. - ---- - -## 2. Test fixes in this commit (all pre-existing isolation bugs, surfaced by the refactor) - -1. **`test_auth_session.py`** — `test_stale_refreshes_and_rewrites` / `test_refresh_timeout_returns_stale_url` - did not patch `load_config`, so they read/wrote the developer's **real** `~/.cloudinary-cli/config.json`. - This had previously **poisoned a real config entry** (`eu-cloud`) with MagicMock garbage. Both now - patch `load_config`. (The poisoned entry was removed from the dev machine.) -2. **`test_cli_config.py::test_cli_config_show_default_no_config`** — asserts the "nothing - configured" path but passed on base only via cross-`invoke` global pollution. Now clears - `CLOUDINARY_*` env (via `patch.dict(..., clear=True)` over a filtered env) so it genuinely tests - the unconfigured path. The new resolver behavior (env re-read on resolve) is *more* correct; - this test just needed a clean env to assert the negative case. -3. **`test_cli_config_oauth.py::TestConfigSecretMasking`** — built `cloudinary.Config()` which - auto-loads the dev env; when `CLOUDINARY_CLOUD_NAME`+`CLOUDINARY_ACCOUNT_URL` are set (common - PyCharm setup), a real `account_url` leaked in, adding a 2nd `echo` call so the assertions - (reading only the last call) missed the masked fields → **4 failures on the developer's machine**. - Fixed by extending `_RestoresSdkConfig` to strip `CLOUDINARY_*` in `setUp` and inheriting it. -4. **`TestEnsureActiveConfigFresh` → `TestSelfRefreshingOAuthToken`** — rewritten for the new model: - presence check (`has_oauth`) does no network; reading `oauth_token` refreshes once; env/`-c`/api-key - never refresh. Plus `test_presence_check_does_not_refresh` as the Caveat-A regression guard. -5. **Skip gating for offline/no-account runs** — `helper_test.CONFIG_PRESENT` / `REQUIRES_CONFIG`, - applied per-method via `@unittest.skipUnless` to the 13 tests that mock HTTP but still need a - resolvable config (`test_cli_url`, `test_cli_utils`, `test_cli_search_api`, `test_cli_api`). They - now **skip** cleanly on a bare machine instead of failing. - -### Test status -- **No config (bare machine):** 191 passed, 21 skipped, 0 failed. -- **Real test cloud (via `tools/allocate_test_cloud.sh`):** 211 passed, 1 skipped (`test_provisioning`, - needs `account_id`), 0 failed. -- **Account-enabled dev env:** the 4 masking failures are fixed; remaining `test_cli_config` failures - there are only because the simulated creds were fake (401) — they pass against a real account. - ---- - -## 3. What is NOT done — for the next agent - -These were identified in the reviews but are **out of scope of this commit**: - -1. **Thread-safety of the refresh under `sync`/`upload-dir` (the original concern in - `OAUTH_REFRESH_CONCURRENCY_REVIEW.md` §2).** The cross-process single-flight (lock + double-check) - is correct and preserved. But the in-process getter, when stale, can have N worker threads enter - `refresh_url_if_stale` together; they serialize on the reentrant `config_lock` and all but one - adopt the peer-refreshed token (correct, but a synchronized stall). Consider a process-local - `threading.Lock` around the getter's stale branch so only one thread per process attempts it. -2. **Refresh-on-401 retry (`OAUTH_REFRESH_CONCURRENCY_REVIEW.md` Fix C).** The lazy property closes - the "stale at request build time" gap, but a token that expires *mid-flight* (between read and - server receipt) still 401s with no retry. A reactive retry at `call_api` would be the robust - complement. -3. **Transactional multi-step config ops (Fix D).** `login` + auto-default and refresh-and-default - still read-decide-write across separate `config_lock` scopes (TOCTOU). Wrap in one lock scope. -4. **"config removed mid-run" message (Fix E).** The getter returns the stale token if the saved - config vanished underneath it; surface a clear "re-login" error instead of a later raw 401. -5. **Chokepoint-completeness is now moot** (the property covers all read paths), but a test that - asserts the active config is always an `OAuthConfig` post-resolve would lock that invariant in. -6. **`reset_config()` audit.** Verified the resolver/install paths, but any *future* direct - `cloudinary.reset_config()` call would replace the OAuthConfig with a plain Config and silently - disable self-refresh. Keep all config installation routed through - `oauth_config.install_oauth_config` / `install_env_config`. - ---- - -## 4. Key files - -| File | Role | -|---|---| -| `cloudinary_cli/auth/oauth_config.py` | **New.** `OAuthConfig`, `has_oauth`, install seams. | -| `cloudinary_cli/utils/config_resolver.py` | Installs OAuthConfig per branch; `ensure_active_config_fresh` deleted. | -| `cloudinary_cli/utils/config_utils.py` | `refresh_cloudinary_config` delegates to install; mtime cache; `is_valid_cloudinary_config` via has_oauth. | -| `cloudinary_cli/utils/config_listing.py` | `config_type_label(obj)` → `has_oauth`. | -| `cloudinary_cli/utils/api_utils.py`, `core/search.py` | Chokepoint calls removed. | -| `test/helper_test.py` | `CONFIG_PRESENT` / `REQUIRES_CONFIG` skip predicate. | - -Run tests: `.venv/bin/python -m pytest test/ -q --ignore=test/test_modules` -(excludes `test/test_modules`, which makes a real Admin API call at import). diff --git a/OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md b/OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md deleted file mode 100644 index fa55e0a..0000000 --- a/OAUTH_LAZY_TOKEN_PROPERTY_DESIGN.md +++ /dev/null @@ -1,196 +0,0 @@ -# Design: self-refreshing `oauth_token` — collapse the scattered refresh hooks - -> Proposal under evaluation: make the SDK config's `oauth_token` resolve through a callable/property -> that refreshes a stale token on read, so we can delete `ensure_active_config_fresh` and every -> chokepoint call (`call_api`, `execute_single_request`, `query_cld_folder`, `cld_folder_exists`). -> Context: access tokens are valid ~5 minutes, so refreshes are frequent and expected. -> -> Verdict: **the idea is sound and the SDK seam exists, but a plain `__getattr__` hook will NOT -> work** because of how the SDK stores the token. A `property`/descriptor approach works, with two -> caveats that must be handled. Details below, traced against the installed SDK. - ---- - -## 1. Why this is the right instinct - -Every place that needs auth reads the token the same way, at call time: - -```python -# cloudinary/api_client/call_api.py:63 -oauth_token = options.pop("oauth_token", cloudinary.config().oauth_token) -# cloudinary/uploader.py:877 -oauth_token = options.get("oauth_token", cloudinary.config().oauth_token) -``` - -Both read **`cloudinary.config().oauth_token`** at the moment of the request. That is a single, -universal seam. If reading that attribute returns a *fresh* token, then: - -- `ensure_active_config_fresh()` and all four chokepoint calls can be **deleted**. -- Search's direct `.execute()` paths, sync's threaded `call_api`, provisioning, and any future API - entry point all get correct tokens for free — no hand-maintained chokepoint list. -- The "stale token mid-bulk-run → 401" gap closes, because the token is re-evaluated per request. - -This directly answers the 5-minute-validity reality: refresh becomes demand-driven at the exact -point of use, once per request, with no proactive probing. - ---- - -## 2. The blocker: the SDK does NOT route `oauth_token` through `__getattr__` - -`cloudinary.Config` (BaseConfig): - -```python -def __getattr__(self, i): - return self.__dict__.get(i) -``` - -`__getattr__` fires **only when the attribute is absent from the instance `__dict__`.** But the -token is written *into* `__dict__`: - -```python -def _setup_from_parsed_url(self, parsed_url): - ... - self.__dict__[k] = v[0] # oauth_token lands here -``` - -So `config().oauth_token` is a normal dict hit and `__getattr__` never runs. **A `__getattr__`-based -hook is dead on arrival.** This is the trap to avoid. - -### What does work - -A **data descriptor** (a `property`) defined on the *class* takes precedence over the instance -`__dict__`, so it intercepts the read even when `oauth_token` is also in `__dict__`. Options, in -order of preference: - -1. **Subclass + property (cleanest, CLI-local, no SDK fork).** Define a `Config` subclass with an - `oauth_token` property whose getter refreshes-if-stale and returns the live access token; the - raw stored URL/refresh-token live in private attributes. Install it as `cloudinary._config` (the - object `cloudinary.config()` returns). The SDK's `config().oauth_token` then hits our property. - - Requires confirming the SDK lets us swap the config singleton (it stores a module-level - `_config`; `reset_config()` rebuilds it — see §4 risk). - -2. **Store the token outside `__dict__` so the existing `__getattr__` fires.** Pop `oauth_token` - out of `__dict__` and serve it from `__getattr__`. Fragile: anything that writes - `config().oauth_token = x` (or `update(oauth_token=...)`) puts it back in `__dict__` and silently - disables the hook. Not recommended. - -3. **Monkeypatch `BaseConfig.oauth_token` as a property at CLI import.** Works (class-level data - descriptor), but mutates SDK global state for the whole process — affects every Config instance, - including the env-derived `cloudinary.Config()` we build in `config_listing`. Has to no-op for - non-OAuth / non-CLI-managed configs. Workable but the broadest blast radius. - -**Recommendation: option 1** (subclass + property installed as the active config). - ---- - -## 3. Two semantic caveats that MUST be handled - -### Caveat A — `oauth_token` is read for *truthiness*, not just for the value - -The CLI itself does this in several places: - -```python -"type": "oauth" if config_obj.oauth_token else "api_key" # config_listing.py x3, core/config.py -if cloudinary.config().cloud_name and cloudinary.config().oauth_token: # is_valid_cloudinary_config -``` - -If `oauth_token` becomes a refresh-on-read property, then **a type check or a validity check would -trigger a network refresh** — exactly the kind of accidental I/O on an offline path (`config -ls`, -`config -s`, `is_valid_cloudinary_config` at the group level) that this whole effort was trying to -remove (the original Finding 1 hang). This is the subtle regression risk. - -Mitigations: -- The property getter must **refresh only when the caller actually needs a live token for a - request** — but a property can't tell "truthiness check" from "real use." So: keep a separate, - non-refreshing attribute for *presence* (e.g. an `is_oauth` flag or a raw `_oauth_token_raw`) and - point the type/validity checks at *that*, leaving the refreshing `oauth_token` only on the - request path. i.e. the property refreshes; the CLI's own introspection reads the raw field. -- Or gate the getter: refresh only if `expires_at` is set AND we're not in a "describe" context. - Context flags are ugly; prefer the separate-presence-field approach. - -### Caveat B — reentrancy and thread safety on read - -A 5-minute token under a multi-threaded sync means many threads may read `oauth_token` near -expiry simultaneously. The getter must reuse the **existing** lock + double-check from -`refresh_url_if_stale` (which is already correct and single-flight across threads and processes — -preserve it verbatim). The getter also must not deadlock: it runs *inside* SDK request code, and -`refresh_url_if_stale` → `update_config` takes the reentrant `config_lock` and then calls -`refresh_cloudinary_config` (which does `reset_config()` + reload of the **global**). Rebuilding the -global config object *from within a getter on that same global* is the dangerous part: - -- Do **not** `reset_config()` inside the getter. Instead, refresh, persist, and update only the - private token fields on the *current* object (no global swap). The getter returns the new access - token; no `cloudinary.reset_config()` on the hot path. This also fixes the latent - reset-then-reload thread race documented in the concurrency review. - ---- - -## 4. Other risks / unknowns to confirm before building - -1. **`reset_config()` rebuilds the singleton.** The SDK's `reset_config()` constructs a fresh - `Config()`. If we installed a subclass instance, any code path that calls `reset_config()` (we do, - in `refresh_cloudinary_config`, and the SDK may internally) would replace our self-refreshing - object with a plain one, silently disabling the property. Audit every `reset_config()` call and - route config installation through one helper that always installs the subclass. - -2. **Env / `-c` configs.** Env-derived and inline-`-c` OAuth URLs currently never refresh (no saved - entry, no refresh token persisted). The property must no-op for configs with no refresh token - (return the static token) — same as `refresh_url_if_stale`'s existing early-out. - -3. **Persistence on rotation.** When the getter refreshes, it must still write the rotated token back - to `config.json` under the lock (so the next *process* benefits and the single-use token isn't - re-burned). That write must use the existing atomic `update_config`. The getter therefore still - needs to know *which saved name* it maps to — i.e. it needs the `_active_name` association, just - carried on the config object instead of a module global. (This is strictly better: the binding - lives with the object, not in resolver module state.) - -4. **`config_to_dict` / masking.** `config_to_dict` iterates `__dict__`; a property is on the class, - not in `__dict__`, so the masking/listing code that enumerates `__dict__` would **miss** - `oauth_token` unless the raw value is still stored as an instance attr. Keep the raw token in - `__dict__` (under a private name) so masking still sees it; expose the live one via the property. - -5. **SDK upgrades.** This couples us to two SDK internals: `config().oauth_token` being read per - request (stable, it's the documented OAuth path) and the descriptor-vs-`__dict__` precedence - (Python language guarantee, safe). The fragile coupling is `reset_config()` behavior (#1). - ---- - -## 5. What gets deleted if this lands - -- `config_resolver.ensure_active_config_fresh` (whole function). -- The 4 call sites: `api_utils.call_api`, `api_utils.query_cld_folder`, `api_utils.cld_folder_exists`, - `core/search.execute_single_request`. -- The module-global `_active_name` *as a refresh input* (still useful for `config -ls` "active" - marker — keep it, or move the active-name onto the config object too). -- The per-call `load_config()` on the hot path. - -What stays (and should be reused verbatim inside the getter): -- `refresh_url_if_stale`'s lock + double-check + atomic persist (the genuinely-correct core). -- The resolver's Phase-A selection/precedence and the offline format check. - ---- - -## 6. Recommended shape - -1. Add a CLI `OAuthConfig(cloudinary.Config)` subclass: - - stores `_raw_url` / `_saved_name` (the saved-config association), - - `oauth_token` is a **property**: if a saved OAuth token, refresh-if-stale (reusing - `refresh_url_if_stale`'s lock/double-check/persist), update the in-object token in place (no - `reset_config`), return the live access token; otherwise return the static stored value, - - keeps the raw token in `__dict__` under a private key for masking/introspection. -2. Route all config installation (resolver, `refresh_cloudinary_config`) through one helper that - installs this subclass and never leaves a plain `Config` as the active global for a saved OAuth - login. Audit `reset_config()`. -3. Point **presence/type/validity** checks (`is_valid_cloudinary_config`, the three - `"oauth" if ... else "api_key"` sites) at the **raw** field / an `is_oauth` flag — NOT the - refreshing property — so offline `config`/`-ls`/`-s` never touch the network (preserves the - Finding-1 fix). -4. Delete `ensure_active_config_fresh` and its four call sites. -5. Tests: (a) reading `oauth_token` on a stale saved config refreshes once and persists; (b) reading - it on a fresh / api-key / env config does no network; (c) `config -ls` / `-s` / - `is_valid_cloudinary_config` do **zero** network even with a stale token (regression guard for - Caveat A); (d) concurrent threaded reads single-flight (one refresh, others adopt). - -This collapses the scattered hooks into one well-placed seam, fixes the mid-run-401 and the -reset-then-reload race, and keeps the offline paths offline — provided Caveat A (truthiness reads) -and the `reset_config` audit (#1/#4) are handled. Those two are the make-or-break items. diff --git a/OAUTH_REFRESH_CONCURRENCY_REVIEW.md b/OAUTH_REFRESH_CONCURRENCY_REVIEW.md deleted file mode 100644 index 983b3c3..0000000 --- a/OAUTH_REFRESH_CONCURRENCY_REVIEW.md +++ /dev/null @@ -1,266 +0,0 @@ -# OAuth lazy-refresh concurrency review — `ensure_active_config_fresh` - -> Scope: the lazy token-refresh design landed in `43fb67a` (`oauth-login` branch). Focuses on -> behavior under (1) high in-process + cross-process parallelism (sync / upload-dir with up to ~30 -> `cld` instances) and (2) one long-running instance while a second instance mutates config -> (changing the default, dropping configs). Every claim below was traced against the code, not the -> design doc. - ---- - -## 1. How refresh is wired today - -`resolve_cli_config` (group callback, Phase A) does **no network**. It records the selected saved -config name in the module global `_active_name`. Then, at each API chokepoint, Phase B runs: - -```python -# config_resolver.ensure_active_config_fresh() -name = _active_name -if name is None: return -url = load_config().get(name) # disk read -if url is None: return -fresh = refresh_url_if_stale(name, url) # may take config_lock + network -if fresh != url: - refresh_cloudinary_config(fresh) # mutates the SDK *process global* -``` - -`refresh_url_if_stale` (auth/__init__.py): - -```python -if not is_oauth_url(url): return url -session = from_cloudinary_url(url) -if (session.is_fresh() and not force) or not session.refresh_token: - return url # FAST PATH: no lock, no I/O beyond the load_config above -with config_lock(): # FileLock on config.json.lock (cross-process, reentrant) - url = load_config().get(name, url) # re-read under lock - session = from_cloudinary_url(url) - if (session.is_fresh() and not force) or not session.refresh_token: - return url # peer already refreshed -> adopt, don't burn - token_response = flow.refresh(...) # NETWORK, single-use refresh token - update_config({name: refreshed_url})# atomic write (tmp + os.replace), reentrant lock - return refreshed_url -``` - -Chokepoints calling `ensure_active_config_fresh`: - -| Site | Frequency | -|---|---| -| `api_utils.call_api` | **once per API call** — and `upload_file` → `call_api`, run under `run_tasks_concurrently` (N worker threads) | -| `core/search.py::execute_single_request` | once per search request | -| `api_utils.query_cld_folder` | once per folder query (before the cursor loop — good) | -| `api_utils.cld_folder_exists` | once per existence check | - -Freshness: `is_fresh()` = `expires_at - 30s > now` (`OAUTH_EXPIRY_SKEW_SECONDS = 30`). - ---- - -## 2. The user's concern #1 — "30 instances all refresh at once" - -### 2.1 What actually happens — it is mostly safe, but the design is wasteful and fragile - -**The good news (correctness):** the refresh itself is *not* a correctness problem under -concurrency. The lock + double-check makes the actual refresh single-flight: - -- Only one process at a time holds `config_lock()`. The first to enter refreshes, rotates the - single-use token, and `os.replace`s the file atomically. Every other process, on acquiring the - lock, re-reads and sees a now-fresh token → adopts it → **does not** burn a second refresh. -- So we will **not** stampede the OAuth server with 30 concurrent refreshes for the same config, - and we will **not** burn 30 refresh tokens. That part of the design is sound. - -**The bad news (this is the poor design you flagged):** - -1. **Per-call fast-path cost, multiplied by every asset and every thread.** `ensure_active_config_fresh` - runs on **every** `call_api`. For a fresh token the cost is `load_config()` (a disk read + - JSON parse) **plus** `from_cloudinary_url` parse, on every single upload/admin call. In a sync - of 50k assets across N threads that is 50k×(disk read + parse) of `config.json`, on the hot - path, purely to discover "still fresh, nothing to do." It is not a hang, but it is real, - repeated, avoidable I/O and lock-adjacent work in the tightest loop the CLI has. - -2. **In-process thread contention on a reentrant lock is invisible but real.** `run_tasks_concurrently` - spins up worker threads; each calls `call_api` → `ensure_active_config_fresh`. On the fast path - they don't take `config_lock`, so threads mostly don't serialize there. **But** the moment the - token actually goes stale mid-run (long sync crossing the `expires_at` boundary), *every* worker - thread simultaneously fails `is_fresh()`, and they pile up on `config_lock()`. One refreshes; the - rest serialize behind it, re-read, adopt. Correct, but a synchronized stall of all workers at the - expiry boundary — and `refresh_cloudinary_config` mutates the **process-global** SDK config (see - §2.2), which those same threads are concurrently reading. - -3. **`refresh_cloudinary_config` mutates global SDK state while other threads make API calls.** - This is the sharpest in-process hazard. `refresh_cloudinary_config` does - `cloudinary.reset_config()` then `_load_from_url(...)`. There is a window where the global config - has been reset but not yet reloaded. A peer worker thread issuing `uploader.upload` during that - window can read a half-cleared global config. The cross-*process* path is safe (separate address - spaces, atomic file); the cross-*thread* path within one process shares one mutable - `cloudinary.config()` singleton with no lock around the reset+reload. This is a latent data race, - not currently covered by any test. - -4. **Cross-process thundering herd at the expiry boundary.** 30 processes started near the token's - expiry will each independently hit the stale branch, then queue on the *file* lock one-by-one. - The first refreshes (network); the other 29 each acquire the lock, re-read, see fresh, release. - That is 29 serialized lock acquisitions + 29 `load_config` re-reads gated on a single - cross-process `FileLock` — a serialization point that all 30 bulk jobs funnel through at the same - instant. No token burn, but a real latency cliff and a single point of contention. - -### 2.2 Summary of #1 - -- **Token burn / OAuth stampede:** safe. Single-flight via lock + double-check works across - processes and threads. -- **Performance:** poor. Per-call `load_config` on the hot path; synchronized stall of all - workers/processes at the expiry boundary; cross-process serialization on one file lock. -- **In-process thread safety of the SDK global:** **unsafe (latent race)** — `reset_config()` + - reload is not atomic w.r.t. concurrent reader threads. - ---- - -## 3. The user's concern #2 — long-running instance vs. a second instance mutating config - -Scenario: instance A is mid-`sync` (minutes long). Instance B runs `cld config -d other`, -`cld config -rm A's-config`, `cld config -ud`, etc. - -### 3.1 Default change (`-d` / `-ud`) — safe, ignored by A - -Instance A resolved its config **once** at startup and cached the selection in its own process -global `_active_name`. A re-reads `config.json` in `ensure_active_config_fresh`, but only to fetch -**A's own** `_active_name` entry's URL — it never re-reads `__default__`. So B changing or clearing -the default has **zero effect** on a running A. Correct and desirable (A shouldn't switch accounts -mid-sync). ✔ - -### 3.2 Dropping the config A is using (`cld config -rm `) — degrades, mostly safe - -- B's `-rm` does `remove_config_keys` under `config_lock()` + atomic write. A's in-flight API calls - are unaffected (A already loaded the URL into its SDK global at resolve time). -- The exposure is **only** inside `ensure_active_config_fresh`: `url = load_config().get(name)`. If - B removed the entry, this returns `None` → A **returns early and does not refresh**. If A's token - was about to expire, A then proceeds with a stale token and the next real API call fails with a - 401 — mid-sync, not a clean error. So: no crash, no corruption, but a removed-out-from-under-you - config can turn a refresh into a silent skip → late 401. -- If A is the default and B removes A's config, `core/config.py` clears `__default__` too — fine, - doesn't affect running A. - -### 3.3 Token rotation interleaving (A refreshes while B refreshes/edits) — safe - -Both go through `config_lock()` + atomic `os.replace`. No torn reads (atomic write guarantees a -reader sees either the old or the new whole file). The reentrant lock means A's -`refresh_url_if_stale` → `update_config` nests safely. ✔ - -### 3.4 The one real cross-process correctness gap — lost update on unrelated keys - -`update_config` is read-modify-write under the lock: - -```python -with config_lock(): - curr = load_config(); curr.update(new_config); save_config(curr) -``` - -Because every writer takes the lock, two writers don't lose each other's updates. **However**, -`set_default_config`, `clear_default_config`, `refresh` and `-rm` are *separate* lock acquisitions, -not one transaction. A multi-step CLI operation that does read-decide-write across two lock scopes -(e.g. `_should_auto_default` reads outside the lock, then `set_default_config` writes inside a new -lock) can interleave with a peer between the two scopes. Concretely: `login` does -`update_config({name: url})` (lock #1), then `_should_auto_default` does `load_config()` (no lock), -then `set_default_config` (lock #2). A peer running `cld config -d X` between #1 and #2 can have its -default silently overwritten by the auto-default, or vice-versa. Low probability, but it is a -genuine TOCTOU across separate lock scopes. ✔ data integrity (no torn file) / ✘ atomicity of the -logical operation. - -### 3.5 Summary of #2 - -| B's action while A runs | A's outcome | Safe? | -|---|---|---| -| `-d` / `-ud` change default | ignored (A cached its selection) | ✔ | -| `-rm` A's config | refresh becomes a no-op → possible late 401 if token expires | ⚠ degraded | -| refresh / rotate token | atomic, single-flight, adopted | ✔ | -| concurrent multi-step config edit | file never torn; logical op can interleave (TOCTOU) | ⚠ | - ---- - -## 4. Why `ensure_active_config_fresh` is the wrong shape - -1. **Wrong altitude.** Freshness is a property of *the session we resolved*, but the check is - re-derived from disk on every API call via a module global. It couples `api_utils` and - `core/search` to resolver internals (`_active_name`) and to `load_config`. -2. **Hot-path I/O.** A `load_config()` (disk + JSON parse) per API call, inside the busiest loops, - to almost always conclude "fresh." -3. **Process-global mutation under concurrency.** `refresh_cloudinary_config` resets the shared SDK - singleton with no guard against concurrent reader threads (§2.2.3). -4. **Chokepoint set is hand-maintained.** Four call sites enumerated by hand; any new API entry - point silently runs on a stale token. No test asserts completeness. -5. **Module-global `_active_name` is not thread-scoped.** Fine for one resolve per process, but it - makes the whole mechanism implicitly single-config-per-process and invisible to readers. - ---- - -## 5. Suggested fixes (in priority order) - -### Fix A — refresh ONCE, eagerly, after resolution; drop the per-call hook (recommended) - -The original eager refresh hung offline commands. The real fix for *that* was to **only refresh -when about to do network work**, not to refresh on *every* call. Move the refresh to a single point: -right after `resolve_cli_config` succeeds **and** the command is known to be a network command -(or lazily, but **once**, guarded by a process-level "already ensured" flag). - -```python -_ensured = False -def ensure_active_config_fresh(): - global _ensured - if _ensured or _active_name is None: - return - _ensured = True # one refresh attempt per process, not per call - url = load_config().get(_active_name) - ... -``` - -- Eliminates per-call `load_config` and per-call parse. -- Eliminates the synchronized all-threads stall at the expiry boundary (only the first call into - any chokepoint pays the cost; the flag short-circuits the rest). -- A token expiring *mid-very-long-sync* is then handled by Fix C, not by re-checking every call. - -### Fix B — make `refresh_cloudinary_config` atomic w.r.t. reader threads - -Guard the `reset_config()` + `_load_from_url()` pair with a process-local `threading.Lock`, and have -the per-thread API path either hold it for the read or only ever swap in a fully-built config. The -cleanest version builds the new `cloudinary.Config` object off to the side and assigns it in one -reference swap rather than reset-then-mutate, so a reader thread never sees a half-cleared global. - -### Fix C — refresh-on-401 retry, not refresh-on-every-call - -The robust pattern for long-running multi-threaded/multi-process jobs is **reactive**: attempt the -API call; on a 401/auth error, take the lock, refresh once (double-checked), reload, and retry the -call exactly once. This: - -- removes all proactive per-call freshness work, -- naturally single-flights across threads and processes (same lock + double-check already in place), -- correctly handles the token expiring mid-run and the "B removed my config" case (the retry can - surface a clear "re-login" error instead of a raw 401). - -Wrap it at `call_api` (which already catches exceptions) and at the two direct `.execute()` sites. - -### Fix D — make logical config operations transactional - -Provide a single `with config_lock(): read; mutate; write` helper for multi-step operations -(`login` + auto-default, refresh-and-default) so read-decide-write happens under **one** lock scope, -closing the §3.4 TOCTOU. At minimum, move `_should_auto_default`'s read and `set_default_config`'s -write into the same lock acquisition. - -### Fix E — handle "config removed mid-run" explicitly - -In the refresh path, distinguish "config gone" (`url is None` after the resolver had a name) from -"nothing to refresh," and surface a clear message (`config '' was removed; please re-login`) -rather than silently skipping and letting a later 401 fall out. - -### Fix F — assert chokepoint completeness - -A test that monkeypatches the refresh entry point to a counter and exercises each top-level network -command, asserting it fired exactly once (pairs with Fix A's once-per-process flag). - ---- - -## 6. Recommendation - -Adopt **Fix A + Fix C** as the core redesign: resolve once, refresh once eagerly *or* reactively on -401, and stop probing the token on every call. Add **Fix B** to close the SDK-global thread race -(required before we trust multi-threaded sync under token rotation). **Fix D/E/F** are smaller -hardening steps. The current code is *correct on token burn* (the lock + double-check is genuinely -good and should be preserved verbatim inside whichever path survives), but the per-call hook and the -unguarded global mutation make it the wrong shape for the 30-instance / N-thread bulk workloads this -CLI is built for. From 0b4b9e602b28d467b0cd7e382c533b580fabf465 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 05:16:42 +0300 Subject: [PATCH 12/23] Make OAuth client id and scopes env-overridable Add CLOUDINARY_OAUTH_CLIENT_ID and CLOUDINARY_OAUTH_SCOPES overrides (mirroring the redirect host/port pattern) for testing against a non-prod authorization server; production defaults are unchanged. Also drop implementation-specific server naming from comments. Co-Authored-By: Claude Opus 4.8 (1M context) --- cloudinary_cli/auth/__init__.py | 2 +- cloudinary_cli/defaults.py | 13 ++++++++----- test/test_auth_region.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/cloudinary_cli/auth/__init__.py b/cloudinary_cli/auth/__init__.py index 7eb5bc2..7df640f 100644 --- a/cloudinary_cli/auth/__init__.py +++ b/cloudinary_cli/auth/__init__.py @@ -118,7 +118,7 @@ def refresh_url_if_stale(name, url, force=False): _refresh_warned.discard(name) # a later success re-arms the warning for this config - # Hydra rotates refresh tokens; keep the old one only if a new one was not returned. + # The authorization server rotates refresh tokens; keep the old one only if a new one was not returned. token_response.setdefault("refresh_token", session.refresh_token) refreshed_url = to_cloudinary_url(session.updated_from(token_response)) update_config({name: refreshed_url}) diff --git a/cloudinary_cli/defaults.py b/cloudinary_cli/defaults.py index 5458053..bec6861 100644 --- a/cloudinary_cli/defaults.py +++ b/cloudinary_cli/defaults.py @@ -28,7 +28,7 @@ # names are rejected as user config names, so this can't collide with a saved config. DEFAULT_CONFIG_KEY = "__default__" -# OAuth (ORY Hydra) configuration for `cld login`. The region string derives both the API and +# OAuth configuration for `cld login`. The region string derives both the API and # OAuth hosts; an unknown region simply fails to resolve. DEFAULT_REGION = 'api' @@ -63,11 +63,14 @@ def oauth_token_url_for_region(region): CLOUDINARY_REGION = normalize_region(os.environ.get('CLOUDINARY_REGION')) -# Public PKCE client (no secret). -OAUTH_CLIENT_ID = 'a920ea9c-531b-4613-9783-1d4f4cc10655' -OAUTH_SCOPES = 'openid offline_access asset_management upload' +# Public PKCE client (no secret). Overridable for testing against a non-prod authorization server +# registered with a different client; production uses the single registered client below. +OAUTH_DEFAULT_CLIENT_ID = 'a920ea9c-531b-4613-9783-1d4f4cc10655' +OAUTH_CLIENT_ID = os.environ.get('CLOUDINARY_OAUTH_CLIENT_ID', OAUTH_DEFAULT_CLIENT_ID) +OAUTH_DEFAULT_SCOPES = 'openid offline_access asset_management upload' +OAUTH_SCOPES = os.environ.get('CLOUDINARY_OAUTH_SCOPES', OAUTH_DEFAULT_SCOPES) -# Hydra requires an exact redirect match, so the port is fixed and must match the registered client. +# The authorization server requires an exact redirect match, so the port is fixed and must match the registered client. OAUTH_DEFAULT_REDIRECT_HOST = '127.0.0.1' OAUTH_REDIRECT_HOST = os.environ.get('CLOUDINARY_OAUTH_REDIRECT_HOST', OAUTH_DEFAULT_REDIRECT_HOST) OAUTH_DEFAULT_REDIRECT_PORT = 49421 diff --git a/test/test_auth_region.py b/test/test_auth_region.py index 86d6bd3..c8ac3c7 100644 --- a/test/test_auth_region.py +++ b/test/test_auth_region.py @@ -1,5 +1,8 @@ +import importlib import unittest +from unittest.mock import patch +import cloudinary_cli.defaults as defaults from cloudinary_cli.defaults import normalize_region, _oauth_host_for, api_host_for_region @@ -32,3 +35,31 @@ def test_oauth_host_central_for_geo_regions(self): def test_oauth_host_dedicated_for_long_suffix(self): # longer suffixes route to their own oauth- host self.assertEqual('oauth-test.cloudinary.com', _oauth_host_for('api-test')) + + +class TestOAuthClientConfig(unittest.TestCase): + """OAUTH_CLIENT_ID / OAUTH_SCOPES are env-overridable (resolved at module import).""" + + def _reload(self, env): + # The values are read at import time, so reload defaults under the patched environment. + with patch.dict("os.environ", env, clear=False): + for key in ("CLOUDINARY_OAUTH_CLIENT_ID", "CLOUDINARY_OAUTH_SCOPES"): + if key not in env: + __import__("os").environ.pop(key, None) + return importlib.reload(defaults) + + def tearDown(self): + importlib.reload(defaults) # restore the unpatched module for other tests + + def test_defaults_when_unset(self): + d = self._reload({}) + self.assertEqual('a920ea9c-531b-4613-9783-1d4f4cc10655', d.OAUTH_CLIENT_ID) + self.assertEqual('openid offline_access asset_management upload', d.OAUTH_SCOPES) + + def test_client_id_override(self): + d = self._reload({"CLOUDINARY_OAUTH_CLIENT_ID": "non-prod-client"}) + self.assertEqual("non-prod-client", d.OAUTH_CLIENT_ID) + + def test_scopes_override(self): + d = self._reload({"CLOUDINARY_OAUTH_SCOPES": "openid upload"}) + self.assertEqual("openid upload", d.OAUTH_SCOPES) From 11efb4c36bd63675d824f47c1dead5ec913cbd35 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 13:05:57 +0300 Subject: [PATCH 13/23] Bump minimum Python to 3.8 The CLI imports dataclasses (3.7+), and the pinned urllib3>=2.2.2 / zipp>=3.19.1 security floors require Python 3.8+, so 3.6 and 3.7 cannot install or run it. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 750bca3..c1e5663 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ It is fully documented at [https://cloudinary.com/documentation/cloudinary_cli]( ## Requirements Your own Cloudinary account. If you don't already have one, sign up at [https://cloudinary.com/users/register/free](https://cloudinary.com/users/register/free). -Python 3.6 or later. You can install Python from [https://www.python.org/](https://www.python.org/). Note that the Python Package Installer (pip) is installed with it. +Python 3.8 or later. You can install Python from [https://www.python.org/](https://www.python.org/). Note that the Python Package Installer (pip) is installed with it. ## Setup and Installation diff --git a/setup.py b/setup.py index d6f1c5a..b2644d7 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ keywords='cloudinary cli pycloudinary image video digital asset management command line interface transformation ' 'friendly easy flexible', license="MIT", - python_requires='>=3.6.0', + python_requires='>=3.8.0', setup_requires=["pytest-runner"], tests_require=["pytest", "mock", "urllib3"], install_requires=requirements, From 9827027a5268c506f89f828e23b25b6d72c18d6b Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Thu, 25 Jun 2026 13:46:04 +0300 Subject: [PATCH 14/23] Revoke OAuth token on logout; report default status on login logout now revokes the refresh token at the authorization server (RFC 7009) before removing the saved configuration; the local entry is removed even if revocation fails. login reports whether the new login became the default and, when it did not, prints the command to make it the default. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 14 +++++++----- cloudinary_cli/auth/__init__.py | 35 ++++++++++++++++++++++++----- cloudinary_cli/auth/flow.py | 12 ++++++++++ cloudinary_cli/core/auth.py | 16 ++++++++++---- cloudinary_cli/defaults.py | 4 ++++ test/test_auth_flow.py | 12 ++++++++++ test/test_cli_config_oauth.py | 39 ++++++++++++++++++++++++++++++--- 7 files changed, 114 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c1e5663..876d3f7 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Python 3.8 or later. You can install Python from [https://www.python.org/](http cld login ``` - This opens your browser to authorize the CLI, then saves the login as a configuration (named after the cloud) and sets it as the default. No API secret is stored on disk — the saved login holds a short-lived token that the CLI refreshes automatically. + This opens your browser to authorize the CLI, then saves the login as a configuration (named after the cloud) and sets it as the default. The CLI refreshes the token automatically, and you can remove the login at any time with `cld logout`. **Option B — Set your CLOUDINARY\_URL environment variable.** For example: * On Mac or Linux:
`export CLOUDINARY_URL=cloudinary://123456789012345:abcdefghijklmnopqrstuvwxyzA@cloud_name` @@ -58,7 +58,7 @@ Usage: cld [cli options] [command] [command options] [method] [method parameters ``` cld --help # Lists available commands. cld login # Logs in to a Cloudinary account via OAuth in your browser. -cld logout # Removes a saved OAuth login. +cld logout # Revokes and removes a saved OAuth login. cld search --help # Shows usage for the Search API. cld admin # Lists Admin API methods. cld uploader # Lists Upload API methods. @@ -255,7 +255,7 @@ Whereas using the saved configuration "accountx": cld -C accountx admin usage ``` -_**Caution:** A saved API-key configuration stores your API secret in a local file. An OAuth login (see below) avoids this by storing a short-lived, auto-refreshed token instead._ +_**Caution:** Creating a saved configuration may put your credentials at risk as they are stored in a local plain text file. This applies to both API-key configurations and OAuth logins._ You can create, delete and list saved configurations using the `config` command. @@ -272,11 +272,13 @@ Instead of saving an API key and secret, you can log in to a Cloudinary account ``` cld login # Log in and save the configuration (named after the cloud). cld login my-account # Save the login under a specific name. -cld logout # Choose a saved OAuth login to remove. -cld logout my-account # Remove a specific saved OAuth login. +cld logout # Choose a saved OAuth login to log out of. +cld logout my-account # Log out of a specific saved OAuth login. ``` -Once saved, an OAuth login is selected with `-C ` just like any other saved configuration. +The first login becomes the default automatically. When other configurations already exist, the new login is saved but not made the default; `cld login` tells you so and prints the command to make it the default. Once saved, an OAuth login is selected with `-C ` just like any other saved configuration. + +`cld logout` revokes the login's token at the server and removes the saved configuration. If the token cannot be revoked (for example, you are offline), the saved configuration is still removed. ### Choosing a default configuration diff --git a/cloudinary_cli/auth/__init__.py b/cloudinary_cli/auth/__init__.py index 7df640f..0dfb3e6 100644 --- a/cloudinary_cli/auth/__init__.py +++ b/cloudinary_cli/auth/__init__.py @@ -37,7 +37,8 @@ def login(region=None, name=None, set_default=False): """ Run the interactive browser login and persist the resulting session as a named config entry. - Returns the saved config name, or None on failure. + Returns (config_name, is_default), where is_default is True when this login was made the default + configuration (explicitly with set_default, or automatically as the sole login). """ if name and is_reserved_config_name(name): raise RuntimeError(f"'{name}' is a reserved configuration name.") @@ -48,9 +49,10 @@ def login(region=None, name=None, set_default=False): config_name = name or _derive_config_name(session.cloud_name, region) update_config({config_name: to_cloudinary_url(session)}) - if set_default or _should_auto_default(config_name): + is_default = bool(set_default or _should_auto_default(config_name)) + if is_default: set_default_config(config_name) - return config_name + return config_name, is_default def _should_auto_default(name): @@ -71,14 +73,37 @@ def _should_auto_default(name): def logout(name): - """Remove a saved OAuth login by name. Returns "removed", "not_found", or "not_oauth".""" + """ + Log out of a saved OAuth login by name: revoke its refresh token at the authorization server, + then remove the saved configuration. The local entry is always removed even if revocation fails + (offline, server error), so logout never leaves a stale entry behind. + + Returns "removed" (revoked and removed), "revoke_failed" (removed locally but the token could not + be revoked), "not_found", or "not_oauth". + """ saved = load_config() if name not in saved: return "not_found" if not is_oauth_url(saved[name]): return "not_oauth" + + revoked = _revoke_login(name, saved[name]) remove_config_keys(name) - return "removed" + return "removed" if revoked else "revoke_failed" + + +def _revoke_login(name, url): + """Best-effort revocation of a saved login's refresh token. Returns True on success (or when + there is nothing to revoke), False if the revoke request failed.""" + session = from_cloudinary_url(url) + if not session.refresh_token: + return True + try: + flow.revoke(session.refresh_token, session.region) + return True + except requests.RequestException as e: + log_exception(e, debug_message=f"Could not revoke the OAuth token for '{name}'") + return False def refresh_url_if_stale(name, url, force=False): diff --git a/cloudinary_cli/auth/flow.py b/cloudinary_cli/auth/flow.py index 8b7811c..a10444e 100644 --- a/cloudinary_cli/auth/flow.py +++ b/cloudinary_cli/auth/flow.py @@ -10,6 +10,7 @@ from cloudinary_cli.defaults import ( oauth_authorize_url_for_region, oauth_token_url_for_region, + oauth_revoke_url_for_region, OAUTH_CLIENT_ID, OAUTH_SCOPES, OAUTH_HTTP_TIMEOUT_SECONDS, @@ -58,3 +59,14 @@ def refresh(refresh_token, region): }, timeout=OAUTH_HTTP_TIMEOUT_SECONDS) resp.raise_for_status() return resp.json() + + +def revoke(token, region, token_type_hint="refresh_token"): + """Revoke a token at the authorization server (RFC 7009). Revoking the refresh token ends the + offline-access grant so it can no longer mint new access tokens.""" + resp = requests.post(oauth_revoke_url_for_region(region), data={ + "token": token, + "token_type_hint": token_type_hint, + "client_id": OAUTH_CLIENT_ID, + }, timeout=OAUTH_HTTP_TIMEOUT_SECONDS) + resp.raise_for_status() diff --git a/cloudinary_cli/core/auth.py b/cloudinary_cli/core/auth.py index e46ab81..9d6b533 100644 --- a/cloudinary_cli/core/auth.py +++ b/cloudinary_cli/core/auth.py @@ -16,17 +16,22 @@ "config is given.") def login(name, region, set_default): try: - config_name = run_login(region=region, name=name, set_default=set_default) + config_name, is_default = run_login(region=region, name=name, set_default=set_default) except Exception as e: log_exception(e, "Login failed") return False logger.info(f"Logged in. Saved as '{config_name}'.") - logger.info(f"Example usage: cld -C {config_name} ") + if is_default: + logger.info(f"This is now the default configuration. Run `cld ` to use it, " + f"or `cld -C {config_name} ` to select it explicitly.") + else: + logger.info(f"Run `cld -C {config_name} ` to use it, " + f"or make it the default with `cld config -d {config_name}`.") return True -@command("logout", help="Log out by removing a saved OAuth configuration. " +@command("logout", help="Log out: revoke a saved OAuth login's token and remove its configuration. " "Run without a name to choose from the saved logins.") @argument("name", required=False) def logout(name): @@ -39,7 +44,10 @@ def logout(name): status = run_logout(name) if status == "removed": - logger.info(f"Logged out of '{name}'.") + logger.info(f"Logged out of '{name}'. Its token was revoked and the saved login removed.") + elif status == "revoke_failed": + logger.warning(f"Removed '{name}', but could not revoke its token at the server " + f"(it may still be valid until it expires).") elif status == "not_oauth": logger.error(f"'{name}' is not an OAuth login; refusing to remove it. " f"Use `config -rm {name}` to delete a saved configuration.") diff --git a/cloudinary_cli/defaults.py b/cloudinary_cli/defaults.py index bec6861..5dcfc6d 100644 --- a/cloudinary_cli/defaults.py +++ b/cloudinary_cli/defaults.py @@ -61,6 +61,10 @@ def oauth_token_url_for_region(region): return f'{oauth_base_url_for_region(region)}/oauth2/token' +def oauth_revoke_url_for_region(region): + return f'{oauth_base_url_for_region(region)}/oauth2/revoke' + + CLOUDINARY_REGION = normalize_region(os.environ.get('CLOUDINARY_REGION')) # Public PKCE client (no secret). Overridable for testing against a non-prod authorization server diff --git a/test/test_auth_flow.py b/test/test_auth_flow.py index d96b48c..d84edf8 100644 --- a/test/test_auth_flow.py +++ b/test/test_auth_flow.py @@ -55,3 +55,15 @@ def test_refresh_posts_refresh_token(self): self.assertEqual("refresh_token", data["grant_type"]) self.assertEqual("rt_abc", data["refresh_token"]) self.assertIn("timeout", post.call_args.kwargs) + + def test_revoke_posts_token_to_revoke_endpoint(self): + resp = MagicMock() + with patch("cloudinary_cli.auth.flow.requests.post", return_value=resp) as post: + flow.revoke("rt_abc", "api-eu") + self.assertEqual("https://oauth.cloudinary.com/oauth2/revoke", post.call_args.args[0]) + data = post.call_args.kwargs["data"] + self.assertEqual("rt_abc", data["token"]) + self.assertEqual("refresh_token", data["token_type_hint"]) + self.assertIn("client_id", data) + self.assertIn("timeout", post.call_args.kwargs) + resp.raise_for_status.assert_called_once() diff --git a/test/test_cli_config_oauth.py b/test/test_cli_config_oauth.py index 3d6e81c..9fb775c 100644 --- a/test/test_cli_config_oauth.py +++ b/test/test_cli_config_oauth.py @@ -42,9 +42,21 @@ def test_removes_oauth_login(self): from cloudinary_cli.auth import logout saved = {"eu-cloud": _oauth_url()} with patch("cloudinary_cli.auth.load_config", return_value=saved), \ - patch("cloudinary_cli.auth.remove_config_keys") as remove: + patch("cloudinary_cli.auth.remove_config_keys") as remove, \ + patch("cloudinary_cli.auth.flow.revoke") as revoke: self.assertEqual("removed", logout("eu-cloud")) remove.assert_called_once_with("eu-cloud") + revoke.assert_called_once_with("rt_secret_value", "api-eu") + + def test_revoke_failure_still_removes_locally(self): + import requests + from cloudinary_cli.auth import logout + saved = {"eu-cloud": _oauth_url()} + with patch("cloudinary_cli.auth.load_config", return_value=saved), \ + patch("cloudinary_cli.auth.remove_config_keys") as remove, \ + patch("cloudinary_cli.auth.flow.revoke", side_effect=requests.ConnectionError()): + self.assertEqual("revoke_failed", logout("eu-cloud")) + remove.assert_called_once_with("eu-cloud") # local entry removed despite revoke failure def test_refuses_non_oauth_config(self): from cloudinary_cli.auth import logout @@ -71,7 +83,8 @@ def test_lists_only_oauth_and_removes_selected(self): saved = {"mykey": "cloudinary://key:secret@cloud", "cloud-a": _oauth_url("cloud-a"), "cloud-b": _oauth_url("cloud-b")} with patch("cloudinary_cli.auth.load_config", return_value=saved), \ - patch("cloudinary_cli.auth.remove_config_keys") as remove: + patch("cloudinary_cli.auth.remove_config_keys") as remove, \ + patch("cloudinary_cli.auth.flow.revoke"): result = self.runner.invoke(cli, ["logout"], input="2\n") self.assertIn("cloud-a", result.output) self.assertIn("cloud-b", result.output) @@ -150,8 +163,17 @@ def test_auto_default_when_sole_config_no_env_no_default(self): with self._patches({"eu-cloud": _oauth_url()}), \ patch("cloudinary_cli.auth.set_default_config") as set_default, \ patch("cloudinary_cli.auth.get_default_config_name", return_value=None): - auth.login(region="eu", name="eu-cloud") + name, is_default = auth.login(region="eu", name="eu-cloud") set_default.assert_called_once_with("eu-cloud") + self.assertEqual(("eu-cloud", True), (name, is_default)) + + def test_returns_not_default_when_other_configs_exist(self): + from cloudinary_cli import auth + with self._patches({"eu-cloud": _oauth_url(), "other": _oauth_url("other")}), \ + patch("cloudinary_cli.auth.set_default_config"), \ + patch("cloudinary_cli.auth.get_default_config_name", return_value=None): + name, is_default = auth.login(region="eu", name="eu-cloud") + self.assertEqual(("eu-cloud", False), (name, is_default)) def test_no_auto_default_when_other_configs_exist(self): from cloudinary_cli import auth @@ -184,6 +206,17 @@ def test_reserved_name_rejected(self): with self.assertRaises(RuntimeError): auth.login(region="eu", name="__default__") + def test_cli_message_when_default(self): + with patch("cloudinary_cli.core.auth.run_login", return_value=("tttt", True)): + result = CliRunner().invoke(cli, ["login", "tttt"]) + self.assertIn("default configuration", result.output) + + def test_cli_message_when_not_default_shows_how_to_default(self): + with patch("cloudinary_cli.core.auth.run_login", return_value=("tttt", False)): + result = CliRunner().invoke(cli, ["login", "tttt"]) + self.assertIn("cld -C tttt", result.output) + self.assertIn("cld config -d tttt", result.output) # how to make it default + class TestConfigSecretMasking(_RestoresSdkConfig): """show_cloudinary_config must never print a secret in the clear.""" From 04805e09115f1bb5de56f277d00974d240b2fbdd Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Sun, 28 Jun 2026 18:02:17 +0300 Subject: [PATCH 15/23] Make OAuth token refresh concurrency-safe Single-flight token rotation under an in-process lock and the existing cross-process file lock, so concurrent workers consume a single-use refresh token at most once. The adopt-vs-rotate decision is keyed on the specific token a worker saw invalid (stale, peer-rotated, or 401), never the clock, avoiding rotation cascades that burn a peer's fresh token. - Recover from AuthorizationRequired via invalidate_token + bounded retry in call_api and upload_file - Detect peer rotation across processes with a config-file mtime check - Read expiry, issued_at and cloud_name from the access token's JWT claims instead of expires_in/local clock; fail loudly on a non-JWT or missing-claim token - Gate SDK response dumps behind should_dump_responses / CLOUDINARY_CLI_LOG_ONLY - Widen expiry skew to 280s Co-Authored-By: Claude Opus 4.8 (1M context) --- cloudinary_cli/auth/__init__.py | 65 +++++-- cloudinary_cli/auth/oauth_config.py | 91 +++++---- cloudinary_cli/auth/session.py | 54 +++--- cloudinary_cli/defaults.py | 4 +- cloudinary_cli/modules/sync.py | 4 +- cloudinary_cli/utils/api_utils.py | 33 +++- cloudinary_cli/utils/config_utils.py | 71 ++++--- cloudinary_cli/utils/utils.py | 10 + test/oauth_helpers.py | 30 +++ test/test_auth_session.py | 92 ++++++--- test/test_cli_config_oauth.py | 11 +- test/test_oauth_multiprocess.py | 210 +++++++++++++++++++++ test/test_oauth_retry.py | 268 +++++++++++++++++++++++++++ test/test_oauth_token_seam.py | 19 +- 14 files changed, 803 insertions(+), 159 deletions(-) create mode 100644 test/oauth_helpers.py create mode 100644 test/test_oauth_multiprocess.py create mode 100644 test/test_oauth_retry.py diff --git a/cloudinary_cli/auth/__init__.py b/cloudinary_cli/auth/__init__.py index 0dfb3e6..0c75262 100644 --- a/cloudinary_cli/auth/__init__.py +++ b/cloudinary_cli/auth/__init__.py @@ -2,6 +2,7 @@ `cloudinary://` entry in `config.json`, and refreshes tokens when a saved login is selected.""" import secrets import webbrowser +from datetime import datetime, timezone import requests @@ -27,12 +28,24 @@ ) from cloudinary_cli.utils.utils import log_exception, is_interactive -# Names already warned about a failed background refresh, so a bulk run (many workers, each reading -# the token) logs the re-login hint once per config instead of once per asset. Mutated only under -# config_lock, so no extra synchronization is needed. +# Configs already warned about a failed refresh, so a bulk run warns once per config, not per asset. _refresh_warned = set() +def _token_hint(token): + """Non-sensitive token fingerprint (trailing chars + length) for debug logs.""" + if not token: + return "" + return f"…{token[-6:]}({len(token)} chars)" + + +def _expiry_hint(epoch): + try: + return datetime.fromtimestamp(int(epoch), tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + except (TypeError, ValueError): + return str(epoch) + + def login(region=None, name=None, set_default=False): """ Run the interactive browser login and persist the resulting session as a named config entry. @@ -106,34 +119,41 @@ def _revoke_login(name, url): return False -def refresh_url_if_stale(name, url, force=False): - """ - Given a saved config value, refresh it if it is a stale OAuth login (rewriting the stored - URL on token rotation). Non-OAuth and still-fresh URLs are returned unchanged. With force=True - a still-fresh token is refreshed too (used by the explicit `config --refresh --force`). +def _should_refresh(session, expected, force): + """Whether `session` should be rotated. `force` rotates any refreshable token; `expected` rotates + only while disk still holds that exact token (else a peer rotated -> adopt); otherwise rotate when + clock-stale.""" + if not session.refresh_token: + return False + if force: + return True + if expected is not None: + return session.access_token == expected + return not session.is_fresh() - The refresh consumes a single-use refresh token, so the whole read-refresh-write runs under - a cross-process lock with the freshness re-checked inside it: a peer that refreshed while we - waited leaves a fresh token we adopt instead of refreshing (and burning) it again. + +def refresh_url_if_stale(name, url, force=False, expected=None): + """ + Refresh a saved config value if its OAuth token should rotate, rewriting the stored URL; other + URLs are returned unchanged. The single-use refresh runs under a cross-process lock, re-checking + the freshly re-read disk token so a peer's rotation is adopted instead of burning another refresh. """ if not is_oauth_url(url): return url - session = from_cloudinary_url(url) - if (session.is_fresh() and not force) or not session.refresh_token: + if not _should_refresh(from_cloudinary_url(url), expected, force): return url with config_lock(): - url = load_config().get(name, url) # re-read: a peer may have refreshed while we waited + url = load_config().get(name, url) # re-read: a peer may have rotated while we waited session = from_cloudinary_url(url) - if (session.is_fresh() and not force) or not session.refresh_token: + if not _should_refresh(session, expected, force): return url try: token_response = flow.refresh(session.refresh_token, session.region) except requests.RequestException as e: - # Serve the stale token (a bulk run survives a transient blip) but make the failure - # visible once per config, not a silent debug line followed by a bare downstream 401. + # Serve the stale token but surface the failure once per config. log_exception(e, debug_message="OAuth token refresh failed") if name not in _refresh_warned: _refresh_warned.add(name) @@ -141,12 +161,17 @@ def refresh_url_if_stale(name, url, force=False): f"token, which may be expired. Re-login with `{relogin_command(name)}`.") return url - _refresh_warned.discard(name) # a later success re-arms the warning for this config + _refresh_warned.discard(name) - # The authorization server rotates refresh tokens; keep the old one only if a new one was not returned. + # Refresh tokens rotate; keep the old one only if a new one was not returned. token_response.setdefault("refresh_token", session.refresh_token) - refreshed_url = to_cloudinary_url(session.updated_from(token_response)) + refreshed = session.updated_from(token_response) + refreshed_url = to_cloudinary_url(refreshed) update_config({name: refreshed_url}) + logger.debug(f"Refreshed OAuth token for '{name}': " + f"access {_token_hint(session.access_token)} -> {_token_hint(refreshed.access_token)}, " + f"refresh {_token_hint(session.refresh_token)} -> {_token_hint(refreshed.refresh_token)}, " + f"expires {_expiry_hint(session.expires_at)} -> {_expiry_hint(refreshed.expires_at)}") return refreshed_url diff --git a/cloudinary_cli/auth/oauth_config.py b/cloudinary_cli/auth/oauth_config.py index 8456c2b..6d0967c 100644 --- a/cloudinary_cli/auth/oauth_config.py +++ b/cloudinary_cli/auth/oauth_config.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +import threading + import cloudinary from cloudinary_cli.auth.session import is_oauth_url, from_cloudinary_url @@ -6,38 +8,40 @@ class OAuthConfig(cloudinary.Config): """ - A Cloudinary config whose `oauth_token` refreshes itself on read, at the moment the SDK builds - a request. Presence/type checks read `has_oauth` instead and never touch the network, so offline - paths (`config -ls`, `config -s`, the group-level validity check) stay offline. - - The raw access token is kept in `__dict__["oauth_token"]` so serialization (config_to_dict, - masking) still sees it; the class-level property shadows it on attribute *read* to - refresh-if-stale. A parsed Session (`_session`) carries expiry/refresh-token so a still-fresh - token short-circuits with no disk read and no lock — only a stale token reads config + refreshes. + A Cloudinary config whose `oauth_token` refreshes itself on read. Presence/type checks read + `has_oauth` instead, which never touches the network, so offline paths stay offline. + + Rotation single-flights: the first worker to see a token invalid (clock-stale, a peer rotated on + disk, or a 401) takes `_refresh_lock` and rotates once; the rest adopt the result. The decision is + keyed on the specific token a worker saw invalid, not the clock, so a token a peer already replaced + is adopted rather than re-rotated (burning a single-use refresh token). Static configs (env / + inline `-c` / api-key, `_session` is None) never refresh. """ + def _init_oauth_state(self, name, session): + from cloudinary_cli.utils.config_utils import config_mtime + self._saved_name = name # None for static configs: they never refresh + self._session = session + self._session_mtime = config_mtime() # a later config mtime = a peer rotated on disk + self._refresh_lock = threading.Lock() + def bind_saved(self, name, url): - # name: the saved-config name this maps to (None for env / inline -c -> never refreshes). - # url: the full cloudinary:// URL, kept parsed so we know expiry without re-reading disk. - self._saved_name = name - self._session = from_cloudinary_url(url) if (name and url and is_oauth_url(url)) else None + session = from_cloudinary_url(url) if (name and url and is_oauth_url(url)) else None + self._init_oauth_state(name, session) @classmethod def from_env(cls): - """An OAuthConfig populated from the environment (CLOUDINARY_URL/CLOUDINARY_*). Static: it is - not bound to a saved name, so reading its oauth_token never refreshes.""" - cfg = cls() # the base Config constructor loads the environment - cfg._saved_name = None - cfg._session = None + """An OAuthConfig from the environment. Static: never refreshes.""" + cfg = cls() + cfg._init_oauth_state(None, None) return cfg @classmethod def from_url(cls, url): - """An OAuthConfig populated from a cloudinary:// URL, not bound to a saved name (static).""" + """An OAuthConfig from a cloudinary:// URL, not bound to a saved name. Static: never refreshes.""" cfg = cls() cfg._load_from_url(url) - cfg._saved_name = None - cfg._session = None + cfg._init_oauth_state(None, None) return cfg @property @@ -45,37 +49,62 @@ def has_oauth(self): """True if this config carries an OAuth token. Cheap, never refreshes.""" return bool(self.__dict__.get("oauth_token")) + def _is_invalid(self, session): + from cloudinary_cli.utils.config_utils import config_mtime + if not session.refresh_token: + return False # unrefreshable: serve it and let it fail + return not session.is_fresh() or config_mtime() > self._session_mtime + @property def oauth_token(self): session = getattr(self, "_session", None) - if session is None: - return self.__dict__.get("oauth_token") # env / -c / api-key: static, no refresh - if session.is_fresh() or not session.refresh_token: - return self.__dict__.get("oauth_token") # still valid (or unrefreshable): no I/O + if session is None or not self._is_invalid(session): + return self.__dict__.get("oauth_token") + + stale_token = self.__dict__.get("oauth_token") + with self._refresh_lock: + if self.__dict__.get("oauth_token") != stale_token: + return self.__dict__.get("oauth_token") # a peer rotated while we waited; adopt it + return self._refresh_locked(stale_token) - # Stale: read the saved URL (a peer may already have refreshed it) and refresh under lock. + def _refresh_locked(self, stale_token): + # Caller holds _refresh_lock. Rotates only while disk still holds `stale_token`, else adopts. from cloudinary_cli.auth import refresh_url_if_stale - from cloudinary_cli.utils.config_utils import load_config + from cloudinary_cli.utils.config_utils import load_config, config_mtime url = load_config().get(self._saved_name) if not url: return self.__dict__.get("oauth_token") # config removed underneath us; serve what we have - fresh_url = refresh_url_if_stale(self._saved_name, url) - self._session = from_cloudinary_url(fresh_url) + url = refresh_url_if_stale(self._saved_name, url, expected=stale_token) + self._session = from_cloudinary_url(url) self.__dict__["oauth_token"] = self._session.access_token + self._session_mtime = config_mtime() return self.__dict__["oauth_token"] @oauth_token.setter def oauth_token(self, value): self.__dict__["oauth_token"] = value + def invalidate_token(self, rejected): + """ + Recover after the server rejected `rejected` (AuthorizationRequired): adopt a peer's rotated + token, else rotate once, keyed on `rejected` so an old-token rejection adopts rather than + re-rotates. Returns True when a usable token is now in place; False for static configs. + """ + if not (getattr(self, "_session", None) and getattr(self, "_saved_name", None)): + return False + with self._refresh_lock: + if self.__dict__.get("oauth_token") != rejected: + return True # a peer already rotated; adopt it + self._refresh_locked(rejected) + return self.__dict__.get("oauth_token") != rejected + def install_oauth_config(cloudinary_url, saved_name=None): """ Load `cloudinary_url` and install it as the active SDK config. The installed object is always an OAuthConfig (so every active config exposes `has_oauth`); it self-refreshes only when bound to a - saved OAuth `saved_name`, and is static for api-key / inline `-c` URLs. The single seam that - swaps the global config object. + saved OAuth `saved_name`, and is static for api-key / inline `-c` URLs. """ cloudinary.reset_config() cfg = OAuthConfig() @@ -87,7 +116,7 @@ def install_oauth_config(cloudinary_url, saved_name=None): def install_env_config(): """Install the environment config as a (static) OAuthConfig, so the active global is always an - OAuthConfig and exposes has_oauth without a refresh. Used for the env fallback branch.""" + OAuthConfig and exposes has_oauth without a refresh.""" cfg = OAuthConfig.from_env() cloudinary._config = cfg return cfg diff --git a/cloudinary_cli/auth/session.py b/cloudinary_cli/auth/session.py index b126ecb..46b6fe3 100644 --- a/cloudinary_cli/auth/session.py +++ b/cloudinary_cli/auth/session.py @@ -8,16 +8,14 @@ from dataclasses import dataclass from cloudinary_cli.defaults import ( - logger, OAUTH_EXPIRY_SKEW_SECONDS, - OAUTH_FALLBACK_EXPIRES_IN_SECONDS, api_host_for_region, ) # Query-string keys that carry the OAuth session inside a cloudinary:// URL. _OAUTH_MARKER = "oauth_token" -_OAUTH_INTERNAL_KEYS = frozenset({"refresh_token", "expires_at", "region", "issuer"}) +_OAUTH_INTERNAL_KEYS = frozenset({"refresh_token", "issued_at", "expires_at", "region", "issuer"}) def strip_oauth_internal_keys(config_dict): @@ -29,6 +27,7 @@ class Session: cloud_name: str access_token: str refresh_token: str = None + issued_at: int = 0 expires_at: int = 0 region: str = "api" issuer: str = None @@ -37,16 +36,22 @@ def is_fresh(self, skew=OAUTH_EXPIRY_SKEW_SECONDS): return int(self.expires_at or 0) - skew > int(time.time()) @classmethod - def from_token_response(cls, token_response, cloud_name=None, region="api", issuer=None): + def from_token_response(cls, token_response, cloud_name=None, region="api"): + # exp/iat come from the token's JWT claims, not the local clock. access_token = token_response["access_token"] - expires_in = int(token_response.get("expires_in") or 0) or OAUTH_FALLBACK_EXPIRES_IN_SECONDS + claims = _decode_jwt_payload(access_token) + cloud_name = cloud_name or _claim_cloud_name(claims) + if not cloud_name: + raise ValueError("OAuth access token has no cloud name (ext.cloud_name claim); " + "cannot perform requests without it") return cls( - cloud_name=cloud_name or decode_cloud_name(access_token), + cloud_name=cloud_name, access_token=access_token, refresh_token=token_response.get("refresh_token"), - expires_at=int(time.time()) + expires_in, + issued_at=_required_claim(claims, "iat"), + expires_at=_required_claim(claims, "exp"), region=region, - issuer=decode_issuer(access_token), + issuer=claims.get("iss"), ) def updated_from(self, token_response): @@ -60,6 +65,7 @@ def to_cloudinary_url(session): params = { "oauth_token": session.access_token, "refresh_token": session.refresh_token or "", + "issued_at": session.issued_at, "expires_at": session.expires_at, "region": session.region, "issuer": session.issuer or "", @@ -76,6 +82,7 @@ def from_cloudinary_url(url): cloud_name=parsed.hostname, access_token=q.get("oauth_token"), refresh_token=q.get("refresh_token") or None, + issued_at=int(q.get("issued_at", 0) or 0), expires_at=int(q.get("expires_at", 0) or 0), region=q.get("region", "api"), issuer=q.get("issuer") or None, @@ -90,24 +97,23 @@ def is_oauth_url(url): def _decode_jwt_payload(access_token): - payload_b64 = access_token.split(".")[1] - payload_b64 += "=" * (-len(payload_b64) % 4) # pad to a multiple of 4 - return json.loads(base64.urlsafe_b64decode(payload_b64)) + """Decode the (unverified) JWT payload of an access token; raises ValueError on a non-JWT token.""" + try: + payload_b64 = access_token.split(".")[1] + payload_b64 += "=" * (-len(payload_b64) % 4) # pad to a multiple of 4 + return json.loads(base64.urlsafe_b64decode(payload_b64)) + except (AttributeError, IndexError, ValueError) as e: + raise ValueError(f"OAuth access token is not a decodable JWT: {e}") from e -def decode_cloud_name(access_token): - """Best-effort extraction of cloud_name from the JWT's `ext` claim.""" +def _required_claim(claims, name): + value = claims.get(name) try: - payload = _decode_jwt_payload(access_token) - return (payload.get("ext") or {}).get("cloud_name") or payload.get("cloud_name") - except Exception as e: - logger.debug(f"Could not decode cloud_name from token: {e}") - return None + return int(value) + except (TypeError, ValueError): + raise ValueError( + f"OAuth access token has a missing or non-numeric '{name}' claim: {value!r}") from None -def decode_issuer(access_token): - try: - return _decode_jwt_payload(access_token).get("iss") - except Exception as e: - logger.debug(f"Could not decode issuer from token: {e}") - return None +def _claim_cloud_name(claims): + return (claims.get("ext") or {}).get("cloud_name") diff --git a/cloudinary_cli/defaults.py b/cloudinary_cli/defaults.py index 5dcfc6d..cba172d 100644 --- a/cloudinary_cli/defaults.py +++ b/cloudinary_cli/defaults.py @@ -82,10 +82,8 @@ def oauth_revoke_url_for_region(region): OAUTH_CALLBACK_PATH = '/callback' OAUTH_CALLBACK_TIMEOUT_SECONDS = 300 -OAUTH_EXPIRY_SKEW_SECONDS = 30 +OAUTH_EXPIRY_SKEW_SECONDS = 280 OAUTH_HTTP_TIMEOUT_SECONDS = 30 -# Fallback when the token response omits expires_in, so it can't pin expires_at to "now". -OAUTH_FALLBACK_EXPIRES_IN_SECONDS = 3600 TEMPLATE_FOLDER_NAME = 'templates' CLOUDINARY_CLI_ROOT = dirname(__file__) diff --git a/cloudinary_cli/modules/sync.py b/cloudinary_cli/modules/sync.py index 483ac4c..f706834 100644 --- a/cloudinary_cli/modules/sync.py +++ b/cloudinary_cli/modules/sync.py @@ -14,7 +14,7 @@ populate_duplicate_name) from cloudinary_cli.utils.json_utils import print_json, read_json_from_file, write_json_to_file from cloudinary_cli.utils.utils import logger, run_tasks_concurrently, get_user_action, invert_dict, chunker, \ - group_params, parse_option_value, duplicate_values + group_params, parse_option_value, duplicate_values, should_dump_responses _DEFAULT_DELETION_BATCH_SIZE = 30 _DEFAULT_CONCURRENT_WORKERS = 30 @@ -83,7 +83,7 @@ def __init__(self, local_dir, remote_dir, include_hidden, concurrent_workers, fo self.sync_meta_file = path.join(self.local_dir, _SYNC_META_FILE) - self.verbose = logger.getEffectiveLevel() < logging.INFO + self.verbose = should_dump_responses() self.local_files = {} self.local_folder_exists = os.path.isdir(path.abspath(self.local_dir)) diff --git a/cloudinary_cli/utils/api_utils.py b/cloudinary_cli/utils/api_utils.py index 3cd0b89..374ea83 100644 --- a/cloudinary_cli/utils/api_utils.py +++ b/cloudinary_cli/utils/api_utils.py @@ -1,9 +1,11 @@ import logging from os import path, makedirs +import cloudinary import requests from click import style, launch from cloudinary import Search, SearchFolders, uploader, api +from cloudinary.exceptions import AuthorizationRequired from cloudinary.utils import cloudinary_url from cloudinary_cli.defaults import logger @@ -12,7 +14,7 @@ populate_duplicate_name) from cloudinary_cli.utils.json_utils import print_json, write_json_to_file from cloudinary_cli.utils.utils import log_exception, confirm_action, get_command_params, merge_responses, \ - normalize_list_params, ConfigurationError, print_api_help, duplicate_values + normalize_list_params, ConfigurationError, print_api_help, duplicate_values, should_dump_responses import re from cloudinary.utils import is_remote_url @@ -158,14 +160,15 @@ def regen_derived_version(public_id, delivery_type, res_type, def upload_file(file_path, options, uploaded=None, failed=None): uploaded = uploaded if uploaded is not None else {} failed = failed if failed is not None else {} - verbose = logger.getEffectiveLevel() < logging.INFO + verbose = should_dump_responses() try: size = 0 if is_remote_url(file_path) else path.getsize(file_path) upload_func = uploader.upload if size > 20000000: upload_func = uploader.upload_large - result = upload_func(file_path, **options) + # Fresh options copy: upload_large mutates it (sets public_id), so a retry stays independent. + result = _call_with_oauth_retry(upload_func, (file_path,), dict(options)) disp_path = _display_path(result) if "batch_id" in result: starting_msg = "Uploading" @@ -290,12 +293,34 @@ def get_folder_mode(): def call_api(func, args, kwargs): try: - return func(*args, **kwargs) + return _call_with_oauth_retry(func, args, kwargs) except Exception as e: log_exception(e, debug_message=f"Failed calling '{func.__name__}' with args: {args} and optional args {kwargs}") raise +# Bounded so a slow call can survive several token rotations without retrying forever. +_OAUTH_RETRY_LIMIT = 3 + + +def _call_with_oauth_retry(func, args, kwargs): + """ + Run an SDK call, recovering from an OAuth token rejection (AuthorizationRequired) by passing the + rejected token to invalidate_token (adopt a peer's rotation, else rotate once) and retrying within + a bounded budget. No-op for static configs (invalidate_token returns False). + """ + config = cloudinary.config() + for attempt in range(_OAUTH_RETRY_LIMIT): + rejected = getattr(config, "oauth_token", None) # the token this request will carry + try: + return func(*args, **kwargs) + except AuthorizationRequired: + last_attempt = attempt == _OAUTH_RETRY_LIMIT - 1 + if last_attempt or not getattr(config, "has_oauth", False) \ + or not config.invalidate_token(rejected): + raise + + def handle_command( params, optional_parameter, diff --git a/cloudinary_cli/utils/config_utils.py b/cloudinary_cli/utils/config_utils.py index b47b786..1928416 100644 --- a/cloudinary_cli/utils/config_utils.py +++ b/cloudinary_cli/utils/config_utils.py @@ -44,6 +44,13 @@ def _config_stat(): return None +def config_mtime(): + """The config file's last-modified time in ns (0 if absent). A cheap cross-process signal for + whether a peer rotated the token.""" + stat = _config_stat() + return stat[0] if stat else 0 + + def _invalidate_config_cache(): global _config_cache, _config_cache_stat _config_cache = None @@ -168,31 +175,47 @@ def _mask_url_secret(url): _ACCOUNT_URL_RE = re.compile(r'^account://([^:/?#]+):([^@]+)@(.+)$') -def _format_account_url(url): - """Render the provisioning account URL as a labeled, secret-masked block (or None if unparsable).""" +def _account_url_fields(url): + """The provisioning account URL as labeled, secret-masked fields (None if unparsable).""" match = _ACCOUNT_URL_RE.match(str(url)) if not match: return None api_key, api_secret, account_id = match.groups() - fields = { + return { "account_id": account_id, "provisioning_api_key": api_key, "provisioning_api_secret": _mask_secret(api_secret), } + + +def _expires_at_fields(value): + """An OAuth expiry epoch expanded into {epoch, utc, expired}, or None if not an int.""" + try: + epoch = int(value) + except (TypeError, ValueError): + return None + return { + "epoch": epoch, + "utc": datetime.fromtimestamp(epoch, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC"), + "expired": epoch <= int(time.time()), + } + + +def _format_account_url(url): + fields = _account_url_fields(url) + if fields is None: + return None width = len(max(fields, key=len)) + 1 template = "{0:" + str(width) + "} {1}" return "\n".join(template.format(f"{k}:", v) for k, v in fields.items()) def _format_expires_at(value): - # OAuth token expiry: show the raw epoch plus a human-readable UTC time and live/expired state. - try: - epoch = int(value) - except (TypeError, ValueError): + parts = _expires_at_fields(value) + if parts is None: return value - human = datetime.fromtimestamp(epoch, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") - state = "expired" if epoch <= int(time.time()) else "valid" - return f"{epoch} ({human}, {state})" + state = "expired" if parts["expired"] else "valid" + return f"{parts['epoch']} ({parts['utc']}, {state})" def show_cloudinary_config(cloudinary_config): @@ -247,11 +270,11 @@ def cloudinary_config_details(cloudinary_config): if key in _SECRET_KEYS: details[key] = _mask_secret(value) elif key == "expires_at": - details[key] = _expires_at_details(value) + details[key] = _expires_at_fields(value) or value else: details[key] = value - account = _account_url_details(account_url) if account_url else None + account = _account_url_fields(account_url) if account_url else None if account is not None: details["account"] = account elif account_url: @@ -260,30 +283,6 @@ def cloudinary_config_details(cloudinary_config): return details -def _expires_at_details(value): - try: - epoch = int(value) - except (TypeError, ValueError): - return value - return { - "epoch": epoch, - "utc": datetime.fromtimestamp(epoch, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC"), - "expired": epoch <= int(time.time()), - } - - -def _account_url_details(url): - match = _ACCOUNT_URL_RE.match(str(url)) - if not match: - return None - api_key, api_secret, account_id = match.groups() - return { - "account_id": account_id, - "provisioning_api_key": api_key, - "provisioning_api_secret": _mask_secret(api_secret), - } - - def _display_value(key, value): if key in _SECRET_KEYS: return _mask_secret(value) diff --git a/cloudinary_cli/utils/utils.py b/cloudinary_cli/utils/utils.py index 8d282a2..5b683b4 100644 --- a/cloudinary_cli/utils/utils.py +++ b/cloudinary_cli/utils/utils.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import builtins import json +import logging import os import sys from collections import OrderedDict @@ -205,6 +206,15 @@ def is_interactive(): return sys.stdin.isatty() +def should_dump_responses(): + """True when full SDK API responses should be echoed (per-asset JSON under upload/sync). On at + DEBUG verbosity, but suppressed by CLOUDINARY_CLI_LOG_ONLY=1 to keep CLI log lines without the + bulky response bodies.""" + if os.environ.get("CLOUDINARY_CLI_LOG_ONLY", "").strip() not in ("", "0", "false", "False"): + return False + return logger.getEffectiveLevel() < logging.INFO + + def prompt_user(message, noninteractive_hint=None): """ Read a line of user input. The single place that calls input(): returns None when no input can diff --git a/test/oauth_helpers.py b/test/oauth_helpers.py new file mode 100644 index 0000000..fb7d350 --- /dev/null +++ b/test/oauth_helpers.py @@ -0,0 +1,30 @@ +"""Build a real (unsigned) JWT access token. Production reads exp/iat/cloud_name/iss from the token's +claims, so fixtures must carry them rather than use opaque strings.""" +import base64 +import json +import time + +_OMIT = object() # sentinel: omit the claim entirely (to test the missing-claim path) + + +def _b64url(data): + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def jwt_access_token(cloud_name="eu-cloud", iat=_OMIT, exp=_OMIT, expires_delta=300, + issuer="https://oauth.cloudinary.com/", tag=None): + """A decodable (unsigned) JWT access token. `iat`/`exp` are absolute epochs, defaulting to + now / now+expires_delta; pass `None` to omit a claim. `tag` varies the signature so successive + rotations produce distinct token strings.""" + now = int(time.time()) + iat = now if iat is _OMIT else iat + exp = (now + expires_delta) if exp is _OMIT else exp + header = _b64url(json.dumps({"alg": "RS256", "typ": "JWT"}).encode()) + claims = {"iss": issuer, "ext": {"cloud_name": cloud_name}} + if iat is not None: + claims["iat"] = iat + if exp is not None: + claims["exp"] = exp + payload = _b64url(json.dumps(claims).encode()) + sig = tag if tag is not None else "sig" + return f"{header}.{payload}.{sig}" diff --git a/test/test_auth_session.py b/test/test_auth_session.py index ee9a484..8323532 100644 --- a/test/test_auth_session.py +++ b/test/test_auth_session.py @@ -19,6 +19,7 @@ is_oauth_url, strip_oauth_internal_keys, ) +from test.oauth_helpers import jwt_access_token def _session(**overrides): @@ -29,6 +30,15 @@ def _session(**overrides): return Session(**base) +# A refresh response carries a real JWT (production reads exp/iat/cloud_name from its claims). Use a +# distinct cloud_name so the resulting access token differs from the stale one being replaced. +_NEW_TOKEN = jwt_access_token(cloud_name="eu-cloud", tag="newsig") + + +def _token_response(access_token=_NEW_TOKEN, refresh_token="rt_new"): + return {"access_token": access_token, "refresh_token": refresh_token, "expires_in": 300} + + class TestSessionCodec(unittest.TestCase): def test_round_trip(self): s = _session() @@ -63,15 +73,47 @@ def test_is_fresh(self): self.assertTrue(_session().is_fresh()) self.assertFalse(_session(expires_at=int(time.time()) - 10).is_fresh()) - def test_missing_expires_in_falls_back_to_fresh(self): - s = Session.from_token_response({"access_token": "eyJ.aaa.bbb"}, cloud_name="c") - self.assertGreater(s.expires_at, int(time.time())) - self.assertTrue(s.is_fresh()) - - def test_zero_expires_in_falls_back_to_fresh(self): - s = Session.from_token_response( - {"access_token": "eyJ.aaa.bbb", "expires_in": 0}, cloud_name="c") - self.assertTrue(s.is_fresh()) + def test_expiry_and_issued_at_come_from_jwt_not_local_clock(self): + # exp/iat are read straight from the token's claims, NOT computed from the local clock or + # expires_in — so a skewed local clock can't distort the stored lifetime. + token = jwt_access_token(cloud_name="c", iat=1000, exp=1300) + s = Session.from_token_response({"access_token": token, "expires_in": 999}, cloud_name="c") + self.assertEqual(1000, s.issued_at) + self.assertEqual(1300, s.expires_at) # 1300 from exp, not 1000+999 and not now+999 + + def test_missing_exp_claim_fails_loudly(self): + token = jwt_access_token(cloud_name="c", iat=1000, exp=None) + with self.assertRaises(ValueError): + Session.from_token_response({"access_token": token}, cloud_name="c") + + def test_non_numeric_exp_claim_fails_loudly(self): + token = jwt_access_token(cloud_name="c", iat=1000, exp="soon") + with self.assertRaises(ValueError): + Session.from_token_response({"access_token": token}, cloud_name="c") + + def test_non_jwt_access_token_fails_loudly(self): + with self.assertRaises(ValueError): + Session.from_token_response({"access_token": "not-a-jwt"}, cloud_name="c") + + def test_cloud_name_read_from_ext_claim_when_not_passed(self): + token = jwt_access_token(cloud_name="from-token", iat=1000, exp=1300) + s = Session.from_token_response({"access_token": token}) # no cloud_name arg + self.assertEqual("from-token", s.cloud_name) + + def test_missing_cloud_name_fails_loudly(self): + # No cloud_name passed and the token carries none -> we cannot address any cloud, so fail. + token = jwt_access_token(cloud_name=None, iat=1000, exp=1300) + with self.assertRaises(ValueError): + Session.from_token_response({"access_token": token}) + + def test_refresh_preserves_cloud_name_without_reading_token(self): + # updated_from passes the existing cloud_name, so a refreshed token need not re-carry it. + original = jwt_access_token(cloud_name="orig", iat=1000, exp=1300) + sess = Session.from_token_response({"access_token": original}) + rotated = jwt_access_token(cloud_name=None, iat=2000, exp=2300, tag="rot") + refreshed = sess.updated_from({"access_token": rotated, "refresh_token": "rt"}) + self.assertEqual("orig", refreshed.cloud_name) + self.assertEqual(2300, refreshed.expires_at) class TestStripOAuthInternalKeys(unittest.TestCase): @@ -108,23 +150,21 @@ def test_fresh_unchanged(self): def test_force_refreshes_fresh_token(self): url = to_cloudinary_url(_session()) # fresh - token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": url}), \ - patch("cloudinary_cli.auth.flow.refresh", return_value=token_response) as refresh, \ + patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()) as refresh, \ patch("cloudinary_cli.auth.update_config"): new_url = refresh_url_if_stale("eu-cloud", url, force=True) refresh.assert_called_once() - self.assertIn("oauth_token=eyJ.new.tok", new_url) + self.assertEqual(_NEW_TOKEN, from_cloudinary_url(new_url).access_token) def test_stale_refreshes_and_rewrites(self): stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) - token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ - patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()), \ patch("cloudinary_cli.auth.update_config") as update_config: new_url = refresh_url_if_stale("eu-cloud", stale_url) - self.assertIn("oauth_token=eyJ.new.tok", new_url) - self.assertIn("refresh_token=rt_new", new_url) + self.assertEqual(_NEW_TOKEN, from_cloudinary_url(new_url).access_token) + self.assertEqual("rt_new", from_cloudinary_url(new_url).refresh_token) update_config.assert_called_once() def test_no_refresh_token_returns_unchanged(self): @@ -163,9 +203,8 @@ def test_refresh_success_rearms_the_warning(self): auth._refresh_warned.add("eu-cloud") self.addCleanup(auth._refresh_warned.discard, "eu-cloud") stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) - token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ - patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()), \ patch("cloudinary_cli.auth.update_config"): refresh_url_if_stale("eu-cloud", stale_url) self.assertNotIn("eu-cloud", auth._refresh_warned) @@ -185,12 +224,11 @@ def test_adopts_peer_refresh_without_calling_refresh(self): def test_refreshes_when_peer_value_still_stale(self): stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) - token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ - patch("cloudinary_cli.auth.flow.refresh", return_value=token_response) as refresh, \ + patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()) as refresh, \ patch("cloudinary_cli.auth.update_config") as update_config: result = refresh_url_if_stale("eu-cloud", stale_url) - self.assertIn("oauth_token=eyJ.new.tok", result) + self.assertEqual(_NEW_TOKEN, from_cloudinary_url(result).access_token) refresh.assert_called_once() update_config.assert_called_once() @@ -220,16 +258,14 @@ def test_fresh_skipped(self): refresh.assert_not_called() def test_stale_refreshed(self): - token_response = {"access_token": "new", "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()), \ - patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()), \ patch("cloudinary_cli.auth.update_config"): self.assertEqual("refreshed", refresh_config("stale")) def test_force_refreshes_fresh(self): - token_response = {"access_token": "new", "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()), \ - patch("cloudinary_cli.auth.flow.refresh", return_value=token_response) as refresh, \ + patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()) as refresh, \ patch("cloudinary_cli.auth.update_config"): self.assertEqual("refreshed", refresh_config("fresh", force=True)) refresh.assert_called_once() @@ -253,9 +289,8 @@ def test_relogin_command_includes_non_default_region(self): self.assertEqual("cld login key", relogin_command("key")) # non-oauth: no region def test_refresh_configs_sweeps_oauth_only(self): - token_response = {"access_token": "new", "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()), \ - patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ + patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()), \ patch("cloudinary_cli.auth.update_config"): results = refresh_configs() self.assertEqual({"stale": "refreshed", "fresh": "fresh"}, results) # "key" not swept @@ -297,7 +332,8 @@ def test_no_browser_but_tty_still_waits(self): patch("cloudinary_cli.auth.webbrowser.open", return_value=False), \ patch("cloudinary_cli.auth.is_interactive", return_value=True), \ patch("cloudinary_cli.auth.wait_for_callback", return_value=("code", "st")) as wait, \ - patch("cloudinary_cli.auth.flow.exchange_code", return_value={"access_token": "x"}): + patch("cloudinary_cli.auth.flow.exchange_code", + return_value={"access_token": jwt_access_token(cloud_name="c")}): # state mismatch is irrelevant here; we only assert it reached the wait (did not fast-fail) with patch("cloudinary_cli.auth.secrets.token_urlsafe", return_value="st"): _run_browser_flow("api-eu") diff --git a/test/test_cli_config_oauth.py b/test/test_cli_config_oauth.py index 9fb775c..2a22706 100644 --- a/test/test_cli_config_oauth.py +++ b/test/test_cli_config_oauth.py @@ -8,6 +8,7 @@ from click.testing import CliRunner from cloudinary_cli.auth.session import Session, to_cloudinary_url +from test.oauth_helpers import jwt_access_token from cloudinary_cli.cli import cli from cloudinary_cli.utils.config_resolver import config_to_api_kwargs, get_cloudinary_config from cloudinary_cli.utils.config_utils import config_to_dict, show_cloudinary_config @@ -558,7 +559,8 @@ def _stale_url(self): def test_reading_oauth_token_refreshes_stale_active_login(self): import cloudinary_cli.utils.config_resolver as resolver saved = {"eu-cloud": self._stale_url()} - token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + new_token = jwt_access_token(cloud_name="eu-cloud", tag="resolver-new") + token_response = {"access_token": new_token, "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ @@ -566,7 +568,7 @@ def test_reading_oauth_token_refreshes_stale_active_login(self): patch("cloudinary_cli.auth.update_config"): resolver.resolve_cli_config(config_saved="eu-cloud") # The read of oauth_token is what triggers the refresh (as the SDK does per request). - self.assertEqual("eyJ.new.tok", cloudinary.config().oauth_token) + self.assertEqual(new_token, cloudinary.config().oauth_token) def test_presence_check_does_not_refresh(self): """has_oauth (used by type/validity/-ls) must NOT touch the network on a stale token.""" @@ -841,7 +843,8 @@ def _stale_url(self): def test_refreshes_stale_target_before_use(self): config = {"eu-cloud": self._stale_url()} - token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + new_token = jwt_access_token(cloud_name="eu-cloud", tag="target-new") + token_response = {"access_token": new_token, "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=config), \ patch("cloudinary_cli.auth.load_config", return_value=config), \ patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ @@ -849,7 +852,7 @@ def test_refreshes_stale_target_before_use(self): patch("cloudinary_cli.utils.config_resolver.ping_cloudinary", return_value=True): target_config = get_cloudinary_config("eu-cloud") self.assertTrue(target_config) - self.assertEqual("eyJ.new.tok", target_config.oauth_token) + self.assertEqual(new_token, target_config.oauth_token) def test_ping_receives_sanitized_config(self): config = {"eu-cloud": _oauth_url()} diff --git a/test/test_oauth_multiprocess.py b/test/test_oauth_multiprocess.py new file mode 100644 index 0000000..83a8c00 --- /dev/null +++ b/test/test_oauth_multiprocess.py @@ -0,0 +1,210 @@ +"""Cross-process OAuth refresh safety: N processes sharing one config.json rotate a single-use +refresh token at most once, mediated by the FileLock and adopt-on-disk check in refresh_url_if_stale. +Workers read oauth_token against a shared token server that counts refresh-token consumption.""" +import json +import multiprocessing +import os +import tempfile +import time +import unittest +from http.server import BaseHTTPRequestHandler, HTTPServer +from threading import Thread +from urllib.parse import parse_qs + +from test.oauth_helpers import jwt_access_token + + +def _make_token(tag): + # Pinned iat/exp so the same tag yields a byte-identical token for cross-process comparison. + return jwt_access_token(cloud_name="proc-cloud", iat=1_700_000_000, exp=2_000_000_000, tag=tag) + + +class _RotatingTokenServer(BaseHTTPRequestHandler): + """Stand-in auth server: each refresh consumes the presented (single-use) token and mints a new + pair; an already-consumed token yields 400. State in class attrs (one server, one process).""" + + valid_refresh = None + generation = 0 + refresh_calls = 0 + rejected_calls = 0 + + def log_message(self, *a): + pass + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length).decode() + params = {k: v[0] for k, v in parse_qs(body).items()} + cls = type(self) + presented = params.get("refresh_token") + cls.refresh_calls += 1 + if presented != cls.valid_refresh: + cls.rejected_calls += 1 + self.send_response(400) + self.end_headers() + self.wfile.write(b'{"error":"invalid_grant"}') + return + cls.generation += 1 + cls.valid_refresh = f"rt_gen{cls.generation}" + resp = { + "access_token": _make_token(f"gen{cls.generation}"), + "refresh_token": cls.valid_refresh, + "expires_in": 300, + } + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(resp).encode()) + + +def _worker(home, token_url, barrier_path, idx, out): + # Point the CLI at the shared config + fake token endpoint, wait at the barrier, then read oauth_token. + os.environ["CLOUDINARY_HOME"] = home + for k in list(os.environ): + if k.startswith("CLOUDINARY_") and k != "CLOUDINARY_HOME": + del os.environ[k] + + import cloudinary + from cloudinary_cli.utils.config_utils import load_config + from cloudinary_cli.auth.oauth_config import install_oauth_config + import cloudinary_cli.auth.flow as flow_mod + + # flow.py imported the URL helper into its own namespace; redirect that binding to the fake server. + flow_mod.oauth_token_url_for_region = lambda region: token_url + + url = load_config()["proc-cloud"] + install_oauth_config(url, saved_name="proc-cloud") + + # cross-process barrier: drop a file, then spin until all are present + open(os.path.join(barrier_path, f"ready-{idx}"), "w").close() + deadline = time.time() + 10 + while len(os.listdir(barrier_path)) < out["n"] and time.time() < deadline: + time.sleep(0.005) + + try: + token = cloudinary.config().oauth_token + out[idx] = token + except Exception as e: # noqa: BLE001 + out[idx] = f"ERROR:{e}" + + +def _worker_401(home, token_url, barrier_path, idx, out): + # Like _worker, but starts clock-fresh and gets one 401: call_api's retry must converge on one rotation. + os.environ["CLOUDINARY_HOME"] = home + for k in list(os.environ): + if k.startswith("CLOUDINARY_") and k != "CLOUDINARY_HOME": + del os.environ[k] + + import cloudinary + from cloudinary.exceptions import AuthorizationRequired + from cloudinary_cli.utils.config_utils import load_config + from cloudinary_cli.utils.api_utils import call_api + from cloudinary_cli.auth.oauth_config import install_oauth_config + import cloudinary_cli.auth.flow as flow_mod + + flow_mod.oauth_token_url_for_region = lambda region: token_url + + url = load_config()["proc-cloud"] + install_oauth_config(url, saved_name="proc-cloud") + + open(os.path.join(barrier_path, f"ready-{idx}"), "w").close() + deadline = time.time() + 10 + while len(os.listdir(barrier_path)) < out["n"] and time.time() < deadline: + time.sleep(0.005) + + state = {"n": 0} + + def api_call(*a, **k): + # Reject the original token once, accept any other. + state["n"] += 1 + token = cloudinary.config().oauth_token + if state["n"] == 1 and token == _make_token("gen0"): + raise AuthorizationRequired("Invalid token [expired]") + return {"ok": token} + + try: + out[idx] = call_api(api_call, (), {})["ok"] + except Exception as e: # noqa: BLE001 + out[idx] = f"ERROR:{e}" + + +class TestCrossProcessSingleFlight(unittest.TestCase): + def setUp(self): + self.home = tempfile.mkdtemp(prefix="cld-mp-home-") + self.barrier = tempfile.mkdtemp(prefix="cld-mp-barrier-") + _RotatingTokenServer.valid_refresh = "rt_gen0" + _RotatingTokenServer.generation = 0 + _RotatingTokenServer.refresh_calls = 0 + _RotatingTokenServer.rejected_calls = 0 + self.server = HTTPServer(("127.0.0.1", 0), _RotatingTokenServer) + self.port = self.server.server_address[1] + self.thread = Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + + def tearDown(self): + self.server.shutdown() + import shutil + shutil.rmtree(self.home, ignore_errors=True) + shutil.rmtree(self.barrier, ignore_errors=True) + + def _write_config(self, expires_delta): + from cloudinary_cli.auth.session import Session, to_cloudinary_url + os.makedirs(self.home, exist_ok=True) + sess = Session( + cloud_name="proc-cloud", access_token=_make_token("gen0"), + refresh_token="rt_gen0", expires_at=int(time.time()) + expires_delta, region="api", + issuer="https://oauth.cloudinary.com/") + with open(os.path.join(self.home, "config.json"), "w") as f: + json.dump({"proc-cloud": to_cloudinary_url(sess)}, f) + + def _write_stale_config(self): + self._write_config(expires_delta=-10) + + def _run_workers(self, worker, n=6): + if multiprocessing.get_start_method(allow_none=True) is None: + multiprocessing.set_start_method("spawn", force=True) + token_url = f"http://127.0.0.1:{self.port}/oauth2/token" + mgr = multiprocessing.Manager() + out = mgr.dict() + out["n"] = n + procs = [multiprocessing.Process(target=worker, + args=(self.home, token_url, self.barrier, i, out)) + for i in range(n)] + for p in procs: + p.start() + for p in procs: + p.join(30) + results = {i: out[i] for i in range(n)} + errors = {i: v for i, v in results.items() if isinstance(v, str) and v.startswith("ERROR")} + self.assertEqual({}, errors, f"workers errored: {errors}") + return results + + def test_n_processes_consume_single_use_refresh_token_once(self): + # Proactive path: all processes start on the same stale token and read oauth_token together. + self._write_stale_config() + results = self._run_workers(_worker) + + self.assertEqual(0, _RotatingTokenServer.rejected_calls, + "a process presented an already-consumed refresh token (rotation cascade)") + self.assertEqual(1, _RotatingTokenServer.generation, + "the single-use refresh token rotated more than once across processes") + distinct = set(results.values()) + self.assertEqual(1, len(distinct), f"workers disagreed on the token: {distinct}") + self.assertEqual(_make_token("gen1"), next(iter(distinct))) + + def test_n_processes_recover_from_401_with_single_rotation(self): + # Reactive path: all processes start clock-fresh, each gets one 401; retry converges on one rotation. + self._write_config(expires_delta=300) + results = self._run_workers(_worker_401) + + self.assertEqual(0, _RotatingTokenServer.rejected_calls, + "a process presented an already-consumed refresh token (rotation cascade)") + self.assertEqual(1, _RotatingTokenServer.generation, + "a 401 on a clock-fresh token rotated more than once across processes") + distinct = set(results.values()) + self.assertEqual({_make_token("gen1")}, distinct, + f"workers did not all recover onto the rotated token: {distinct}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_oauth_retry.py b/test/test_oauth_retry.py new file mode 100644 index 0000000..7982153 --- /dev/null +++ b/test/test_oauth_retry.py @@ -0,0 +1,268 @@ +"""In-process single-flight refresh and 401 retry/adopt behavior for the OAuth token seam.""" +import threading +import time +import unittest +from unittest.mock import patch, MagicMock + +import cloudinary +from cloudinary.exceptions import AuthorizationRequired + +from cloudinary_cli.auth.oauth_config import OAuthConfig, install_oauth_config, install_env_config +from cloudinary_cli.auth.session import Session, to_cloudinary_url, from_cloudinary_url +from cloudinary_cli.utils.api_utils import call_api, _OAUTH_RETRY_LIMIT + +from test.oauth_helpers import jwt_access_token + + +def _url(cloud="eu-cloud", token="eyJ.tok", refresh="rt", region="api-eu", expires_delta=300): + return to_cloudinary_url(Session( + cloud_name=cloud, access_token=token, refresh_token=refresh, + expires_at=int(time.time()) + expires_delta, region=region, + issuer="https://oauth.cloudinary.com/")) + + +class _RestoresSdkConfig(unittest.TestCase): + def setUp(self): + import os + self._env_snapshot = dict(os.environ) + for key in [k for k in os.environ if k.startswith("CLOUDINARY_")]: + del os.environ[key] + cloudinary.reset_config() + self.addCleanup(self._restore) + + def _restore(self): + import os + os.environ.clear() + os.environ.update(self._env_snapshot) + cloudinary.reset_config() + + +class TestSingleFlightRefresh(_RestoresSdkConfig): + """Concurrent stale reads refresh once under the in-process lock; losers adopt the result.""" + + def test_concurrent_stale_reads_refresh_once(self): + saved = {"eu-cloud": _url(token="eyJ.old", refresh="rt_old", expires_delta=-10)} + new_token = jwt_access_token(cloud_name="eu-cloud", tag="single-flight-new") + token_response = {"access_token": new_token, "refresh_token": "rt_new", "expires_in": 300} + + def slow_refresh(refresh_token, region): + time.sleep(0.02) # widen the window so threads pile on the lock + return dict(token_response) + + with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh", side_effect=slow_refresh) as refresh, \ + patch("cloudinary_cli.auth.update_config"): + install_oauth_config(saved["eu-cloud"], saved_name="eu-cloud") + config = cloudinary.config() + + results = [] + + def read_token(): + results.append(config.oauth_token) + + threads = [threading.Thread(target=read_token) for _ in range(20)] + for t in threads: + t.start() + for t in threads: + t.join() + + refresh.assert_called_once() # one rotation, not 20 + self.assertEqual([new_token] * 20, results) + + +class TestRetryOn401(_RestoresSdkConfig): + """call_api marks the token invalid on AuthorizationRequired and retries via the refresh seam.""" + + def test_retries_after_peer_rotation_and_succeeds(self): + # Config holds the rejected token but a peer wrote a new one: retry adopts it, no rotation. + install_oauth_config(_url(token="eyJ.old", refresh="rt_old", expires_delta=300), + saved_name="eu-cloud") + saved = {"eu-cloud": _url(token="eyJ.new", refresh="rt_new", expires_delta=300)} + calls = {"n": 0} + + def func(*a, **k): + calls["n"] += 1 + if calls["n"] == 1: + raise AuthorizationRequired("Invalid token [expired]") + return {"public_id": "ok"} + + with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh") as refresh, \ + patch("cloudinary_cli.auth.update_config"): + result = call_api(func, ("file.mp4",), {}) + + self.assertEqual({"public_id": "ok"}, result) + self.assertEqual(2, calls["n"]) # original + one retry + refresh.assert_not_called() # adopted peer's token, no rotation + self.assertEqual("eyJ.new", cloudinary.config().oauth_token) + + def test_401_on_clock_fresh_token_forces_one_refresh_then_succeeds(self): + # No peer rotated: the rejected token is still clock-fresh on disk, so the retry forces one rotation. + install_oauth_config(_url(token="eyJ.old", refresh="rt_old", expires_delta=300), + saved_name="eu-cloud") + saved = {"eu-cloud": _url(token="eyJ.old", refresh="rt_old", expires_delta=300)} + new_token = jwt_access_token(cloud_name="eu-cloud", tag="clock-fresh-new") + calls = {"n": 0} + + def func(*a, **k): + calls["n"] += 1 + if calls["n"] == 1: + raise AuthorizationRequired("Invalid token [expired]") + return {"public_id": "ok"} + + with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh", + return_value={"access_token": new_token, "refresh_token": "rt_new", + "expires_in": 300}) as refresh, \ + patch("cloudinary_cli.auth.update_config"): + result = call_api(func, ("file.mp4",), {}) + + self.assertEqual({"public_id": "ok"}, result) + self.assertEqual(2, calls["n"]) + refresh.assert_called_once() # forced past the clock by the 401 + self.assertEqual(new_token, cloudinary.config().oauth_token) + + def test_revoked_token_fails_fast(self): + # flow.refresh fails too: no new token to adopt, so propagate after the first attempt. + import requests + install_oauth_config(_url(token="eyJ.old", refresh="rt_old", expires_delta=300), + saved_name="eu-cloud") + saved = {"eu-cloud": _url(token="eyJ.old", refresh="rt_old", expires_delta=300)} + calls = {"n": 0} + + def func(*a, **k): + calls["n"] += 1 + raise AuthorizationRequired("Invalid token [expired]") + + with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh", + side_effect=requests.RequestException("refresh token revoked")), \ + patch("cloudinary_cli.auth.update_config"): + with self.assertRaises(AuthorizationRequired): + call_api(func, ("file.mp4",), {}) + + self.assertEqual(1, calls["n"]) # nothing to adopt -> fail fast + + def test_retry_is_bounded_under_repeated_rotation(self): + # Every retry's token is rejected again: recovery stops after _OAUTH_RETRY_LIMIT attempts. + install_oauth_config(_url(token="eyJ.t0", refresh="rt0", expires_delta=300), + saved_name="eu-cloud") + saved = {"eu-cloud": _url(token="eyJ.t0", refresh="rt0", expires_delta=300)} + calls = {"n": 0} + + def func(*a, **k): + calls["n"] += 1 + raise AuthorizationRequired("Invalid token [expired]") + + def ever_new_token(refresh_token, region): + return {"access_token": jwt_access_token(cloud_name="eu-cloud", tag=f"t{calls['n']}"), + "refresh_token": f"rt{calls['n']}", "expires_in": 300} + + with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh", side_effect=ever_new_token), \ + patch("cloudinary_cli.auth.update_config"): + with self.assertRaises(AuthorizationRequired): + call_api(func, ("file.mp4",), {}) + + self.assertEqual(_OAUTH_RETRY_LIMIT, calls["n"]) # bounded + + def test_non_oauth_config_propagates_immediately(self): + install_oauth_config("cloudinary://key:secret@cloud", saved_name=None) # api-key: has_oauth False + calls = {"n": 0} + + def func(*a, **k): + calls["n"] += 1 + raise AuthorizationRequired("nope") + + with self.assertRaises(AuthorizationRequired): + call_api(func, ("x",), {}) + self.assertEqual(1, calls["n"]) # no adopt attempt on a non-OAuth config + + def test_env_config_propagates_immediately(self): + import os + os.environ["CLOUDINARY_URL"] = ( + "cloudinary://env_cloud?oauth_token=eyJ.env&refresh_token=rt&" + f"expires_at={int(time.time()) - 10}®ion=api") + cloudinary.reset_config() + install_env_config() # static: _saved_name is None -> invalidate_token returns False + calls = {"n": 0} + + def func(*a, **k): + calls["n"] += 1 + raise AuthorizationRequired("expired") + + with self.assertRaises(AuthorizationRequired): + call_api(func, ("x",), {}) + self.assertEqual(1, calls["n"]) + + def test_success_passes_through_without_refresh(self): + install_oauth_config(_url(), saved_name="eu-cloud") + sentinel = MagicMock(return_value={"public_id": "p"}) + result = call_api(sentinel, ("file",), {"folder": "f"}) + self.assertEqual({"public_id": "p"}, result) + sentinel.assert_called_once_with("file", folder="f") # no retry, args forwarded verbatim + + +class TestRefreshDecision(_RestoresSdkConfig): + """refresh_url_if_stale's rotate-vs-adopt rule for `force`, `expected`, and the proactive sweep.""" + + def _refresh(self, **kwargs): + from cloudinary_cli.auth import refresh_url_if_stale + return refresh_url_if_stale("eu-cloud", self.url, **kwargs) + + def test_expected_matches_disk_rotates_even_when_clock_fresh(self): + # 401 path: token clock-fresh but rejected; disk still holds it -> rotate once. + self.url = _url(token="eyJ.cur", refresh="rt", expires_delta=300) + saved = {"eu-cloud": self.url} + new_token = jwt_access_token(cloud_name="eu-cloud", tag="expected-matches-new") + with patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh", + return_value={"access_token": new_token, "refresh_token": "rt2", + "expires_in": 300}) as refresh, \ + patch("cloudinary_cli.auth.update_config"): + new_url = self._refresh(expected="eyJ.cur") + refresh.assert_called_once() + self.assertEqual(new_token, from_cloudinary_url(new_url).access_token) + + def test_expected_differs_from_disk_adopts_without_refresh(self): + # Peer already rotated: disk token != expected -> adopt, no network. + self.url = _url(token="eyJ.new", refresh="rt2", expires_delta=300) # what disk now holds + saved = {"eu-cloud": self.url} + with patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh") as refresh, \ + patch("cloudinary_cli.auth.update_config"): + new_url = self._refresh(expected="eyJ.old") # we were sent the OLD token + refresh.assert_not_called() + self.assertEqual(self.url, new_url) + + def test_force_refreshes_a_clock_fresh_token_user_path(self): + # `config --refresh --force`: rotate even a perfectly fresh token. + self.url = _url(token="eyJ.cur", refresh="rt", expires_delta=300) + saved = {"eu-cloud": self.url} + forced_token = jwt_access_token(cloud_name="eu-cloud", tag="forced-new") + with patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh", + return_value={"access_token": forced_token, "refresh_token": "rt2", + "expires_in": 300}) as refresh, \ + patch("cloudinary_cli.auth.update_config"): + new_url = self._refresh(force=True) + refresh.assert_called_once() + self.assertEqual(forced_token, from_cloudinary_url(new_url).access_token) + + def test_no_expected_no_force_uses_clock_freshness(self): + # The proactive sweep with no specific token: a fresh token is left untouched. + self.url = _url(token="eyJ.cur", refresh="rt", expires_delta=300) + saved = {"eu-cloud": self.url} + with patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.flow.refresh") as refresh, \ + patch("cloudinary_cli.auth.update_config"): + new_url = self._refresh() + refresh.assert_not_called() + self.assertEqual(self.url, new_url) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_oauth_token_seam.py b/test/test_oauth_token_seam.py index b3a1558..b665774 100644 --- a/test/test_oauth_token_seam.py +++ b/test/test_oauth_token_seam.py @@ -11,6 +11,8 @@ from cloudinary_cli.auth.oauth_config import OAuthConfig, install_oauth_config, install_env_config from cloudinary_cli.auth.session import Session, to_cloudinary_url +from test.oauth_helpers import jwt_access_token + def _url(cloud="eu-cloud", token="eyJ.tok", refresh="rt", region="api-eu", expires_delta=300): return to_cloudinary_url(Session( @@ -44,7 +46,8 @@ def _saved_stale(self): def test_call_api_authorize_path_refreshes_stale_token(self): saved = self._saved_stale() - token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + new_token = jwt_access_token(cloud_name="eu-cloud", tag="call-api-new") + token_response = {"access_token": new_token, "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ @@ -54,11 +57,12 @@ def test_call_api_authorize_path_refreshes_stale_token(self): # time (call_api.py:63): options.pop("oauth_token", cloudinary.config().oauth_token). options = {} oauth_token = options.pop("oauth_token", cloudinary.config().oauth_token) - self.assertEqual("eyJ.new.tok", oauth_token) + self.assertEqual(new_token, oauth_token) def test_uploader_header_path_refreshes_stale_token(self): saved = self._saved_stale() - token_response = {"access_token": "eyJ.fresh.tok", "refresh_token": "rt_new", "expires_in": 300} + fresh_token = jwt_access_token(cloud_name="eu-cloud", tag="uploader-fresh") + token_response = {"access_token": fresh_token, "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ @@ -69,11 +73,12 @@ def test_uploader_header_path_refreshes_stale_token(self): import cloudinary.uploader # noqa: F401 (ensures the seam module is importable) options = {} token = options.get("oauth_token", cloudinary.config().oauth_token) - self.assertEqual("eyJ.fresh.tok", token) + self.assertEqual(fresh_token, token) def test_seam_read_refreshes_only_once_then_serves_cached(self): saved = self._saved_stale() - token_response = {"access_token": "eyJ.new.tok", "refresh_token": "rt_new", "expires_in": 300} + new_token = jwt_access_token(cloud_name="eu-cloud", tag="cached-new") + token_response = {"access_token": new_token, "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", return_value=token_response) as refresh, \ @@ -81,8 +86,8 @@ def test_seam_read_refreshes_only_once_then_serves_cached(self): install_oauth_config(saved["eu-cloud"], saved_name="eu-cloud") first = cloudinary.config().oauth_token second = cloudinary.config().oauth_token - self.assertEqual("eyJ.new.tok", first) - self.assertEqual("eyJ.new.tok", second) + self.assertEqual(new_token, first) + self.assertEqual(new_token, second) refresh.assert_called_once() # the now-fresh _session short-circuits the second read From ec335d5456a7432f07748e3bc33bc1c531003cb1 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Mon, 29 Jun 2026 16:47:44 +0300 Subject: [PATCH 16/23] Improve OAuth retry --- cloudinary_cli/auth/__init__.py | 141 ++++----------------------- cloudinary_cli/auth/flow.py | 30 ++++++ cloudinary_cli/auth/oauth_config.py | 34 +++++-- cloudinary_cli/auth/refresh.py | 123 +++++++++++++++++++++++ cloudinary_cli/core/config.py | 2 +- cloudinary_cli/modules/sync.py | 4 +- cloudinary_cli/utils/api_utils.py | 41 ++++---- cloudinary_cli/utils/config_utils.py | 23 +++++ cloudinary_cli/utils/utils.py | 15 +++ test/test_auth_flow.py | 67 +++++++++++++ test/test_auth_session.py | 103 ++++++++++++------- test/test_cli_config_oauth.py | 55 +++++++++-- test/test_modules/test_cli_sync.py | 3 + test/test_oauth_retry.py | 69 +++++++++---- test/test_oauth_token_seam.py | 12 +-- 15 files changed, 500 insertions(+), 222 deletions(-) create mode 100644 cloudinary_cli/auth/refresh.py diff --git a/cloudinary_cli/auth/__init__.py b/cloudinary_cli/auth/__init__.py index 0c75262..f5f7e54 100644 --- a/cloudinary_cli/auth/__init__.py +++ b/cloudinary_cli/auth/__init__.py @@ -1,8 +1,7 @@ -"""OAuth login façade: runs the PKCE loopback flow, persists each login as a named -`cloudinary://` entry in `config.json`, and refreshes tokens when a saved login is selected.""" +"""OAuth login façade: runs the PKCE loopback flow and persists each login as a named +`cloudinary://` entry in `config.json`. Token refresh lives in `auth.refresh`, re-exported here.""" import secrets import webbrowser -from datetime import datetime, timezone import requests @@ -14,12 +13,18 @@ from_cloudinary_url, is_oauth_url, ) +from cloudinary_cli.auth.refresh import ( + refresh_url_if_stale, + refresh_config, + refresh_configs, + relogin_command, + list_oauth_login_names, +) from cloudinary_cli.defaults import logger, normalize_region, DEFAULT_REGION, CLOUDINARY_REGION from cloudinary_cli.utils.config_utils import ( load_config, update_config, remove_config_keys, - config_lock, user_config_names, get_default_config_name, set_default_config, @@ -28,22 +33,15 @@ ) from cloudinary_cli.utils.utils import log_exception, is_interactive -# Configs already warned about a failed refresh, so a bulk run warns once per config, not per asset. -_refresh_warned = set() - - -def _token_hint(token): - """Non-sensitive token fingerprint (trailing chars + length) for debug logs.""" - if not token: - return "" - return f"…{token[-6:]}({len(token)} chars)" - - -def _expiry_hint(epoch): - try: - return datetime.fromtimestamp(int(epoch), tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") - except (TypeError, ValueError): - return str(epoch) +__all__ = [ + "login", + "logout", + "refresh_url_if_stale", + "refresh_config", + "refresh_configs", + "relogin_command", + "list_oauth_login_names", +] def login(region=None, name=None, set_default=False): @@ -119,109 +117,6 @@ def _revoke_login(name, url): return False -def _should_refresh(session, expected, force): - """Whether `session` should be rotated. `force` rotates any refreshable token; `expected` rotates - only while disk still holds that exact token (else a peer rotated -> adopt); otherwise rotate when - clock-stale.""" - if not session.refresh_token: - return False - if force: - return True - if expected is not None: - return session.access_token == expected - return not session.is_fresh() - - -def refresh_url_if_stale(name, url, force=False, expected=None): - """ - Refresh a saved config value if its OAuth token should rotate, rewriting the stored URL; other - URLs are returned unchanged. The single-use refresh runs under a cross-process lock, re-checking - the freshly re-read disk token so a peer's rotation is adopted instead of burning another refresh. - """ - if not is_oauth_url(url): - return url - - if not _should_refresh(from_cloudinary_url(url), expected, force): - return url - - with config_lock(): - url = load_config().get(name, url) # re-read: a peer may have rotated while we waited - session = from_cloudinary_url(url) - if not _should_refresh(session, expected, force): - return url - - try: - token_response = flow.refresh(session.refresh_token, session.region) - except requests.RequestException as e: - # Serve the stale token but surface the failure once per config. - log_exception(e, debug_message="OAuth token refresh failed") - if name not in _refresh_warned: - _refresh_warned.add(name) - logger.warning(f"Could not refresh the OAuth token for '{name}'; using the existing " - f"token, which may be expired. Re-login with `{relogin_command(name)}`.") - return url - - _refresh_warned.discard(name) - - # Refresh tokens rotate; keep the old one only if a new one was not returned. - token_response.setdefault("refresh_token", session.refresh_token) - refreshed = session.updated_from(token_response) - refreshed_url = to_cloudinary_url(refreshed) - update_config({name: refreshed_url}) - logger.debug(f"Refreshed OAuth token for '{name}': " - f"access {_token_hint(session.access_token)} -> {_token_hint(refreshed.access_token)}, " - f"refresh {_token_hint(session.refresh_token)} -> {_token_hint(refreshed.refresh_token)}, " - f"expires {_expiry_hint(session.expires_at)} -> {_expiry_hint(refreshed.expires_at)}") - return refreshed_url - - -def refresh_config(name, force=False): - """ - Refresh a single saved OAuth config by name and report the outcome. Returns one of: - "not_found", "not_oauth", "fresh" (skipped, still valid), "refreshed", or "failed" - ("failed" = stale/forced but no refresh token, or the network refresh did not rotate it). - """ - cfg = load_config() - if name not in user_config_names(cfg): - return "not_found" - url = cfg[name] - if not is_oauth_url(url): - return "not_oauth" - - session = from_cloudinary_url(url) - if session.is_fresh() and not force: - return "fresh" - if not session.refresh_token: - return "failed" - - new_url = refresh_url_if_stale(name, url, force=force) - return "refreshed" if new_url != url else "failed" - - -def refresh_configs(force=False): - """Refresh every saved OAuth config. Returns {name: outcome} (see refresh_config).""" - return {name: refresh_config(name, force=force) for name in list_oauth_login_names()} - - -def relogin_command(name): - """ - Build the `cld login` command to re-authenticate a saved OAuth config, preserving its region - (a non-default region must be passed explicitly so the right OAuth host is used). - """ - cmd = f"cld login {name}" - url = load_config().get(name) - region = from_cloudinary_url(url).region if url and is_oauth_url(url) else None - if region and region != DEFAULT_REGION: - cmd += f" --region {region}" - return cmd - - -def list_oauth_login_names(): - """Return the names of all saved OAuth logins.""" - cfg = load_config() - return [name for name in user_config_names(cfg) if is_oauth_url(cfg[name])] - - def _run_browser_flow(region): verifier, challenge = flow.generate_pkce_pair() state = secrets.token_urlsafe(16) diff --git a/cloudinary_cli/auth/flow.py b/cloudinary_cli/auth/flow.py index a10444e..f7fcbf5 100644 --- a/cloudinary_cli/auth/flow.py +++ b/cloudinary_cli/auth/flow.py @@ -61,6 +61,36 @@ def refresh(refresh_token, region): return resp.json() +_MAX_OAUTH_DESCRIPTION = 80 + + +def oauth_error_body(exc): + """The raw response body text from a failed token request, or None if no response is attached. + Logged verbatim at debug for investigation - it carries the full server error_description.""" + resp = getattr(exc, "response", None) + return resp.text if resp is not None else None + + +def oauth_error_detail(exc): + """The server's OAuth error code from a failed token request (RFC 6749 §5.2), or None when the + response carries no parseable OAuth error body. The error_description is appended only when it is + short; the endpoint often returns a multi-sentence boilerplate paragraph that is noise in a log.""" + resp = getattr(exc, "response", None) + if resp is None: + return None + try: + body = resp.json() + except ValueError: + return None + error = body.get("error") + if not error: + return None + description = body.get("error_description") + if description and len(description) <= _MAX_OAUTH_DESCRIPTION: + return f"{error}: {description}" + return error + + def revoke(token, region, token_type_hint="refresh_token"): """Revoke a token at the authorization server (RFC 7009). Revoking the refresh token ends the offline-access grant so it can no longer mint new access tokens.""" diff --git a/cloudinary_cli/auth/oauth_config.py b/cloudinary_cli/auth/oauth_config.py index 6d0967c..5189f8d 100644 --- a/cloudinary_cli/auth/oauth_config.py +++ b/cloudinary_cli/auth/oauth_config.py @@ -2,8 +2,13 @@ import threading import cloudinary +from cloudinary.exceptions import AuthorizationRequired from cloudinary_cli.auth.session import is_oauth_url, from_cloudinary_url +from cloudinary_cli.auth.refresh import refresh_url_if_stale +from cloudinary_cli.defaults import logger +from cloudinary_cli.utils import config_utils +from cloudinary_cli.utils.utils import token_hint class OAuthConfig(cloudinary.Config): @@ -19,12 +24,27 @@ class OAuthConfig(cloudinary.Config): """ def _init_oauth_state(self, name, session): - from cloudinary_cli.utils.config_utils import config_mtime self._saved_name = name # None for static configs: they never refresh self._session = session - self._session_mtime = config_mtime() # a later config mtime = a peer rotated on disk + self._session_mtime = config_utils.config_mtime() # a later config mtime = a peer rotated on disk self._refresh_lock = threading.Lock() + @property + def oauth_token_refresh_callback(self): + # SDK hook (uploader.upload_large): rotate on a chunk 401, retrying the chunk to resume. + # A property (not a __dict__ entry) so it stays out of config serialization, which dumps + # the public keys of __dict__. + return self._refresh_for_sdk + + def _refresh_for_sdk(self, rejected): + # invalidate_token returns False when no usable token results (static config, dead refresh); + # the SDK contract signals that by raising, so the rejected token is not retried. + logger.debug(f"Upload chunk got a 401 on token {token_hint(rejected)}; attempting OAuth refresh") + if not self.invalidate_token(rejected): + logger.debug(f"OAuth refresh did not yield a new token for {token_hint(rejected)}; chunk upload fails") + raise AuthorizationRequired("OAuth token refresh produced no usable token") + logger.debug(f"OAuth token refreshed to {token_hint(self.__dict__.get('oauth_token'))}; retrying upload chunk") + def bind_saved(self, name, url): session = from_cloudinary_url(url) if (name and url and is_oauth_url(url)) else None self._init_oauth_state(name, session) @@ -50,10 +70,9 @@ def has_oauth(self): return bool(self.__dict__.get("oauth_token")) def _is_invalid(self, session): - from cloudinary_cli.utils.config_utils import config_mtime if not session.refresh_token: return False # unrefreshable: serve it and let it fail - return not session.is_fresh() or config_mtime() > self._session_mtime + return not session.is_fresh() or config_utils.config_mtime() > self._session_mtime @property def oauth_token(self): @@ -69,16 +88,13 @@ def oauth_token(self): def _refresh_locked(self, stale_token): # Caller holds _refresh_lock. Rotates only while disk still holds `stale_token`, else adopts. - from cloudinary_cli.auth import refresh_url_if_stale - from cloudinary_cli.utils.config_utils import load_config, config_mtime - - url = load_config().get(self._saved_name) + url = config_utils.load_config().get(self._saved_name) if not url: return self.__dict__.get("oauth_token") # config removed underneath us; serve what we have url = refresh_url_if_stale(self._saved_name, url, expected=stale_token) self._session = from_cloudinary_url(url) self.__dict__["oauth_token"] = self._session.access_token - self._session_mtime = config_mtime() + self._session_mtime = config_utils.config_mtime() return self.__dict__["oauth_token"] @oauth_token.setter diff --git a/cloudinary_cli/auth/refresh.py b/cloudinary_cli/auth/refresh.py new file mode 100644 index 0000000..4974f19 --- /dev/null +++ b/cloudinary_cli/auth/refresh.py @@ -0,0 +1,123 @@ +"""Non-interactive OAuth token refresh: rotates saved tokens on read/401 under a cross-process lock.""" +import requests + +from cloudinary_cli.auth import flow +from cloudinary_cli.auth.session import from_cloudinary_url, to_cloudinary_url, is_oauth_url +from cloudinary_cli.defaults import logger, DEFAULT_REGION +from cloudinary_cli.utils.config_utils import ( + load_config, + update_config, + config_lock, + user_config_names, +) +from cloudinary_cli.utils.utils import token_hint, expiry_hint + +# Configs already warned about a failed refresh, so a bulk run warns once per config, not per asset. +_refresh_warned = set() + + +def _should_refresh(session, expected, force): + """Whether `session` should be rotated. `force` rotates any refreshable token; `expected` rotates + only while disk still holds that exact token (else a peer rotated -> adopt); otherwise rotate when + clock-stale.""" + if not session.refresh_token: + return False + if force: + return True + if expected is not None: + return session.access_token == expected + return not session.is_fresh() + + +def refresh_url_if_stale(name, url, force=False, expected=None): + """ + Refresh a saved config value if its OAuth token should rotate, rewriting the stored URL; other + URLs are returned unchanged. The single-use refresh runs under a cross-process lock, re-checking + the freshly re-read disk token so a peer's rotation is adopted instead of burning another refresh. + """ + if not is_oauth_url(url): + return url + + if not _should_refresh(from_cloudinary_url(url), expected, force): + return url + + with config_lock(): + url = load_config().get(name, url) # re-read: a peer may have rotated while we waited + session = from_cloudinary_url(url) + if not _should_refresh(session, expected, force): + return url + + try: + token_response = flow.refresh(session.refresh_token, session.region) + except requests.RequestException as e: + body = flow.oauth_error_body(e) + logger.debug(f"OAuth token refresh failed for '{name}': {e}" + + (f"; response body: {body}" if body else ""), exc_info=True) + if name not in _refresh_warned: + _refresh_warned.add(name) + detail = flow.oauth_error_detail(e) + reason = f" ({detail})" if detail else "" + logger.warning(f"Could not refresh the OAuth token for '{name}'{reason}; using the " + f"existing token, which may be expired. Re-login with " + f"`{relogin_command(name)}`.") + return url + + _refresh_warned.discard(name) + + # Refresh tokens rotate; keep the old one only if a new one was not returned. + token_response.setdefault("refresh_token", session.refresh_token) + refreshed = session.updated_from(token_response) + refreshed_url = to_cloudinary_url(refreshed) + update_config({name: refreshed_url}) + logger.debug(f"Refreshed OAuth token for '{name}': " + f"access {token_hint(session.access_token)} -> {token_hint(refreshed.access_token)}, " + f"refresh {token_hint(session.refresh_token)} -> {token_hint(refreshed.refresh_token)}, " + f"expires {expiry_hint(session.expires_at)} -> {expiry_hint(refreshed.expires_at)}") + return refreshed_url + + +def refresh_config(name, force=False): + """ + Refresh a single saved OAuth config by name and report the outcome. Returns one of: + "not_found", "not_oauth", "fresh" (skipped, still valid), "refreshed", or "failed" + ("failed" = stale/forced but no refresh token, or the network refresh did not rotate it). + """ + cfg = load_config() + if name not in user_config_names(cfg): + return "not_found" + url = cfg[name] + if not is_oauth_url(url): + return "not_oauth" + + session = from_cloudinary_url(url) + if session.is_fresh() and not force: + return "fresh" + if not session.refresh_token: + return "failed" + + new_url = refresh_url_if_stale(name, url, force=force) + return "refreshed" if new_url != url else "failed" + + +def refresh_configs(force=False): + """Refresh every saved OAuth config. Returns {name: outcome} (see refresh_config).""" + return {name: refresh_config(name, force=force) for name in list_oauth_login_names()} + + +def relogin_command(name): + """ + Build the `cld login` command to re-authenticate a saved OAuth config, preserving its region + (a non-default region must be passed explicitly so the right OAuth host is used). + """ + cmd = f"cld login {name}" + url = load_config().get(name) + region = from_cloudinary_url(url).region if url and is_oauth_url(url) else None + if region and region != DEFAULT_REGION: + cmd += f" --region {region}" + return cmd + + +def list_oauth_login_names(): + """Return the names of all saved OAuth logins.""" + cfg = load_config() + return [name for name in user_config_names(cfg) if is_oauth_url(cfg[name])] diff --git a/cloudinary_cli/core/config.py b/cloudinary_cli/core/config.py index 258e118..c0daaea 100644 --- a/cloudinary_cli/core/config.py +++ b/cloudinary_cli/core/config.py @@ -137,7 +137,7 @@ def config_command(new, ls, as_json, show, rm, from_url, default, set_default, u "not_oauth": ("info", "'{name}' is an api-key config; nothing to refresh."), "fresh": ("info", "'{name}' token is still fresh; nothing to refresh (use --force to refresh anyway)."), "refreshed": ("info", "Refreshed '{name}'."), - "failed": ("error", "Could not refresh '{name}'. Please re-login with `{relogin}`."), + "failed": ("error", "'{name}' could not be refreshed; re-login with `{relogin}`."), } diff --git a/cloudinary_cli/modules/sync.py b/cloudinary_cli/modules/sync.py index f706834..56af6d4 100644 --- a/cloudinary_cli/modules/sync.py +++ b/cloudinary_cli/modules/sync.py @@ -6,6 +6,7 @@ from os import path, remove from click import command, argument, option, style, UsageError, Choice +import cloudinary from cloudinary import api from cloudinary_cli.utils.api_utils import query_cld_folder, upload_file, download_file, get_folder_mode, \ @@ -177,7 +178,8 @@ def push(self): logger.info(f"{file}") return True - logger.info(f"Uploading {len(files_to_push)} items to Cloudinary folder '{self.user_friendly_remote_dir}'") + logger.info(f"Uploading {len(files_to_push)} items to Cloudinary folder '{self.user_friendly_remote_dir}' " + f"in cloud '{cloudinary.config().cloud_name}'") options = { **get_default_upload_options(self.folder_mode), diff --git a/cloudinary_cli/utils/api_utils.py b/cloudinary_cli/utils/api_utils.py index 374ea83..bf0fdf1 100644 --- a/cloudinary_cli/utils/api_utils.py +++ b/cloudinary_cli/utils/api_utils.py @@ -164,11 +164,13 @@ def upload_file(file_path, options, uploaded=None, failed=None): try: size = 0 if is_remote_url(file_path) else path.getsize(file_path) - upload_func = uploader.upload - if size > 20000000: - upload_func = uploader.upload_large # Fresh options copy: upload_large mutates it (sets public_id), so a retry stays independent. - result = _call_with_oauth_retry(upload_func, (file_path,), dict(options)) + if size > 20000000: + # upload_large recovers from a token 401 per chunk (SDK oauth_token_refresh_callback), + # resuming the upload; a whole-file retry here would restart from byte 0. + result = uploader.upload_large(file_path, **dict(options)) + else: + result = _call_with_oauth_retry(uploader.upload, (file_path,), dict(options)) disp_path = _display_path(result) if "batch_id" in result: starting_msg = "Uploading" @@ -299,26 +301,27 @@ def call_api(func, args, kwargs): raise -# Bounded so a slow call can survive several token rotations without retrying forever. -_OAUTH_RETRY_LIMIT = 3 - - def _call_with_oauth_retry(func, args, kwargs): """ Run an SDK call, recovering from an OAuth token rejection (AuthorizationRequired) by passing the - rejected token to invalidate_token (adopt a peer's rotation, else rotate once) and retrying within - a bounded budget. No-op for static configs (invalidate_token returns False). + rejected token to invalidate_token (adopt a peer's rotation, else rotate once) and retrying once. + No-op for static configs (invalidate_token returns False). + + The token is pinned: the value read here is the value passed to the SDK and to invalidate_token, + so the token handed to invalidate_token is provably the one the request carried. Without pinning, + the SDK re-reads the self-refreshing oauth_token property independently, so a peer rotation between + the two reads makes invalidate_token mis-decide "adopt" for a token the wire never sent, and the + retry re-fails. """ config = cloudinary.config() - for attempt in range(_OAUTH_RETRY_LIMIT): - rejected = getattr(config, "oauth_token", None) # the token this request will carry - try: - return func(*args, **kwargs) - except AuthorizationRequired: - last_attempt = attempt == _OAUTH_RETRY_LIMIT - 1 - if last_attempt or not getattr(config, "has_oauth", False) \ - or not config.invalidate_token(rejected): - raise + token = getattr(config, "oauth_token", None) # the token this request will carry + pinned = dict(kwargs, oauth_token=token) if token else kwargs + try: + return func(*args, **pinned) + except AuthorizationRequired: + if not getattr(config, "has_oauth", False) or not config.invalidate_token(token): + raise + return func(*args, **kwargs) # retry unpinned: the SDK re-reads the freshly rotated token def handle_command( diff --git a/cloudinary_cli/utils/config_utils.py b/cloudinary_cli/utils/config_utils.py index 1928416..e5a5926 100644 --- a/cloudinary_cli/utils/config_utils.py +++ b/cloudinary_cli/utils/config_utils.py @@ -210,6 +210,25 @@ def _format_account_url(url): return "\n".join(template.format(f"{k}:", v) for k, v in fields.items()) +def _issued_at_fields(value): + """An OAuth issued-at epoch expanded into {epoch, utc}, or None if not an int.""" + try: + epoch = int(value) + except (TypeError, ValueError): + return None + return { + "epoch": epoch, + "utc": datetime.fromtimestamp(epoch, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC"), + } + + +def _format_epoch(value): + parts = _issued_at_fields(value) + if parts is None: + return value + return f"{parts['epoch']} ({parts['utc']})" + + def _format_expires_at(value): parts = _expires_at_fields(value) if parts is None: @@ -271,6 +290,8 @@ def cloudinary_config_details(cloudinary_config): details[key] = _mask_secret(value) elif key == "expires_at": details[key] = _expires_at_fields(value) or value + elif key == "issued_at": + details[key] = _issued_at_fields(value) or value else: details[key] = value @@ -288,6 +309,8 @@ def _display_value(key, value): return _mask_secret(value) if key == "expires_at": return _format_expires_at(value) + if key == "issued_at": + return _format_epoch(value) return value diff --git a/cloudinary_cli/utils/utils.py b/cloudinary_cli/utils/utils.py index 5b683b4..a4bd68c 100644 --- a/cloudinary_cli/utils/utils.py +++ b/cloudinary_cli/utils/utils.py @@ -6,6 +6,7 @@ import sys from collections import OrderedDict from csv import DictWriter +from datetime import datetime, timezone from functools import reduce from hashlib import md5 from inspect import signature, getfullargspec @@ -72,6 +73,20 @@ def print_api_help(api, block_list=not_callable, allow_list=()): logger.info(get_help_str(api, block_list=block_list, allow_list=allow_list)) +def token_hint(token): + """Non-sensitive token fingerprint (trailing chars + length) for debug logs.""" + if not token: + return "" + return f"…{token[-6:]}({len(token)} chars)" + + +def expiry_hint(epoch): + try: + return datetime.fromtimestamp(int(epoch), tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + except (TypeError, ValueError): + return str(epoch) + + def log_exception(e, message=None, debug_message=None): message = f"{message}, error: {str(e)}" if message is not None else str(e) debug_message = debug_message or message diff --git a/test/test_auth_flow.py b/test/test_auth_flow.py index d84edf8..3304ede 100644 --- a/test/test_auth_flow.py +++ b/test/test_auth_flow.py @@ -4,9 +4,26 @@ from unittest.mock import patch, MagicMock from urllib.parse import urlparse, parse_qs +import requests + from cloudinary_cli.auth import flow +def _http_error(body=None, no_response=False, not_json=False): + e = requests.HTTPError("400 Client Error: Bad Request for url: https://oauth.cloudinary.com/oauth2/token") + if no_response: + return e + resp = MagicMock() + if not_json: + resp.json.side_effect = ValueError("no json") + resp.text = body if body is not None else "500" + else: + resp.json.return_value = body + resp.text = str(body) + e.response = resp + return e + + class TestAuthFlow(unittest.TestCase): def test_pkce_pair_s256_no_padding(self): verifier, challenge = flow.generate_pkce_pair() @@ -67,3 +84,53 @@ def test_revoke_posts_token_to_revoke_endpoint(self): self.assertIn("client_id", data) self.assertIn("timeout", post.call_args.kwargs) resp.raise_for_status.assert_called_once() + + +class TestOAuthErrorDetail(unittest.TestCase): + """flow.oauth_error_detail extracts the RFC 6749 error code, appending a short description but + never the multi-sentence boilerplate the token endpoint returns for invalid_grant.""" + + def test_short_description_is_appended(self): + e = _http_error({"error": "invalid_client", "error_description": "Unknown client"}) + self.assertEqual("invalid_client: Unknown client", flow.oauth_error_detail(e)) + + def test_long_description_is_suppressed(self): + # The real invalid_grant body is a >80-char paragraph; only the code should surface. + long_desc = ("The provided authorization grant or refresh token is invalid, expired, " + "revoked, or was issued to another client. The refresh token is malformed.") + e = _http_error({"error": "invalid_grant", "error_description": long_desc}) + self.assertEqual("invalid_grant", flow.oauth_error_detail(e)) + + def test_error_only_no_description(self): + e = _http_error({"error": "invalid_grant"}) + self.assertEqual("invalid_grant", flow.oauth_error_detail(e)) + + def test_description_exactly_at_limit_is_kept(self): + desc = "x" * 80 + e = _http_error({"error": "invalid_request", "error_description": desc}) + self.assertEqual(f"invalid_request: {desc}", flow.oauth_error_detail(e)) + + def test_description_one_over_limit_is_dropped(self): + e = _http_error({"error": "invalid_request", "error_description": "x" * 81}) + self.assertEqual("invalid_request", flow.oauth_error_detail(e)) + + def test_no_error_key_returns_none(self): + self.assertIsNone(flow.oauth_error_detail(_http_error({"foo": "bar"}))) + + def test_non_json_body_returns_none(self): + self.assertIsNone(flow.oauth_error_detail(_http_error(not_json=True))) + + def test_no_response_returns_none(self): + self.assertIsNone(flow.oauth_error_detail(_http_error(no_response=True))) + + +class TestOAuthErrorBody(unittest.TestCase): + """flow.oauth_error_body returns the raw response text verbatim for debug logging.""" + + def test_returns_raw_text(self): + raw = '{"error":"invalid_grant","error_description":"long boilerplate here"}' + e = _http_error(not_json=True, body=raw) + self.assertEqual(raw, flow.oauth_error_body(e)) + + def test_no_response_returns_none(self): + self.assertIsNone(flow.oauth_error_body(_http_error(no_response=True))) diff --git a/test/test_auth_session.py b/test/test_auth_session.py index 8323532..bdef819 100644 --- a/test/test_auth_session.py +++ b/test/test_auth_session.py @@ -150,18 +150,18 @@ def test_fresh_unchanged(self): def test_force_refreshes_fresh_token(self): url = to_cloudinary_url(_session()) # fresh - with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": url}), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"eu-cloud": url}), \ patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()) as refresh, \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): new_url = refresh_url_if_stale("eu-cloud", url, force=True) refresh.assert_called_once() self.assertEqual(_NEW_TOKEN, from_cloudinary_url(new_url).access_token) def test_stale_refreshes_and_rewrites(self): stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) - with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"eu-cloud": stale_url}), \ patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()), \ - patch("cloudinary_cli.auth.update_config") as update_config: + patch("cloudinary_cli.auth.refresh.update_config") as update_config: new_url = refresh_url_if_stale("eu-cloud", stale_url) self.assertEqual(_NEW_TOKEN, from_cloudinary_url(new_url).access_token) self.assertEqual("rt_new", from_cloudinary_url(new_url).refresh_token) @@ -174,9 +174,9 @@ def test_no_refresh_token_returns_unchanged(self): def test_refresh_timeout_returns_stale_url(self): import requests stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) - with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"eu-cloud": stale_url}), \ patch("cloudinary_cli.auth.flow.refresh", side_effect=requests.Timeout()), \ - patch("cloudinary_cli.auth.update_config") as update_config: + patch("cloudinary_cli.auth.refresh.update_config") as update_config: self.assertEqual(stale_url, refresh_url_if_stale("eu-cloud", stale_url)) update_config.assert_not_called() @@ -184,39 +184,76 @@ def test_refresh_failure_warns_once_per_config(self): # A3a: a failed background refresh must surface a re-login hint (not just a debug line), but # only once per config so a bulk run does not log it per asset. import requests - import cloudinary_cli.auth as auth - auth._refresh_warned.discard("eu-cloud") - self.addCleanup(auth._refresh_warned.discard, "eu-cloud") + import cloudinary_cli.auth.refresh as refresh_mod + refresh_mod._refresh_warned.discard("eu-cloud") + self.addCleanup(refresh_mod._refresh_warned.discard, "eu-cloud") stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) - with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"eu-cloud": stale_url}), \ patch("cloudinary_cli.auth.flow.refresh", side_effect=requests.ConnectionError()), \ - patch("cloudinary_cli.auth.update_config"), \ - patch("cloudinary_cli.auth.logger.warning") as warn: + patch("cloudinary_cli.auth.refresh.update_config"), \ + patch("cloudinary_cli.auth.refresh.logger.warning") as warn: refresh_url_if_stale("eu-cloud", stale_url) refresh_url_if_stale("eu-cloud", stale_url) # second stale read in the same run warn.assert_called_once() self.assertIn("cld login eu-cloud", warn.call_args[0][0]) + def test_refresh_failure_warning_includes_oauth_error_code(self): + # An HTTPError carrying an OAuth body surfaces the server's error code in the warning. + import requests + import cloudinary_cli.auth.refresh as refresh_mod + refresh_mod._refresh_warned.discard("eu-cloud") + self.addCleanup(refresh_mod._refresh_warned.discard, "eu-cloud") + resp = mock.MagicMock() + resp.json.return_value = {"error": "invalid_grant", "error_description": "x" * 200} + http_error = requests.HTTPError("400 Client Error") + http_error.response = resp + stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"eu-cloud": stale_url}), \ + patch("cloudinary_cli.auth.flow.refresh", side_effect=http_error), \ + patch("cloudinary_cli.auth.refresh.update_config"), \ + patch("cloudinary_cli.auth.refresh.logger.warning") as warn: + refresh_url_if_stale("eu-cloud", stale_url) + msg = warn.call_args[0][0] + self.assertIn("(invalid_grant)", msg) # code surfaced + self.assertNotIn("x" * 200, msg) # long boilerplate not dumped into the warning + self.assertIn("cld login eu-cloud", msg) + + def test_refresh_failure_warning_without_response_omits_detail(self): + # A bare connection error (no response body) still warns, just without an error code paren. + import requests + import cloudinary_cli.auth.refresh as refresh_mod + refresh_mod._refresh_warned.discard("eu-cloud") + self.addCleanup(refresh_mod._refresh_warned.discard, "eu-cloud") + stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"eu-cloud": stale_url}), \ + patch("cloudinary_cli.auth.flow.refresh", side_effect=requests.ConnectionError()), \ + patch("cloudinary_cli.auth.refresh.update_config"), \ + patch("cloudinary_cli.auth.refresh.logger.warning") as warn: + refresh_url_if_stale("eu-cloud", stale_url) + msg = warn.call_args[0][0] + self.assertNotIn("(", msg.split("using the")[0]) # no error-code paren before the hint + self.assertIn("cld login eu-cloud", msg) + def test_refresh_success_rearms_the_warning(self): # After a successful refresh the warning is re-armed, so a later failure warns again. - import cloudinary_cli.auth as auth - auth._refresh_warned.add("eu-cloud") - self.addCleanup(auth._refresh_warned.discard, "eu-cloud") + import cloudinary_cli.auth.refresh as refresh_mod + refresh_mod._refresh_warned.add("eu-cloud") + self.addCleanup(refresh_mod._refresh_warned.discard, "eu-cloud") stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) - with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"eu-cloud": stale_url}), \ patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()), \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): refresh_url_if_stale("eu-cloud", stale_url) - self.assertNotIn("eu-cloud", auth._refresh_warned) + self.assertNotIn("eu-cloud", refresh_mod._refresh_warned) def test_adopts_peer_refresh_without_calling_refresh(self): # Peer already rewrote the saved URL to a fresh token while we waited for the lock. stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) peer_fresh_url = to_cloudinary_url(_session( access_token="eyJ.peer.tok", expires_at=int(time.time()) + 300)) - with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": peer_fresh_url}), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"eu-cloud": peer_fresh_url}), \ patch("cloudinary_cli.auth.flow.refresh") as refresh, \ - patch("cloudinary_cli.auth.update_config") as update_config: + patch("cloudinary_cli.auth.refresh.update_config") as update_config: result = refresh_url_if_stale("eu-cloud", stale_url) self.assertEqual(peer_fresh_url, result) refresh.assert_not_called() # we did not burn the (already-rotated) refresh token @@ -224,9 +261,9 @@ def test_adopts_peer_refresh_without_calling_refresh(self): def test_refreshes_when_peer_value_still_stale(self): stale_url = to_cloudinary_url(_session(expires_at=int(time.time()) - 10)) - with patch("cloudinary_cli.auth.load_config", return_value={"eu-cloud": stale_url}), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"eu-cloud": stale_url}), \ patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()) as refresh, \ - patch("cloudinary_cli.auth.update_config") as update_config: + patch("cloudinary_cli.auth.refresh.update_config") as update_config: result = refresh_url_if_stale("eu-cloud", stale_url) self.assertEqual(_NEW_TOKEN, from_cloudinary_url(result).access_token) refresh.assert_called_once() @@ -244,36 +281,36 @@ def _cfg(self, **extra): return cfg def test_not_found(self): - with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()): + with patch("cloudinary_cli.auth.refresh.load_config", return_value=self._cfg()): self.assertEqual("not_found", refresh_config("ghost")) def test_not_oauth(self): - with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()): + with patch("cloudinary_cli.auth.refresh.load_config", return_value=self._cfg()): self.assertEqual("not_oauth", refresh_config("key")) def test_fresh_skipped(self): - with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value=self._cfg()), \ patch("cloudinary_cli.auth.flow.refresh") as refresh: self.assertEqual("fresh", refresh_config("fresh")) refresh.assert_not_called() def test_stale_refreshed(self): - with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value=self._cfg()), \ patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()), \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): self.assertEqual("refreshed", refresh_config("stale")) def test_force_refreshes_fresh(self): - with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value=self._cfg()), \ patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()) as refresh, \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): self.assertEqual("refreshed", refresh_config("fresh", force=True)) refresh.assert_called_once() def test_failed_when_no_refresh_token(self): cfg = self._cfg(stale=to_cloudinary_url(_session( cloud_name="stale", expires_at=int(time.time()) - 10, refresh_token=None))) - with patch("cloudinary_cli.auth.load_config", return_value=cfg): + with patch("cloudinary_cli.auth.refresh.load_config", return_value=cfg): self.assertEqual("failed", refresh_config("stale")) def test_relogin_command_includes_non_default_region(self): @@ -283,15 +320,15 @@ def test_relogin_command_includes_non_default_region(self): "stg": to_cloudinary_url(_session(cloud_name="stg", region="api-staging")), "key": "cloudinary://k:s@kc", } - with patch("cloudinary_cli.auth.load_config", return_value=cfg): + with patch("cloudinary_cli.auth.refresh.load_config", return_value=cfg): self.assertEqual("cld login global", relogin_command("global")) self.assertEqual("cld login stg --region api-staging", relogin_command("stg")) self.assertEqual("cld login key", relogin_command("key")) # non-oauth: no region def test_refresh_configs_sweeps_oauth_only(self): - with patch("cloudinary_cli.auth.load_config", return_value=self._cfg()), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value=self._cfg()), \ patch("cloudinary_cli.auth.flow.refresh", return_value=_token_response()), \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): results = refresh_configs() self.assertEqual({"stale": "refreshed", "fresh": "fresh"}, results) # "key" not swept diff --git a/test/test_cli_config_oauth.py b/test/test_cli_config_oauth.py index 2a22706..108e789 100644 --- a/test/test_cli_config_oauth.py +++ b/test/test_cli_config_oauth.py @@ -84,6 +84,7 @@ def test_lists_only_oauth_and_removes_selected(self): saved = {"mykey": "cloudinary://key:secret@cloud", "cloud-a": _oauth_url("cloud-a"), "cloud-b": _oauth_url("cloud-b")} with patch("cloudinary_cli.auth.load_config", return_value=saved), \ + patch("cloudinary_cli.auth.refresh.load_config", return_value=saved), \ patch("cloudinary_cli.auth.remove_config_keys") as remove, \ patch("cloudinary_cli.auth.flow.revoke"): result = self.runner.invoke(cli, ["logout"], input="2\n") @@ -93,7 +94,7 @@ def test_lists_only_oauth_and_removes_selected(self): remove.assert_called_once_with("cloud-b") def test_no_oauth_logins(self): - with patch("cloudinary_cli.auth.load_config", + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"mykey": "cloudinary://key:secret@cloud"}), \ patch("cloudinary_cli.auth.remove_config_keys") as remove: result = self.runner.invoke(cli, ["logout"], input="\n") @@ -101,14 +102,14 @@ def test_no_oauth_logins(self): remove.assert_not_called() def test_cancel_on_empty_input(self): - with patch("cloudinary_cli.auth.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ patch("cloudinary_cli.auth.remove_config_keys") as remove: result = self.runner.invoke(cli, ["logout"], input="\n") remove.assert_not_called() self.assertEqual(0, result.exit_code) def test_invalid_non_numeric_errors(self): - with patch("cloudinary_cli.auth.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ patch("cloudinary_cli.auth.remove_config_keys") as remove: result = self.runner.invoke(cli, ["logout"], input="sdfdsf\n", standalone_mode=False) self.assertIn("Invalid selection", result.output) @@ -116,7 +117,7 @@ def test_invalid_non_numeric_errors(self): remove.assert_not_called() def test_out_of_range_errors(self): - with patch("cloudinary_cli.auth.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ patch("cloudinary_cli.auth.remove_config_keys") as remove: result = self.runner.invoke(cli, ["logout"], input="5\n", standalone_mode=False) self.assertIn("Invalid selection", result.output) @@ -127,7 +128,7 @@ def test_noninteractive_stdin_errors_with_hint(self): # Closed stdin (no input at all): the selection cannot be made, so error with the # non-interactive form (`cld logout `) and exit non-zero, not a silent no-op. import builtins - with patch("cloudinary_cli.auth.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value={"cloud-a": _oauth_url("cloud-a")}), \ patch("cloudinary_cli.auth.remove_config_keys") as remove, \ patch.object(builtins, "input", side_effect=EOFError()): result = self.runner.invoke(cli, ["logout"], standalone_mode=False) @@ -289,6 +290,26 @@ def test_expires_at_human_readable_and_state(self): self.assertIn("2026-06-24", out) # human-readable date self.assertIn("expired", out) + def test_issued_at_human_readable_no_state(self): + config = cloudinary.Config() + config.update(cloud_name="c", oauth_token="eyJ.tok", issued_at=1782310673) + with patch("cloudinary_cli.utils.config_utils.echo") as echo: + show_cloudinary_config(config) + out = echo.call_args[0][0] + self.assertIn("1782310673", out) # raw epoch kept + self.assertIn("2026-06-24", out) # human-readable date + self.assertIn("UTC", out) + self.assertNotIn("valid", out) # issuance has no validity state + self.assertNotIn("expired", out) + + def test_issued_at_non_numeric_left_as_is(self): + config = cloudinary.Config() + config.update(cloud_name="c", oauth_token="eyJ.tok", issued_at="not-an-epoch") + with patch("cloudinary_cli.utils.config_utils.echo") as echo: + show_cloudinary_config(config) + out = echo.call_args[0][0] + self.assertIn("not-an-epoch", out) + def test_account_url_shown_as_structured_section(self): config = cloudinary.Config() config.update(cloud_name="c", api_key="k", api_secret="abcdefghIJKLMNOP", @@ -418,6 +439,22 @@ def test_show_json_includes_meta_and_masks_secrets(self): self.assertIn("epoch", data["expires_at"]) self.assertIn("expired", data["expires_at"]) + def test_details_expands_issued_at_without_state(self): + from cloudinary_cli.utils.config_utils import cloudinary_config_details + config = cloudinary.Config() + config.update(cloud_name="c", oauth_token="eyJ.tok", issued_at=1782310673) + details = cloudinary_config_details(config) + self.assertEqual(1782310673, details["issued_at"]["epoch"]) + self.assertIn("2026-06-24", details["issued_at"]["utc"]) + self.assertNotIn("expired", details["issued_at"]) # issuance has no validity state + + def test_details_issued_at_non_numeric_left_as_is(self): + from cloudinary_cli.utils.config_utils import cloudinary_config_details + config = cloudinary.Config() + config.update(cloud_name="c", oauth_token="eyJ.tok", issued_at="bogus") + details = cloudinary_config_details(config) + self.assertEqual("bogus", details["issued_at"]) + def test_bare_config_json_matches_show_json(self): cfg = {"__default__": "eu-cloud", "eu-cloud": _oauth_url()} bare = self._show_json(['config', '--json'], cfg) @@ -562,10 +599,10 @@ def test_reading_oauth_token_refreshes_stale_active_login(self): new_token = jwt_access_token(cloud_name="eu-cloud", tag="resolver-new") token_response = {"access_token": new_token, "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=dict(saved)), \ - patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): resolver.resolve_cli_config(config_saved="eu-cloud") # The read of oauth_token is what triggers the refresh (as the SDK does per request). self.assertEqual(new_token, cloudinary.config().oauth_token) @@ -846,9 +883,9 @@ def test_refreshes_stale_target_before_use(self): new_token = jwt_access_token(cloud_name="eu-cloud", tag="target-new") token_response = {"access_token": new_token, "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.utils.config_resolver.load_config", return_value=config), \ - patch("cloudinary_cli.auth.load_config", return_value=config), \ + patch("cloudinary_cli.auth.refresh.load_config", return_value=config), \ patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ - patch("cloudinary_cli.auth.update_config"), \ + patch("cloudinary_cli.auth.refresh.update_config"), \ patch("cloudinary_cli.utils.config_resolver.ping_cloudinary", return_value=True): target_config = get_cloudinary_config("eu-cloud") self.assertTrue(target_config) diff --git a/test/test_modules/test_cli_sync.py b/test/test_modules/test_cli_sync.py index 9d785ba..a3f7a25 100644 --- a/test/test_modules/test_cli_sync.py +++ b/test/test_modules/test_cli_sync.py @@ -43,6 +43,9 @@ def test_cli_sync_push(self): self.assertEqual(0, result.exit_code) self.assertIn("Synced | 12", result.output) self.assertIn("Done!", result.output) + # the upload banner names both the destination folder and the active cloud + self.assertIn(f"to Cloudinary folder '{self.CLD_SYNC_DIR}'", result.output) + self.assertIn("in cloud '", result.output) def test_cli_sync_push_non_existing_folder(self): non_existing_dir = self.LOCAL_SYNC_PULL_DIR + "non_existing" diff --git a/test/test_oauth_retry.py b/test/test_oauth_retry.py index 7982153..62a7257 100644 --- a/test/test_oauth_retry.py +++ b/test/test_oauth_retry.py @@ -9,7 +9,7 @@ from cloudinary_cli.auth.oauth_config import OAuthConfig, install_oauth_config, install_env_config from cloudinary_cli.auth.session import Session, to_cloudinary_url, from_cloudinary_url -from cloudinary_cli.utils.api_utils import call_api, _OAUTH_RETRY_LIMIT +from cloudinary_cli.utils.api_utils import call_api from test.oauth_helpers import jwt_access_token @@ -50,9 +50,9 @@ def slow_refresh(refresh_token, region): return dict(token_response) with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ - patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", side_effect=slow_refresh) as refresh, \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): install_oauth_config(saved["eu-cloud"], saved_name="eu-cloud") config = cloudinary.config() @@ -88,8 +88,9 @@ def func(*a, **k): return {"public_id": "ok"} with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh") as refresh, \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): result = call_api(func, ("file.mp4",), {}) self.assertEqual({"public_id": "ok"}, result) @@ -112,10 +113,11 @@ def func(*a, **k): return {"public_id": "ok"} with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", return_value={"access_token": new_token, "refresh_token": "rt_new", "expires_in": 300}) as refresh, \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): result = call_api(func, ("file.mp4",), {}) self.assertEqual({"public_id": "ok"}, result) @@ -136,17 +138,17 @@ def func(*a, **k): raise AuthorizationRequired("Invalid token [expired]") with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ - patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", side_effect=requests.RequestException("refresh token revoked")), \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): with self.assertRaises(AuthorizationRequired): call_api(func, ("file.mp4",), {}) self.assertEqual(1, calls["n"]) # nothing to adopt -> fail fast - def test_retry_is_bounded_under_repeated_rotation(self): - # Every retry's token is rejected again: recovery stops after _OAUTH_RETRY_LIMIT attempts. + def test_one_refresh_and_retry_then_propagates(self): + # Refresh succeeds (new token) but the server rejects it too: one retry, then propagate. install_oauth_config(_url(token="eyJ.t0", refresh="rt0", expires_delta=300), saved_name="eu-cloud") saved = {"eu-cloud": _url(token="eyJ.t0", refresh="rt0", expires_delta=300)} @@ -161,13 +163,13 @@ def ever_new_token(refresh_token, region): "refresh_token": f"rt{calls['n']}", "expires_in": 300} with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ - patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", side_effect=ever_new_token), \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): with self.assertRaises(AuthorizationRequired): call_api(func, ("file.mp4",), {}) - self.assertEqual(_OAUTH_RETRY_LIMIT, calls["n"]) # bounded + self.assertEqual(2, calls["n"]) # original + one retry, no unbounded rotation def test_non_oauth_config_propagates_immediately(self): install_oauth_config("cloudinary://key:secret@cloud", saved_name=None) # api-key: has_oauth False @@ -203,7 +205,32 @@ def test_success_passes_through_without_refresh(self): sentinel = MagicMock(return_value={"public_id": "p"}) result = call_api(sentinel, ("file",), {"folder": "f"}) self.assertEqual({"public_id": "p"}, result) - sentinel.assert_called_once_with("file", folder="f") # no retry, args forwarded verbatim + # no retry; args forwarded verbatim with the active token pinned so the wire token == rejected + sentinel.assert_called_once_with("file", folder="f", oauth_token="eyJ.tok") + + def test_token_pinned_so_wire_token_equals_invalidate_arg(self): + # The token sent to the SDK and the token handed to invalidate_token must be identical, even if + # a peer rotates the config between our read and the SDK's own read. Pinning closes that gap. + config = install_oauth_config(_url(token="eyJ.pin", refresh="rt", expires_delta=300), + saved_name="eu-cloud") + sent = {} + + def func(*a, **k): + sent["oauth_token"] = k.get("oauth_token") + raise AuthorizationRequired("Invalid token [expired]") + + seen = {} + + def spy(rejected): + seen["rejected"] = rejected + return False # stop after one attempt; we only care about the pinned value + + with patch.object(config, "invalidate_token", side_effect=spy): + with self.assertRaises(AuthorizationRequired): + call_api(func, ("file.mp4",), {}) + + self.assertEqual("eyJ.pin", sent["oauth_token"]) # the value the SDK would send + self.assertEqual(sent["oauth_token"], seen["rejected"]) # == what invalidate_token is told class TestRefreshDecision(_RestoresSdkConfig): @@ -218,11 +245,11 @@ def test_expected_matches_disk_rotates_even_when_clock_fresh(self): self.url = _url(token="eyJ.cur", refresh="rt", expires_delta=300) saved = {"eu-cloud": self.url} new_token = jwt_access_token(cloud_name="eu-cloud", tag="expected-matches-new") - with patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", return_value={"access_token": new_token, "refresh_token": "rt2", "expires_in": 300}) as refresh, \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): new_url = self._refresh(expected="eyJ.cur") refresh.assert_called_once() self.assertEqual(new_token, from_cloudinary_url(new_url).access_token) @@ -231,9 +258,9 @@ def test_expected_differs_from_disk_adopts_without_refresh(self): # Peer already rotated: disk token != expected -> adopt, no network. self.url = _url(token="eyJ.new", refresh="rt2", expires_delta=300) # what disk now holds saved = {"eu-cloud": self.url} - with patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh") as refresh, \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): new_url = self._refresh(expected="eyJ.old") # we were sent the OLD token refresh.assert_not_called() self.assertEqual(self.url, new_url) @@ -243,11 +270,11 @@ def test_force_refreshes_a_clock_fresh_token_user_path(self): self.url = _url(token="eyJ.cur", refresh="rt", expires_delta=300) saved = {"eu-cloud": self.url} forced_token = jwt_access_token(cloud_name="eu-cloud", tag="forced-new") - with patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", return_value={"access_token": forced_token, "refresh_token": "rt2", "expires_in": 300}) as refresh, \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): new_url = self._refresh(force=True) refresh.assert_called_once() self.assertEqual(forced_token, from_cloudinary_url(new_url).access_token) @@ -256,9 +283,9 @@ def test_no_expected_no_force_uses_clock_freshness(self): # The proactive sweep with no specific token: a fresh token is left untouched. self.url = _url(token="eyJ.cur", refresh="rt", expires_delta=300) saved = {"eu-cloud": self.url} - with patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + with patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh") as refresh, \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): new_url = self._refresh() refresh.assert_not_called() self.assertEqual(self.url, new_url) diff --git a/test/test_oauth_token_seam.py b/test/test_oauth_token_seam.py index b665774..2ae6ae6 100644 --- a/test/test_oauth_token_seam.py +++ b/test/test_oauth_token_seam.py @@ -49,9 +49,9 @@ def test_call_api_authorize_path_refreshes_stale_token(self): new_token = jwt_access_token(cloud_name="eu-cloud", tag="call-api-new") token_response = {"access_token": new_token, "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ - patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): install_oauth_config(saved["eu-cloud"], saved_name="eu-cloud") # Reproduce verbatim the read cloudinary.api_client.call_api performs at request build # time (call_api.py:63): options.pop("oauth_token", cloudinary.config().oauth_token). @@ -64,9 +64,9 @@ def test_uploader_header_path_refreshes_stale_token(self): fresh_token = jwt_access_token(cloud_name="eu-cloud", tag="uploader-fresh") token_response = {"access_token": fresh_token, "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ - patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", return_value=token_response), \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): install_oauth_config(saved["eu-cloud"], saved_name="eu-cloud") # The uploader reads the same attribute to set the Bearer header (uploader.py:877): # oauth_token = options.get("oauth_token", cloudinary.config().oauth_token). @@ -80,9 +80,9 @@ def test_seam_read_refreshes_only_once_then_serves_cached(self): new_token = jwt_access_token(cloud_name="eu-cloud", tag="cached-new") token_response = {"access_token": new_token, "refresh_token": "rt_new", "expires_in": 300} with patch("cloudinary_cli.utils.config_utils.load_config", return_value=dict(saved)), \ - patch("cloudinary_cli.auth.load_config", return_value=dict(saved)), \ + patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh", return_value=token_response) as refresh, \ - patch("cloudinary_cli.auth.update_config"): + patch("cloudinary_cli.auth.refresh.update_config"): install_oauth_config(saved["eu-cloud"], saved_name="eu-cloud") first = cloudinary.config().oauth_token second = cloudinary.config().oauth_token From a355b15cdb1f32f43bd4e6d7e703019e1ba49718 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Mon, 29 Jun 2026 18:05:17 +0300 Subject: [PATCH 17/23] Route all SDK calls through the OAuth 401-retry boundary Several first-class command paths bypassed the OAuth retry helper, so an expired or server-rejected token recovered for admin commands but failed for sync, clone, search, migrate, get_folder_mode, and derived regeneration. Collapse _call_with_oauth_retry and call_api into a single call_api(func, *args, **kwargs) and route every previously-bypassing SDK call through it: Search/SearchFolders.execute (query_cld_folder, cld_folder_exists, execute_single_request for search/clone), uploader.explicit, api.config, api.delete_resources, and api.upload_mapping. The pinned oauth_token option is consumed by both function-style APIs and Search builders, so one function covers both call shapes. call_api now logs at debug and re-raises; handle_command and handle_api_command log-and-return False so each failure surfaces exactly one error line. Co-Authored-By: Claude Opus 4.8 (1M context) --- cloudinary_cli/core/search.py | 3 +- cloudinary_cli/modules/migrate.py | 3 +- cloudinary_cli/modules/sync.py | 5 +-- cloudinary_cli/utils/api_utils.py | 60 +++++++++++++++---------------- test/test_oauth_multiprocess.py | 2 +- test/test_oauth_retry.py | 16 ++++----- 6 files changed, 44 insertions(+), 45 deletions(-) diff --git a/cloudinary_cli/core/search.py b/cloudinary_cli/core/search.py index 55cb6b3..d6c1c1b 100644 --- a/cloudinary_cli/core/search.py +++ b/cloudinary_cli/core/search.py @@ -6,6 +6,7 @@ from cloudinary_cli.utils.json_utils import write_json_to_file, print_json from cloudinary_cli.utils.utils import write_json_list_to_csv, confirm_action, whitelist_keys, \ normalize_list_params +from cloudinary_cli.utils.api_utils import call_api from cloudinary_cli.utils.search_utils import parse_aggregate DEFAULT_MAX_RESULTS = 500 @@ -134,7 +135,7 @@ def _perform_search(query, with_field, fields, sort_by, aggregate, max_results, def execute_single_request(expression, fields_to_keep, result_field='resources'): - res = expression.execute() + res = call_api(expression.execute) if fields_to_keep: res[result_field] = whitelist_keys(res[result_field], fields_to_keep) diff --git a/cloudinary_cli/modules/migrate.py b/cloudinary_cli/modules/migrate.py index ad97b1c..ea53e61 100644 --- a/cloudinary_cli/modules/migrate.py +++ b/cloudinary_cli/modules/migrate.py @@ -6,6 +6,7 @@ from cloudinary.utils import cloudinary_url from requests import head +from cloudinary_cli.utils.api_utils import call_api from cloudinary_cli.utils.utils import logger, log_exception @@ -30,7 +31,7 @@ def migrate(upload_mapping, file, delimiter, verbose): return False try: - mapping = api.upload_mapping(upload_mapping) + mapping = call_api(api.upload_mapping, upload_mapping) except Error as e: log_exception(e, f"Failed retrieving upload mapping: '{upload_mapping}'") return False diff --git a/cloudinary_cli/modules/sync.py b/cloudinary_cli/modules/sync.py index 56af6d4..4f0be0f 100644 --- a/cloudinary_cli/modules/sync.py +++ b/cloudinary_cli/modules/sync.py @@ -10,7 +10,7 @@ from cloudinary import api from cloudinary_cli.utils.api_utils import query_cld_folder, upload_file, download_file, get_folder_mode, \ - get_default_upload_options, get_destination_folder_options, cld_folder_exists + get_default_upload_options, get_destination_folder_options, cld_folder_exists, call_api from cloudinary_cli.utils.file_utils import (walk_dir, delete_empty_dirs, normalize_file_extension, posix_rel_path, populate_duplicate_name) from cloudinary_cli.utils.json_utils import print_json, read_json_from_file, write_json_to_file @@ -371,7 +371,8 @@ def _handle_unique_remote_files(self): logger.info(f"Dry run mode enabled. Would delete {len(deletion_batch)} resources:\n" + "\n".join(deletion_batch)) continue - res = api.delete_resources(deletion_batch, invalidate=True, resource_type=attrs[0], type=attrs[1]) + res = call_api(api.delete_resources, deletion_batch, invalidate=True, + resource_type=attrs[0], type=attrs[1]) num_deleted = Counter(res['deleted'].values())["deleted"] if self.verbose: print_json(res) diff --git a/cloudinary_cli/utils/api_utils.py b/cloudinary_cli/utils/api_utils.py index bf0fdf1..3dbd6ca 100644 --- a/cloudinary_cli/utils/api_utils.py +++ b/cloudinary_cli/utils/api_utils.py @@ -69,7 +69,7 @@ def query_cld_folder(folder, folder_mode, status=None): next_cursor = True while next_cursor: - res = search.execute() + res = call_api(search.execute) for asset in res['resources']: rel_path = _relative_path(asset, folder) @@ -106,7 +106,7 @@ def cld_folder_exists(folder): if not folder: return True # root folder - res = SearchFolders().expression(f"path=\"{folder}\"").execute() + res = call_api(SearchFolders().expression(f"path=\"{folder}\"").execute) return res.get("total_count", 0) > 0 @@ -146,7 +146,7 @@ def regen_derived_version(public_id, delivery_type, res_type, "eager_notification_url": eager_notification_url, "overwrite": True, "invalidate": True} try: - exp_res = uploader.explicit(public_id, **options) + exp_res = call_api(uploader.explicit, public_id, **options) derived_url = f'{exp_res.get("eager")[0].get("secure_url")}' msg = ('Processing' if options.get('eager_async') else 'Regenerated') + f' {derived_url}' logger.info(style(msg, fg="green")) @@ -170,7 +170,7 @@ def upload_file(file_path, options, uploaded=None, failed=None): # resuming the upload; a whole-file retry here would restart from byte 0. result = uploader.upload_large(file_path, **dict(options)) else: - result = _call_with_oauth_retry(uploader.upload, (file_path,), dict(options)) + result = call_api(uploader.upload, file_path, **dict(options)) disp_path = _display_path(result) if "batch_id" in result: starting_msg = "Uploading" @@ -283,7 +283,7 @@ def get_folder_mode(): :return: String representing folder mode. Can be "fixed" or "dynamic". """ try: - config_res = api.config(settings="true") + config_res = call_api(api.config, settings="true") mode = config_res["settings"]["folder_mode"] logger.debug(f"Using {mode} folder mode") except Exception as e: @@ -293,35 +293,26 @@ def get_folder_mode(): return mode -def call_api(func, args, kwargs): - try: - return _call_with_oauth_retry(func, args, kwargs) - except Exception as e: - log_exception(e, debug_message=f"Failed calling '{func.__name__}' with args: {args} and optional args {kwargs}") - raise - - -def _call_with_oauth_retry(func, args, kwargs): +def call_api(func, *args, **kwargs): """ - Run an SDK call, recovering from an OAuth token rejection (AuthorizationRequired) by passing the - rejected token to invalidate_token (adopt a peer's rotation, else rotate once) and retrying once. - No-op for static configs (invalidate_token returns False). - - The token is pinned: the value read here is the value passed to the SDK and to invalidate_token, - so the token handed to invalidate_token is provably the one the request carried. Without pinning, - the SDK re-reads the self-refreshing oauth_token property independently, so a peer rotation between - the two reads makes invalidate_token mis-decide "adopt" for a token the wire never sent, and the - retry re-fails. + Run an SDK call (function-style API or Search().execute), retrying once on an OAuth 401 after + invalidating the rejected token, then log at debug and re-raise on failure. """ config = cloudinary.config() token = getattr(config, "oauth_token", None) # the token this request will carry + # Pin the token so the value sent is provably the value handed to invalidate_token: without it the + # SDK re-reads the self-refreshing oauth_token and a peer rotation between reads breaks the decision. pinned = dict(kwargs, oauth_token=token) if token else kwargs try: - return func(*args, **pinned) - except AuthorizationRequired: - if not getattr(config, "has_oauth", False) or not config.invalidate_token(token): - raise - return func(*args, **kwargs) # retry unpinned: the SDK re-reads the freshly rotated token + try: + return func(*args, **pinned) + except AuthorizationRequired: + if not getattr(config, "has_oauth", False) or not config.invalidate_token(token): + raise + return func(*args, **kwargs) # retry unpinned: the SDK re-reads the freshly rotated token + except Exception: + logger.debug(f"Failed calling '{func.__name__}' with args: {args} and optional args {kwargs}", exc_info=True) + raise def handle_command( @@ -337,7 +328,11 @@ def handle_command( log_exception(e) return False - return call_api(func, args, kwargs) + try: + return call_api(func, *args, **kwargs) + except Exception as e: + log_exception(e) + return False def handle_api_command( @@ -377,8 +372,9 @@ def handle_api_command( raise ConfigurationError("No Cloudinary configuration found.") try: - res = call_api(func, args, kwargs) - except Exception: + res = call_api(func, *args, **kwargs) + except Exception as e: + log_exception(e) return False if auto_paginate: @@ -421,7 +417,7 @@ def handle_auto_pagination(res, func, args, kwargs, force, filter_fields): pagination_field = None while res.get(cursor_field, None): kwargs[cursor_field] = res.get(cursor_field, None) - res = call_api(func, args, kwargs) + res = call_api(func, *args, **kwargs) all_results, pagination_field = merge_responses(all_results, res, fields_to_keep=fields_to_keep, pagination_field=pagination_field) diff --git a/test/test_oauth_multiprocess.py b/test/test_oauth_multiprocess.py index 83a8c00..614f949 100644 --- a/test/test_oauth_multiprocess.py +++ b/test/test_oauth_multiprocess.py @@ -123,7 +123,7 @@ def api_call(*a, **k): return {"ok": token} try: - out[idx] = call_api(api_call, (), {})["ok"] + out[idx] = call_api(api_call)["ok"] except Exception as e: # noqa: BLE001 out[idx] = f"ERROR:{e}" diff --git a/test/test_oauth_retry.py b/test/test_oauth_retry.py index 62a7257..7f00b4b 100644 --- a/test/test_oauth_retry.py +++ b/test/test_oauth_retry.py @@ -91,7 +91,7 @@ def func(*a, **k): patch("cloudinary_cli.auth.refresh.load_config", return_value=dict(saved)), \ patch("cloudinary_cli.auth.flow.refresh") as refresh, \ patch("cloudinary_cli.auth.refresh.update_config"): - result = call_api(func, ("file.mp4",), {}) + result = call_api(func, "file.mp4") self.assertEqual({"public_id": "ok"}, result) self.assertEqual(2, calls["n"]) # original + one retry @@ -118,7 +118,7 @@ def func(*a, **k): return_value={"access_token": new_token, "refresh_token": "rt_new", "expires_in": 300}) as refresh, \ patch("cloudinary_cli.auth.refresh.update_config"): - result = call_api(func, ("file.mp4",), {}) + result = call_api(func, "file.mp4") self.assertEqual({"public_id": "ok"}, result) self.assertEqual(2, calls["n"]) @@ -143,7 +143,7 @@ def func(*a, **k): side_effect=requests.RequestException("refresh token revoked")), \ patch("cloudinary_cli.auth.refresh.update_config"): with self.assertRaises(AuthorizationRequired): - call_api(func, ("file.mp4",), {}) + call_api(func, "file.mp4") self.assertEqual(1, calls["n"]) # nothing to adopt -> fail fast @@ -167,7 +167,7 @@ def ever_new_token(refresh_token, region): patch("cloudinary_cli.auth.flow.refresh", side_effect=ever_new_token), \ patch("cloudinary_cli.auth.refresh.update_config"): with self.assertRaises(AuthorizationRequired): - call_api(func, ("file.mp4",), {}) + call_api(func, "file.mp4") self.assertEqual(2, calls["n"]) # original + one retry, no unbounded rotation @@ -180,7 +180,7 @@ def func(*a, **k): raise AuthorizationRequired("nope") with self.assertRaises(AuthorizationRequired): - call_api(func, ("x",), {}) + call_api(func, "x") self.assertEqual(1, calls["n"]) # no adopt attempt on a non-OAuth config def test_env_config_propagates_immediately(self): @@ -197,13 +197,13 @@ def func(*a, **k): raise AuthorizationRequired("expired") with self.assertRaises(AuthorizationRequired): - call_api(func, ("x",), {}) + call_api(func, "x") self.assertEqual(1, calls["n"]) def test_success_passes_through_without_refresh(self): install_oauth_config(_url(), saved_name="eu-cloud") sentinel = MagicMock(return_value={"public_id": "p"}) - result = call_api(sentinel, ("file",), {"folder": "f"}) + result = call_api(sentinel, "file", folder="f") self.assertEqual({"public_id": "p"}, result) # no retry; args forwarded verbatim with the active token pinned so the wire token == rejected sentinel.assert_called_once_with("file", folder="f", oauth_token="eyJ.tok") @@ -227,7 +227,7 @@ def spy(rejected): with patch.object(config, "invalidate_token", side_effect=spy): with self.assertRaises(AuthorizationRequired): - call_api(func, ("file.mp4",), {}) + call_api(func, "file.mp4") self.assertEqual("eyJ.pin", sent["oauth_token"]) # the value the SDK would send self.assertEqual(sent["oauth_token"], seen["rejected"]) # == what invalidate_token is told From d78a9f19945091213cd5051836f34ba07c05be6b Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Tue, 30 Jun 2026 03:00:47 +0300 Subject: [PATCH 18/23] Render a branded, escaped OAuth login callback page The loopback callback wrote the OAuth error from the redirect query string straight into the HTML body. Escape it (html.escape) and move the page into auth/callback_page.py, which renders a self-contained Cloudinary-branded page (inline logo + CSS, no network fetch) for both the success and failure states. Co-Authored-By: Claude Opus 4.8 (1M context) --- cloudinary_cli/auth/callback_page.py | 94 ++++++++++++++++++++++++++ cloudinary_cli/auth/loopback_server.py | 7 +- test/test_auth_loopback.py | 26 ++++++- 3 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 cloudinary_cli/auth/callback_page.py diff --git a/cloudinary_cli/auth/callback_page.py b/cloudinary_cli/auth/callback_page.py new file mode 100644 index 0000000..8cda5aa --- /dev/null +++ b/cloudinary_cli/auth/callback_page.py @@ -0,0 +1,94 @@ +"""HTML for the local OAuth callback page shown in the browser after `cld login`. Kept apart from +the server logic because of the inline logo/CSS. The page is fully self-contained (inline SVG + CSS, +no network fetch) so it renders even offline or right after an auth failure.""" +import html + +# Official Cloudinary wordmark (cloudinary.com header logo), inlined so the page needs no network. +# Single brand-blue fill; sized via CSS on the wrapper, currentColor left untouched. +_LOGO_SVG = ( + '' +) + +_BRAND_BLUE = "#3448c5" + +_PAGE_TEMPLATE = """ + + + + +Cloudinary CLI · {title} + + + +
+ {logo} +
{badge}
+

{heading}

+

{message}

+ {reason} +
+ +""" + + +def callback_page(auth_error): + """Branded HTML for the OAuth callback. auth_error comes from the redirect query string + (untrusted), so it is HTML-escaped before rendering; the raw reason also reaches the terminal.""" + if auth_error: + reason = f'
{html.escape(auth_error)}
' + return _PAGE_TEMPLATE.format( + title="Login failed", brand=_BRAND_BLUE, logo=_LOGO_SVG, + badge_class="err", badge="×", heading="Login failed", + message="Return to the terminal for details, then try again.", reason=reason, + ) + return _PAGE_TEMPLATE.format( + title="Login successful", brand=_BRAND_BLUE, logo=_LOGO_SVG, + badge_class="ok", badge="✓", heading="Login successful", + message="You can close this tab and return to the terminal.", reason="", + ) diff --git a/cloudinary_cli/auth/loopback_server.py b/cloudinary_cli/auth/loopback_server.py index 873b5fc..79eb58b 100644 --- a/cloudinary_cli/auth/loopback_server.py +++ b/cloudinary_cli/auth/loopback_server.py @@ -4,6 +4,7 @@ import urllib.parse from http.server import BaseHTTPRequestHandler, HTTPServer +from cloudinary_cli.auth.callback_page import callback_page from cloudinary_cli.defaults import ( OAUTH_REDIRECT_HOST, OAUTH_REDIRECT_PORT, @@ -32,11 +33,7 @@ def do_GET(self): # noqa: N802 (http.server API) self.send_response(200) self.send_header("Content-Type", "text/html; charset=utf-8") self.end_headers() - if self.server.auth_error: - body = f"

Login failed

{self.server.auth_error}

" - else: - body = "

Login successful

You can close this tab and return to the terminal.

" - self.wfile.write(f"{body}".encode("utf-8")) + self.wfile.write(callback_page(self.server.auth_error).encode("utf-8")) def log_message(self, *args): pass # silence the default stderr request logging diff --git a/test/test_auth_loopback.py b/test/test_auth_loopback.py index 8985e0e..232a71c 100644 --- a/test/test_auth_loopback.py +++ b/test/test_auth_loopback.py @@ -4,7 +4,12 @@ from http.server import HTTPServer from unittest.mock import patch -from cloudinary_cli.auth.loopback_server import _CallbackHandler, start_callback_server, wait_for_callback +from cloudinary_cli.auth.callback_page import callback_page +from cloudinary_cli.auth.loopback_server import ( + _CallbackHandler, + start_callback_server, + wait_for_callback, +) class TestLoopbackServer(unittest.TestCase): @@ -67,6 +72,25 @@ def run(): self.assertIn("access_denied", str(error["e"])) +class TestCallbackPage(unittest.TestCase): + """auth_error comes from the redirect query string (untrusted) and must be HTML-escaped + before being rendered into the callback page.""" + + def test_error_is_html_escaped(self): + page = callback_page("") + self.assertNotIn("", page) + self.assertIn("<script>alert(1)</script>", page) + + def test_normal_error_rendered(self): + page = callback_page("access_denied") + self.assertIn("Login failed", page) + self.assertIn("access_denied", page) + + def test_success_page(self): + page = callback_page(None) + self.assertIn("Login successful", page) + + class TestStartCallbackServerPortBusy(unittest.TestCase): """A2: a failed bind (e.g. busy redirect port) must surface a clear RuntimeError, not a raw OSError. The bind is mocked to fail so the test is deterministic across OSes (Windows does not From 765f10dbc4916021b8ffca4f38b0b52625da25c4 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Tue, 30 Jun 2026 03:00:53 +0300 Subject: [PATCH 19/23] Clarify default-config messages on login and config -d cld login into a config that was already the default told the user to 'make it the default with cld config -d ', suggesting a no-op. login() now returns a 3-state default status (made/already/no) captured before the config write, so the message distinguishes 'now the default' from 'is the default'. config -d / --set-default now also tell the user how to use the config (cld vs cld -C ). Co-Authored-By: Claude Opus 4.8 (1M context) --- cloudinary_cli/auth/__init__.py | 17 ++++++++++++----- cloudinary_cli/core/auth.py | 7 +++++-- cloudinary_cli/core/config.py | 6 ++++-- test/test_cli_config_oauth.py | 34 +++++++++++++++++++++++++-------- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/cloudinary_cli/auth/__init__.py b/cloudinary_cli/auth/__init__.py index f5f7e54..c0d6fc6 100644 --- a/cloudinary_cli/auth/__init__.py +++ b/cloudinary_cli/auth/__init__.py @@ -48,8 +48,11 @@ def login(region=None, name=None, set_default=False): """ Run the interactive browser login and persist the resulting session as a named config entry. - Returns (config_name, is_default), where is_default is True when this login was made the default - configuration (explicitly with set_default, or automatically as the sole login). + Returns (config_name, default_status), where default_status is: + "made" - this login just became the default (explicit --set-default, or auto-defaulted as + the sole login), + "already" - the re-logged-into config was already the stored default, + "no" - it is not the default. """ if name and is_reserved_config_name(name): raise RuntimeError(f"'{name}' is a reserved configuration name.") @@ -58,12 +61,16 @@ def login(region=None, name=None, set_default=False): if not session.cloud_name: raise RuntimeError("Login token did not include a cloud name; cannot save this login.") config_name = name or _derive_config_name(session.cloud_name, region) + + was_default = get_default_config_name() == config_name # before we touch the config update_config({config_name: to_cloudinary_url(session)}) - is_default = bool(set_default or _should_auto_default(config_name)) - if is_default: + if was_default: + return config_name, "already" + if set_default or _should_auto_default(config_name): set_default_config(config_name) - return config_name, is_default + return config_name, "made" + return config_name, "no" def _should_auto_default(name): diff --git a/cloudinary_cli/core/auth.py b/cloudinary_cli/core/auth.py index 9d6b533..6fb4402 100644 --- a/cloudinary_cli/core/auth.py +++ b/cloudinary_cli/core/auth.py @@ -16,15 +16,18 @@ "config is given.") def login(name, region, set_default): try: - config_name, is_default = run_login(region=region, name=name, set_default=set_default) + config_name, default_status = run_login(region=region, name=name, set_default=set_default) except Exception as e: log_exception(e, "Login failed") return False logger.info(f"Logged in. Saved as '{config_name}'.") - if is_default: + if default_status == "made": logger.info(f"This is now the default configuration. Run `cld ` to use it, " f"or `cld -C {config_name} ` to select it explicitly.") + elif default_status == "already": + logger.info(f"This is the default configuration. Run `cld ` to use it, " + f"or `cld -C {config_name} ` to select it explicitly.") else: logger.info(f"Run `cld -C {config_name} ` to use it, " f"or make it the default with `cld config -d {config_name}`.") diff --git a/cloudinary_cli/core/config.py b/cloudinary_cli/core/config.py index c0daaea..0c9897d 100644 --- a/cloudinary_cli/core/config.py +++ b/cloudinary_cli/core/config.py @@ -87,13 +87,15 @@ def config_command(new, ls, as_json, show, rm, from_url, default, set_default, u if set_default: set_default_config(config_name) - logger.info(f"Default set to '{config_name}'.") + logger.info(f"Default set to '{config_name}'. Run `cld ` to use it, " + f"or `cld -C {config_name} ` to select it explicitly.") elif default: if default not in user_config_names(load_config()): raise BadParameter(f"Configuration {default} does not exist, " f"use -ls to list available configurations.") set_default_config(default) - logger.info(f"Default set to '{default}'.") + logger.info(f"Default set to '{default}'. Run `cld ` to use it, " + f"or `cld -C {default} ` to select it explicitly.") elif unset_default: clear_default_config() logger.info("Default configuration cleared.") diff --git a/test/test_cli_config_oauth.py b/test/test_cli_config_oauth.py index 108e789..5573c7f 100644 --- a/test/test_cli_config_oauth.py +++ b/test/test_cli_config_oauth.py @@ -165,17 +165,28 @@ def test_auto_default_when_sole_config_no_env_no_default(self): with self._patches({"eu-cloud": _oauth_url()}), \ patch("cloudinary_cli.auth.set_default_config") as set_default, \ patch("cloudinary_cli.auth.get_default_config_name", return_value=None): - name, is_default = auth.login(region="eu", name="eu-cloud") + name, default_status = auth.login(region="eu", name="eu-cloud") set_default.assert_called_once_with("eu-cloud") - self.assertEqual(("eu-cloud", True), (name, is_default)) + self.assertEqual(("eu-cloud", "made"), (name, default_status)) def test_returns_not_default_when_other_configs_exist(self): from cloudinary_cli import auth with self._patches({"eu-cloud": _oauth_url(), "other": _oauth_url("other")}), \ patch("cloudinary_cli.auth.set_default_config"), \ patch("cloudinary_cli.auth.get_default_config_name", return_value=None): - name, is_default = auth.login(region="eu", name="eu-cloud") - self.assertEqual(("eu-cloud", False), (name, is_default)) + name, default_status = auth.login(region="eu", name="eu-cloud") + self.assertEqual(("eu-cloud", "no"), (name, default_status)) + + def test_relogin_into_existing_default_reports_already(self): + """Re-login into a config that is already the stored default must NOT set it again and must + report "already" so the CLI doesn't tell the user to make it the default.""" + from cloudinary_cli import auth + with self._patches({"eu-cloud": _oauth_url(), "other": _oauth_url("other")}), \ + patch("cloudinary_cli.auth.set_default_config") as set_default, \ + patch("cloudinary_cli.auth.get_default_config_name", return_value="eu-cloud"): + name, default_status = auth.login(region="eu", name="eu-cloud") + set_default.assert_not_called() + self.assertEqual(("eu-cloud", "already"), (name, default_status)) def test_no_auto_default_when_other_configs_exist(self): from cloudinary_cli import auth @@ -208,15 +219,22 @@ def test_reserved_name_rejected(self): with self.assertRaises(RuntimeError): auth.login(region="eu", name="__default__") - def test_cli_message_when_default(self): - with patch("cloudinary_cli.core.auth.run_login", return_value=("tttt", True)): + def test_cli_message_when_made_default(self): + with patch("cloudinary_cli.core.auth.run_login", return_value=("tttt", "made")): + result = CliRunner().invoke(cli, ["login", "tttt"]) + self.assertIn("now the default configuration", result.output) + + def test_cli_message_when_already_default_does_not_suggest_setting_it(self): + with patch("cloudinary_cli.core.auth.run_login", return_value=("tttt", "already")): result = CliRunner().invoke(cli, ["login", "tttt"]) - self.assertIn("default configuration", result.output) + self.assertIn("This is the default configuration", result.output) + self.assertNotIn("config -d tttt", result.output) # don't suggest a no-op def test_cli_message_when_not_default_shows_how_to_default(self): - with patch("cloudinary_cli.core.auth.run_login", return_value=("tttt", False)): + with patch("cloudinary_cli.core.auth.run_login", return_value=("tttt", "no")): result = CliRunner().invoke(cli, ["login", "tttt"]) self.assertIn("cld -C tttt", result.output) + self.assertIn("config -d tttt", result.output) self.assertIn("cld config -d tttt", result.output) # how to make it default From 84be16c83288f8964b75adac72ec2433fe8ff3e6 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Tue, 30 Jun 2026 03:02:20 +0300 Subject: [PATCH 20/23] Isolate the developer's CLI config file from the test suite Add an autouse conftest fixture that redirects CLOUDINARY_CLI_CONFIG_FILE and the config lock to a per-test tmp_path, so a saved __default__ or saved accounts in ~/.cloudinary-cli/config.json can no longer leak into tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/conftest.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 test/conftest.py diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..fc68033 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,16 @@ +from unittest.mock import patch + +import pytest +from filelock import FileLock + +from cloudinary_cli.utils import config_utils + + +@pytest.fixture(autouse=True) +def isolate_cli_config(tmp_path): + """Redirect the CLI config file to a fresh per-test path so the developer's real + ~/.cloudinary-cli/config.json (saved accounts and __default__) never leaks into tests.""" + config_file = str(tmp_path / "config.json") + with patch.object(config_utils, "CLOUDINARY_CLI_CONFIG_FILE", config_file), \ + patch.object(config_utils, "_config_lock", FileLock(config_file + ".lock")): + yield From ad51e368ad5b311ce4016ef3b63d33e9fe882688 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Tue, 30 Jun 2026 03:02:20 +0300 Subject: [PATCH 21/23] Require cloudinary>=1.44.4 1.44.4 raises a clear ValueError when signing without an api_secret (the OAuth-only signing guard the CLI relies on) and carries the large-upload OAuth chunk-retry callback contract. Co-Authored-By: Claude Opus 4.8 (1M context) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7576a44..856c21d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -cloudinary>=1.42.2 +cloudinary>=1.44.4 pygments jinja2 click From 11b281808a91763eabf80fb6379374a3b86fda61 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Tue, 30 Jun 2026 03:15:10 +0300 Subject: [PATCH 22/23] Document pipx, uv, and pip install options in the README Replace the single pip3 line with an Installation section covering pipx (recommended), uv/uvx, pip (venv and --user), and Docker, plus a Verify step and a Troubleshooting subsection for a missing `cld` command. All commands were run and verified; the package installs the `cld` command, so the uvx one-off is `uvx --from cloudinary-cli cld`. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 120 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 100 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 876d3f7..c759290 100644 --- a/README.md +++ b/README.md @@ -14,36 +14,116 @@ Your own Cloudinary account. If you don't already have one, sign up at [https:/ Python 3.8 or later. You can install Python from [https://www.python.org/](https://www.python.org/). Note that the Python Package Installer (pip) is installed with it. -## Setup and Installation +## Installation -1. To install this package, run: `pip3 install cloudinary-cli` -2. Point your `cld` commands at a Cloudinary account using **either** of the following: +The CLI is published on PyPI as [`cloudinary-cli`](https://pypi.org/project/cloudinary-cli/). The package name (`cloudinary-cli`) is what you install; the command it provides is **`cld`** (it also installs a `cloudinary` alias). Pick the method that fits your setup. If you just want a working `cld` command and aren't sure, use **pipx** or **uv** — they install the CLI in its own isolated environment, so it won't conflict with other Python packages and you don't need to manage a virtual environment yourself. - **Option A — Log in with OAuth (recommended).** Run: +### Option 1 — pipx (recommended) - ``` - cld login - ``` +[pipx](https://pipx.pypa.io) installs Python CLI tools into isolated environments and puts the `cld` command on your `PATH` automatically. - This opens your browser to authorize the CLI, then saves the login as a configuration (named after the cloud) and sets it as the default. The CLI refreshes the token automatically, and you can remove the login at any time with `cld logout`. +```sh +# Install pipx if you don't have it: +# macOS: brew install pipx && pipx ensurepath +# Debian/Ubuntu: sudo apt install pipx && pipx ensurepath +# Any platform: python3 -m pip install --user pipx && python3 -m pipx ensurepath + +pipx install cloudinary-cli + +# Upgrade later with: +pipx upgrade cloudinary-cli +``` + +After `pipx ensurepath`, open a new terminal so the updated `PATH` takes effect. + +### Option 2 — uv + +[uv](https://docs.astral.sh/uv/) is a fast Python package manager. Its `uv tool` command installs CLIs in isolation, like pipx: + +```sh +uv tool install cloudinary-cli + +# Upgrade later with: +uv tool upgrade cloudinary-cli +``` + +Or run it once without installing. The package's command is `cld`, so name it with `--from`: + +```sh +uvx --from cloudinary-cli cld --help # uvx is shorthand for `uv tool run` +``` + +### Option 3 — pip + +A plain `pip` install also works. Prefer a virtual environment so the CLI and its dependencies don't collide with your system or other projects: + +```sh +python3 -m venv ~/.venvs/cloudinary-cli +source ~/.venvs/cloudinary-cli/bin/activate # Windows: .\.venvs\cloudinary-cli\Scripts\activate +pip install cloudinary-cli +``` + +To install without a virtual environment, use a per-user install (avoids needing `sudo` and keeps it out of system Python): + +```sh +python3 -m pip install --user cloudinary-cli +``` + +If `cld` is not found afterwards, the user scripts directory is not on your `PATH`. See [Troubleshooting](#troubleshooting-the-cld-command). + +### Option 4 — Docker (no Python needed) + +If you'd rather not install Python at all, run the CLI from the official Docker image. See [Docker Usage](#docker-usage) below. + +### Verify the installation + +```sh +cld --version +``` + +### Troubleshooting the `cld` command + +If your shell reports `cld: command not found` after installing: + +- **pipx / uv:** run `pipx ensurepath` (or `uv tool update-shell`), then open a new terminal. +- **pip `--user` install:** the user scripts directory is not on your `PATH`. Find it with `python3 -m site --user-base` (the scripts live in its `bin` subdirectory on macOS/Linux, or `Scripts` on Windows) and add that to your `PATH`. For example, on macOS/Linux add this to `~/.zshrc` or `~/.bash_profile`: + + ```sh + export PATH="$PATH:$(python3 -m site --user-base)/bin" + ``` - **Option B — Set your CLOUDINARY\_URL environment variable.** For example: - * On Mac or Linux:
`export CLOUDINARY_URL=cloudinary://123456789012345:abcdefghijklmnopqrstuvwxyzA@cloud_name` - * On Windows (cmd.exe):
`set CLOUDINARY_URL=cloudinary://123456789012345:abcdefghijklmnopqrstuvwxyzA@cloud_name` - * On Windows (PowerShell):
`$Env:CLOUDINARY_URL="cloudinary://123456789012345:abcdefghijklmnopqrstuvwxyzA@cloud_name"` +- As a fallback, you can always invoke the CLI through Python: `python3 -m cloudinary_cli.cli `. + +## Configuration + +Once installed, point your `cld` commands at a Cloudinary account using **either** of the following. + +**Option A — Log in with OAuth (recommended).** Run: + +```sh +cld login +``` + +This opens your browser to authorize the CLI, then saves the login as a configuration (named after the cloud) and sets it as the default. The CLI refreshes the token automatically, and you can remove the login at any time with `cld logout`. + +**Option B — Set your `CLOUDINARY_URL` environment variable.** For example: + +* On Mac or Linux:
`export CLOUDINARY_URL=cloudinary://123456789012345:abcdefghijklmnopqrstuvwxyzA@cloud_name` +* On Windows (cmd.exe):
`set CLOUDINARY_URL=cloudinary://123456789012345:abcdefghijklmnopqrstuvwxyzA@cloud_name` +* On Windows (PowerShell):
`$Env:CLOUDINARY_URL="cloudinary://123456789012345:abcdefghijklmnopqrstuvwxyzA@cloud_name"` _**Note:** you can copy and paste your account environment variable from the Account Details section of the Dashboard page in the Cloudinary console._ -3. Check your configuration by running `cld config`. A response of the following form is returned: +Then check your configuration by running `cld config`. A response of the following form is returned: - ``` - cloud_name: - api_key: - api_secret: *************** - private_cdn: - ``` +``` +cloud_name: +api_key: +api_secret: *************** +private_cdn: +``` - If you get an error message when running `cld config`, you may need to add your Python installation to your $PATH. To do so, you can run `PATH="$PATH:/Library/Python/Versions/3.8/bin"` in your terminal, and add `export PATH="$PATH:/Library/Python/Versions/3.8/bin"` to your `/.bash_profile` or `~/.zshrc`. +If `cld` itself is not found, see [Troubleshooting the `cld` command](#troubleshooting-the-cld-command). ## Quickstart From 8c1cba5eda2ba27f6911c8cb8f90c9821e14c2c6 Mon Sep 17 00:00:00 2001 From: Constantine Nathanson Date: Tue, 30 Jun 2026 03:24:05 +0300 Subject: [PATCH 23/23] Fix tests Spawn worker processes in test_oauth_multiprocess so they re-import and honor CLOUDINARY_HOME (fork inherited the parent's frozen config path and missed proc-cloud, failing on Linux Python <3.14). Also guard the CI job so a same-repo push and its PR don't run the suite twice. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/cloudinary-cli-test.yml | 6 ++++++ test/test_oauth_multiprocess.py | 11 ++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cloudinary-cli-test.yml b/.github/workflows/cloudinary-cli-test.yml index dfef328..eeb590f 100644 --- a/.github/workflows/cloudinary-cli-test.yml +++ b/.github/workflows/cloudinary-cli-test.yml @@ -8,6 +8,12 @@ on: jobs: build: + # Run on every branch push, but avoid duplicate runs when a same-repo PR + # exists: same-repo changes run via the push event, while fork PRs (which + # can't trigger a push in this repo) run via the pull_request event. + if: >- + (github.event_name == 'push' && !github.event.pull_request.head.repo.fork) || + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork) runs-on: ${{ matrix.os }} strategy: diff --git a/test/test_oauth_multiprocess.py b/test/test_oauth_multiprocess.py index 614f949..3da224a 100644 --- a/test/test_oauth_multiprocess.py +++ b/test/test_oauth_multiprocess.py @@ -161,14 +161,15 @@ def _write_stale_config(self): self._write_config(expires_delta=-10) def _run_workers(self, worker, n=6): - if multiprocessing.get_start_method(allow_none=True) is None: - multiprocessing.set_start_method("spawn", force=True) + # Spawn so workers re-import and honor CLOUDINARY_HOME; fork would inherit the parent's + # frozen config path and miss "proc-cloud". + ctx = multiprocessing.get_context("spawn") token_url = f"http://127.0.0.1:{self.port}/oauth2/token" - mgr = multiprocessing.Manager() + mgr = ctx.Manager() out = mgr.dict() out["n"] = n - procs = [multiprocessing.Process(target=worker, - args=(self.home, token_url, self.barrier, i, out)) + procs = [ctx.Process(target=worker, + args=(self.home, token_url, self.barrier, i, out)) for i in range(n)] for p in procs: p.start()