From 9c84b593e0a97d03b57b37ffb9758d5d5fee18a1 Mon Sep 17 00:00:00 2001 From: elkaix Date: Tue, 18 Aug 2026 18:04:26 -0400 Subject: [PATCH 1/6] build: package for PyPI as watermark_remover Rename the installed import root from the generic top-level skills.* namespace to a single watermark_remover package. The on-disk Claude-skill layout is unchanged; only package-dir mapping and the console-script targets move. Add the metadata a PyPI project page needs: readme, urls, keywords. Rewrite README links to absolute URLs so they resolve off-GitHub, and lead the quick start with pip install and the wm commands rather than a clone. Replace the PEP 639-deprecated license table and classifier with an SPDX license expression plus license-files. Add release-pypi.yml, publishing via Trusted Publishing on v* tags so no API token is stored. Correct the stale pythoughts-labs org name to PyModel across the workflows, compose file, and docs, since Trusted Publishing binds to the exact current repository owner. --- .github/workflows/release-images.yml | 4 +- .github/workflows/release-pypi.yml | 35 +++++++++ .gitignore | 1 + CODE_OF_CONDUCT.md | 2 +- README.md | 71 ++++++++++++------- SECURITY.md | 2 +- compose.yaml | 6 +- docs/windows-autostart.md | 2 +- pyproject.toml | 45 ++++++++---- skills/remove-ai-marks/__init__.py | 7 +- .../references/service-mode.md | 2 +- 11 files changed, 126 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/release-pypi.yml diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index 111ff9a..ce478fe 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -27,7 +27,7 @@ jobs: id: tags run: | TAG="${{ github.ref_name }}" - echo "tags=ghcr.io/pythoughts-labs/watermark-remover:${TAG},ghcr.io/pythoughts-labs/watermark-remover:latest" >> "$GITHUB_OUTPUT" + echo "tags=ghcr.io/pymodel/watermark-remover:${TAG},ghcr.io/pymodel/watermark-remover:latest" >> "$GITHUB_OUTPUT" echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: @@ -66,7 +66,7 @@ jobs: id: tags run: | TAG="${{ github.ref_name }}" - echo "tags=ghcr.io/pythoughts-labs/watermark-remover:${{ matrix.tag }}-${TAG},ghcr.io/pythoughts-labs/watermark-remover:${{ matrix.tag }}-latest" >> "$GITHUB_OUTPUT" + echo "tags=ghcr.io/pymodel/watermark-remover:${{ matrix.tag }}-${TAG},ghcr.io/pymodel/watermark-remover:${{ matrix.tag }}-latest" >> "$GITHUB_OUTPUT" - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: . diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml new file mode 100644 index 0000000..59500bc --- /dev/null +++ b/.github/workflows/release-pypi.yml @@ -0,0 +1,35 @@ +name: release-pypi + +on: + push: + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: read + +jobs: + publish: + name: Build and publish to PyPI + runs-on: ubuntu-latest + environment: pypi + permissions: + contents: read + # Trusted Publishing: mints a short-lived OIDC token, so no API token + # is stored anywhere. Bound on PyPI to PyModel/watermark-remover. + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Build sdist and wheel + run: | + python -m pip install --upgrade build + python -m build + - name: Verify the tag matches the packaged version + env: + TAG: ${{ github.ref_name }} + run: test -f "dist/watermark_remover-${TAG#v}.tar.gz" + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 diff --git a/.gitignore b/.gitignore index 501e9e1..6bf0ed4 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ CLAUDE.md # Python build artifacts build/ +dist/ *.egg-info/ # Secrets / local env diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index a7fa381..88cca80 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -61,7 +61,7 @@ representative at an online or offline event. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement through GitHub by opening a private security advisory or contacting the repository maintainers -via [GitHub](https://github.com/Pythoughts-labs/watermark-remover) or by email at +via [GitHub](https://github.com/PyModel/watermark-remover) or by email at . All complaints will be reviewed and investigated promptly and fairly. diff --git a/README.md b/README.md index 23bb0db..ac4c7d1 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -![watermark-remover](assets/banner.svg) +![watermark-remover](https://raw.githubusercontent.com/PyModel/watermark-remover/main/assets/banner.svg)

- CI - Release - Stars - License: MIT - Visitors + CI + Release + Stars + License: MIT + Visitors

Tools for finding and removing AI provenance signals from files you own. Four channels are covered: hidden Unicode in text, statistical token watermarks, visible marks burned into images, and metadata such as C2PA, EXIF, and XMP. @@ -38,7 +38,26 @@ Optional edges, none required by the core: ## Quick start ```bash -git clone https://github.com/Pythoughts-labs/watermark-remover.git +pip install watermark-remover + +# Unified clean. The source is never modified without --in-place. +wm draft.md -o draft.cleaned.md +wm image.png -o image.cleaned.png + +# Machine-readable result, plus a JSON record of what was removed +wm draft.md -o draft.cleaned.md --json --audit +``` + +The install pulls no dependencies — the core is standard library only. Extras +are opt-in: `watermark-remover[visible]` for image inpainting, +`[quality]` for scoring, `[ai]` for the torch-backed adapters, `[provenance]` +for C2PA, or `[all]`. Four commands are installed: `wm`, `wm-serve`, +`wm-audit-dir`, `wm-audit-site`. + +### From a clone + +```bash +git clone https://github.com/PyModel/watermark-remover.git cd watermark-remover SCRIPTS=skills/remove-ai-marks/scripts @@ -145,7 +164,7 @@ This is an opt-in anti-watermark transform inspired by 2026 character-perturbati ## Visible image marks -[`morphomod.py`](skills/remove-ai-marks/scripts/morphomod.py) never guesses a watermark region. Supply one of: +[`morphomod.py`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/scripts/morphomod.py) never guesses a watermark region. Supply one of: - `--mask mask.pgm|mask.png` (white means remove) - `--box X,Y,W,H` @@ -318,7 +337,7 @@ docker compose --profile harness --profile heavy up --build -d ./compose-check.sh ``` -See [`skills/remove-ai-marks/references/service-mode.md`](skills/remove-ai-marks/references/service-mode.md) for the thin-client curl flow and [`docs/windows-autostart.md`](docs/windows-autostart.md) for a Windows login task. +See [`skills/remove-ai-marks/references/service-mode.md`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/references/service-mode.md) for the thin-client curl flow and [`docs/windows-autostart.md`](https://github.com/PyModel/watermark-remover/blob/main/docs/windows-autostart.md) for a Windows login task. ### Heavy backends (external checkouts, never bundled) @@ -370,25 +389,25 @@ Bootstrap any of them with `setup_ctrlregen.sh`, `setup_synthid.sh`, `setup_mark ## Interesting techniques -**Context-aware Unicode scrubbing.** [`text_unicode.py`](skills/remove-ai-marks/scripts/text_unicode.py) classifies every hidden carrier rather than deleting a blocklist. Zero-width joiners hold emoji sequences together and variation selectors change glyph form, so those survive. Tag characters and bidi overrides do not. Bidi handling follows the same directional model browsers expose through [`unicode-bidi`](https://developer.mozilla.org/en-US/docs/Web/CSS/unicode-bidi), and cleaned text is normalized to NFC, the form described under [`String.prototype.normalize()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/normalize). +**Context-aware Unicode scrubbing.** [`text_unicode.py`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/scripts/text_unicode.py) classifies every hidden carrier rather than deleting a blocklist. Zero-width joiners hold emoji sequences together and variation selectors change glyph form, so those survive. Tag characters and bidi overrides do not. Bidi handling follows the same directional model browsers expose through [`unicode-bidi`](https://developer.mozilla.org/en-US/docs/Web/CSS/unicode-bidi), and cleaned text is normalized to NFC, the form described under [`String.prototype.normalize()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/normalize). -**NSGA-II inside a text rewriter.** [`tsapa.py`](skills/remove-ai-marks/scripts/tsapa.py) is a genuine multi-objective loop, not a prompt template: population generation, non-dominated sorting, crowding distance, crossover at sentence boundaries, mutation targeted at the weakest sentence, and Pareto knee selection. Attack strength and semantic fidelity are optimized as two separate objectives so neither silently wins. +**NSGA-II inside a text rewriter.** [`tsapa.py`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/scripts/tsapa.py) is a genuine multi-objective loop, not a prompt template: population generation, non-dominated sorting, crowding distance, crossover at sentence boundaries, mutation targeted at the weakest sentence, and Pareto knee selection. Attack strength and semantic fidelity are optimized as two separate objectives so neither silently wins. **Scoring that names its own fallback.** Pseudo-log-likelihood comes from an OpenAI-compatible endpoint with logprobs, and cosine similarity from an embeddings endpoint. Either can fail on its own and drop to a standard-library proxy, which is reported as a proxy rather than passed off as the real measurement. -**Non-cascading dilation in pure Python.** [`morphomod.py`](skills/remove-ai-marks/scripts/morphomod.py) dilates a mask in a single pass against the original buffer, so growth does not compound across iterations. It runs in O(width × height) with no array library. +**Non-cascading dilation in pure Python.** [`morphomod.py`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/scripts/morphomod.py) dilates a mask in a single pass against the original buffer, so growth does not compound across iterations. It runs in O(width × height) with no array library. **Patch-based inpainting with no model.** The default visible backend searches nearby patches by boundary error, picks the best match, and fully replaces the refined mask so watermark pixels cannot bleed through. It is not LaMa quality and is not sold as such, but it needs nothing installed. -**In-place ISO-BMFF neutralization.** [`heif_meta.py`](skills/remove-ai-marks/scripts/heif_meta.py) overwrites JUMBF and C2PA UUID boxes and direct-file Exif and XMP extents without moving anything, so every byte offset in the file stays valid. Layouts it cannot prove safe fail closed instead of being rewritten. +**In-place ISO-BMFF neutralization.** [`heif_meta.py`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/scripts/heif_meta.py) overwrites JUMBF and C2PA UUID boxes and direct-file Exif and XMP extents without moving anything, so every byte offset in the file stays valid. Layouts it cannot prove safe fail closed instead of being rewritten. -**PNG chunk surgery.** [`image_meta.py`](skills/remove-ai-marks/scripts/image_meta.py) walks the chunk stream, removes the private ancillary `caBX` chunk that C2PA actually uses in PNG, and recomputes CRCs with `zlib.crc32`. +**PNG chunk surgery.** [`image_meta.py`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/scripts/image_meta.py) walks the chunk stream, removes the private ancillary `caBX` chunk that C2PA actually uses in PNG, and recomputes CRCs with `zlib.crc32`. -**Markup-aware container cleaning.** [`container_meta.py`](skills/remove-ai-marks/scripts/container_meta.py) targets SVG [``](https://developer.mozilla.org/en-US/docs/Web/SVG/Element/metadata) elements, HTML [``](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta) tags, JSON-LD provenance blocks, and [`data-*` attributes](https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes/data-*) matching `data-ai*`. DOCX `customXml` parts are dropped and the dangling relationships and Content-Type overrides are pruned, because customXml can re-carry provenance data; this is a provenance tool, so pruning keeps the package valid instead of leaving orphaned parts behind. +**Markup-aware container cleaning.** [`container_meta.py`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/scripts/container_meta.py) targets SVG [``](https://developer.mozilla.org/en-US/docs/Web/SVG/Element/metadata) elements, HTML [``](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta) tags, JSON-LD provenance blocks, and [`data-*` attributes](https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes/data-*) matching `data-ai*`. DOCX `customXml` parts are dropped and the dangling relationships and Content-Type overrides are pruned, because customXml can re-carry provenance data; this is a provenance tool, so pruning keeps the package valid instead of leaving orphaned parts behind. **Fallback chains that degrade loudly.** PDF cleaning tries `exiftool`, then a full-document `pypdf` clone, then a byte-exact copy with an explicit residual warning. The third case still returns a file, and still tells you nothing was removed. -**The banner is drawn, not exported.** [`assets/banner.svg`](assets/banner.svg) uses two overlapping [``](https://developer.mozilla.org/en-US/docs/Web/SVG/Element/clipPath) regions over one duplicated block of text, so the same codepoints render dim on the left and lit on the right. That puts the scrub line in the middle with no gradient mask and no raster asset. +**The banner is drawn, not exported.** [`assets/banner.svg`](https://github.com/PyModel/watermark-remover/blob/main/assets/banner.svg) uses two overlapping [``](https://developer.mozilla.org/en-US/docs/Web/SVG/Element/clipPath) regions over one duplicated block of text, so the same codepoints render dim on the left and lit on the right. That puts the scrub line in the middle with no gradient mask and no raster asset. ## Technologies and libraries @@ -396,7 +415,7 @@ Nothing below is required to run the core. - [pypdf](https://github.com/py-pdf/pypdf) for a structural PDF clone that keeps outlines, forms, and attachments while dropping docinfo and XMP. - [ExifTool](https://exiftool.org/), [qpdf](https://qpdf.sourceforge.io/), and [c2patool](https://github.com/contentauth/c2patool) as system binaries, used when present. -- [Gradio](https://www.gradio.app/) for [`demo.py`](demo.py), which calls the same modules as the CLI rather than reimplementing them. +- [Gradio](https://www.gradio.app/) for [`demo.py`](https://github.com/PyModel/watermark-remover/blob/main/demo.py), which calls the same modules as the CLI rather than reimplementing them. - [Ollama](https://ollama.com/) or any OpenAI-compatible endpoint for Layer B execution. Tests inject a fake callable instead. - [LaMa](https://github.com/advimman/lama) and [MI-GAN](https://github.com/Picsart-AI-Research/MI-GAN) as external inpainting commands behind the `external` backend. - [reverse-SynthID](https://github.com/aloshdenny/reverse-SynthID) for optional pixel scoring. Not bundled, and non-commercial upstream. @@ -448,9 +467,9 @@ watermark-remover/ └── requirements-test.txt ``` -[`skills/remove-ai-marks/`](skills/remove-ai-marks) is the whole product. It is laid out as an agent skill so it can be symlinked into `.grok/skills` or `~/.grok/skills` and invoked directly, but [`scripts/`](skills/remove-ai-marks/scripts) is a set of ordinary CLIs that work on their own. [`references/`](skills/remove-ai-marks/references) holds the source notes the parsers were built from, including [`ethics.md`](skills/remove-ai-marks/references/ethics.md) and the vendor behavior notes. +[`skills/remove-ai-marks/`](https://github.com/PyModel/watermark-remover/tree/main/skills/remove-ai-marks) is the whole product. It is laid out as an agent skill so it can be symlinked into `.grok/skills` or `~/.grok/skills` and invoked directly, but [`scripts/`](https://github.com/PyModel/watermark-remover/tree/main/skills/remove-ai-marks/scripts) is a set of ordinary CLIs that work on their own. [`references/`](https://github.com/PyModel/watermark-remover/tree/main/skills/remove-ai-marks/references) holds the source notes the parsers were built from, including [`ethics.md`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/references/ethics.md) and the vendor behavior notes. -`research/` is intentionally gitignored local evidence and dogfood material; the durable claims discipline is captured in [`DESIGN.md`](DESIGN.md). [`assets/`](assets) holds repository images. [`tests/fixtures/`](tests/fixtures) holds the small binary files the format parsers are tested against. +`research/` is intentionally gitignored local evidence and dogfood material; the durable claims discipline is captured in [`DESIGN.md`](https://github.com/PyModel/watermark-remover/blob/main/DESIGN.md). [`assets/`](https://github.com/PyModel/watermark-remover/tree/main/assets) holds repository images. [`tests/fixtures/`](https://github.com/PyModel/watermark-remover/tree/main/tests/fixtures) holds the small binary files the format parsers are tested against. ## Coverage and limits @@ -467,7 +486,7 @@ No public universal text detector exists, and a detector miss does not prove eve ## Ethics -Built for privacy, hygiene, accessibility, and research on content you own. Not for academic fraud, evading disclosure requirements, or claiming output is proven human-written. See [`ethics.md`](skills/remove-ai-marks/references/ethics.md). +Built for privacy, hygiene, accessibility, and research on content you own. Not for academic fraud, evading disclosure requirements, or claiming output is proven human-written. See [`ethics.md`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/references/ethics.md). --- @@ -490,10 +509,10 @@ make check ## Documentation -- [`DESIGN.md`](DESIGN.md), architecture, seams, guarantee classes, roadmap -- [`SKILL.md`](skills/remove-ai-marks/SKILL.md), agent workflow -- [`mark-classes.md`](skills/remove-ai-marks/references/mark-classes.md), [`removal-matrix.md`](skills/remove-ai-marks/references/removal-matrix.md), [`vendor-notes.md`](skills/remove-ai-marks/references/vendor-notes.md) -- [`CONTRIBUTING.md`](CONTRIBUTING.md) and [`SECURITY.md`](SECURITY.md) +- [`DESIGN.md`](https://github.com/PyModel/watermark-remover/blob/main/DESIGN.md), architecture, seams, guarantee classes, roadmap +- [`SKILL.md`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/SKILL.md), agent workflow +- [`mark-classes.md`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/references/mark-classes.md), [`removal-matrix.md`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/references/removal-matrix.md), [`vendor-notes.md`](https://github.com/PyModel/watermark-remover/blob/main/skills/remove-ai-marks/references/vendor-notes.md) +- [`CONTRIBUTING.md`](https://github.com/PyModel/watermark-remover/blob/main/CONTRIBUTING.md) and [`SECURITY.md`](https://github.com/PyModel/watermark-remover/blob/main/SECURITY.md) # Legal Disclaimer and Responsible Use @@ -545,11 +564,11 @@ If you use watermark-remover in research papers or published work, consider disc ## Report Problems -Security concerns or misuse reports: see [SECURITY.md](SECURITY.md). +Security concerns or misuse reports: see [SECURITY.md](https://github.com/PyModel/watermark-remover/blob/main/SECURITY.md). ## License -MIT, see [LICENSE](LICENSE). +MIT, see [LICENSE](https://github.com/PyModel/watermark-remover/blob/main/LICENSE). ## Primary references diff --git a/SECURITY.md b/SECURITY.md index 5de4443..be9cc08 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,7 +11,7 @@ GitHub Release (when releases exist). Older tags are not maintained. Please report vulnerabilities privately using one of: -1. **[GitHub Security Advisories](https://github.com/Pythoughts-labs/watermark-remover/security/advisories/new)** +1. **[GitHub Security Advisories](https://github.com/PyModel/watermark-remover/security/advisories/new)** (preferred) — "Report a vulnerability" on the repository Security tab 2. Email the maintainer at diff --git a/compose.yaml b/compose.yaml index 1cf73a0..0092d5e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -21,7 +21,7 @@ services: build: context: . dockerfile: Dockerfile - image: ghcr.io/pythoughts-labs/watermark-remover:latest + image: ghcr.io/pymodel/watermark-remover:latest ports: - "127.0.0.1:8765:8765" environment: @@ -46,7 +46,7 @@ services: build: context: . dockerfile: Dockerfile.markllm - image: ghcr.io/pythoughts-labs/watermark-remover:markllm-latest + image: ghcr.io/pymodel/watermark-remover:markllm-latest # One-shot CLI: `up` just confirms the image; run real jobs with # `docker compose run --rm wr-markllm detect /data/wm.txt --scheme kgw`. command: ["--help"] @@ -66,7 +66,7 @@ services: build: context: . dockerfile: Dockerfile.markdiffusion - image: ghcr.io/pythoughts-labs/watermark-remover:markdiffusion-latest + image: ghcr.io/pymodel/watermark-remover:markdiffusion-latest command: ["--help"] read_only: true tmpfs: diff --git a/docs/windows-autostart.md b/docs/windows-autostart.md index 65a4f54..f3a4366 100644 --- a/docs/windows-autostart.md +++ b/docs/windows-autostart.md @@ -10,7 +10,7 @@ Replace `` below with wherever you want the repo to live (e.g. path consistently in every step below. ```powershell -git clone https://github.com/Pythoughts-labs/watermark-remover.git +git clone https://github.com/PyModel/watermark-remover.git ``` ## 2. Create a silent launcher script diff --git a/pyproject.toml b/pyproject.toml index 01d439f..8593c32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,30 @@ [build-system] -requires = ["setuptools>=68.0", "wheel"] +requires = ["setuptools>=77.0", "wheel"] build-backend = "setuptools.build_meta" [project] name = "watermark-remover" version = "0.4.0" description = "Pure-Python AI watermark remover for text, images, and documents" +readme = "README.md" requires-python = ">=3.10" -license = {text = "MIT"} +license = "MIT" +license-files = ["LICENSE"] authors = [{name = "Mohamed Elkholy"}] +keywords = [ + "watermark", + "watermark-removal", + "ai-detection", + "c2pa", + "provenance", + "unicode", + "steganography", + "exif", + "metadata", +] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -23,6 +35,12 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] +[project.urls] +Homepage = "https://github.com/PyModel/watermark-remover" +Repository = "https://github.com/PyModel/watermark-remover" +Issues = "https://github.com/PyModel/watermark-remover/issues" +Changelog = "https://github.com/PyModel/watermark-remover/releases" + [project.optional-dependencies] visible = [ "numpy", @@ -47,22 +65,23 @@ all = [ ] [project.scripts] -wm = "skills.remove_ai_marks.scripts.clean_file:main" -wm-serve = "skills.remove_ai_marks.scripts.server:main" -wm-audit-dir = "skills.remove_ai_marks.scripts.audit_dir:main" -wm-audit-site = "skills.remove_ai_marks.scripts.audit_website:main" +wm = "watermark_remover.scripts.clean_file:main" +wm-serve = "watermark_remover.scripts.server:main" +wm-audit-dir = "watermark_remover.scripts.audit_dir:main" +wm-audit-site = "watermark_remover.scripts.audit_website:main" [tool.setuptools] -# The on-disk directory uses a hyphen (skills/remove-ai-marks) while the -# importable package uses an underscore; map them explicitly. -package-dir = {"skills.remove_ai_marks" = "skills/remove-ai-marks"} -packages = ["skills.remove_ai_marks", "skills.remove_ai_marks.scripts"] +# The on-disk directory keeps its Claude-skill layout (skills/remove-ai-marks); +# the installed package is a single top-level `watermark_remover` root. Mapping +# them here avoids claiming a generic `skills` namespace in site-packages. +package-dir = {"watermark_remover" = "skills/remove-ai-marks"} +packages = ["watermark_remover", "watermark_remover.scripts"] [tool.setuptools.package-data] # Ship the skill documentation and bootstrap assets so a wheel install is # self-sufficient for the CLI surfaces that read them at runtime. -"skills.remove_ai_marks" = ["SKILL.md", "references/*.md"] -"skills.remove_ai_marks.scripts" = ["*.txt", "setup_*.sh", "setup_*.ps1"] +"watermark_remover" = ["SKILL.md", "references/*.md"] +"watermark_remover.scripts" = ["*.txt", "setup_*.sh", "setup_*.ps1"] [tool.ruff] line-length = 100 diff --git a/skills/remove-ai-marks/__init__.py b/skills/remove-ai-marks/__init__.py index 3ba0fa4..cea0224 100644 --- a/skills/remove-ai-marks/__init__.py +++ b/skills/remove-ai-marks/__init__.py @@ -1,8 +1,9 @@ """Watermark-remover agent skill package. -The on-disk directory uses a hyphen (``skills/remove-ai-marks/``) while the -importable package is ``skills.remove_ai_marks``. pyproject.toml maps the two -through ``package-dir``; this file makes the mapped directory a real package. +The on-disk directory keeps its Claude-skill layout +(``skills/remove-ai-marks/``) while the installed package is a single top-level +``watermark_remover`` root. pyproject.toml maps the two through +``package-dir``; this file makes the mapped directory a real package. """ from __future__ import annotations diff --git a/skills/remove-ai-marks/references/service-mode.md b/skills/remove-ai-marks/references/service-mode.md index 4ded816..a945908 100644 --- a/skills/remove-ai-marks/references/service-mode.md +++ b/skills/remove-ai-marks/references/service-mode.md @@ -15,7 +15,7 @@ WM="${WATERMARKS_SERVICE_URL:-http://127.0.0.1:8765}" ``` Start it either with Docker/compose (`docker compose up -d`, or the published -GHCR image `ghcr.io/pythoughts-labs/watermark-remover`) or locally +GHCR image `ghcr.io/pymodel/watermark-remover`) or locally (`make serve` / `wm-serve`). **Always check it first**, and stop with a clear message if it is unreachable: From c65ed3a56bd36b7af0d251fd4736913694d0eb09 Mon Sep 17 00:00:00 2001 From: elkaix Date: Tue, 18 Aug 2026 18:11:08 -0400 Subject: [PATCH 2/6] fix: harden release paths against traversal, injection, and stale refs Validate uploaded file paths in the demo: resolve strictly, require the file to live under the system temp dir, reject symlinks and non-regular files, and sanitize the stem and suffix used to build the output name. Pass the tag through a step-level environment variable in the image release workflow and validate its format before use, so a crafted ref name cannot be interpolated into the shell. Re-pin existing upstream checkouts in the MarkLLM and MarkDiffusion setup scripts: an existing .git directory previously short-circuited fetch, detached checkout, sparse-checkout, and HEAD verification, so a stale or tampered tree was installed as-is. Guard os.fchmod behind a hasattr check. It does not exist on Windows, which was failing the Windows CI job on any in-place operation. --- .github/workflows/release-images.yml | 16 +- demo.py | 34 +- .../clean-user-facing-text/scripts/common.py | 95 ++++- skills/remove-ai-marks/scripts/clean_asset.py | 5 +- skills/remove-ai-marks/scripts/common.py | 6 +- .../scripts/setup_markdiffusion.sh | 22 +- .../remove-ai-marks/scripts/setup_markllm.sh | 38 +- tests/test_clean_asset.py | 60 ++- tests/test_clean_text.py | 19 + tests/test_external_command.py | 3 + tests/test_image_degrade.py | 4 +- tests/test_release_hardening.py | 355 ++++++++++++++++++ 12 files changed, 599 insertions(+), 58 deletions(-) create mode 100644 tests/test_release_hardening.py diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index ce478fe..cef7686 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -25,8 +25,14 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Compute image tags id: tags + env: + REF_NAME: ${{ github.ref_name }} run: | - TAG="${{ github.ref_name }}" + if [[ ! "$REF_NAME" =~ ^v[0-9][0-9A-Za-z.+-]*$ ]]; then + echo "invalid release ref: $REF_NAME" >&2 + exit 1 + fi + TAG="$REF_NAME" echo "tags=ghcr.io/pymodel/watermark-remover:${TAG},ghcr.io/pymodel/watermark-remover:latest" >> "$GITHUB_OUTPUT" echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 @@ -64,8 +70,14 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Compute image tags id: tags + env: + REF_NAME: ${{ github.ref_name }} run: | - TAG="${{ github.ref_name }}" + if [[ ! "$REF_NAME" =~ ^v[0-9][0-9A-Za-z.+-]*$ ]]; then + echo "invalid release ref: $REF_NAME" >&2 + exit 1 + fi + TAG="$REF_NAME" echo "tags=ghcr.io/pymodel/watermark-remover:${{ matrix.tag }}-${TAG},ghcr.io/pymodel/watermark-remover:${{ matrix.tag }}-latest" >> "$GITHUB_OUTPUT" - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: diff --git a/demo.py b/demo.py index 0cb1fa4..5d0b442 100644 --- a/demo.py +++ b/demo.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +import re import sys import tempfile from pathlib import Path @@ -29,17 +30,44 @@ def clean_upload(file_obj, keep_non_ai: bool, layer_b: bool, strength: str): """Gradio handler. Returns (report markdown, cleaned file path, rewrite prompt).""" if file_obj is None: return "Upload a file first.", None, "" - src = Path(file_obj.name if hasattr(file_obj, "name") else str(file_obj)) + requested_src = Path(file_obj.name if hasattr(file_obj, "name") else str(file_obj)) workdir = Path(tempfile.mkdtemp(prefix="wmr-")) - dest = workdir / f"{src.stem}.cleaned{src.suffix}" try: + if requested_src.is_symlink(): + raise ValueError("uploaded path is a symlink") + src = requested_src.resolve(strict=True) + if not src.is_file(): + raise ValueError("uploaded path is not a regular file") + configured_temporary_root = Path(tempfile.gettempdir()).absolute() + temporary_root = configured_temporary_root.resolve(strict=True) + if not src.is_relative_to(temporary_root): + raise ValueError("uploaded path is outside the system temporary directory") + + requested_absolute = requested_src.absolute() + for candidate_root in (configured_temporary_root, temporary_root): + try: + relative_source = requested_absolute.relative_to(candidate_root) + break + except ValueError: + continue + else: + raise ValueError("uploaded path reaches the temporary directory through a symlink") + current = candidate_root + for component in relative_source.parts: + current /= component + if current.is_symlink(): + raise ValueError(f"uploaded path contains a symlink: {current}") + + safe_stem = re.sub(r"[^A-Za-z0-9._-]", "_", src.stem) or "upload" + safe_suffix = re.sub(r"[^A-Za-z0-9._-]", "_", src.suffix) + dest = workdir / f"{safe_stem}.cleaned{safe_suffix}" kind = classify_asset(src) result = clean_asset( src, dest, CleanPlan(forced_kind=kind, strip_all_metadata=not keep_non_ai) ).to_dict() except Exception as e: - return f"**Error cleaning {src.name}:** `{e}`", None, "" + return f"**Error cleaning {requested_src.name}:** `{e}`", None, "" prompt = "" if layer_b and kind == "text": diff --git a/skills/clean-user-facing-text/scripts/common.py b/skills/clean-user-facing-text/scripts/common.py index 550064d..5455406 100644 --- a/skills/clean-user-facing-text/scripts/common.py +++ b/skills/clean-user-facing-text/scripts/common.py @@ -4,8 +4,10 @@ import json import os +import stat import sys import tempfile +from contextlib import suppress from pathlib import Path from typing import Any @@ -126,28 +128,99 @@ def _default_file_mode() -> int: def safe_write_bytes(path: str | Path, data: bytes) -> None: destination = Path(path) destination.parent.mkdir(parents=True, exist_ok=True) + parent = destination.parent.resolve(strict=True) + destination = parent / destination.name if destination.is_symlink(): raise OSError(f"refusing to write through symlink: {destination}") - fd, temporary = tempfile.mkstemp( - prefix=f".{destination.name}.", - suffix=".tmp", - dir=str(destination.parent), - ) + parent_fd: int | None = None + if os.name != "nt": + flags = os.O_RDONLY + if hasattr(os, "O_DIRECTORY"): + flags |= os.O_DIRECTORY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + parent_fd = os.open(parent, flags) + + parent_identity = os.fstat(parent_fd) if parent_fd is not None else os.lstat(parent) + fd = -1 + temporary: Path | None = None + + def validate_paths(staged_identity: os.stat_result) -> None: + current_parent = os.lstat(parent) + if stat.S_ISLNK(current_parent.st_mode) or not os.path.samestat( + parent_identity, current_parent + ): + raise OSError(f"destination directory changed while writing: {parent}") + + if parent_fd is not None: + staged = os.stat(temporary.name, dir_fd=parent_fd, follow_symlinks=False) + try: + current_destination = os.stat( + destination.name, dir_fd=parent_fd, follow_symlinks=False + ) + except FileNotFoundError: + current_destination = None + else: + staged = os.lstat(temporary) + try: + current_destination = os.lstat(destination) + except FileNotFoundError: + current_destination = None + + if not os.path.samestat(staged_identity, staged): + raise OSError(f"temporary path changed while writing: {temporary}") + if current_destination is None: + return + if stat.S_ISLNK(current_destination.st_mode): + raise OSError(f"refusing to write through symlink: {destination}") + if os.path.samestat(staged_identity, current_destination): + raise OSError(f"destination aliases temporary file: {destination}") + try: + fd, temporary_name = tempfile.mkstemp( + prefix=f".{destination.name}.", + suffix=".tmp", + dir=str(parent), + ) + temporary = Path(temporary_name) + staged_identity = os.fstat(fd) + validate_paths(staged_identity) if hasattr(os, "fchmod"): os.fchmod(fd, _default_file_mode()) - with os.fdopen(fd, "wb") as handle: + handle = os.fdopen(fd, "wb") + fd = -1 + with handle: handle.write(data) handle.flush() os.fsync(handle.fileno()) - os.replace(temporary, destination) + validate_paths(staged_identity) + if parent_fd is not None: + os.replace( + temporary.name, + destination.name, + src_dir_fd=parent_fd, + dst_dir_fd=parent_fd, + ) + else: + os.replace(temporary, destination) + temporary = None except BaseException: - try: - os.unlink(temporary) - except OSError: - pass + if fd >= 0: + with suppress(OSError): + os.close(fd) + if temporary is not None: + try: + if parent_fd is not None: + os.unlink(temporary.name, dir_fd=parent_fd) + else: + os.unlink(temporary) + except OSError: + pass raise + finally: + if parent_fd is not None: + os.close(parent_fd) def write_text_output(text: str, path: str | None) -> None: diff --git a/skills/remove-ai-marks/scripts/clean_asset.py b/skills/remove-ai-marks/scripts/clean_asset.py index 5055f3e..98d1c2d 100644 --- a/skills/remove-ai-marks/scripts/clean_asset.py +++ b/skills/remove-ai-marks/scripts/clean_asset.py @@ -106,7 +106,10 @@ def _publish_image_artifacts( backup_file.write(source_snapshot.data) backup_file.flush() os.fsync(backup_fd) - os.fchmod(backup_fd, source_snapshot.mode) + if hasattr(os, "fchmod"): + os.fchmod(backup_fd, source_snapshot.mode) + else: + os.chmod(backup_temp, source_snapshot.mode) finally: os.close(backup_fd) diff --git a/skills/remove-ai-marks/scripts/common.py b/skills/remove-ai-marks/scripts/common.py index 49352c0..dbb06e3 100755 --- a/skills/remove-ai-marks/scripts/common.py +++ b/skills/remove-ai-marks/scripts/common.py @@ -298,7 +298,11 @@ def create_backup(source: Path) -> Path: dest_file.write(chunk) dest_file.flush() os.fsync(dest_fd) - os.fchmod(dest_fd, stat.S_IMODE(source_stat.st_mode)) + mode = stat.S_IMODE(source_stat.st_mode) + if hasattr(os, "fchmod"): + os.fchmod(dest_fd, mode) + else: + os.chmod(dest, mode) except Exception: if dest_fd is not None: os.close(dest_fd) diff --git a/skills/remove-ai-marks/scripts/setup_markdiffusion.sh b/skills/remove-ai-marks/scripts/setup_markdiffusion.sh index ba91ddb..9e53e4f 100755 --- a/skills/remove-ai-marks/scripts/setup_markdiffusion.sh +++ b/skills/remove-ai-marks/scripts/setup_markdiffusion.sh @@ -81,16 +81,24 @@ if [[ "$CHECKOUT" -eq 1 ]]; then echo "Cloning THU-BPM/MarkDiffusion into $DIR (pinned ref: $REF)" git clone --depth 1 --filter=blob:none --sparse \ https://github.com/THU-BPM/MarkDiffusion.git "$DIR" - git -C "$DIR" fetch --depth 1 origin "$REF" - git -C "$DIR" checkout --detach "$REF" - HEAD_SHA="$(git -C "$DIR" rev-parse HEAD)" - if [[ "$HEAD_SHA" != "$REF" ]]; then - echo "error: expected pinned ref $REF, got $HEAD_SHA" >&2 - exit 1 - fi else echo "Using existing checkout: $DIR" fi + if ! git -C "$DIR" fetch --depth 1 origin "$REF"; then + if ! git -C "$DIR" rev-parse --verify "${REF}^{commit}" >/dev/null 2>&1; then + echo "error: could not fetch pinned ref $REF and it is not available locally" >&2 + exit 1 + fi + echo "warning: fetch failed; using locally available pinned ref $REF" >&2 + fi + git -C "$DIR" checkout --detach "$REF" + git -C "$DIR" sparse-checkout reapply + EXPECTED_SHA="$(git -C "$DIR" rev-parse --verify "${REF}^{commit}")" + HEAD_SHA="$(git -C "$DIR" rev-parse HEAD)" + if [[ "$HEAD_SHA" != "$EXPECTED_SHA" ]]; then + echo "error: expected pinned ref $REF ($EXPECTED_SHA), got $HEAD_SHA" >&2 + exit 1 + fi fi if [[ ! -x "$DIR/.venv/bin/python" ]]; then diff --git a/skills/remove-ai-marks/scripts/setup_markllm.sh b/skills/remove-ai-marks/scripts/setup_markllm.sh index fc9ae70..793c74f 100755 --- a/skills/remove-ai-marks/scripts/setup_markllm.sh +++ b/skills/remove-ai-marks/scripts/setup_markllm.sh @@ -71,25 +71,33 @@ if [[ ! -d "$DIR/.git" ]]; then echo "Cloning THU-BPM/MarkLLM into $DIR (pinned ref: $REF)" git clone --depth 1 --filter=blob:none --sparse \ https://github.com/THU-BPM/MarkLLM.git "$DIR" - git -C "$DIR" fetch --depth 1 origin "$REF" - git -C "$DIR" checkout --detach "$REF" - git -C "$DIR" sparse-checkout set --no-cone \ - '/watermark/' \ - '/config/' \ - '/utils/' \ - '/exceptions/' \ - '/evaluation/dataset.py' \ - '/LICENSE' \ - '/README.md' - HEAD_SHA="$(git -C "$DIR" rev-parse HEAD)" - if [[ "$HEAD_SHA" != "$REF" ]]; then - echo "error: expected pinned ref $REF, got $HEAD_SHA" >&2 - exit 1 - fi else echo "Using existing checkout: $DIR" fi +if ! git -C "$DIR" fetch --depth 1 origin "$REF"; then + if ! git -C "$DIR" rev-parse --verify "${REF}^{commit}" >/dev/null 2>&1; then + echo "error: could not fetch pinned ref $REF and it is not available locally" >&2 + exit 1 + fi + echo "warning: fetch failed; using locally available pinned ref $REF" >&2 +fi +git -C "$DIR" checkout --detach "$REF" +git -C "$DIR" sparse-checkout set --no-cone \ + '/watermark/' \ + '/config/' \ + '/utils/' \ + '/exceptions/' \ + '/evaluation/dataset.py' \ + '/LICENSE' \ + '/README.md' +EXPECTED_SHA="$(git -C "$DIR" rev-parse --verify "${REF}^{commit}")" +HEAD_SHA="$(git -C "$DIR" rev-parse HEAD)" +if [[ "$HEAD_SHA" != "$EXPECTED_SHA" ]]; then + echo "error: expected pinned ref $REF ($EXPECTED_SHA), got $HEAD_SHA" >&2 + exit 1 +fi + if [[ ! -x "$DIR/.venv/bin/python" ]]; then echo "Creating venv at $DIR/.venv" "$PYTHON" -m venv "$DIR/.venv" diff --git a/tests/test_clean_asset.py b/tests/test_clean_asset.py index 11fb817..009e864 100644 --- a/tests/test_clean_asset.py +++ b/tests/test_clean_asset.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import stat from dataclasses import FrozenInstanceError, replace from pathlib import Path @@ -190,6 +191,30 @@ def test_late_in_place_failure_preserves_source_without_backup( assert not source.with_suffix(".png.bak").exists() +def test_image_in_place_publication_without_fchmod_preserves_backup_bytes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "input.png" + original = encode_png(Raster(8, 8, 3, bytearray([10, 20, 30] * 64))) + source.write_bytes(original) + monkeypatch.delattr(clean_asset_module.os, "fchmod", raising=False) + + result = clean_asset( + source, + source, + CleanPlan( + forced_kind="image", + in_place=True, + degrade=ImageDegradePlan(strategy="jpeg"), + ), + ) + + assert source.with_suffix(".png.bak").read_bytes() == original + assert result.output == source + assert result.to_dict()["degrade"]["strategy"] == "jpeg" + assert source.read_bytes().startswith(b"\x89PNG\r\n\x1a\n") + + def _inject_failure_after_atomic_publish( monkeypatch: pytest.MonkeyPatch, failure_target: Path, @@ -251,9 +276,10 @@ def test_image_publication_failure_rolls_back_destination_and_mask( clean_asset(source, destination, _visible_plan(mask_output)) assert destination.read_bytes() == b"prior destination" - assert stat.S_IMODE(destination.stat().st_mode) == 0o640 assert mask_output.read_bytes() == b"prior mask" - assert stat.S_IMODE(mask_output.stat().st_mode) == 0o604 + if os.name == "posix": + assert stat.S_IMODE(destination.stat().st_mode) == 0o640 + assert stat.S_IMODE(mask_output.stat().st_mode) == 0o604 def test_mask_publication_failure_rolls_back_destination_and_mask( @@ -273,9 +299,10 @@ def test_mask_publication_failure_rolls_back_destination_and_mask( clean_asset(source, destination, _visible_plan(mask_output)) assert destination.read_bytes() == b"prior destination" - assert stat.S_IMODE(destination.stat().st_mode) == 0o640 assert mask_output.read_bytes() == b"prior mask" - assert stat.S_IMODE(mask_output.stat().st_mode) == 0o604 + if os.name == "posix": + assert stat.S_IMODE(destination.stat().st_mode) == 0o640 + assert stat.S_IMODE(mask_output.stat().st_mode) == 0o604 def test_backup_publication_failure_rolls_back_all_artifacts( @@ -289,27 +316,23 @@ def test_backup_publication_failure_rolls_back_all_artifacts( mask_output.write_bytes(b"prior mask") mask_output.chmod(0o604) backup = source.with_suffix(".png.bak") - original_fchmod = clean_asset_module.os.fchmod - failed = False - def fail_once_after_chmod(fd: int, mode: int) -> None: - nonlocal failed - original_fchmod(fd, mode) - if not failed: - failed = True + def fail_backup_link(_source: Path, target: Path, **_kwargs) -> None: + if target == backup: raise OSError(f"injected publication failure: {backup.name}") - monkeypatch.setattr(clean_asset_module.os, "fchmod", fail_once_after_chmod) + monkeypatch.setattr(clean_asset_module.os, "link", fail_backup_link) plan = _visible_plan(mask_output) with pytest.raises(OSError, match="publication failure"): clean_asset(source, source, replace(plan, in_place=True)) assert source.read_bytes() == original - assert stat.S_IMODE(source.stat().st_mode) == 0o640 assert mask_output.read_bytes() == b"prior mask" - assert stat.S_IMODE(mask_output.stat().st_mode) == 0o604 assert not backup.exists() + if os.name == "posix": + assert stat.S_IMODE(source.stat().st_mode) == 0o640 + assert stat.S_IMODE(mask_output.stat().st_mode) == 0o604 def test_backup_fstat_failure_closes_fd_and_removes_partial_backup( @@ -365,9 +388,10 @@ def track_backup_close(fd: int) -> None: assert fstat_failed is True assert fd_closed is True assert source.read_bytes() == original - assert stat.S_IMODE(source.stat().st_mode) == 0o640 assert not backup.exists() assert not list(tmp_path.glob(f".{backup.name}.*")) + if os.name == "posix": + assert stat.S_IMODE(source.stat().st_mode) == 0o640 def test_competing_backup_created_during_publication_is_preserved( @@ -403,8 +427,9 @@ def create_competing_backup_before_link(source_path: Path, target_path: Path, ** ) assert source.read_bytes() == original - assert stat.S_IMODE(source.stat().st_mode) == 0o640 assert backup.read_bytes() == competing_data + if os.name == "posix": + assert stat.S_IMODE(source.stat().st_mode) == 0o640 def test_keyboard_interrupt_after_in_place_source_publish_restores_source( @@ -452,8 +477,9 @@ def interrupt_once_after_write(target: Path, data: bytes) -> None: assert caught.value is interruption assert publication_targets[:2] == [backup, source] assert source.read_bytes() == original - assert stat.S_IMODE(source.stat().st_mode) == 0o640 assert not backup.exists() + if os.name == "posix": + assert stat.S_IMODE(source.stat().st_mode) == 0o640 def test_final_bytes_and_outside_mask_failure_are_reported(tmp_path: Path) -> None: diff --git a/tests/test_clean_text.py b/tests/test_clean_text.py index 3a9e42e..f29eee7 100644 --- a/tests/test_clean_text.py +++ b/tests/test_clean_text.py @@ -6,10 +6,14 @@ import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] SCRIPTS = ROOT / "skills" / "remove-ai-marks" / "scripts" sys.path.insert(0, str(SCRIPTS)) +import clean_text as clean_text_cli +import common from text_unicode import clean_text, inspect_text @@ -143,6 +147,21 @@ def test_cli_roundtrips_invalid_utf8_and_backup_is_byte_exact(tmp_path: Path): assert src.read_bytes() == b"abc\xffdef" +def test_text_in_place_without_fchmod_preserves_backup_bytes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + src = tmp_path / "input.txt" + original = b"hello\xe2\x80\x8bworld" + src.write_bytes(original) + monkeypatch.delattr(common.os, "fchmod", raising=False) + monkeypatch.setattr(sys, "argv", [str(SCRIPTS / "clean_text.py"), str(src), "--in-place"]) + + assert clean_text_cli.main() == 0 + + assert src.with_suffix(".txt.bak").read_bytes() == original + assert src.read_bytes() == b"helloworld" + + def test_aggressive_confusable(): # Cyrillic 'а' (U+0430) looks like Latin 'a' raw = "p\u0430y" # p + cyrillic a + y diff --git a/tests/test_external_command.py b/tests/test_external_command.py index 125ef2d..270a717 100644 --- a/tests/test_external_command.py +++ b/tests/test_external_command.py @@ -267,6 +267,9 @@ def __init__(self, *_args, **_kwargs): def wait(self, *, timeout: float) -> int: return 0 + def poll(self) -> int: + return 0 + monkeypatch.setattr(external_command.subprocess, "Popen", FakePopen) with pytest.raises(RuntimeError, match="failed to read external command output"): diff --git a/tests/test_image_degrade.py b/tests/test_image_degrade.py index 5b290ab..6163466 100644 --- a/tests/test_image_degrade.py +++ b/tests/test_image_degrade.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import subprocess import sys from pathlib import Path @@ -80,7 +81,8 @@ def test_degrade_preserves_output_file_mode(tmp_path: Path) -> None: dest, CleanPlan(degrade=ImageDegradePlan(strategy="median")), ) - assert dest.stat().st_mode & 0o777 == 0o640 + if os.name == "posix": + assert dest.stat().st_mode & 0o777 == 0o640 def test_degrade_rejects_non_png_output(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_release_hardening.py b/tests/test_release_hardening.py new file mode 100644 index 0000000..5ecad6b --- /dev/null +++ b/tests/test_release_hardening.py @@ -0,0 +1,355 @@ +"""Focused security regressions for release hardening.""" + +from __future__ import annotations + +import importlib.util +import os +import re +import shutil +import stat +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "skills" / "remove-ai-marks" / "scripts" +LIGHTWEIGHT_COMMON = ROOT / "skills" / "clean-user-facing-text" / "scripts" / "common.py" + +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(SCRIPTS)) + +import demo + + +def _load_lightweight_common(): + spec = importlib.util.spec_from_file_location( + "lightweight_common_release_hardening", LIGHTWEIGHT_COMMON + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_demo_rejects_upload_symlink(tmp_path: Path) -> None: + target = tmp_path / "target.txt" + target.write_text("hello\u200bworld", encoding="utf-8") + source = tmp_path / "upload.txt" + try: + source.symlink_to(target) + except OSError as error: + pytest.skip(f"symlinks unavailable: {error}") + + report, output, prompt = demo.clean_upload(str(source), False, False, "paraphrase") + + assert report.startswith("**Error cleaning upload.txt:**") + assert "symlink" in report.lower() + assert output is None + assert prompt == "" + assert target.read_text(encoding="utf-8") == "hello\u200bworld" + + +def test_demo_rejects_upload_through_symlinked_directory(tmp_path: Path) -> None: + actual = tmp_path / "actual" + actual.mkdir() + source = actual / "upload.txt" + source.write_text("hello\u200bworld", encoding="utf-8") + linked = tmp_path / "linked" + try: + linked.symlink_to(actual, target_is_directory=True) + except OSError as error: + pytest.skip(f"symlinks unavailable: {error}") + + report, output, prompt = demo.clean_upload( + str(linked / source.name), False, False, "paraphrase" + ) + + assert report.startswith("**Error cleaning upload.txt:**") + assert "symlink" in report.lower() + assert output is None + assert prompt == "" + + +def test_demo_rejects_upload_outside_system_temp( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + trusted_temp = tmp_path / "trusted" + trusted_temp.mkdir() + source = tmp_path / "outside.txt" + source.write_text("hello", encoding="utf-8") + monkeypatch.setattr(demo.tempfile, "gettempdir", lambda: str(trusted_temp)) + + report, output, prompt = demo.clean_upload(str(source), False, False, "paraphrase") + + assert report.startswith("**Error cleaning outside.txt:**") + assert "temporary directory" in report + assert output is None + assert prompt == "" + + +def test_demo_rejects_non_regular_upload(tmp_path: Path) -> None: + source = tmp_path / "folder.txt" + source.mkdir() + + report, output, prompt = demo.clean_upload(str(source), False, False, "paraphrase") + + assert report.startswith("**Error cleaning folder.txt:**") + assert "regular file" in report + assert output is None + assert prompt == "" + + +def test_demo_sanitizes_cleaned_output_name(tmp_path: Path) -> None: + source = tmp_path / "report .txt" + source.write_text("hello\u200bworld", encoding="utf-8") + + report, output, prompt = demo.clean_upload(str(source), False, False, "paraphrase") + + assert "**Kind:** `text`" in report + assert output is not None + cleaned = Path(output) + assert re.fullmatch(r"[A-Za-z0-9._-]+", cleaned.name) + assert cleaned.read_text(encoding="utf-8") == "helloworld" + assert prompt == "" + + +def test_safe_write_bytes_rejects_destination_symlink(tmp_path: Path) -> None: + common = _load_lightweight_common() + victim = tmp_path / "victim.txt" + victim.write_bytes(b"preserve") + destination = tmp_path / "output.txt" + try: + destination.symlink_to(victim) + except OSError as error: + pytest.skip(f"symlinks unavailable: {error}") + + with pytest.raises(OSError, match="symlink"): + common.safe_write_bytes(destination, b"replacement") + + assert destination.is_symlink() + assert victim.read_bytes() == b"preserve" + + +def test_safe_write_bytes_preserves_parent_bytes_and_mode( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + common = _load_lightweight_common() + if not hasattr(common.os, "fchmod"): + pytest.skip("fchmod unavailable") + destination = tmp_path / "created" / "output.txt" + monkeypatch.setattr(common, "_default_file_mode", lambda: 0o640) + payload = b"exact\x00bytes\xff" + + common.safe_write_bytes(destination, payload) + + assert destination.read_bytes() == payload + assert stat.S_IMODE(destination.stat().st_mode) == 0o640 + + +def test_safe_write_bytes_rejects_destination_symlink_race( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + common = _load_lightweight_common() + victim = tmp_path / "victim.txt" + victim.write_bytes(b"preserve") + destination = tmp_path / "output.txt" + original_mkstemp = common.tempfile.mkstemp + + def replace_destination_with_symlink(*args, **kwargs): + fd, temporary = original_mkstemp(*args, **kwargs) + try: + destination.symlink_to(victim) + except OSError: + os.close(fd) + os.unlink(temporary) + raise + return fd, temporary + + monkeypatch.setattr(common.tempfile, "mkstemp", replace_destination_with_symlink) + + with pytest.raises(OSError, match="symlink"): + common.safe_write_bytes(destination, b"replacement") + + assert destination.is_symlink() + assert victim.read_bytes() == b"preserve" + assert not list(tmp_path.glob(".output.txt.*.tmp")) + + +def test_safe_write_bytes_rejects_temporary_destination_alias( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + common = _load_lightweight_common() + destination = tmp_path / "output.txt" + original_mkstemp = common.tempfile.mkstemp + + def alias_destination_to_temporary(*args, **kwargs): + fd, temporary = original_mkstemp(*args, **kwargs) + try: + os.link(temporary, destination) + except OSError: + os.close(fd) + os.unlink(temporary) + raise + return fd, temporary + + monkeypatch.setattr(common.tempfile, "mkstemp", alias_destination_to_temporary) + + with pytest.raises(OSError, match="aliases"): + common.safe_write_bytes(destination, b"replacement") + + assert destination.read_bytes() == b"" + assert not list(tmp_path.glob(".output.txt.*.tmp")) + + +def test_safe_write_bytes_rejects_parent_directory_race( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + if os.name == "nt": + pytest.skip("POSIX directory-descriptor hardening") + common = _load_lightweight_common() + parent = tmp_path / "output" + parent.mkdir() + moved_parent = tmp_path / "moved-output" + attacker = tmp_path / "attacker" + attacker.mkdir() + destination = parent / "output.txt" + original_mkstemp = common.tempfile.mkstemp + + def redirect_parent_after_staging(*args, **kwargs): + fd, temporary = original_mkstemp(*args, **kwargs) + parent.rename(moved_parent) + parent.symlink_to(attacker, target_is_directory=True) + return fd, temporary + + monkeypatch.setattr(common.tempfile, "mkstemp", redirect_parent_after_staging) + + with pytest.raises(OSError, match="directory changed"): + common.safe_write_bytes(destination, b"replacement") + + assert not (attacker / destination.name).exists() + assert not list(moved_parent.glob(".output.txt.*.tmp")) + + +def test_release_image_tags_validate_ref_from_environment() -> None: + workflow = (ROOT / ".github" / "workflows" / "release-images.yml").read_text(encoding="utf-8") + + assert workflow.count("REF_NAME: ${{ github.ref_name }}") == 2 + assert 'TAG="${{ github.ref_name }}"' not in workflow + assert workflow.count('[[ ! "$REF_NAME" =~ ^v[0-9][0-9A-Za-z.+-]*$ ]]') == 2 + assert workflow.count('TAG="$REF_NAME"') == 2 + assert ( + "tags=ghcr.io/pymodel/watermark-remover:${TAG},ghcr.io/pymodel/watermark-remover:latest" + in workflow + ) + assert ( + "tags=ghcr.io/pymodel/watermark-remover:${{ matrix.tag }}-${TAG}," + "ghcr.io/pymodel/watermark-remover:${{ matrix.tag }}-latest" + ) in workflow + assert "VERSION=${{ steps.tags.outputs.version }}" in workflow + + +@pytest.mark.parametrize( + ("script_name", "extra_args", "sparse_command"), + [ + ( + "setup_markllm.sh", + [], + "sparse-checkout set --no-cone /watermark/ /config/ /utils/ /exceptions/ " + "/evaluation/dataset.py /LICENSE /README.md", + ), + ("setup_markdiffusion.sh", ["--checkout"], "sparse-checkout reapply"), + ], +) +@pytest.mark.parametrize("head_matches", [True, False], ids=["matching-head", "stale-head"]) +def test_existing_checkout_is_repinned_before_install( + tmp_path: Path, + script_name: str, + extra_args: list[str], + sparse_command: str, + head_matches: bool, +) -> None: + checkout = tmp_path / "checkout" + (checkout / ".git").mkdir(parents=True) + python = checkout / ".venv" / "bin" / "python" + python.parent.mkdir(parents=True) + command_log = tmp_path / "commands.log" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + + _write_executable( + fake_bin / "git", + """#!/bin/sh +printf 'git %s\n' "$*" >> "$COMMAND_LOG" +case " $* " in + *" fetch "*) exit 1 ;; + *" rev-parse HEAD "*) printf '%s\n' "$HEAD_REF" ;; + *" rev-parse "*) printf '%s\n' "$EXPECTED_REF" ;; +esac +""", + ) + _write_executable( + fake_bin / "realpath", + """#!/bin/sh +if [ "$1" = "-m" ]; then shift; fi +printf '%s\n' "$1" +""", + ) + _write_executable( + python, + """#!/bin/sh +printf 'python %s\n' "$*" >> "$COMMAND_LOG" +""", + ) + + ref = "0123456789abcdef0123456789abcdef01234567" + env = os.environ.copy() + env.update( + { + "COMMAND_LOG": str(command_log), + "EXPECTED_REF": ref, + "HEAD_REF": ref if head_matches else "0000000000000000000000000000000000000000", + "PATH": f"{fake_bin}:/usr/bin:/bin", + } + ) + bash = shutil.which("bash") + assert bash is not None + result = subprocess.run( + [ + bash, + str(SCRIPTS / script_name), + "--dir", + str(checkout), + "--ref", + ref, + *extra_args, + ], + capture_output=True, + text=True, + env=env, + check=False, + timeout=10, + ) + + commands = command_log.read_text(encoding="utf-8").splitlines() + fetch_index = next(i for i, command in enumerate(commands) if " fetch " in command) + checkout_index = next( + i for i, command in enumerate(commands) if f"checkout --detach {ref}" in command + ) + sparse_index = next(i for i, command in enumerate(commands) if sparse_command in command) + verify_index = next(i for i, command in enumerate(commands) if "rev-parse HEAD" in command) + assert fetch_index < checkout_index < sparse_index < verify_index + if not head_matches: + assert result.returncode == 1 + assert not any(command.startswith("python ") for command in commands) + return + + assert result.returncode == 0, result.stderr + install_index = next(i for i, command in enumerate(commands) if command.startswith("python ")) + assert verify_index < install_index + + +def _write_executable(path: Path, content: str) -> None: + path.write_text(content, encoding="utf-8") + path.chmod(0o755) From b7bfd792002c762b52fe9a0b5066a3a04c69f44a Mon Sep 17 00:00:00 2001 From: elkaix Date: Tue, 18 Aug 2026 18:19:34 -0400 Subject: [PATCH 3/6] fix(asset-kind): route HEIF magic bytes to the image pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit image_meta.detect_format reports the whole HEIF/HEIC family as "heif" (via heif_meta.detect_heif) and never returns "heic", but classify_bytes and classify only matched "heic" — HEIF bytes without a known extension classified as "unknown" and were refused by the server handlers. Unify all three sniffers on one _IMAGE_FORMAT_NAMES constant so the lists cannot drift apart again. --- skills/remove-ai-marks/scripts/asset_kind.py | 21 ++++++++------------ tests/test_format_dispatch.py | 19 ++++++++++++++++++ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/skills/remove-ai-marks/scripts/asset_kind.py b/skills/remove-ai-marks/scripts/asset_kind.py index 16c420a..32aedd4 100644 --- a/skills/remove-ai-marks/scripts/asset_kind.py +++ b/skills/remove-ai-marks/scripts/asset_kind.py @@ -50,6 +50,11 @@ _ASSET_KINDS: tuple[AssetKind, ...] = ("text", "image", "container", "unknown") _SNIFF_BYTES = 4096 # HEIF brand detection scans at most the first 4 KiB. +#: image_meta.detect_format values that route to the image pipeline. The +#: HEIF/HEIC family is reported as "heif" (see heif_meta.detect_heif), never +#: "heic" — keep one copy so the three sniffers below cannot drift apart. +_IMAGE_FORMAT_NAMES = frozenset({"png", "jpeg", "webp", "avif", "heif", "bmp", "gif", "tiff"}) + #: Bytes read for header-only sniffing. Every supported image/container #: magic lives in the prefix; zip-based containers (docx/odt/...) need the #: full central directory, which sits at the end of the archive, so only a @@ -66,7 +71,7 @@ def classify_bytes(data: bytes, suffix: str | None = None) -> AssetKind: return "container" if ext in _TEXT_EXTENSIONS: return "text" - if detect_image_format(data) in ("png", "jpeg", "webp", "avif", "heic", "bmp", "gif", "tiff"): + if detect_image_format(data) in _IMAGE_FORMAT_NAMES: return "image" if data: sniff_path = Path("input") if not ext else Path(f"input{ext}") @@ -86,7 +91,7 @@ def classify(path: Path) -> AssetKind: return "text" with path.open("rb") as fh: head = fh.read(CLASSIFY_HEADER_BYTES) - if detect_image_format(head) in ("png", "jpeg", "webp", "avif", "heic", "bmp", "gif", "tiff"): + if detect_image_format(head) in _IMAGE_FORMAT_NAMES: return "image" if head: data = path.read_bytes() if head[:4] == b"PK" else head @@ -118,17 +123,7 @@ def classify_asset(path: Path, *, forced_kind: str = "auto") -> AssetKind: with path.open("rb") as source: prefix = source.read(_SNIFF_BYTES) - if detect_image_format(prefix) in ( - "png", - "jpeg", - "webp", - "avif", - "heic", - "heif", - "bmp", - "gif", - "tiff", - ): + if detect_image_format(prefix) in _IMAGE_FORMAT_NAMES: return "image" if detect_container_format(path, prefix) != "unknown": return "container" diff --git a/tests/test_format_dispatch.py b/tests/test_format_dispatch.py index a165ac2..435130d 100644 --- a/tests/test_format_dispatch.py +++ b/tests/test_format_dispatch.py @@ -64,3 +64,22 @@ def test_classify_bytes_unknown_fallback(): assert classify_bytes(b"random bytes \x00\xff", None) == "unknown" assert classify_bytes(b"\x89PNG\r\n\x1a\nrest", None) == "image" assert classify_bytes(b"PK\x03\x04rest", None) == "unknown" # not a full zip + + +def _ftyp(brand: bytes) -> bytes: + payload = brand + b"\x00\x00\x00\x00" + brand + return (len(payload) + 8).to_bytes(4, "big") + b"ftyp" + payload + + +def test_classify_bytes_heif_magic_is_image(): + # detect_format reports the whole HEIF/HEIC family as "heif" (never + # "heic"); routing must accept that token or HEIF bytes land in "unknown". + assert classify_bytes(_ftyp(b"heic"), None) == "image" + assert classify_bytes(_ftyp(b"heic"), "") == "image" + assert classify_bytes(_ftyp(b"heic"), ".bin") == "image" + + +def test_classify_heif_file_without_known_extension(tmp_path): + f = tmp_path / "photo.blob" + f.write_bytes(_ftyp(b"heic") + b"\x00" * 64) + assert classify(f) == "image" From 272ece81f4b53016b0c2c43c5bcc2818f60b8afb Mon Sep 17 00:00:00 2001 From: elkaix Date: Tue, 18 Aug 2026 18:19:41 -0400 Subject: [PATCH 4/6] fix(pdf): fall back to pypdf when exiftool runs but fails clean_pdf only reached the pypdf path when exiftool was absent. When exiftool was installed but exited non-zero or raised, it published the original unstripped bytes under mode "exiftool" with no degraded flag. Now a failed exiftool hands off to clean_pdf_pypdf (skip_exiftool avoids re-invoking the same failing command), which reports mode/degraded honestly and never silently ships marked bytes. --- .../remove-ai-marks/scripts/container_meta.py | 23 ++++++-- tests/test_pdf_structural_rewrite.py | 52 ++++++++++++++++++- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/skills/remove-ai-marks/scripts/container_meta.py b/skills/remove-ai-marks/scripts/container_meta.py index a04967a..2e9e3bc 100644 --- a/skills/remove-ai-marks/scripts/container_meta.py +++ b/skills/remove-ai-marks/scripts/container_meta.py @@ -1621,8 +1621,14 @@ def inspect_pdf(path: Path, data: bytes) -> tuple[bool, bool, list[str], dict]: return has_c2pa, has_ai or has_c2pa, findings, {"tools": tools} -def clean_pdf_pypdf(path: Path, dest: Path) -> tuple[list[str], dict]: - """Clean PDF metadata. exiftool > full-document pypdf clone > unchanged copy.""" +def clean_pdf_pypdf( + path: Path, dest: Path, *, skip_exiftool: bool = False +) -> tuple[list[str], dict]: + """Clean PDF metadata. exiftool > full-document pypdf clone > unchanged copy. + + *skip_exiftool* is used by clean_pdf when exiftool already ran and failed, + so the fallback does not invoke the same failing command a second time. + """ actions: list[str] = [] data = path.read_bytes() dest.parent.mkdir(parents=True, exist_ok=True) @@ -1630,7 +1636,7 @@ def clean_pdf_pypdf(path: Path, dest: Path) -> tuple[list[str], dict]: exiftool = which("exiftool") # Strategy 1: exiftool (most reliable) - if exiftool: + if exiftool and not skip_exiftool: dest.write_bytes(data) try: result = external_command.run_command( @@ -1740,6 +1746,7 @@ def clean_pdf(path: Path, dest: Path) -> tuple[list[str], dict]: exiftool = which("exiftool") if exiftool: atomic_write_bytes(dest, data) + exiftool_ok = False try: r = run_command( [exiftool, "-all=", "-overwrite_original", str(dest)], @@ -1747,10 +1754,18 @@ def clean_pdf(path: Path, dest: Path) -> tuple[list[str], dict]: output_limit=2 * 1024 * 1024, ) actions.append(f"exiftool -all= (rc={r.returncode})") - if r.returncode != 0: + exiftool_ok = r.returncode == 0 + if not exiftool_ok: actions.append(f"exiftool degraded (rc={r.returncode})") except Exception as e: actions.append(f"exiftool failed: {e}") + if not exiftool_ok: + # exiftool ran but did not strip; dest still holds the original + # bytes. Hand off to the pypdf path rather than publishing + # unstripped output under mode "exiftool" with no degraded flag. + actions.append("trying pypdf fallback") + fallback_actions, fallback_meta = clean_pdf_pypdf(path, dest, skip_exiftool=True) + return actions + fallback_actions, fallback_meta rewritten = _pdf_structural_rewrite(dest, actions) c2patool = which("c2patool") if c2patool: diff --git a/tests/test_pdf_structural_rewrite.py b/tests/test_pdf_structural_rewrite.py index 9c6d2b6..fc11fd7 100644 --- a/tests/test_pdf_structural_rewrite.py +++ b/tests/test_pdf_structural_rewrite.py @@ -2,6 +2,7 @@ from __future__ import annotations +import builtins import shutil import sys from pathlib import Path @@ -39,7 +40,14 @@ def _ai_pdf() -> bytes: return bytes(out) -def _fake_tools(monkeypatch, *, qpdf: bool, qpdf_rc: int = 0, qpdf_writes: bool = True): +def _fake_tools( + monkeypatch, + *, + qpdf: bool, + qpdf_rc: int = 0, + qpdf_writes: bool = True, + exiftool_rc: int = 0, +): seen: list[list[str]] = [] def fake_which(cmd: str): @@ -55,7 +63,7 @@ def fake_run(cmd, **kwargs): if qpdf_writes: Path(cmd[-1]).write_bytes(b"%PDF-1.4\n% rebuilt\n%%EOF\n") return SimpleNamespace(returncode=qpdf_rc, stdout=b"", stderr=b"") - return SimpleNamespace(returncode=0, stdout=b"", stderr=b"") + return SimpleNamespace(returncode=exiftool_rc, stdout=b"", stderr=b"") monkeypatch.setattr(container_meta, "which", fake_which) monkeypatch.setattr(container_meta, "run_command", fake_run) @@ -109,6 +117,46 @@ def test_qpdf_failure_keeps_exiftool_output_and_warns(monkeypatch, tmp_path: Pat assert not list(tmp_path.glob("*.qpdf-tmp")) +def test_exiftool_failure_falls_back_to_pypdf(monkeypatch, tmp_path: Path): + # Regression: exiftool present but failing used to publish the original + # bytes under mode "exiftool" with no degraded flag and no fallback. + src = tmp_path / "in.pdf" + dest = tmp_path / "out.pdf" + src.write_bytes(_ai_pdf()) + seen = _fake_tools(monkeypatch, qpdf=True, exiftool_rc=1) + + actions, meta = clean_pdf(src, dest) + + assert meta == {"mode": "pypdf", "degraded": False} + assert any("exiftool degraded (rc=1)" in a for a in actions), actions + assert any("pypdf" in a for a in actions), actions + assert dest.read_bytes().startswith(b"%PDF") + # The qpdf structural rewrite belongs to the exiftool path only. + assert not any(c[0].endswith("qpdf") for c in seen) + + +def test_exiftool_failure_without_pypdf_copies_degraded(monkeypatch, tmp_path: Path): + src = tmp_path / "in.pdf" + dest = tmp_path / "out.pdf" + original = _ai_pdf() + src.write_bytes(original) + _fake_tools(monkeypatch, qpdf=True, exiftool_rc=1) + + real_import = builtins.__import__ + + def no_pypdf(name, *args, **kwargs): + if name == "pypdf": + raise ImportError("simulated missing optional dependency") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", no_pypdf) + actions, meta = clean_pdf(src, dest) + + assert meta == {"mode": "copy", "degraded": True} + assert dest.read_bytes() == original + assert any("copied unchanged" in a for a in actions), actions + + @pytest.mark.skipif( shutil.which("exiftool") is None or shutil.which("qpdf") is None, reason="needs real exiftool and qpdf", From c8225190465bc8faf503dc03f2f978eb90af29ce Mon Sep 17 00:00:00 2001 From: elkaix Date: Tue, 18 Aug 2026 18:28:12 -0400 Subject: [PATCH 5/6] test(hardening): skip POSIX-only fixtures on Windows test_demo_sanitizes_cleaned_output_name creates 'report .txt', which Windows forbids, and test_existing_checkout_is_repinned_before_install drives the POSIX .sh setup scripts (Windows ships .ps1 variants). Both now skip on os.name == 'nt', matching the existing guard in test_safe_write_bytes_rejects_parent_directory_race. Fixes the red windows-latest CI job. --- tests/test_release_hardening.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_release_hardening.py b/tests/test_release_hardening.py index 5ecad6b..5971acb 100644 --- a/tests/test_release_hardening.py +++ b/tests/test_release_hardening.py @@ -102,6 +102,8 @@ def test_demo_rejects_non_regular_upload(tmp_path: Path) -> None: def test_demo_sanitizes_cleaned_output_name(tmp_path: Path) -> None: + if os.name == "nt": + pytest.skip("Windows forbids < and > in filenames") source = tmp_path / "report .txt" source.write_text("hello\u200bworld", encoding="utf-8") @@ -270,6 +272,8 @@ def test_existing_checkout_is_repinned_before_install( sparse_command: str, head_matches: bool, ) -> None: + if os.name == "nt": + pytest.skip("setup shell scripts are POSIX-only (Windows uses the .ps1 variants)") checkout = tmp_path / "checkout" (checkout / ".git").mkdir(parents=True) python = checkout / ".venv" / "bin" / "python" From 286c065897d6c7d576cbbf4ae6395b4c8dd81425 Mon Sep 17 00:00:00 2001 From: elkaix Date: Tue, 18 Aug 2026 18:33:50 -0400 Subject: [PATCH 6/6] fix: address CodeRabbit review on release prep - demo: remove the per-upload workdir on failure paths instead of leaking it (success keeps it alive; Gradio serves the cleaned file from there) - pdf: treat truncated exiftool output as a failed strategy in clean_pdf, matching clean_pdf_pypdf's contract, so it also falls back to pypdf - release-pypi: checkout with persist-credentials: false; the job runs no git commands after checkout --- .github/workflows/release-pypi.yml | 3 +++ demo.py | 4 ++++ skills/remove-ai-marks/scripts/container_meta.py | 9 +++++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 59500bc..f9e55aa 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -21,6 +21,9 @@ jobs: id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # No git operations after checkout; don't leave GITHUB_TOKEN in .git/config. + persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" diff --git a/demo.py b/demo.py index 5d0b442..634c98d 100644 --- a/demo.py +++ b/demo.py @@ -14,6 +14,7 @@ import json import re +import shutil import sys import tempfile from pathlib import Path @@ -67,6 +68,9 @@ def clean_upload(file_obj, keep_non_ai: bool, layer_b: bool, strength: str): src, dest, CleanPlan(forced_kind=kind, strip_all_metadata=not keep_non_ai) ).to_dict() except Exception as e: + # Success must keep workdir alive (Gradio serves dest from it); on + # failure nothing references it, so don't leak a directory per upload. + shutil.rmtree(workdir, ignore_errors=True) return f"**Error cleaning {requested_src.name}:** `{e}`", None, "" prompt = "" diff --git a/skills/remove-ai-marks/scripts/container_meta.py b/skills/remove-ai-marks/scripts/container_meta.py index 2e9e3bc..72d090d 100644 --- a/skills/remove-ai-marks/scripts/container_meta.py +++ b/skills/remove-ai-marks/scripts/container_meta.py @@ -1754,9 +1754,14 @@ def clean_pdf(path: Path, dest: Path) -> tuple[list[str], dict]: output_limit=2 * 1024 * 1024, ) actions.append(f"exiftool -all= (rc={r.returncode})") - exiftool_ok = r.returncode == 0 - if not exiftool_ok: + truncated = bool( + getattr(r, "stdout_truncated", False) or getattr(r, "stderr_truncated", False) + ) + exiftool_ok = r.returncode == 0 and not truncated + if r.returncode != 0: actions.append(f"exiftool degraded (rc={r.returncode})") + elif truncated: + actions.append("exiftool output exceeded safety limit") except Exception as e: actions.append(f"exiftool failed: {e}") if not exiftool_ok: