diff --git a/.github/workflows/preview_create.yml b/.github/workflows/preview_create.yml index f60be71702..c82feae220 100644 --- a/.github/workflows/preview_create.yml +++ b/.github/workflows/preview_create.yml @@ -30,6 +30,9 @@ jobs: steps: - name: Checkout branch uses: actions/checkout@v5 + with: + # Full history so the PR-time link check can diff against the base branch. + fetch-depth: 0 - name: Setup Node.js uses: actions/setup-node@v5 @@ -73,4 +76,33 @@ jobs: - name: Update pointer to current run output run: | - aws s3api put-object --bucket tiny-cloud-antora-docs-preview --key ${PR}/index.html --body .github/workflows/resources/empty.html --content-type text/html --metadata pointer=${RUN} \ No newline at end of file + aws s3api put-object --bucket tiny-cloud-antora-docs-preview --key ${PR}/index.html --body .github/workflows/resources/empty.html --content-type text/html --metadata pointer=${RUN} + + # Link check runs last so it never blocks the preview deployment. It + # validates only the external links this PR adds or changes in .adoc source + # (pages or partials) — the changed link is checked once, regardless of how + # many pages include it. Non-blocking at rollout (fail: false); flip to + # fail: true once the baseline is clean. + - name: Extract external links changed in this PR + id: changed_links + run: | + base="origin/${{ github.event.pull_request.base.ref }}" + git fetch --no-tags origin "${{ github.event.pull_request.base.ref }}" + git diff "$base...HEAD" -- 'modules/**/*.adoc' \ + | awk '/^\+[^+]/' \ + | grep -oE 'https?://[^][<>"^ '"'"']+' \ + | sed -E 's/[.,;:]+$//' | sort -u > changed-links.txt || true + count=$(wc -l < changed-links.txt | tr -d '[:space:]') + echo "count=${count:-0}" >> "$GITHUB_OUTPUT" + echo "Changed external links to validate ($count):"; cat changed-links.txt + + - name: Check changed links + if: steps.changed_links.outputs.count != '0' + uses: lycheeverse/lychee-action@v2 + with: + args: --config .lychee.toml --no-progress changed-links.txt + format: markdown + output: lychee/results.md + token: ${{ secrets.GITHUB_TOKEN }} + fail: false + jobSummary: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9e6c80874f..61be885d87 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,10 @@ vendor # Antora tmp files _site/ +# lychee link checker (local cache + CI report output) +.lycheecache +lychee/ + # Cursor setup (local only, do not push) .cursor/ AGENTS.md diff --git a/.lychee.toml b/.lychee.toml new file mode 100644 index 0000000000..ca70b9c49c --- /dev/null +++ b/.lychee.toml @@ -0,0 +1,58 @@ +# lychee link-checker configuration +# https://github.com/lycheeverse/lychee +# +# Policy: a link is only "broken" if it fails to resolve or returns a genuine +# error (4xx/5xx). Redirects and rate-limiting are treated as healthy. Hosts +# that block automated checkers (403 to bots) are curated in .lycheeignore. + +# --- Request behaviour --- +timeout = 30 # seconds before a request is considered timed out +max_retries = 3 # retry transient failures (covers flaky 5xx / network blips) +retry_wait_time = 2 # seconds between retries +max_concurrency = 16 +max_redirects = 10 # follow redirect chains to their final destination + +# A browser-like user agent reduces false 403s from bot-hostile hosts. +user_agent = "Mozilla/5.0 (compatible; TinyMCE-Docs-Link-Checker/1.0; +https://www.tiny.cloud/docs)" + +# Resolve root-relative links (leading "/") against the live domain instead of +# erroring on them. The site is served from https://www.tiny.cloud/docs/, so a +# link like "/docs/_/css/site.css" or "/roadmap/" becomes an absolute URL that +# is HTTP-checked. Relative page-to-page links are still resolved locally on +# disk, so internal cross-reference integrity is still verified. +# NOTE: an `exclude = ["^/"]` pattern does NOT suppress these — lychee raises the +# "cannot resolve root-relative link" error before the exclude filter runs, so a +# base_url is the correct mechanism. +base_url = "https://www.tiny.cloud" + +# --- What counts as success --- +# Redirects are followed, so the final status is what matters; the 3xx codes are +# listed defensively for chains that exceed max_redirects. 429 = rate-limited, +# not dead. 403 is NOT accepted globally (that would hide genuinely dead pages) — +# bot-hostile hosts are curated per-URL in .lycheeignore instead. +accept = ["200..=206", "301", "302", "307", "308", "429"] + +# --- Caching (keeps PR-time runs fast) --- +cache = true +max_cache_age = "2d" + +# --- Structural excludes (not link rot) --- +include_mail = false # skip mailto: links +exclude = [ + # The site's own domains: self-referential links, UI-bundle chrome (CSS, + # favicons, footer images) and canonical URLs. base_url resolves root-relative + # links to these, and this filter then drops them. They are validated by the + # build/deploy pipeline; HTTP-checking them here would hammer our own + # production site (a rate-limit storm that fails thousands of links) and 404 on + # pages that a PR has added but not yet deployed. Internal .adoc cross-links are + # still verified locally (relative links resolve on disk) and by Antora at build. + '^https?://([a-z0-9-]+\.)*tiny\.cloud', + # Loopback / local addresses that only exist during a build or in examples + '^https?://localhost', + '^https?://127\.0\.0\.1', + '^https?://0\.0\.0\.0', + # GitHub action URLs that require auth and are not real navigational links + '^https://github\.com/[^/]+/[^/]+/(edit|new|delete)/', + # Internal Jira referenced in tickets/notes, not public docs + '^https://[a-z]+\.atlassian\.net', +] diff --git a/.lycheeignore b/.lycheeignore new file mode 100644 index 0000000000..c728a31347 --- /dev/null +++ b/.lycheeignore @@ -0,0 +1,37 @@ +# .lycheeignore — curated allow-list of URLs/patterns lychee should skip. +# Works like .gitignore, but for links. One URL or regex per line. +# +# Add an entry here ONLY after manually confirming the link works in a browser. +# This is the escape hatch for hosts that return 403/429 to automated checkers +# but are perfectly valid for readers (LinkedIn, Instagram, some CDNs, etc.). +# +# Seed entries below are commented out; uncomment / extend as the baseline crawl +# surfaces real false positives. Keep this list minimal — genuinely dead links +# must NOT be parked here. + +# Verified alive in a browser but rejected by the automated checker +# (bot detection / rate limiting): +https://www.stylemanual.gov.au/ +# Footer "join our Slack" link: tinyurl -> join.slack.com invite. Valid, but +# tinyurl/Slack return 403 to bots. Appears on every page. +https://www.tinyurl.com/tinyslack + +# Hosts that serve a JS/WAF challenge or 403/999 to automated checkers but load +# fine in a browser. Verified reachable manually; cannot be checked by any bot, +# so checking them only produces false failures. +https://www.npmjs.com/ +https://codepen.io/ +https://dotnet.microsoft.com/ +https://platform.openai.com/ +https://help.openai.com/ +https://www.researchgate.net/ +https://researchgate.net/ +https://benmarshall.me/ +https://malavkhatri.com/ + +# Common social hosts that block automated checkers — uncomment if they surface: +# https://www.linkedin.com/ +# https://www.instagram.com/ +# https://www.facebook.com/ +# https://twitter.com/ +# https://x.com/ diff --git a/package.json b/package.json index 3994f1bc67..c7409e1ffd 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "scripts": { "build": "./-scripts/api-reference.sh", "build-local-ref": "./-scripts/api-reference-local.sh", + "check-links": "lychee --config .lychee.toml --no-progress './build/site/**/*.html'", "clean": "rm -rf ./build", "generate-llm-files": "node ./-scripts/generate-llm-files.js", "generate-llm-files-from-url": "node ./-scripts/generate-llm-files.js https://www.tiny.cloud/docs/antora-sitemap.xml",