docs: PDF output + docs CI validation (Phase 4+5 of N)

.github/workflows/docs.yml

@@ -0,0 +1,194 @@
+# Documentation validation.
+#
+# Builds the modernized docs (docs-src/build.py: Markdown -> HTML + man + PDF)
+# and runs the validators that lock in the reverse-DocBook migration's
+# guarantees. Modeled on ci.yml/fuzz.yml conventions: hard gates block PRs,
+# advisory tiers are continue-on-error, and heavy work (PDF/TeX-free but slow,
+# external link check) is scheduled-only or best-effort so per-PR jobs stay
+# fast.
+#
+# Tooling comes from the flake dev shell (nix develop) so CI matches local
+# exactly -- pandoc, weasyprint, poppler-utils, mandoc, codespell, lychee and
+# write-good are all pinned there.
+#
+# HARD gates (fail the PR):     build (HTML+man), no-loss, completeness, spelling,
+#                               internal link-check, man-lint.
+# ADVISORY (continue-on-error): prose (write-good), external link-check.
+# BEST-EFFORT (scheduled / continue-on-error): PDF build + validation (slow).
+
+name: Docs
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    paths:
+      - 'docs-src/**'
+      - '.github/workflows/docs.yml'
+      - 'flake.nix'
+      - 'dist/RELEASE'
+  workflow_dispatch:
+  schedule:
+    # Weekly (Mon 05:23 UTC): the full run including the slow PDF build and the
+    # external link check, which are best-effort/skipped on per-PR runs.
+    - cron: '23 5 * * 1'
+
+concurrency:
+  group: docs-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  # ----------------------------------------------------------------------------
+  # Build HTML + man (fast, always) and run the hard gates that depend on the
+  # generated output. PDF is built here only on schedule/dispatch (see the
+  # `pdf` job) so a PR isn't gated on the ~4-minute weasyprint pass.
+  # ----------------------------------------------------------------------------
+  build:
+    name: build + gates (html, man, no-loss, completeness, spelling, links, man-lint)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Nix (flakes enabled)
+        uses: cachix/install-nix-action@v27
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      # 1. BUILD (hard): HTML + man with 0 errors. --no-pdf keeps the PR fast;
+      #    the PDF path is exercised by the `pdf` job (scheduled/best-effort).
+      - name: Build HTML + man
+        run: nix develop --command bash -c 'cd docs-src && python3 build.py --no-pdf'
+
+      # build.py self-check guards the md->man reshape, PDF book discovery, and
+      # the .md->.html link rewrite (unit-level, no external tools).
+      - name: build.py self-check
+        run: nix develop --command bash -c 'cd docs-src && python3 build.py --selfcheck'
+
+      # 2. NO-LOSS GATE (hard): every migrated tree still retains its source
+      #    content (word-multiset retention + no code/section drop). Locks the
+      #    "nothing lost" guarantee against future edits.
+      - name: No-loss gate (all trees)
+        run: nix develop --command bash -c 'python3 docs-src/_migrate/verify_all.py'
+
+      # 3. COMPLETENESS GATE (hard): every public db.h function is documented
+      #    (28/28), and every uncovered method is on the frozen allowlist -- a
+      #    NEW undocumented API fails.
+      - name: Completeness gate (API coverage)
+        run: nix develop --command bash -c 'python3 docs-src/_migrate/man_coverage.py --ci'
+
+      # 4. SPELLING (hard on NEW typos): codespell, baselined against the legacy
+      #    typo backlog so only newly-introduced typos fail.
+      - name: Spelling gate (codespell, baselined)
+        run: nix develop --command bash -c 'python3 docs-src/_migrate/spellcheck.py'
+
+      # 6. INTERNAL LINK CHECK (hard): every link into migrated content must
+      #    resolve. Deferred-tree + un-migrated-asset links are excluded (see
+      #    docs-src/_migrate/lychee.toml). External links are the advisory job.
+      - name: Internal link check (lychee, offline)
+        run: |
+          nix develop --command bash -c \
+            'shopt -s globstar; lychee --offline --config docs-src/_migrate/lychee.toml --no-progress "docs-build/html/**/*.html"'
+
+      # 7. MAN-LINT (hard): 0 ERRORS from mandoc across every generated .3
+      #    (STYLE/WARNING are fine).
+      - name: Man-lint (mandoc -Tlint, 0 ERRORS)
+        run: |
+          nix develop --command bash -c '
+            e=0
+            for f in docs-build/man/man3/*.3; do
+              if mandoc -Tlint "$f" 2>&1 | grep -q "ERROR"; then
+                echo "ERRORS in $f:"; mandoc -Tlint "$f" 2>&1 | grep "ERROR" | head -3
+                e=$((e+1))
+              fi
+            done
+            echo "man pages with ERRORs: $e"
+            test "$e" -eq 0'
+
+  # ----------------------------------------------------------------------------
+  # 5. PROSE (advisory): write-good passive-voice / wordiness counts. This is
+  # decades-old technical prose -- surface the numbers, never gate.
+  # ----------------------------------------------------------------------------
+  prose:
+    name: prose advisory (write-good)
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Nix (flakes enabled)
+        uses: cachix/install-nix-action@v27
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+      - name: write-good suggestion counts (per tree)
+        run: |
+          nix develop --command bash -c '
+            shopt -s globstar
+            total=0
+            for d in docs-src/api/* docs-src/guides/*; do
+              [ -d "$d" ] || continue
+              n=$(write-good "$d"/**/*.md 2>/dev/null | grep -c "on line" || true)
+              printf "%-40s %5s suggestions\n" "$d" "$n"
+              total=$((total+n))
+            done
+            echo "::notice title=Prose (write-good)::$total advisory suggestion(s) across docs-src"'
+
+  # ----------------------------------------------------------------------------
+  # PDF (best-effort / scheduled): weasyprint renders one PDF per book (~4 min
+  # over all 13). Not a per-PR gate -- runs on schedule/dispatch, or on a PR
+  # that touches build.py's PDF path. continue-on-error so a rendering hiccup
+  # informs without blocking.
+  # ----------------------------------------------------------------------------
+  pdf:
+    name: pdf build + validate (best-effort)
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'push'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Nix (flakes enabled)
+        uses: cachix/install-nix-action@v27
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+      - name: Build all outputs (incl. PDF)
+        run: nix develop --command bash -c 'cd docs-src && timeout 900 python3 build.py'
+      - name: Validate PDFs (non-empty, page count, title-page version)
+        run: nix develop --command bash -c 'python3 docs-src/_migrate/validate_pdf.py'
+      - name: Upload PDFs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs-pdf
+          path: docs-build/pdf/*.pdf
+          if-no-files-found: ignore
+
+  # ----------------------------------------------------------------------------
+  # External link check (advisory / scheduled): lychee WITH network, so it
+  # depends on the reachability of third-party sites. Never gates; scheduled
+  # and dispatch only so PRs don't wait on the network.
+  # ----------------------------------------------------------------------------
+  external-links:
+    name: external link check (advisory)
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Nix (flakes enabled)
+        uses: cachix/install-nix-action@v27
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+      - name: Build HTML
+        run: nix develop --command bash -c 'cd docs-src && python3 build.py --no-pdf'
+      - name: Check external links (network, advisory)
+        run: |
+          nix develop --command bash -c '
+            shopt -s globstar
+            lychee --no-progress --scheme http --scheme https \
+              --exclude "localhost" --max-concurrency 8 \
+              "docs-build/html/**/*.html" || true'

docs-src/PLAN.md

@@ -126,12 +126,32 @@ the site (and can attach to GitHub releases).
 3. **Man pages** (DONE): `build_man()` -> 787 `*.3` (785 C+STL refentry pages +
    `libdb.3` overview + utilities), `mandoc -Tlint` = 0 errors. `man_coverage.py`
    reports API coverage (measure-only; the CI gate is phase 5).
-4. **PDF** (TODO): pandoc per book (programmer_reference, the GSGs, api_reference)
-   with a shared LaTeX header. `build.py build_pdf()` is the stubbed seam.
-5. **CI** (TODO): `docs.yml` with all validators + the completeness gate (wire
-   `verify.py` per tree + `man_coverage.py` as hard gates).
+4. **PDF** (DONE): `build_pdf()` renders ONE PDF per book (13 books: 2 API refs
+   + 9 guides + articles' 2 sub-books) via pandoc(html)->**weasyprint** -- no TeX
+   toolchain, deterministic, ~3.5 min for all 13. Title page (project + live
+   version + copyright) + running header/footer via `_templates/pdf-print.css`
+   (CSS paged-media). Output `docs-build/pdf/<book>.pdf`; `validate_pdf.py`
+   asserts non-empty + sane page count + version on the title page. Page counts:
+   api_c 655, programmer_reference 370, api_stl 257, installation 168,
+   upgrading 164, gsg_txn 119, gsg 96, collections 93, gsg_db_rep 66, bdb-sql 47,
+   mssgtxt 41, inmemory 20, porting 16.
+5. **CI** (DONE): `.github/workflows/docs.yml` (nix devShell for tool parity).
+   HARD gates: build (html+man, 0 errors), no-loss (`verify_all.py`, all 13
+   trees), completeness (`man_coverage.py --ci`: 28/28 functions + allowlisted
+   methods), spelling (`spellcheck.py`, codespell baselined to legacy typos),
+   internal link-check (`lychee` + `lychee.toml`), man-lint (mandoc, 0 ERRORS).
+   ADVISORY: prose (write-good). BEST-EFFORT/scheduled: PDF build+validate,
+   external link-check. The 2 genuinely-undocumented APIs
+   (`db_env_set_func_assert`, `db_env_set_win_security`) got real stub pages, so
+   the function gate is a hard 100%.
 6. **Publish** (TODO): wire gh-pages to the generated HTML; update the landing
-   page.
+   page. Needs: a `pages` job (or step) that runs `build.py`, uploads
+   `docs-build/html` via `actions/upload-pages-artifact` + `deploy-pages`
+   (permissions: `pages: write`, `id-token: write`); a top-level `index.md`
+   linking the 12 book landing pages + the man/PDF outputs; and a decision on
+   whether PDFs/man are published alongside HTML. The deferred CXX/TCL/java/
+   csharp trees stay out until regenerated from their native doc tools (their
+   inbound links are excluded in `lychee.toml`).
 
 ### Phase 2 retention (verify.py, mean word retention; 0 hard drops on all)
 

docs-src/_migrate/codespell-baseline.txt

@@ -0,0 +1,153 @@
+docs-src/api/c/db_heap_rid.md	indx
+docs-src/api/c/db_sql_codegen.md	requre
+docs-src/api/c/dbcompact.md	re-use
+docs-src/api/c/dbget.md	retun
+docs-src/api/c/dbset_flags.md	ACI
+docs-src/api/c/dbset_partition.md	implimented
+docs-src/api/c/dbset_partition.md	simultaniously
+docs-src/api/c/envclose.md	unncessary
+docs-src/api/c/envget_create_dir.md	ponter
+docs-src/api/c/envlog_get_config.md	ACI
+docs-src/api/c/envlog_set_config.md	ACI
+docs-src/api/c/envset_flags.md	ACI
+docs-src/api/c/envset_mp_mtxcount.md	defualt
+docs-src/api/c/envset_thread_id.md	re-use
+docs-src/api/c/mempfget.md	exlusive
+docs-src/api/c/mutexget_init.md	inital
+docs-src/api/c/mutexset_init.md	inital
+docs-src/api/c/txnbegin.md	ACI
+docs-src/api/c/txncommit.md	ACI
+docs-src/api/c/txncommit.md	possiblity
+docs-src/api/stl/DbstlDbt.md	neccessary
+docs-src/api/stl/DbstlDbt.md	refered
+docs-src/api/stl/DbstlElemTraits.md	compatiable
+docs-src/api/stl/DbstlElemTraits.md	contigous
+docs-src/api/stl/DbstlElemTraits.md	funcitons
+docs-src/api/stl/DbstlElemTraits.md	singeleton
+docs-src/api/stl/ElementHolder.md	Wappers
+docs-src/api/stl/ElementHolder.md	unerlying
+docs-src/api/stl/ElementRef.md	Wappers
+docs-src/api/stl/Element_wrappers.md	Wappers
+docs-src/api/stl/db_container.md	proctected
+docs-src/api/stl/db_map.md	unequality
+docs-src/api/stl/db_multimap.md	unequality
+docs-src/api/stl/db_vector.md	contaienr
+docs-src/api/stl/db_vector.md	databse
+docs-src/api/stl/db_vector_base_iterator.md	explictily
+docs-src/api/stl/db_vector_iterator.md	explictily
+docs-src/api/stl/dbstl_global_functions.md	exisiting
+docs-src/api/stl/dbstl_helper_classes.md	Wappers
+docs-src/api/stl/stlDbstlDbtoperator_assign.md	neccessary
+docs-src/api/stl/stlDbstlElemTraitscompare.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitscompare.md	funcitons
+docs-src/api/stl/stlDbstlElemTraitscopy.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitscopy.md	funcitons
+docs-src/api/stl/stlDbstlElemTraitseof.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitseof.md	funcitons
+docs-src/api/stl/stlDbstlElemTraitseq.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitseq.md	funcitons
+docs-src/api/stl/stlDbstlElemTraitseq_int_type.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitseq_int_type.md	funcitons
+docs-src/api/stl/stlDbstlElemTraitsfind.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitsfind.md	funcitons
+docs-src/api/stl/stlDbstlElemTraitsinstance.md	singeleton
+docs-src/api/stl/stlDbstlElemTraitslength.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitslength.md	funcitons
+docs-src/api/stl/stlDbstlElemTraitslt.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitslt.md	funcitons
+docs-src/api/stl/stlDbstlElemTraitsmove.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitsmove.md	funcitons
+docs-src/api/stl/stlDbstlElemTraitsnot_eof.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitsnot_eof.md	funcitons
+docs-src/api/stl/stlDbstlElemTraitsto_char_type.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitsto_char_type.md	funcitons
+docs-src/api/stl/stlDbstlElemTraitsto_int_type.md	compatiable
+docs-src/api/stl/stlDbstlElemTraitsto_int_type.md	funcitons
+docs-src/api/stl/stlElementHolderoperator__aa.md	bahavior
+docs-src/api/stl/stlElementHolderoperator__ma.md	bahavior
+docs-src/api/stl/stlElementHolderoperator_assign.md	bahavior
+docs-src/api/stl/stlElementHolderoperator_da.md	bahavior
+docs-src/api/stl/stlElementHolderoperator_decr.md	bahavior
+docs-src/api/stl/stlElementHolderoperator_gt_ge.md	bahavior
+docs-src/api/stl/stlElementHolderoperator_ia.md	bahavior
+docs-src/api/stl/stlElementHolderoperator_incr.md	bahavior
+docs-src/api/stl/stlElementHolderoperator_lt_le.md	bahavior
+docs-src/api/stl/stlElementHolderoperator_modasg.md	bahavior
+docs-src/api/stl/stlElementHolderoperator_oa.md	bahavior
+docs-src/api/stl/stlElementHolderoperator_sa.md	bahavior
+docs-src/api/stl/stlElementHolderoperator_xa.md	bahavior
+docs-src/api/stl/stlElementRefElementRef.md	unerlying
+docs-src/api/stl/stldb_mapinsert.md	similiar
+docs-src/api/stl/stldb_mapoperator_ueq.md	unequality
+docs-src/api/stl/stldb_multimapoperator_ueq.md	unequality
+docs-src/api/stl/stldb_vector_base_iteratordstr_db_vector_base_iterator.md	explictily
+docs-src/api/stl/stldb_vector_base_iteratoroperator_sub.md	substract
+docs-src/api/stl/stldb_vector_iteratordstr_db_vector_iterator.md	explictily
+docs-src/api/stl/stldb_vector_iteratoroperator_sub.md	substract
+docs-src/api/stl/stldb_vectorassign.md	requirs
+docs-src/api/stl/stldb_vectorunique.md	dertermine
+docs-src/api/stl/stldbstl_global_functionscommit_txn.md	funcion
+docs-src/api/stl/stldbstl_global_functionsset_current_txn_handle.md	commiting
+docs-src/api/stl/stldbstl_global_functionsset_global_dbfile_suffix_number.md	exisiting
+docs-src/guides/articles/inmemory/index.md	desireable
+docs-src/guides/articles/mssgtxt/index.md	commiting
+docs-src/guides/articles/mssgtxt/index.md	compresssion
+docs-src/guides/articles/mssgtxt/index.md	connnect
+docs-src/guides/articles/mssgtxt/index.md	indx
+docs-src/guides/articles/mssgtxt/index.md	paritions
+docs-src/guides/articles/mssgtxt/index.md	spcified
+docs-src/guides/articles/mssgtxt/index.md	traget
+docs-src/guides/bdb-sql/sqlrep.md	operatons
+docs-src/guides/gsg_db_rep/elections.md	desireable
+docs-src/guides/gsg_db_rep/fwrkmasterreplica.md	applicaton
+docs-src/guides/gsg_db_rep/rep_init_code.md	peformed
+docs-src/guides/installation/build_android_jdbc.md	exisits
+docs-src/guides/installation/build_unix_conf.md	Documenation
+docs-src/guides/installation/changelog_4_8.md	Millenium
+docs-src/guides/installation/changelog_4_8.md	hearbeat
+docs-src/guides/installation/changelog_4_8.md	partically
+docs-src/guides/installation/changelog_4_8.md	redefinitons
+docs-src/guides/installation/changelog_5_0.md	Millenium
+docs-src/guides/installation/changelog_5_0.md	datbase
+docs-src/guides/installation/changelog_5_0.md	eqivalent
+docs-src/guides/installation/changelog_5_0.md	mulitple
+docs-src/guides/installation/changelog_5_0.md	resouces
+docs-src/guides/installation/changelog_5_0.md	segementation
+docs-src/guides/installation/changelog_5_0.md	simulatenously
+docs-src/guides/installation/changelog_5_0.md	teh
+docs-src/guides/installation/changelog_5_0.md	unitialized
+docs-src/guides/installation/changelog_5_1.md	datbase
+docs-src/guides/installation/changelog_5_1.md	explict
+docs-src/guides/installation/changelog_5_1.md	numer
+docs-src/guides/installation/changelog_5_1.md	operaton
+docs-src/guides/installation/changelog_5_1.md	segementation
+docs-src/guides/installation/changelog_5_2.md	re-use
+docs-src/guides/installation/changelog_5_3.md	Enhaced
+docs-src/guides/installation/changelog_5_3.md	begining
+docs-src/guides/installation/changelog_5_3.md	dependant
+docs-src/guides/installation/introduction.md	infomation
+docs-src/guides/installation/upgrade_11gr2_52_repmgr_channels.md	asychronous
+docs-src/guides/installation/upgrade_11gr2_52_repmgr_channels.md	sychronous
+docs-src/guides/installation/upgrade_11gr2_52_xa.md	Applictions
+docs-src/guides/installation/upgrade_11gr2_remsupp.md	Millenium
+docs-src/guides/porting/certport.md	thrid
+docs-src/guides/porting/certport.md	warninigs
+docs-src/guides/porting/modscope.md	envrionment
+docs-src/guides/porting/modscope.md	platfrom
+docs-src/guides/programmer_reference/arch_apis.md	extention
+docs-src/guides/programmer_reference/bt_conf.md	preceeding
+docs-src/guides/programmer_reference/ch13s02.md	pre-emptive
+docs-src/guides/programmer_reference/csharp.md	libaries
+docs-src/guides/programmer_reference/embedded.md	seemlessly
+docs-src/guides/programmer_reference/env_encrypt.md	Documenation
+docs-src/guides/programmer_reference/intro_products.md	informaion
+docs-src/guides/programmer_reference/lock_max.md	enviroment
+docs-src/guides/programmer_reference/mp_warm.md	intialize
+docs-src/guides/programmer_reference/program_perfmon.md	stap
+docs-src/guides/programmer_reference/program_ram.md	re-used
+docs-src/guides/programmer_reference/stl_examples.md	squre
+docs-src/guides/programmer_reference/stl_usecase.md	prefered
+docs-src/guides/programmer_reference/transapp_atomicity.md	ACI
+docs-src/guides/programmer_reference/transapp_throughput.md	ACI
+docs-src/guides/programmer_reference/txn_config.md	ACI
+docs-src/guides/upgrading/changelog_4_7.md	invalide