eval_ci.yml

name: CI

# Two jobs, two purposes:
#   unit-tests       -> the PR GATE. Fast, free, deterministic, no LLM keys.
#                       Runs on every pull request and push to main.
#   faithfulness-eval -> LLM-based RAGAS eval. Runs ONLY nightly or on manual
#                        trigger, never on PRs, so it does not burn LLM quota
#                        on every push (and won't fail PRs on a stray 429).

on:
  pull_request:
    branches: [main]
  push:
    branches: [main]
  schedule:
    - cron: "0 3 * * *"          # nightly 03:00 UTC
  workflow_dispatch:              # manual "Run workflow" button

jobs:
  # ── Gate: free, deterministic offline tests on every PR/push ──────────────
  unit-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - uses: actions/cache@v4
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
      - run: pip install -r requirements.txt
      - run: pytest tests/unit -q

  # ── Faithfulness eval: nightly / manual only (no quota burn on PRs) ───────
  faithfulness-eval:
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    # Free-tier LLM eval is informational, not a hard gate — don't fail the
    # workflow on a flaky 429/413. The deterministic unit-tests job is the gate.
    continue-on-error: true
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - uses: actions/cache@v4
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
      - uses: actions/cache@v4
        with:
          path: ~/.cache/huggingface
          key: hf-models-bge-small-reranker-base
      - run: pip install -r requirements.txt
      - name: Run faithfulness evaluation
        env:
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}   # generation
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}       # eval judge
          TOKENIZERS_PARALLELISM: "false"
        run: python app/services/evaluation/eval.py --subset 5