diff --git a/.github/workflows/_decide.yml b/.github/workflows/_decide.yml new file mode 100644 index 0000000000..62f6dfa07c --- /dev/null +++ b/.github/workflows/_decide.yml @@ -0,0 +1,72 @@ +name: _decide + +# Resolve the lane + backend from the triggering event. Shared by the +# per-platform entry workflows (ci-linux-x86_64 / ci-windows / ci-sbsa) so the +# rules live in exactly one place. `github.*` here refers to the caller's event. +# +# lane: fast (PR push) | full (approval / `ci: full` / main push) | +# nightly (schedule) | skip (non-approval review) +# backend: standard | rtx | both — from the `backend: TensorRT[-RTX]` labels +# (or the workflow_dispatch `backend` input) + +on: + workflow_call: + outputs: + lane: + description: "fast | full | nightly | skip" + value: ${{ jobs.decide.outputs.lane }} + backend: + description: "standard | rtx | both" + value: ${{ jobs.decide.outputs.backend }} + +jobs: + decide: + runs-on: ubuntu-latest + outputs: + lane: ${{ steps.pick.outputs.lane }} + backend: ${{ steps.pick.outputs.backend }} + steps: + - id: pick + env: + EVENT: ${{ github.event_name }} + REVIEW_STATE: ${{ github.event.review.state }} + # Lane labels (consolidated): `ci: full` runs all tiers (L0+L1+L2); + # `ci: nightly` also adds the nightly-only suites. Neither → fast (L0 + # smoke) on PR pushes. (The old `Force All Tests[L0+L1+L2]` label is gone.) + HAS_FULL_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full') }}" + HAS_NIGHTLY_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'ci: nightly') }}" + # Backend labels narrow the engine; on a full/nightly run their absence + # means BOTH. Exact match: 'backend: TensorRT' != 'backend: TensorRT-RTX'. + HAS_RTX_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'backend: TensorRT-RTX') }}" + HAS_STD_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'backend: TensorRT') }}" + DISPATCH_LANE: ${{ github.event.inputs.lane }} + DISPATCH_BACKEND: ${{ github.event.inputs.backend }} + run: | + set -euo pipefail + case "$EVENT" in + schedule) lane=nightly ;; + workflow_dispatch) lane="${DISPATCH_LANE:-full}" ;; + push) lane=full ;; # main canary + pull_request_review) + [ "$REVIEW_STATE" = "approved" ] && lane=full || lane=skip ;; + pull_request) + if [ "$HAS_NIGHTLY_LABEL" = "true" ]; then lane=nightly + elif [ "$HAS_FULL_LABEL" = "true" ]; then lane=full + else lane=fast; fi ;; + *) lane=fast ;; + esac + echo "lane=$lane" >> "$GITHUB_OUTPUT" + # Backend: explicit labels win; otherwise a fast PR push stays + # standard-only (cheap), while full/nightly default to BOTH engines. + case "$EVENT" in + workflow_dispatch) backend="${DISPATCH_BACKEND:-both}" ;; + pull_request) + if [ "$HAS_RTX_LABEL" = "true" ] && [ "$HAS_STD_LABEL" = "true" ]; then backend=both + elif [ "$HAS_RTX_LABEL" = "true" ]; then backend=rtx + elif [ "$HAS_STD_LABEL" = "true" ]; then backend=standard + elif [ "$lane" = "fast" ]; then backend=standard + else backend=both; fi ;; + *) backend=both ;; # push / approval / schedule + esac + echo "backend=$backend" >> "$GITHUB_OUTPUT" + echo "Resolved lane='$lane' backend='$backend' (event=$EVENT)." diff --git a/.github/workflows/_linux-x86_64-core.yml b/.github/workflows/_linux-x86_64-core.yml deleted file mode 100644 index c0011470b2..0000000000 --- a/.github/workflows/_linux-x86_64-core.yml +++ /dev/null @@ -1,671 +0,0 @@ -# Reusable core for the Linux x86_64 build+test pipeline. -# -# Both build-test-linux-x86_64.yml (standard TensorRT) and -# build-test-linux-x86_64_rtx.yml (TensorRT-RTX) call this with use-rtx -# false/true. Keep ALL build/test logic here so a change lands once. -# -# RTX differences are expressed two ways: -# * Jobs that only exist for standard TRT (torchscript, distributed, -# executorch) are gated with ``if: ${{ !inputs.use-rtx && ... }}``. -# * Jobs whose pytest scope differs branch inside the script on the -# ``$USE_TRT_RTX`` env var that linux-test.yml exports. -# -# The per-job pass/fail map is bubbled up via the ``results`` output so the -# thin caller workflows can render a single ``ci-rollup`` status check whose -# name (e.g. "CI / Linux x86_64") stays stable for branch protection. -name: _linux-x86_64-core - -on: - workflow_call: - inputs: - use-rtx: - description: "Build/test against TensorRT-RTX instead of standard TensorRT" - type: boolean - default: false - name-prefix: - description: "Display-name prefix for the build job (e.g. 'RTX - ')" - type: string - default: "" - outputs: - results: - description: "JSON map of {job-id: {result, outputs}} for every build/test job" - value: ${{ jobs.collect-results.outputs.results }} - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - LIMIT_PR=${{ github.event_name == 'pull_request' && 'true' || 'false' }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx ${{ inputs.use-rtx }} --limit-pr-builds "${LIMIT_PR}" --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: ${{ inputs.name-prefix }}Build Linux x86_64 torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "x86_64" - use-rtx: ${{ inputs.use-rtx }} - pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - - # Standard-TRT only: ExecuTorch static build is not part of the RTX matrix. - executorch-static-build: - needs: [filter-matrix, build] - if: ${{ !inputs.use-rtx }} - uses: ./.github/workflows/executorch-static-linux.yml - with: - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - - L0-dynamo-converter-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L0 dynamo converter tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L0-dynamo-converter-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l0_converter - L0-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L0 dynamo core tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L0-dynamo-core-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l0_core - L0-py-core-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L0 core python tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L0-py-core-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l0_py_core - # Standard-TRT only: TorchScript frontend is not exercised on RTX. - L0-torchscript-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ !inputs.use-rtx && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L0 torchscript tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L0-torchscript-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l0_torchscript - L1-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L0-dynamo-converter-tests, - L0-dynamo-core-tests, - L0-py-core-tests, - L0-torchscript-tests, - ] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L1 dynamo core tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L1-dynamo-core-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l1_dynamo_core - L1-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L0-dynamo-converter-tests, - L0-dynamo-core-tests, - L0-py-core-tests, - L0-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L1 dynamo compile tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L1-dynamo-compile-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l1_dynamo_compile - L1-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L0-dynamo-converter-tests, - L0-dynamo-core-tests, - L0-py-core-tests, - L0-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L1 torch compile tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L1-torch-compile-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l1_torch_compile - # Standard-TRT only: TorchScript frontend is not exercised on RTX. - L1-torchscript-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L0-dynamo-core-tests, - L0-dynamo-converter-tests, - L0-py-core-tests, - L0-torchscript-tests, - ] - if: "${{ !inputs.use-rtx && !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L1 torch script tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L1-torchscript-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l1_torchscript - L2-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-torch-compile-tests, - L1-dynamo-compile-tests, - L1-dynamo-core-tests, - L1-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 torch compile tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-torch-compile-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_torch_compile - L2-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-dynamo-compile-tests, - L1-dynamo-core-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci: run-l2') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 dynamo compile tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-dynamo-compile-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_dynamo_compile - L2-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-dynamo-core-tests, - L1-dynamo-compile-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci: run-l2') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 dynamo core tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-dynamo-core-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_dynamo_core - L2-dynamo-plugin-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-dynamo-core-tests, - L1-dynamo-compile-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - ] - if: "${{ !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci: run-l2') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 dynamo plugin tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-dynamo-plugin-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_plugin - # Standard-TRT only: TorchScript frontend is not exercised on RTX. - L2-torchscript-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-dynamo-core-tests, - L1-dynamo-compile-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - ] - if: "${{ !inputs.use-rtx && !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci: run-l2') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 torch script tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-torchscript-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_torchscript - # Standard-TRT only: distributed tests need a multi-GPU runner not in the RTX matrix. - L2-dynamo-distributed-tests: - name: ${{ matrix.display-name }} - needs: - [ - filter-matrix, - build, - L1-dynamo-core-tests, - L1-dynamo-compile-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - ] - if: "${{ !inputs.use-rtx && !contains(github.event.pull_request.labels.*.name, 'ci: only-l0') && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci: run-l2') || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && ((github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success()) }}" - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: L2 dynamo distributed tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: L2-dynamo-distributed-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: ${{ inputs.use-rtx }} - runner: linux.g4dn.12xlarge.nvidia.gpu - script: | - set -euo pipefail - # Tier definition lives in tests/py/utils/ci_helpers.sh (single source of - # truth, shared with the local `just` recipes). USE_TRT_RTX is exported - # by this workflow's caller. - source tests/py/utils/ci_helpers.sh - trt_tier_l2_distributed - # Gather every build/test job's result into a single JSON map and expose it - # as the ``results`` workflow output. The thin caller workflows render this - # into one ``ci-rollup`` status check. ``if: always()`` so the map is - # produced even when upstream jobs failed / were skipped / were cancelled. - collect-results: - if: ${{ always() }} - permissions: {} - needs: - [ - build, - L0-dynamo-converter-tests, - L0-dynamo-core-tests, - L0-py-core-tests, - L0-torchscript-tests, - L1-dynamo-core-tests, - L1-dynamo-compile-tests, - L1-torch-compile-tests, - L1-torchscript-tests, - L2-torch-compile-tests, - L2-dynamo-compile-tests, - L2-dynamo-core-tests, - L2-dynamo-plugin-tests, - L2-torchscript-tests, - L2-dynamo-distributed-tests, - ] - runs-on: ubuntu-latest - outputs: - results: ${{ steps.collect.outputs.results }} - steps: - - name: Collect job results - id: collect - env: - RESULTS: ${{ toJSON(needs) }} - run: | - set -euo pipefail - { - echo "results<> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_test-linux.yml b/.github/workflows/_test-linux.yml new file mode 100644 index 0000000000..9f4a2c56b9 --- /dev/null +++ b/.github/workflows/_test-linux.yml @@ -0,0 +1,148 @@ +name: _test-linux + +# Manifest-driven build (+ optional test) for ONE Linux channel. Covers both: +# * x86_64 — os=linux, architecture=x86_64, run-tests=true (build + suites) +# * SBSA — os=linux-aarch64, architecture=aarch64, run-tests=false (build-only; +# there are no aarch64 GPU test runners) +# +# One reusable serves both because the build (build_linux.yml) and test +# (linux-test.yml) reusables are the same — only inputs differ. (GitHub requires +# `uses:` to be a literal, so Windows, which needs build_windows/windows-test, +# has its own _test-windows.yml.) + +on: + workflow_call: + inputs: + lane: + description: "fast | full | nightly" + required: true + type: string + use-rtx: + description: "Test against TensorRT-RTX (x86_64 only)" + type: boolean + default: false + name-prefix: + description: "Display-name prefix (e.g. 'RTX - ', 'SBSA ')" + type: string + default: "" + raw-matrix: + description: "Raw generate_binary_build_matrix output. Generated ONCE by the + caller and shared across channels — generate_binary_build_matrix has a + concurrency group keyed on (workflow, PR, os) with cancel-in-progress, so + calling it per-channel makes the channels cancel each other's matrix job." + required: true + type: string + architecture: + description: "x86_64 | aarch64" + type: string + default: x86_64 + run-tests: + description: "Run the suite matrix after building (false = build-only, e.g. SBSA)" + type: boolean + default: true + python-only: + description: "Build the PYTHON_ONLY=1 wheel (no C++ runtime) via env_vars_python_only.txt" + type: boolean + default: false + +jobs: + # The build matrix is generated ONCE by the caller (see raw-matrix input) and + # passed in; here we only filter it per-variant (a plain script job — no + # test-infra concurrency group, so it's safe to run per-channel). + filter-matrix: + outputs: + matrix: ${{ steps.generate.outputs.matrix }} + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - uses: actions/checkout@v6 + with: + repository: pytorch/tensorrt + - name: Generate matrix + id: generate + run: | + set -eou pipefail + MATRIX_BLOB=${{ toJSON(inputs.raw-matrix) }} + # Build one representative wheel for PR-triggered runs (any lane); + # the full python×cuda matrix only on push-to-main and nightly. + LIMIT_PR=${{ (github.event_name == 'push' || github.event_name == 'schedule') && 'false' || 'true' }} + MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx ${{ inputs.use-rtx }} --limit-pr-builds "${LIMIT_PR}" --matrix "${MATRIX_BLOB}")" + echo "${MATRIX_BLOB}" + echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" + + build: + needs: filter-matrix + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/tensorrt + pre-script: packaging/pre_build_script.sh + env-var-script: packaging/env_vars.txt + post-script: packaging/post_build_script.sh + smoke-test-script: packaging/smoke_test_script.sh + package-name: torch_tensorrt + display-name: ${{ inputs.name-prefix }}Build Linux ${{ inputs.architecture }} torch-tensorrt whl package + name: ${{ matrix.display-name }} + uses: ./.github/workflows/build_linux.yml + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + # python-only swaps in the env_vars_python_only.txt (sets PYTHON_ONLY=1). + env-var-script: ${{ inputs.python-only && 'packaging/env_vars_python_only.txt' || matrix.env-var-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + architecture: ${{ inputs.architecture }} + use-rtx: ${{ inputs.use-rtx }} + pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" + + # Suite list from the manifest. Skipped for build-only channels (SBSA). + suite-matrix: + if: ${{ inputs.run-tests }} + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.gen.outputs.matrix }} + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - id: gen + run: | + set -euo pipefail + variant=${{ inputs.use-rtx && 'rtx' || 'standard' }} + json="$(python -m tests.ci matrix --platform linux-x86_64 --lane '${{ inputs.lane }}' --variant "${variant}")" + echo "matrix=${json}" >> "$GITHUB_OUTPUT" + echo "Lane '${{ inputs.lane }}' (${variant}) suites:"; echo "${json}" | python -m json.tool + + # One test job per suite (auto-skips when suite-matrix is skipped, i.e. SBSA). + test: + needs: [filter-matrix, build, suite-matrix] + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.suite-matrix.outputs.matrix) }} + uses: ./.github/workflows/linux-test.yml + with: + job-name: ${{ matrix.suite }}-${{ matrix.variant }} + repository: "pytorch/tensorrt" + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + pre-script: packaging/pre_build_script.sh + use-rtx: ${{ inputs.use-rtx }} + fail-on-empty: true + script: | + set -euo pipefail + python -m tests.ci run "${{ matrix.suite }}" --variant "${{ matrix.variant }}" diff --git a/.github/workflows/_test-windows.yml b/.github/workflows/_test-windows.yml new file mode 100644 index 0000000000..77174ceeb2 --- /dev/null +++ b/.github/workflows/_test-windows.yml @@ -0,0 +1,139 @@ +name: _test-windows + +# Manifest-driven build + test for the Windows channel. Mirrors _test-linux.yml +# but uses build_windows.yml + windows-test.yml, and wraps the suite run in +# vc_env_helper.bat (the MSVC env). Suite SELECTION is identical to Linux — the +# manifest is platform-agnostic (`ci matrix --platform windows`); only the build +# reusable and the execution wrapper differ. + +on: + workflow_call: + inputs: + lane: + description: "fast | full | nightly" + required: true + type: string + use-rtx: + description: "Test against TensorRT-RTX" + type: boolean + default: false + name-prefix: + description: "Display-name prefix (e.g. 'RTX - ')" + type: string + default: "" + raw-matrix: + description: "Raw generate_binary_build_matrix output, generated ONCE by the + caller and shared across channels (generate_binary_build_matrix's concurrency + group would otherwise make per-channel calls cancel each other)." + required: true + type: string + +jobs: + # The build matrix is generated ONCE by the caller (raw-matrix); here we only + # filter it per-variant (a plain script job — no test-infra concurrency group). + filter-matrix: + outputs: + matrix: ${{ steps.generate.outputs.matrix }} + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - uses: actions/checkout@v6 + with: + repository: pytorch/tensorrt + - name: Generate matrix + id: generate + run: | + set -eou pipefail + MATRIX_BLOB=${{ toJSON(inputs.raw-matrix) }} + # Build one representative wheel for PR-triggered runs (any lane); + # the full python×cuda matrix only on push-to-main and nightly. + LIMIT_PR=${{ (github.event_name == 'push' || github.event_name == 'schedule') && 'false' || 'true' }} + MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx ${{ inputs.use-rtx }} --limit-pr-builds "${LIMIT_PR}" --matrix "${MATRIX_BLOB}")" + echo "${MATRIX_BLOB}" + echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" + + # Swap the build runner label for a GPU test runner (same as the legacy windows flow). + substitute-runner: + needs: filter-matrix + outputs: + matrix: ${{ steps.substitute.outputs.matrix }} + runs-on: ubuntu-latest + steps: + - name: Substitute runner + id: substitute + run: | + echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> "${GITHUB_OUTPUT}" + + build: + needs: substitute-runner + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/tensorrt + pre-script: packaging/pre_build_script_windows.sh + env-script: packaging/vc_env_helper.bat + smoke-test-script: packaging/smoke_test_windows.py + package-name: torch_tensorrt + display-name: ${{ inputs.name-prefix }}Build Windows torch-tensorrt whl package + name: ${{ matrix.display-name }} + uses: ./.github/workflows/build_windows.yml + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + env-script: ${{ matrix.env-script }} + smoke-test-script: ${{ matrix.smoke-test-script }} + package-name: ${{ matrix.package-name }} + trigger-event: ${{ github.event_name }} + # Build the RTX wheel when requested (matches the original RTX-windows + # entry); empty params + use-rtx=false for the standard build. + use-rtx: ${{ inputs.use-rtx }} + wheel-build-params: ${{ inputs.use-rtx && '--use-rtx' || '' }} + timeout: 120 + + suite-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.gen.outputs.matrix }} + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - id: gen + run: | + set -euo pipefail + variant=${{ inputs.use-rtx && 'rtx' || 'standard' }} + json="$(python -m tests.ci matrix --platform windows --lane '${{ inputs.lane }}' --variant "${variant}")" + echo "matrix=${json}" >> "$GITHUB_OUTPUT" + echo "Lane '${{ inputs.lane }}' (${variant}) windows suites:"; echo "${json}" | python -m json.tool + + test: + needs: [substitute-runner, build, suite-matrix] + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.suite-matrix.outputs.matrix) }} + uses: ./.github/workflows/windows-test.yml + with: + job-name: ${{ matrix.suite }}-${{ matrix.variant }} + repository: "pytorch/tensorrt" + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + pre-script: packaging/driver_upgrade.bat + use-rtx: ${{ inputs.use-rtx }} + # vc_env_helper.bat sets the MSVC env, then runs the manifest runner; the + # inner `python -m pytest` it spawns inherits that env. + script: | + set -euo pipefail + packaging/vc_env_helper.bat python -m tests.ci run "${{ matrix.suite }}" --variant "${{ matrix.variant }}" diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml deleted file mode 100644 index d5bdf0ed95..0000000000 --- a/.github/workflows/blossom-ci.yml +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A workflow to trigger ci on hybrid infra (github + self hosted runner) -name: Blossom-CI -on: - issue_comment: - types: [created] - workflow_dispatch: - inputs: - platform: - description: 'runs-on argument' - required: false - args: - description: 'argument' - required: false -jobs: - Authorization: - name: Authorization - runs-on: blossom - outputs: - args: ${{ env.args }} - - # This job only runs for pull request comments - if: | - contains( 'andi4191, narendasan, peri044, bowang007,', format('{0},', github.actor)) && - github.event.comment.body == '/blossom-ci' - steps: - - name: Check if comment is issued by authorized person - run: blossom-ci - env: - OPERATION: 'AUTH' - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }} - - Vulnerability-scan: - name: Vulnerability scan - needs: [Authorization] - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - repository: ${{ fromJson(needs.Authorization.outputs.args).repo }} - ref: ${{ fromJson(needs.Authorization.outputs.args).ref }} - lfs: 'true' - - # repo specific steps - #- name: Setup java - # uses: actions/setup-java@v5 - # with: - # java-version: 1.8 - - # add blackduck properties https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631308372/Methods+for+Configuring+Analysis#Using-a-configuration-file - #- name: Setup blackduck properties - # run: | - # PROJECTS=$(mvn -am dependency:tree | grep maven-dependency-plugin | awk '{ out="com.nvidia:"$(NF-1);print out }' | grep rapids | xargs | sed -e 's/ /,/g') - # echo detect.maven.build.command="-pl=$PROJECTS -am" >> application.properties - # echo detect.maven.included.scopes=compile >> application.properties - - - name: Run blossom action - uses: NVIDIA/blossom-action@main - env: - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }} - with: - args1: ${{ fromJson(needs.Authorization.outputs.args).args1 }} - args2: ${{ fromJson(needs.Authorization.outputs.args).args2 }} - args3: ${{ fromJson(needs.Authorization.outputs.args).args3 }} - - Job-trigger: - name: Start ci job - needs: [Vulnerability-scan] - runs-on: blossom - steps: - - name: Start ci job - run: blossom-ci - env: - OPERATION: 'START-CI-JOB' - CI_SERVER: ${{ secrets.CI_SERVER }} - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - Upload-Log: - name: Upload log - runs-on: blossom - if : github.event_name == 'workflow_dispatch' - steps: - - name: Jenkins log for pull request ${{ fromJson(github.event.inputs.args).pr }} (click here) - run: blossom-ci - env: - OPERATION: 'POST-PROCESSING' - CI_SERVER: ${{ secrets.CI_SERVER }} - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-tensorrt-linux.yml b/.github/workflows/build-tensorrt-linux.yml deleted file mode 100644 index 11555e9f8f..0000000000 --- a/.github/workflows/build-tensorrt-linux.yml +++ /dev/null @@ -1,222 +0,0 @@ -name: Build Torch-TensorRT wheel on Linux with Future TensorRT Versions - -on: - workflow_call: - inputs: - repository: - description: 'Repository to checkout, defaults to ""' - default: "" - type: string - ref: - description: 'Reference to checkout, defaults to "nightly"' - default: "nightly" - type: string - test-infra-repository: - description: "Test infra repository to use" - default: "pytorch/test-infra" - type: string - test-infra-ref: - description: "Test infra reference to use" - default: "" - type: string - build-matrix: - description: "Build matrix to utilize" - default: "" - type: string - pre-script: - description: "Pre script to run prior to build" - default: "" - type: string - post-script: - description: "Post script to run prior to build" - default: "" - type: string - smoke-test-script: - description: "Script for Smoke Test for a specific domain" - default: "" - type: string - env-var-script: - description: "Script that sets Domain-Specific Environment Variables" - default: "" - type: string - package-name: - description: "Name of the actual python package that is imported" - default: "" - type: string - trigger-event: - description: "Trigger Event in caller that determines whether or not to upload" - default: "" - type: string - cache-path: - description: "The path(s) on the runner to cache or restore. The path is relative to repository." - default: "" - type: string - cache-key: - description: "The key created when saving a cache and the key used to search for a cache." - default: "" - type: string - architecture: - description: Architecture to build for x86_64 for default Linux, or aarch64 for Linux aarch64 builds - required: false - type: string - default: x86_64 - submodules: - description: Works as stated in actions/checkout, but the default value is recursive - required: false - type: string - default: recursive - setup-miniconda: - description: Set to true if setup-miniconda is needed - required: false - type: boolean - default: true - -permissions: - id-token: write - contents: read - -jobs: - build: - strategy: - fail-fast: false - matrix: ${{ fromJSON(inputs.build-matrix) }} - env: - PYTHON_VERSION: ${{ matrix.python_version }} - PACKAGE_TYPE: wheel - REPOSITORY: ${{ inputs.repository }} - REF: ${{ inputs.ref }} - CU_VERSION: ${{ matrix.desired_cuda }} - UPLOAD_TO_BASE_BUCKET: ${{ matrix.upload_to_base_bucket }} - ARCH: ${{ inputs.architecture }} - TENSORRT_STRIP_PREFIX: ${{ matrix.tensorrt.strip_prefix }} - TENSORRT_VERSION: ${{ matrix.tensorrt.version }} - TENSORRT_URLS: ${{ matrix.tensorrt.urls }} - TENSORRT_SHA256: ${{ matrix.tensorrt.sha256 }} - UPLOAD_ARTIFACT_NAME: pytorch_tensorrt_${{ matrix.tensorrt.version }}_${{ matrix.python_version }}_${{ matrix.desired_cuda }}_${{ inputs.architecture }} - name: build_tensorrt${{ matrix.tensorrt.version }}_py${{matrix.python_version}}_${{matrix.desired_cuda}} - runs-on: ${{ matrix.validation_runner }} - container: - image: ${{ matrix.container_image }} - options: ${{ matrix.gpu_arch_type == 'cuda' && '--gpus all' || ' ' }} - # If a build is taking longer than 120 minutes on these runners we need - # to have a conversation - timeout-minutes: 120 - - steps: - - name: Clean workspace - shell: bash -l {0} - run: | - set -x - echo "::group::Cleanup debug output" - rm -rf "${GITHUB_WORKSPACE}" - mkdir -p "${GITHUB_WORKSPACE}" - if [[ "${{ inputs.architecture }}" = "aarch64" ]]; then - rm -rf "${RUNNER_TEMP}/*" - fi - echo "::endgroup::" - - uses: actions/checkout@v6 - with: - # Support the use case where we need to checkout someone's fork - repository: ${{ inputs.test-infra-repository }} - ref: ${{ inputs.test-infra-ref }} - path: test-infra - - uses: actions/checkout@v6 - if: ${{ env.ARCH == 'aarch64' }} - with: - # Support the use case where we need to checkout someone's fork - repository: "pytorch/builder" - ref: "main" - path: builder - - name: Set linux aarch64 CI - if: ${{ inputs.architecture == 'aarch64' }} - shell: bash -l {0} - env: - DESIRED_PYTHON: ${{ matrix.python_version }} - run: | - set +e - # TODO: This is temporary aarch64 setup script, this should be integrated into aarch64 docker. - ${GITHUB_WORKSPACE}/builder/aarch64_linux/aarch64_ci_setup.sh - echo "/opt/conda/bin" >> $GITHUB_PATH - set -e - - uses: ./test-infra/.github/actions/set-channel - - name: Set PYTORCH_VERSION - if: ${{ env.CHANNEL == 'test' }} - run: | - # When building RC, set the version to be the current candidate version, - # otherwise, leave it alone so nightly will pick up the latest - echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}" - - uses: ./test-infra/.github/actions/setup-binary-builds - env: - PLATFORM: ${{ inputs.architecture == 'aarch64' && 'linux-aarch64' || ''}} - with: - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - submodules: ${{ inputs.submodules }} - setup-miniconda: ${{ inputs.setup-miniconda }} - python-version: ${{ env.PYTHON_VERSION }} - cuda-version: ${{ env.CU_VERSION }} - arch: ${{ env.ARCH }} - - name: Combine Env Var and Build Env Files - if: ${{ inputs.env-var-script != '' }} - working-directory: ${{ inputs.repository }} - shell: bash -l {0} - run: | - cat "${{ inputs.env-var-script }}" >> "${BUILD_ENV_FILE}" - - name: Install torch dependency - shell: bash -l {0} - run: | - set -x - # shellcheck disable=SC1090 - source "${BUILD_ENV_FILE}" - # shellcheck disable=SC2086 - ${CONDA_RUN} ${PIP_INSTALL_TORCH} - - name: Run Pre-Script with Caching - if: ${{ inputs.pre-script != '' }} - uses: ./test-infra/.github/actions/run-script-with-cache - with: - cache-path: ${{ inputs.cache-path }} - cache-key: ${{ inputs.cache-key }} - repository: ${{ inputs.repository }} - script: ${{ inputs.pre-script }} - - name: Build clean - working-directory: ${{ inputs.repository }} - shell: bash -l {0} - run: | - set -x - source "${BUILD_ENV_FILE}" - ${CONDA_RUN} python setup.py clean - - name: Build the wheel (bdist_wheel) - working-directory: ${{ inputs.repository }} - shell: bash -l {0} - run: | - set -x - source "${BUILD_ENV_FILE}" - ${CONDA_RUN} python setup.py bdist_wheel - - - name: Run Post-Script - if: ${{ inputs.post-script != '' }} - uses: ./test-infra/.github/actions/run-script-with-cache - with: - repository: ${{ inputs.repository }} - script: ${{ inputs.post-script }} - - name: Smoke Test - shell: bash -l {0} - env: - PACKAGE_NAME: ${{ inputs.package-name }} - SMOKE_TEST_SCRIPT: ${{ inputs.smoke-test-script }} - run: | - set -x - source "${BUILD_ENV_FILE}" - # TODO: add smoke test for the auditwheel tarball built - - # NB: Only upload to GitHub after passing smoke tests - - name: Upload wheel to GitHub - continue-on-error: true - uses: actions/upload-artifact@v6 - with: - name: ${{ env.UPLOAD_ARTIFACT_NAME }} - path: ${{ inputs.repository }}/dist - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true \ No newline at end of file diff --git a/.github/workflows/build-tensorrt-windows.yml b/.github/workflows/build-tensorrt-windows.yml deleted file mode 100644 index 3c69427a22..0000000000 --- a/.github/workflows/build-tensorrt-windows.yml +++ /dev/null @@ -1,232 +0,0 @@ -name: Build Torch-TensorRT wheel on Windows with Future TensorRT Versions - -on: - workflow_call: - inputs: - repository: - description: 'Repository to checkout, defaults to ""' - default: "" - type: string - ref: - description: 'Reference to checkout, defaults to "nightly"' - default: "nightly" - type: string - test-infra-repository: - description: "Test infra repository to use" - default: "pytorch/test-infra" - type: string - test-infra-ref: - description: "Test infra reference to use" - default: "" - type: string - build-matrix: - description: "Build matrix to utilize" - default: "" - type: string - pre-script: - description: "Pre script to run prior to build" - default: "" - type: string - env-script: - description: "Script to setup environment variables for the build" - default: "" - type: string - wheel-build-params: - description: "Additional parameters for bdist_wheel" - default: "" - type: string - post-script: - description: "Post script to run prior to build" - default: "" - type: string - smoke-test-script: - description: "Script for Smoke Test for a specific domain" - default: "" - type: string - package-name: - description: "Name of the actual python package that is imported" - default: "" - type: string - trigger-event: - description: "Trigger Event in caller that determines whether or not to upload" - default: "" - type: string - cache-path: - description: "The path(s) on the runner to cache or restore. The path is relative to repository." - default: "" - type: string - cache-key: - description: "The key created when saving a cache and the key used to search for a cache." - default: "" - type: string - submodules: - description: "Works as stated in actions/checkout, but the default value is recursive" - required: false - type: string - default: recursive - timeout: - description: 'Timeout for the job (in minutes)' - default: 60 - type: number - -permissions: - id-token: write - contents: read - -jobs: - build: - strategy: - fail-fast: false - matrix: ${{ fromJSON(inputs.build-matrix) }} - env: - PYTHON_VERSION: ${{ matrix.python_version }} - PACKAGE_TYPE: wheel - REPOSITORY: ${{ inputs.repository }} - REF: ${{ inputs.ref }} - CU_VERSION: ${{ matrix.desired_cuda }} - UPLOAD_TO_BASE_BUCKET: ${{ matrix.upload_to_base_bucket }} - ARCH: win_amd64 - TENSORRT_STRIP_PREFIX: ${{ matrix.tensorrt.strip_prefix }} - TENSORRT_VERSION: ${{ matrix.tensorrt.version }} - TENSORRT_URLS: ${{ matrix.tensorrt.urls }} - TENSORRT_SHA256: ${{ matrix.tensorrt.sha256 }} - UPLOAD_ARTIFACT_NAME: pytorch_tensorrt_${{ matrix.tensorrt.version }}_${{ matrix.python_version }}_${{ matrix.desired_cuda }}_win_amd64 - name: build_tensorrt${{ matrix.tensorrt.version }}_py${{matrix.python_version}}_${{matrix.desired_cuda}} - runs-on: ${{ matrix.validation_runner }} - defaults: - run: - shell: bash -l {0} - # If a build is taking longer than 60 minutes on these runners we need - # to have a conversation - timeout-minutes: 120 - steps: - - uses: actions/checkout@v6 - with: - # Support the use case where we need to checkout someone's fork - repository: ${{ inputs.test-infra-repository }} - ref: ${{ inputs.test-infra-ref }} - path: test-infra - - uses: ./test-infra/.github/actions/setup-ssh - name: Setup SSH - with: - github-secret: ${{ secrets.GITHUB_TOKEN }} - activate-with-label: false - instructions: "SSH with rdesktop using ssh -L 3389:localhost:3389 %%username%%@%%hostname%%" - - name: Add Conda scripts to GitHub path - run: | - echo "C:/Jenkins/Miniconda3/Scripts" >> $GITHUB_PATH - - uses: ./test-infra/.github/actions/set-channel - - name: Set PYTORCH_VERSION - if: ${{ env.CHANNEL == 'test' }} - run: | - # When building RC, set the version to be the current candidate version, - # otherwise, leave it alone so nightly will pick up the latest - echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}" - - uses: ./test-infra/.github/actions/setup-binary-builds - with: - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - submodules: ${{ inputs.submodules }} - setup-miniconda: false - python-version: ${{ env.PYTHON_VERSION }} - cuda-version: ${{ env.CU_VERSION }} - arch: ${{ env.ARCH }} - - name: Shorten Conda environment path - run: | - bash "${REPOSITORY}/.github/scripts/shorten-conda-env-windows.sh" - - name: Install XPU support package - if: ${{ matrix.gpu_arch_type == 'xpu' }} - run: | - cmd //c .\\test-infra\\.github\\scripts\\install_xpu.bat - - name: Install torch dependency - run: | - source "${BUILD_ENV_FILE}" - # shellcheck disable=SC2086 - ${CONDA_RUN} ${PIP_INSTALL_TORCH} - - name: Run Pre-Script with Caching - if: ${{ inputs.pre-script != '' }} - uses: ./test-infra/.github/actions/run-script-with-cache - with: - cache-path: ${{ inputs.cache-path }} - cache-key: ${{ inputs.cache-key }} - repository: ${{ inputs.repository }} - script: ${{ inputs.pre-script }} - is_windows: 'enabled' - - name: Build clean - working-directory: ${{ inputs.repository }} - env: - ENV_SCRIPT: ${{ inputs.env-script }} - run: | - source "${BUILD_ENV_FILE}" - if [[ -z "${ENV_SCRIPT}" ]]; then - ${CONDA_RUN} python setup.py clean - else - if [[ ! -f ${ENV_SCRIPT} ]]; then - echo "::error::Specified env-script file (${ENV_SCRIPT}) not found" - exit 1 - else - ${CONDA_RUN} ${ENV_SCRIPT} python setup.py clean - fi - fi - - name: Build the wheel (bdist_wheel) - working-directory: ${{ inputs.repository }} - env: - ENV_SCRIPT: ${{ inputs.env-script }} - BUILD_PARAMS: ${{ inputs.wheel-build-params }} - run: | - source "${BUILD_ENV_FILE}" - - if [[ "$CU_VERSION" == "cpu" ]]; then - # CUDA and CPU are ABI compatible on the CPU-only parts, so strip - # in this case - export PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" - else - export PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version: *//')" - fi - - if [[ -z "${ENV_SCRIPT}" ]]; then - ${CONDA_RUN} python setup.py bdist_wheel - else - ${CONDA_RUN} ${ENV_SCRIPT} python setup.py bdist_wheel ${BUILD_PARAMS} - fi - - name: Run post-script - working-directory: ${{ inputs.repository }} - env: - POST_SCRIPT: ${{ inputs.post-script }} - ENV_SCRIPT: ${{ inputs.env-script }} - if: ${{ inputs.post-script != '' }} - run: | - set -euxo pipefail - source "${BUILD_ENV_FILE}" - ${CONDA_RUN} ${ENV_SCRIPT} ${POST_SCRIPT} - - name: Smoke Test - env: - ENV_SCRIPT: ${{ inputs.env-script }} - PACKAGE_NAME: ${{ inputs.package-name }} - SMOKE_TEST_SCRIPT: ${{ inputs.smoke-test-script }} - run: | - source "${BUILD_ENV_FILE}" - WHEEL_NAME=$(ls "${{ inputs.repository }}/dist/") - echo "$WHEEL_NAME" - ${CONDA_RUN} pip install "${{ inputs.repository }}/dist/$WHEEL_NAME" - if [[ ! -f "${{ inputs.repository }}"/${SMOKE_TEST_SCRIPT} ]]; then - echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} not found" - ${CONDA_RUN} "${{ inputs.repository }}/${ENV_SCRIPT}" python -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)" - else - echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" - ${CONDA_RUN} "${{ inputs.repository }}/${ENV_SCRIPT}" python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" - fi - # NB: Only upload to GitHub after passing smoke tests - - name: Upload wheel to GitHub - continue-on-error: true - uses: actions/upload-artifact@v6 - with: - name: ${{ env.UPLOAD_ARTIFACT_NAME }} - path: ${{ inputs.repository }}/dist/ - - uses: ./test-infra/.github/actions/teardown-windows - if: always() - name: Teardown Windows - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true \ No newline at end of file diff --git a/.github/workflows/build-test-linux-aarch64-jetpack.yml b/.github/workflows/build-test-linux-aarch64-jetpack.yml deleted file mode 100644 index 5cda6acec5..0000000000 --- a/.github/workflows/build-test-linux-aarch64-jetpack.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Build and test Linux aarch64 wheels for Jetpack - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux-aarch64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.filter.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Filter matrix - id: filter - env: - LIMIT_PR_BUILDS: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ciflow/binaries/all') }} - run: | - set -eou pipefail - echo "LIMIT_PR_BUILDS=${LIMIT_PR_BUILDS}" - echo '${{ github.event_name }}' - echo '${{ github.event.ref}}' - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}" --jetpack true --limit-pr-builds "${LIMIT_PR_BUILDS}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: Build Jetpack torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "aarch64" - is-jetpack: true - - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true \ No newline at end of file diff --git a/.github/workflows/build-test-linux-aarch64-python-only.yml b/.github/workflows/build-test-linux-aarch64-python-only.yml deleted file mode 100644 index 9900877d82..0000000000 --- a/.github/workflows/build-test-linux-aarch64-python-only.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Python-only build Linux aarch64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -permissions: - contents: read - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux-aarch64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.filter.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Filter matrix - id: filter - env: - LIMIT_PR_BUILDS: ${{ github.event_name == 'pull_request' && !contains( github.event.pull_request.labels.*.name, 'ciflow/binaries/all') }} - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars_python_only.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: Python-only build Linux aarch64 torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "aarch64" - use-rtx: false - pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-python-only-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-linux-aarch64.yml b/.github/workflows/build-test-linux-aarch64.yml deleted file mode 100644 index eea7ec01fa..0000000000 --- a/.github/workflows/build-test-linux-aarch64.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Build and test Linux aarch64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux-aarch64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.filter.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Filter matrix - id: filter - env: - LIMIT_PR_BUILDS: ${{ github.event_name == 'pull_request' && !contains( github.event.pull_request.labels.*.name, 'ciflow/binaries/all') }} - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: Build SBSA torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "aarch64" - pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true \ No newline at end of file diff --git a/.github/workflows/build-test-linux-x86_64-python-only.yml b/.github/workflows/build-test-linux-x86_64-python-only.yml deleted file mode 100644 index c79d4157bb..0000000000 --- a/.github/workflows/build-test-linux-x86_64-python-only.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: Python-only build and test Linux x86_64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -permissions: - contents: read - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars_python_only.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: Python-only build Linux x86_64 torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "x86_64" - use-rtx: false - pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - - dynamo-runtime-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: Python-only dynamo runtime tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: python-only-dynamo-runtime-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/runtime - python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/python_only_dynamo_runtime_tests_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-python-only-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-linux-x86_64.yml b/.github/workflows/build-test-linux-x86_64.yml deleted file mode 100644 index 89b3ee3a32..0000000000 --- a/.github/workflows/build-test-linux-x86_64.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Build and test Linux x86_64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - # All build/test logic lives in the shared reusable core; this workflow - # just selects standard TensorRT (use-rtx: false). See - # .github/workflows/_linux-x86_64-core.yml. - core: - permissions: - id-token: write - contents: read - uses: ./.github/workflows/_linux-x86_64-core.yml - with: - use-rtx: false - - # Single rollup status that summarises every build/test job. Mark this one - # as the required check in branch protection — reviewers see a single - # ✅/❌ instead of 50 matrix entries. Click-through still surfaces the - # individual job logs. - # - # ``if: always()`` makes the rollup run even if the core jobs failed, - # were skipped, or were cancelled (so we always render a check). The body - # fails the rollup iff any build/test job ended in 'failure'; 'skipped' - # (label-gated) and 'success' both count as healthy. - ci-rollup: - name: CI / Linux x86_64 - if: ${{ always() }} - permissions: {} - needs: [core] - runs-on: ubuntu-latest - steps: - - name: Aggregate job results - env: - RESULTS: ${{ needs.core.outputs.results }} - # Safety net: if the core reusable call concluded - # failure/cancelled but its results output came back empty - # (a known edge case), still fail the rollup rather than - # silently reporting green. - CORE_RESULT: ${{ needs.core.result }} - WORKFLOW_LABEL: "Linux x86_64" - run: | - set -euo pipefail - # Emit two surfaces: - # * stdout / job exit code → drives the green/red rollup - # status that branch protection keys on. - # * $GITHUB_STEP_SUMMARY → the markdown that renders - # on the workflow run page, with a per-job result table. - python3 - <<'PY' - import json, os, sys - raw = os.environ.get("RESULTS") or "{}" - core_result = os.environ.get("CORE_RESULT", "") - try: - needs = json.loads(raw) - except json.JSONDecodeError: - needs = {} - label = os.environ.get("WORKFLOW_LABEL", "Linux x86_64") - by_result = {"success": [], "failure": [], "skipped": [], "cancelled": []} - for name, info in needs.items(): - by_result.setdefault(info.get("result") or "unknown", []).append(name) - failed = sorted(by_result["failure"]) - passed = sorted(by_result["success"]) - skipped = sorted(by_result["skipped"]) - cancelled = sorted(by_result["cancelled"]) - - # --- stdout: short pass/fail summary for the log tab --- - print(f"PASS: {len(passed)}") - print(f"FAIL: {len(failed)}") - print(f"SKIPPED: {len(skipped)} (label-gated or never started)") - print(f"CANCELLED: {len(cancelled)}") - if failed: - print() - print("Failed jobs:") - for name in failed: - print(f" - {name}") - - # --- step summary: markdown table for reviewers --- - summary_path = os.environ.get("GITHUB_STEP_SUMMARY") - if summary_path: - icon = {"success": "✅", "failure": "❌", "skipped": "⏭️", "cancelled": "🚫"} - with open(summary_path, "a", encoding="utf-8") as f: - f.write(f"# CI / {label} — rollup\n\n") - f.write( - f"**{len(passed)}** passed · " - f"**{len(failed)}** failed · " - f"**{len(skipped)}** skipped · " - f"**{len(cancelled)}** cancelled\n\n" - ) - f.write("| Result | Job |\n|---|---|\n") - for status in ("failure", "cancelled", "skipped", "success"): - for name in sorted(by_result.get(status, [])): - f.write(f"| {icon.get(status, '?')} {status} | `{name}` |\n") - if failed: - f.write( - "\n> Click into a failed job above to see " - "the rendered test table (via `pytest-results-action`) " - "and the `::warning::Reproduce locally with: ...` hint " - "near the bottom of the log.\n" - ) - - # Fail if any job failed, OR the core call did not succeed - # (covers an empty/missing results payload). - if failed or core_result not in ("success", "skipped", ""): - if core_result not in ("success", "skipped", "") and not failed: - print(f"\nCore reusable workflow concluded '{core_result}' " - f"with no per-job results; failing rollup defensively.") - sys.exit(1) - PY - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-linux-x86_64_rtx-python-only.yml b/.github/workflows/build-test-linux-x86_64_rtx-python-only.yml deleted file mode 100644 index 27e6a21f69..0000000000 --- a/.github/workflows/build-test-linux-x86_64_rtx-python-only.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: RTX - Python-only build and test Linux x86_64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -permissions: - contents: read - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx true --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars_python_only.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - display-name: RTX - Python-only build Linux x86_64 torch-tensorrt-rtx whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - architecture: "x86_64" - use-rtx: true - pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - - dynamo-runtime-tests: - name: ${{ matrix.display-name }} - needs: [filter-matrix, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - display-name: RTX - Python-only dynamo runtime tests - uses: ./.github/workflows/linux-test.yml - with: - job-name: rtx-python-only-dynamo-runtime-tests - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/runtime - python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/python_only_dynamo_runtime_tests_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-rtx-python-only-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-linux-x86_64_rtx.yml b/.github/workflows/build-test-linux-x86_64_rtx.yml deleted file mode 100644 index aed838a50b..0000000000 --- a/.github/workflows/build-test-linux-x86_64_rtx.yml +++ /dev/null @@ -1,109 +0,0 @@ -name: RTX - Build and test Linux x86_64 wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - # All build/test logic lives in the shared reusable core; this workflow - # just selects TensorRT-RTX (use-rtx: true). See - # .github/workflows/_linux-x86_64-core.yml. - core: - permissions: - id-token: write - contents: read - uses: ./.github/workflows/_linux-x86_64-core.yml - with: - use-rtx: true - name-prefix: "RTX - " - - # Single rollup status for the RTX matrix; mirror the non-RTX workflow's - # ci-rollup so branch protection can require one check per workflow. - ci-rollup: - name: CI / Linux x86_64 (RTX) - if: ${{ always() }} - permissions: {} - needs: [core] - runs-on: ubuntu-latest - steps: - - name: Aggregate job results - env: - RESULTS: ${{ needs.core.outputs.results }} - # Safety net: fail the rollup if the core call concluded - # failure/cancelled even when its results output came back empty. - CORE_RESULT: ${{ needs.core.result }} - # Surface a label so the markdown summary disambiguates RTX vs standard. - WORKFLOW_LABEL: "Linux x86_64 (RTX)" - run: | - set -euo pipefail - # Same logic as the non-RTX rollup: stdout for the rollup status, - # $GITHUB_STEP_SUMMARY for the reviewer-facing markdown table. - python3 - <<'PY' - import json, os, sys - raw = os.environ.get("RESULTS") or "{}" - core_result = os.environ.get("CORE_RESULT", "") - try: - needs = json.loads(raw) - except json.JSONDecodeError: - needs = {} - label = os.environ.get("WORKFLOW_LABEL", "Linux x86_64") - by_result = {"success": [], "failure": [], "skipped": [], "cancelled": []} - for name, info in needs.items(): - by_result.setdefault(info.get("result") or "unknown", []).append(name) - failed = sorted(by_result["failure"]) - passed = sorted(by_result["success"]) - skipped = sorted(by_result["skipped"]) - cancelled = sorted(by_result["cancelled"]) - - print(f"PASS: {len(passed)}") - print(f"FAIL: {len(failed)}") - print(f"SKIPPED: {len(skipped)} (label-gated or never started)") - print(f"CANCELLED: {len(cancelled)}") - if failed: - print() - print("Failed jobs:") - for name in failed: - print(f" - {name}") - - summary_path = os.environ.get("GITHUB_STEP_SUMMARY") - if summary_path: - icon = {"success": "✅", "failure": "❌", "skipped": "⏭️", "cancelled": "🚫"} - with open(summary_path, "a", encoding="utf-8") as f: - f.write(f"# CI / {label} — rollup\n\n") - f.write( - f"**{len(passed)}** passed · " - f"**{len(failed)}** failed · " - f"**{len(skipped)}** skipped · " - f"**{len(cancelled)}** cancelled\n\n" - ) - f.write("| Result | Job |\n|---|---|\n") - for status in ("failure", "cancelled", "skipped", "success"): - for name in sorted(by_result.get(status, [])): - f.write(f"| {icon.get(status, '?')} {status} | `{name}` |\n") - if failed: - f.write( - "\n> Click into a failed job above to see the " - "rendered test table (via `pytest-results-action`) " - "and the `::warning::Reproduce locally with: ...` " - "hint near the bottom of the log.\n" - ) - - if failed or core_result not in ("success", "skipped", ""): - if core_result not in ("success", "skipped", "") and not failed: - print(f"\nCore reusable workflow concluded '{core_result}' " - f"with no per-job results; failing rollup defensively.") - sys.exit(1) - PY - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-tensorrt-linux.yml b/.github/workflows/build-test-tensorrt-linux.yml deleted file mode 100644 index 2b1a08c45a..0000000000 --- a/.github/workflows/build-test-tensorrt-linux.yml +++ /dev/null @@ -1,336 +0,0 @@ -name: Build and Test Torch-TensorRT on Linux with Future TensorRT Versions - -on: - workflow_dispatch: - schedule: - - cron: '0 0 * * 0' # Runs at 00:00 UTC every Sunday (minute hour day-of-month month-of-year day-of-week) - -permissions: - id-token: write - contents: read - packages: write - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - python-versions: '["3.11"]' - - generate-tensorrt-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate tensorrt matrix - id: generate - run: | - set -eou pipefail - python -m pip install --upgrade pip - pip install requests - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/generate-tensorrt-test-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - build: - needs: [generate-tensorrt-matrix] - name: Build torch-tensorrt whl package - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_vars.txt - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - package-name: torch_tensorrt - uses: ./.github/workflows/build-tensorrt-linux.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-var-script: ${{ matrix.env-var-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - - tests-py-torchscript-fe: - name: Test torchscript frontend [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-torchscript-fe - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_api_test_results.xml api/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_models_test_results.xml models/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_integrations_test_results.xml integrations/ - popd - - tests-py-dynamo-converters: - name: Test dynamo converters [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-converters - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/ - popd - - tests-py-dynamo-fe: - name: Test dynamo frontend [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-fe - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ - popd - - tests-py-dynamo-serde: - name: Test dynamo export serde [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-serde - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py - popd - - tests-py-torch-compile-be: - name: Test torch compile backend [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-torch-compile-be - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py - popd - - tests-py-dynamo-core: - name: Test dynamo core [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-core - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ - popd - - tests-py-dynamo-cudagraphs: - name: Test dynamo cudagraphs [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-cudagraphs - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - nvidia-smi - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py || true - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py || true - popd - - tests-py-core: - name: Test core [Python] - needs: [generate-tensorrt-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-core - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py/core - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-tensorrt-windows.yml b/.github/workflows/build-test-tensorrt-windows.yml deleted file mode 100644 index 6c66c8d7c6..0000000000 --- a/.github/workflows/build-test-tensorrt-windows.yml +++ /dev/null @@ -1,320 +0,0 @@ -name: Build and Test Torch-TensorRT on Windows with Future TensorRT Versions - -on: - workflow_dispatch: - schedule: - - cron: '0 0 * * 0' # Runs at 00:00 UTC every Sunday (minute hour day-of-month month-of-year day-of-week) - -permissions: - id-token: write - contents: read - packages: write - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - python-versions: '["3.11"]' - - generate-tensorrt-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate tensorrt matrix - id: generate - run: | - set -eou pipefail - python -m pip install --upgrade pip - pip install requests - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/generate-tensorrt-test-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - substitute-runner: - needs: generate-tensorrt-matrix - outputs: - matrix: ${{ steps.substitute.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - name: Substitute runner - id: substitute - run: | - echo matrix="$(echo '${{ needs.generate-tensorrt-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> ${GITHUB_OUTPUT} - - build: - needs: substitute-runner - name: Build torch-tensorrt whl package - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script_windows.sh - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/smoke_test_windows.py - package-name: torch_tensorrt - uses: ./.github/workflows/build-tensorrt-windows.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} - trigger-event: ${{ github.event_name }} - timeout: 120 - - tests-py-torchscript-fe: - name: Test torchscript frontend [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-torchscript-fe - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_api_test_results.xml api/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_models_test_results.xml models/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_integrations_test_results.xml integrations/ - popd - - tests-py-dynamo-converters: - name: Test dynamo converters [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-converters - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/ - popd - - tests-py-dynamo-fe: - name: Test dynamo frontend [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-fe - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ - popd - - tests-py-dynamo-serde: - name: Test dynamo export serde [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-serde - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py - popd - - tests-py-torch-compile-be: - name: Test torch compile backend [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-torch-compile-be - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py - popd - - tests-py-dynamo-core: - name: Test dynamo core [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-core - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ - popd - - tests-py-dynamo-cudagraphs: - name: Test dynamo cudagraphs [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-cudagraphs - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py - popd - - tests-py-core: - name: Test core [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-core - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py/core - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-windows-python-only.yml b/.github/workflows/build-test-windows-python-only.yml deleted file mode 100644 index c48a054479..0000000000 --- a/.github/workflows/build-test-windows-python-only.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: Python-only build and test Windows wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -permissions: - contents: read - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - substitute-runner: - needs: filter-matrix - outputs: - matrix: ${{ steps.substitute.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - name: Substitute runner - id: substitute - run: | - echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> ${GITHUB_OUTPUT} - - build: - needs: substitute-runner - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script_windows.sh - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/smoke_test_windows.py - package-name: torch_tensorrt - display-name: Python-only build Windows torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_windows.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - wheel-build-params: "--py-only" - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} - trigger-event: ${{ github.event_name }} - timeout: 120 - - dynamo-runtime-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: Python-only dynamo runtime tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: python-only-dynamo-runtime-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/runtime - ../../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/python_only_dynamo_runtime_tests_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-python-only-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-windows.yml b/.github/workflows/build-test-windows.yml deleted file mode 100644 index f092e5a7d0..0000000000 --- a/.github/workflows/build-test-windows.yml +++ /dev/null @@ -1,461 +0,0 @@ -name: Build and test Windows wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - substitute-runner: - needs: filter-matrix - outputs: - matrix: ${{ steps.substitute.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - name: Substitute runner - id: substitute - run: | - echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> ${GITHUB_OUTPUT} - - build: - needs: substitute-runner - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script_windows.sh - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/smoke_test_windows.py - package-name: torch_tensorrt - display-name: Build Windows torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_windows.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} - trigger-event: ${{ github.event_name }} - timeout: 120 - - L0-dynamo-converter-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 dynamo converter tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-dynamo-converter-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_converter_tests_results.xml --dist=loadscope --maxfail=20 conversion/ - popd - - L0-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_runtime_tests_results.xml runtime/test_000_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_partitioning_tests_results.xml partitioning/test_000_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_lowering_tests_results.xml lowering/ - popd - - L0-py-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 core python tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-core-python-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/core - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_py_core_tests_results.xml . - popd - - L0-torchscript-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 torchscript tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-torchscript-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_ts_api_tests_results.xml api/ - popd - - L1-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests, L0-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_core_tests_results.xml runtime/test_001_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_core_partitioning_tests_results.xml partitioning/test_001_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_hlo_tests_results.xml hlo/ - popd - - L1-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests, L0-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 dynamo compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-dynamo-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_compile_tests_results.xml models/ - popd - - L1-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests, L0-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 torch compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-torch-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_be_tests_results.xml backend/ - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_models_tests_results.xml --ir torch_compile models/test_models.py - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_dyn_models_tests_results.xml --ir torch_compile models/test_dyn_models.py - popd - - L1-torchscript-tests: - name: L1 torchscript tests - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests, L0-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-torchscript-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_ts_models_tests_results.xml models/ - popd - - L2-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests, L1-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 torch compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-torch-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_torch_compile_models_tests_results.xml --ir torch_compile models/test_models.py - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_torch_compile_dyn_models_tests_results.xml --ir torch_compile models/test_dyn_models.py - popd - - L2-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests, L1-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_compile_tests_results.xml models/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_compile_llm_tests_results.xml llm/ - popd - - L2-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests, L1-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_core_tests_results.xml -k "not test_000_ and not test_001_" runtime/* - popd - - L2-dynamo-plugin-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests, L1-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo plugin tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-plugin-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_plugins_tests_results.xml automatic_plugin/ - popd - - L2-torchscript-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests, L1-torchscript-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 torchscript tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-torchscript-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_ts_integrations_tests_results.xml integrations/ - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-windows_rtx-python-only.yml b/.github/workflows/build-test-windows_rtx-python-only.yml deleted file mode 100644 index ab4678caa4..0000000000 --- a/.github/workflows/build-test-windows_rtx-python-only.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: RTX - Python-only build and test Windows wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -permissions: - contents: read - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx true --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - substitute-runner: - needs: filter-matrix - outputs: - matrix: ${{ steps.substitute.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - name: Substitute runner - id: substitute - run: | - echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> ${GITHUB_OUTPUT} - - build: - needs: substitute-runner - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script_windows.sh - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/smoke_test_windows.py - package-name: torch_tensorrt - display-name: RTX - Python-only build Windows torch-tensorrt-rtx whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_windows.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - wheel-build-params: "--py-only --use-rtx" - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} - trigger-event: ${{ github.event_name }} - timeout: 120 - use-rtx: true - - dynamo-runtime-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: RTX - Python-only dynamo runtime tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: rtx-python-only-dynamo-runtime-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/runtime - ../../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/python_only_dynamo_runtime_tests_results.xml . - popd - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-rtx-python-only-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/build-test-windows_rtx.yml b/.github/workflows/build-test-windows_rtx.yml deleted file mode 100644 index 53d2bd6a35..0000000000 --- a/.github/workflows/build-test-windows_rtx.yml +++ /dev/null @@ -1,381 +0,0 @@ -name: RTX - Build and test Windows wheels - -on: - pull_request: - push: - branches: - - main - - nightly - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-rocm: false - with-cpu: false - - filter-matrix: - needs: [generate-matrix] - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - - uses: actions/checkout@v6 - with: - repository: pytorch/tensorrt - - name: Generate matrix - id: generate - run: | - set -eou pipefail - MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx true --matrix "${MATRIX_BLOB}")" - echo "${MATRIX_BLOB}" - echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" - - substitute-runner: - needs: filter-matrix - outputs: - matrix: ${{ steps.substitute.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - name: Substitute runner - id: substitute - run: | - echo matrix="$(echo '${{ needs.filter-matrix.outputs.matrix }}' | sed -e 's/windows.g4dn.xlarge/windows.g5.4xlarge.nvidia.gpu/g')" >> ${GITHUB_OUTPUT} - - build: - needs: substitute-runner - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - pre-script: packaging/pre_build_script_windows.sh - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/smoke_test_windows.py - package-name: torch_tensorrt - display-name: RTX - Build Windows torch-tensorrt whl package - name: ${{ matrix.display-name }} - uses: ./.github/workflows/build_windows.yml - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} - trigger-event: ${{ github.event_name }} - wheel-build-params: "--use-rtx" - use-rtx: true - timeout: 120 - - L0-dynamo-converter-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 dynamo converter tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-dynamo-converter-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_converter_tests_results.xml --dist=loadscope --maxfail=20 conversion/ - popd - - L0-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_runtime_tests_results.xml runtime/test_000_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_partitioning_tests_results.xml partitioning/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_dynamo_core_lowering_tests_results.xml lowering/ - popd - - L0-py-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L0 core python tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L0-core-python-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/core - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l0_py_core_tests_results.xml . - popd - - L1-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_core_tests_results.xml runtime/test_001_* - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 8 --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_hlo_tests_results.xml hlo/ - popd - - L1-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 dynamo compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-dynamo-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_dynamo_compile_tests_results.xml models/ - popd - - L1-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L1 torch compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L1-torch-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_be_tests_results.xml backend/ - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_models_tests_results.xml --ir torch_compile models/test_models.py - ../../../packaging/vc_env_helper.bat python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l1_torch_compile_dyn_models_tests_results.xml --ir torch_compile models/test_dyn_models.py - popd - - L2-torch-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 torch compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-torch-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_torch_compile_models_tests_results.xml --ir torch_compile models/test_models.py - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_torch_compile_dyn_models_tests_results.xml --ir torch_compile models/test_dyn_models.py - popd - - L2-dynamo-compile-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo compile tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-compile-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo/ - ../../../packaging/vc_env_helper.bat python -m pytest -m "not critical" -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_compile_tests_results.xml models/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_compile_llm_tests_results.xml llm/ - popd - - L2-dynamo-core-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo core tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-core-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_core_tests_results.xml -k "not test_000_ and not test_001_" runtime/* - popd - - L2-dynamo-plugin-tests: - name: ${{ matrix.display-name }} - needs: [substitute-runner, build, L1-torch-compile-tests, L1-dynamo-compile-tests, L1-dynamo-core-tests] - if: ${{ (github.ref_name == 'main' || github.ref_name == 'nightly' || startsWith(github.ref_name, 'release/') || (startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-rc')) || contains(github.event.pull_request.labels.*.name, 'Force All Tests[L0+L1+L2]')) && always() || success() }} - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - display-name: L2 dynamo plugin tests - uses: ./.github/workflows/windows-test.yml - with: - job-name: L2-dynamo-plugin-tests - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - pushd . - cd tests/py/dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_plugins_tests_results.xml automatic_plugin/ - popd - - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-rtx-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} - cancel-in-progress: true diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml index 90333aedf8..de62327eaf 100644 --- a/.github/workflows/build_linux.yml +++ b/.github/workflows/build_linux.yml @@ -254,6 +254,20 @@ jobs: cache-key: ${{ inputs.cache-key }} repository: ${{ inputs.repository }} script: ${{ inputs.pre-script }} + # Persist bazel's action cache across runs so the C++/CUDA compile — the + # serial long pole — reuses unchanged action outputs instead of rebuilding + # from scratch. Lives under RUNNER_TEMP (the "Clean workspace" step wipes + # GITHUB_WORKSPACE). setup.py picks up BAZEL_DISK_CACHE (exported below). + # Key is COARSE (toolchain/deps, not the commit) so most PRs restore a warm + # cache; the restore-keys prefix makes even a dep bump an incremental build. + - name: Cache the bazel C++/CUDA build + if: ${{ inputs.architecture != 'aarch64' }} + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/trt-bazel-disk-cache + key: trt-bazel-${{ inputs.architecture }}-${{ matrix.desired_cuda }}-rtx${{ inputs.use-rtx }}-${{ hashFiles('**/MODULE.bazel', '**/WORKSPACE', '**/.bazelrc', '**/dev_dep_versions.yml') }} + restore-keys: | + trt-bazel-${{ inputs.architecture }}-${{ matrix.desired_cuda }}-rtx${{ inputs.use-rtx }}- - name: Build the wheel (python-build-package) if: ${{ inputs.build-platform == 'python-build-package' }} working-directory: ${{ inputs.repository }} @@ -272,6 +286,8 @@ jobs: run: | set -x source "${BUILD_ENV_FILE}" + # Reuse the cached bazel action outputs (see "Cache the bazel..." step). + export BAZEL_DISK_CACHE="${RUNNER_TEMP}/trt-bazel-disk-cache" export PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" ${CONDA_RUN} python setup.py clean echo "Successfully ran `python setup.py clean`" @@ -413,5 +429,10 @@ jobs: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{inputs.is-release-wheel}}-${{inputs.is-release-tarball}}-${{inputs.use-rtx}}-${{inputs.architecture}}-${{inputs.is-jetpack}}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'no-tag' }}-${{ inputs.use-rtx }} + # env-var-script distinguishes the python-only build (env_vars_python_only.txt) + # from the standard build (env_vars.txt). Without it, the standard and + # python-only channels of the SAME run share this group and cancel-in-progress + # kills one of them — which is why "run everything" never completed. (Local + # addition to the test-infra-synced key; keep on re-sync.) + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{inputs.is-release-wheel}}-${{inputs.is-release-tarball}}-${{inputs.use-rtx}}-${{inputs.architecture}}-${{inputs.is-jetpack}}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'no-tag' }}-${{ inputs.use-rtx }}-${{ inputs.env-var-script }} cancel-in-progress: true diff --git a/.github/workflows/ci-linux-x86_64.yml b/.github/workflows/ci-linux-x86_64.yml new file mode 100644 index 0000000000..27f497eb11 --- /dev/null +++ b/.github/workflows/ci-linux-x86_64.yml @@ -0,0 +1,147 @@ +name: CI Linux x86_64 + +# Per-platform entry (one small, independent run — not the old 8-channel mega-run). +# Runs the linux-x86_64 channels: standard, RTX, and python-only. Lane + backend +# come from the shared _decide reusable. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + pull_request_review: + types: [submitted] + push: + branches: [main] + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + inputs: + lane: + description: "Which lane to run" + type: choice + options: [fast, full, nightly] + default: full + backend: + description: "Which backend(s) to test" + type: choice + options: [standard, rtx, both] + default: both + +concurrency: + # Only NON-CI label events (component:*, cla signed, …) get their own per-label + # group, so auto-label churn never cancels the running pipeline. CI-control labels + # (ci:full / ci:nightly / backend:*) and real pushes share the PR group so they + # SUPERSEDE cleanly — exactly ONE pipeline per commit. Coexisting runs would + # otherwise collide inside build_linux.yml's own concurrency and cancel a + # half-built wheel (that is what cancelled the SBSA aarch64 build). + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"ci: nightly\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +jobs: + decide: + # A label event only warrants running the pipeline when the label controls CI; + # auto-labels (component:*, cla signed) must not spawn work — and would each + # start a full run once ci:full is present. Non-label events always proceed. + if: >- + github.event.action != 'labeled' || + contains(fromJSON('["ci: full", "ci: nightly", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) + uses: ./.github/workflows/_decide.yml + + # Generate the build matrix ONCE. generate_binary_build_matrix's concurrency + # group is keyed on (workflow, PR, os) with cancel-in-progress, so calling it + # per-channel makes the channels cancel each other's matrix job. All channels + # share this one output and filter it per-variant. + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-rocm: false + with-cpu: false + + # Standard — the only channel on the fast lane (every PR push). + standard: + needs: [decide, generate-matrix] + if: needs.decide.outputs.lane != 'skip' && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-linux.yml + with: + lane: ${{ needs.decide.outputs.lane }} + use-rtx: false + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + rtx: + needs: [decide, generate-matrix] + if: needs.decide.outputs.lane != 'skip' && needs.decide.outputs.backend != 'standard' + uses: ./.github/workflows/_test-linux.yml + with: + lane: ${{ needs.decide.outputs.lane }} + use-rtx: true + name-prefix: "RTX - " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + python-only: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-linux.yml + with: + lane: python-only + python-only: true + name-prefix: "Python-only " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + # python-only against TensorRT-RTX (so backend=both runs BOTH python-only variants). + python-only-rtx: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard' + uses: ./.github/workflows/_test-linux.yml + with: + lane: python-only + python-only: true + use-rtx: true + name-prefix: "Python-only RTX " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + # Required check. Fails ONLY on a genuine failure; skipped/cancelled don't block. + gate: + needs: [decide, standard, rtx, python-only, python-only-rtx] + if: always() + runs-on: ubuntu-latest + steps: + - name: Gate + run: | + set -euo pipefail + if [ "${{ contains(needs.*.result, 'failure') }}" = "true" ]; then + echo "::error::a Linux x86_64 channel failed — see the channel jobs above" + exit 1 + fi + echo "Linux x86_64 gate OK (cancelled/skipped channels ignored)." + + # Consolidated, agent-friendly report over this run's suites (informational). + report: + needs: [standard, rtx, python-only, python-only-rtx] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - uses: actions/download-artifact@v7 + with: + pattern: junit-* + path: all-results + merge-multiple: true + - name: Consolidated report + run: | + set -uo pipefail + mkdir -p all-results + if [ -z "$(find all-results -name '*.xml' 2>/dev/null)" ]; then + echo "No JUnit results uploaded." >> "$GITHUB_STEP_SUMMARY"; exit 0 + fi + python tests/py/utils/junit_summary.py all-results --agent >> "$GITHUB_STEP_SUMMARY" || true + python tests/py/utils/junit_summary.py all-results || true diff --git a/.github/workflows/ci-sbsa.yml b/.github/workflows/ci-sbsa.yml new file mode 100644 index 0000000000..a4be4e2ce1 --- /dev/null +++ b/.github/workflows/ci-sbsa.yml @@ -0,0 +1,99 @@ +name: CI SBSA + +# Per-platform entry — SBSA (linux-aarch64) is BUILD-ONLY (no aarch64 GPU test +# runners), so this just validates the wheel builds (standard + python-only) on +# the full/nightly lanes. No test/report jobs since nothing runs. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + pull_request_review: + types: [submitted] + push: + branches: [main] + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + inputs: + lane: + description: "Which lane to run" + type: choice + options: [fast, full, nightly] + default: full + backend: + description: "Which backend(s) to test" + type: choice + options: [standard, rtx, both] + default: both + +concurrency: + # Only NON-CI label events (component:*, cla signed, …) get their own per-label + # group, so auto-label churn never cancels the running pipeline. CI-control labels + # (ci:full / ci:nightly / backend:*) and real pushes share the PR group so they + # SUPERSEDE cleanly — exactly ONE pipeline per commit. Coexisting runs would + # otherwise collide inside build_linux.yml's own concurrency and cancel a + # half-built wheel (that is what cancelled the SBSA aarch64 build). + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"ci: nightly\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +jobs: + decide: + # A label event only warrants running the pipeline when the label controls CI; + # auto-labels (component:*, cla signed) must not spawn work — and would each + # start a full run once ci:full is present. Non-label events always proceed. + if: >- + github.event.action != 'labeled' || + contains(fromJSON('["ci: full", "ci: nightly", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) + uses: ./.github/workflows/_decide.yml + + # Generate the aarch64 build matrix once (shared by both build channels). + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux-aarch64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-rocm: false + with-cpu: false + + build: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-linux.yml + with: + lane: ${{ needs.decide.outputs.lane }} + architecture: aarch64 + run-tests: false + name-prefix: "SBSA " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + python-only: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-linux.yml + with: + lane: python-only + python-only: true + architecture: aarch64 + run-tests: false + name-prefix: "Python-only SBSA " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + gate: + needs: [decide, build, python-only] + if: always() + runs-on: ubuntu-latest + steps: + - name: Gate + run: | + set -euo pipefail + if [ "${{ contains(needs.*.result, 'failure') }}" = "true" ]; then + echo "::error::an SBSA build failed — see the build jobs above" + exit 1 + fi + echo "SBSA gate OK (cancelled/skipped channels ignored)." diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml new file mode 100644 index 0000000000..1220df1279 --- /dev/null +++ b/.github/workflows/ci-windows.yml @@ -0,0 +1,141 @@ +name: CI Windows + +# Per-platform entry — runs the Windows channels (standard, RTX, python-only) as +# one small independent run. Windows is heavier, so it runs on the full/nightly +# lanes only (never the fast PR-push lane). + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + pull_request_review: + types: [submitted] + push: + branches: [main] + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + inputs: + lane: + description: "Which lane to run" + type: choice + options: [fast, full, nightly] + default: full + backend: + description: "Which backend(s) to test" + type: choice + options: [standard, rtx, both] + default: both + +concurrency: + # Only NON-CI label events (component:*, cla signed, …) get their own per-label + # group, so auto-label churn never cancels the running pipeline. CI-control labels + # (ci:full / ci:nightly / backend:*) and real pushes share the PR group so they + # SUPERSEDE cleanly — exactly ONE pipeline per commit. Coexisting runs would + # otherwise collide inside build_linux.yml's own concurrency and cancel a + # half-built wheel (that is what cancelled the SBSA aarch64 build). + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"ci: nightly\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}" + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +jobs: + decide: + # A label event only warrants running the pipeline when the label controls CI; + # auto-labels (component:*, cla signed) must not spawn work — and would each + # start a full run once ci:full is present. Non-label events always proceed. + if: >- + github.event.action != 'labeled' || + contains(fromJSON('["ci: full", "ci: nightly", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name) + uses: ./.github/workflows/_decide.yml + + # Generate the windows build matrix once (shared across channels; + # generate_binary_build_matrix's cancel-in-progress concurrency would otherwise + # make per-channel calls cancel each other). + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: windows + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-rocm: false + with-cpu: false + + standard: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-windows.yml + with: + lane: ${{ needs.decide.outputs.lane }} + use-rtx: false + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + rtx: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard' + uses: ./.github/workflows/_test-windows.yml + with: + lane: ${{ needs.decide.outputs.lane }} + use-rtx: true + name-prefix: "RTX - " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + python-only: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx' + uses: ./.github/workflows/_test-windows.yml + with: + lane: python-only + name-prefix: "Python-only " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + # python-only against TensorRT-RTX (so backend=both runs BOTH python-only variants). + python-only-rtx: + needs: [decide, generate-matrix] + if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard' + uses: ./.github/workflows/_test-windows.yml + with: + lane: python-only + use-rtx: true + name-prefix: "Python-only RTX " + raw-matrix: ${{ needs.generate-matrix.outputs.matrix }} + + gate: + needs: [decide, standard, rtx, python-only, python-only-rtx] + if: always() + runs-on: ubuntu-latest + steps: + - name: Gate + run: | + set -euo pipefail + if [ "${{ contains(needs.*.result, 'failure') }}" = "true" ]; then + echo "::error::a Windows channel failed — see the channel jobs above" + exit 1 + fi + echo "Windows gate OK (cancelled/skipped channels ignored)." + + report: + needs: [standard, rtx, python-only, python-only-rtx] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - uses: actions/download-artifact@v7 + with: + pattern: junit-* + path: all-results + merge-multiple: true + - name: Consolidated report + run: | + set -uo pipefail + mkdir -p all-results + if [ -z "$(find all-results -name '*.xml' 2>/dev/null)" ]; then + echo "No JUnit results uploaded." >> "$GITHUB_STEP_SUMMARY"; exit 0 + fi + python tests/py/utils/junit_summary.py all-results --agent >> "$GITHUB_STEP_SUMMARY" || true + python tests/py/utils/junit_summary.py all-results || true diff --git a/.github/workflows/linux-test.yml b/.github/workflows/linux-test.yml index d055276143..4537d4d010 100644 --- a/.github/workflows/linux-test.yml +++ b/.github/workflows/linux-test.yml @@ -174,6 +174,17 @@ jobs: display-options: fEs fail-on-empty: ${{ inputs.fail-on-empty }} + # Upload the raw JUnit so a downstream job can aggregate every suite's + # results into one consolidated report (junit_summary.py). Additive: the + # per-job pmeier summary above is unchanged. + - name: Upload JUnit results + if: always() + uses: actions/upload-artifact@v6 + with: + name: junit-${{ inputs.job-name }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }} + path: ${{ env.RUNNER_TEST_RESULTS_DIR }}/**/*.xml + if-no-files-found: ignore + - name: Prepare artifacts for upload working-directory: ${{ inputs.repository }} id: check-artifacts diff --git a/.github/workflows/retrigger-ci.yml b/.github/workflows/retrigger-ci.yml index da3fd921b5..8a184ad65a 100644 --- a/.github/workflows/retrigger-ci.yml +++ b/.github/workflows/retrigger-ci.yml @@ -1,12 +1,22 @@ name: Retrigger CI -# Re-run CI on a PR without pushing a new commit. +# Re-run or cancel CI on a PR without pushing a new commit. # # Usage — post a comment on any PR: -# /rerun re-run only failed / cancelled jobs -# /rerun all re-run all jobs from scratch +# /rerun re-run only failed / cancelled jobs (current commit) +# /rerun all re-run all jobs from scratch (current commit) +# /cancel cancel stale "zombie" runs — anything still in-flight on an +# OLD commit of this PR (leaves the current commit's run alone) +# /cancel all cancel EVERY in-flight run for this PR (full stop) +# /test [backend] +# dispatch a fresh run on this PR's branch — lane ∈ +# fast|full|nightly, backend ∈ standard|rtx|both (default both). +# e.g. `/test full rtx`, `/test nightly`, `/test full` # # The comment author must have write access to the repo. +# +# NOTE: issue_comment workflows always run from the DEFAULT branch, so changes to +# this file only take effect once merged to main. on: issue_comment: @@ -17,16 +27,17 @@ permissions: pull-requests: read jobs: - retrigger: + command: if: | github.event.issue.pull_request != null && ( - startsWith(github.event.comment.body, '/rerun') + startsWith(github.event.comment.body, '/rerun') || + startsWith(github.event.comment.body, '/cancel') || + startsWith(github.event.comment.body, '/test') ) runs-on: ubuntu-latest steps: - name: Check commenter has write access - id: auth uses: actions/github-script@v7 with: script: | @@ -35,68 +46,86 @@ jobs: repo: context.repo.repo, username: context.actor, }); - const level = perm.permission; - if (!["admin", "write"].includes(level)) { - core.setFailed(`${context.actor} has permission '${level}' — write access required to retrigger CI`); + if (!["admin", "write"].includes(perm.permission)) { + core.setFailed(`${context.actor} has permission '${perm.permission}' — write access required to control CI`); } - return level; - - name: Parse command - id: cmd + - name: Run command uses: actions/github-script@v7 with: script: | const body = context.payload.comment.body.trim(); - const rerunAll = body === '/rerun all'; - core.setOutput('rerun_all', rerunAll ? 'true' : 'false'); - return rerunAll; - - - name: Re-run CI - uses: actions/github-script@v7 - with: - script: | - const rerunAll = '${{ steps.cmd.outputs.rerun_all }}' === 'true'; - + const { owner, repo } = context.repo; const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, + owner, repo, pull_number: context.issue.number, }); - const sha = pr.head.sha; + const headSha = pr.head.sha; + // ── /test [backend] — dispatch a run with an exact lane ── + if (body.startsWith('/test')) { + const parts = body.split(/\s+/).slice(1); + const lane = ['fast', 'full', 'nightly'].includes(parts[0]) ? parts[0] : 'full'; + const backend = ['standard', 'rtx', 'both'].includes(parts[1]) ? parts[1] : 'both'; + let dispatched = 0; + for (const wf of ['ci-linux-x86_64.yml', 'ci-windows.yml', 'ci-sbsa.yml']) { + try { + await github.rest.actions.createWorkflowDispatch({ + owner, repo, workflow_id: wf, ref: pr.head.ref, + inputs: { lane, backend }, + }); + dispatched++; + } catch (e) { + core.info(`Skipped ${wf}: ${e.message}`); + } + } + core.notice(`Dispatched ${dispatched} workflow(s) — lane=${lane} backend=${backend} on ${pr.head.ref}`); + return; + } + + // ── /cancel [all] — kill zombie runs ─────────────────────────── + if (body.startsWith('/cancel')) { + const all = body === '/cancel all'; + const { data: { workflow_runs: runs } } = + await github.rest.actions.listWorkflowRunsForRepo({ + owner, repo, branch: pr.head.ref, per_page: 100, + }); + let cancelled = 0; + for (const run of runs) { + if (run.status === 'completed') continue; // already done + if (!all && run.head_sha === headSha) continue; // keep current commit + try { + await github.rest.actions.cancelWorkflowRun({ owner, repo, run_id: run.id }); + cancelled++; + } catch (e) { + core.info(`Skipped ${run.id} (${run.name}): ${e.message}`); + } + } + core.notice(`Cancelled ${cancelled} ${all ? 'in-flight' : 'stale (old-commit)'} run(s).`); + return; + } + + // ── /rerun [all] ─────────────────────────────────────────────── + const rerunAll = body === '/rerun all'; const { data: { workflow_runs: runs } } = await github.rest.actions.listWorkflowRunsForRepo({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: sha, - per_page: 50, + owner, repo, head_sha: headSha, per_page: 50, }); - if (runs.length === 0) { - core.warning(`No workflow runs found for SHA ${sha}`); + core.warning(`No workflow runs found for SHA ${headSha}`); return; } - let retriggered = 0; for (const run of runs) { try { if (rerunAll) { - await github.rest.actions.reRunWorkflow({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: run.id, - }); + await github.rest.actions.reRunWorkflow({ owner, repo, run_id: run.id }); } else { - await github.rest.actions.reRunWorkflowFailedJobs({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: run.id, - }); + await github.rest.actions.reRunWorkflowFailedJobs({ owner, repo, run_id: run.id }); } retriggered++; } catch (e) { - // A run that is still in progress can't be re-run; skip it. + // A run still in progress can't be re-run; skip it. core.info(`Skipped run ${run.id} (${run.name}): ${e.message}`); } } - core.info(`Retriggered ${retriggered}/${runs.length} workflow runs`); + core.notice(`Retriggered ${retriggered}/${runs.length} workflow runs.`); diff --git a/.github/workflows/windows-test.yml b/.github/workflows/windows-test.yml index 0d919ca3c3..89c2e83aab 100644 --- a/.github/workflows/windows-test.yml +++ b/.github/workflows/windows-test.yml @@ -153,6 +153,15 @@ jobs: summary: true display-options: fEs fail-on-empty: true + # Upload raw JUnit so the consolidated report (junit_summary.py) can + # aggregate Windows results alongside Linux. Additive. + - name: Upload JUnit results + if: always() + uses: actions/upload-artifact@v6 + with: + name: junit-${{ inputs.job-name }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }} + path: ${{ env.RUNNER_TEST_RESULTS_DIR }}/*.xml + if-no-files-found: ignore - name: Teardown Windows if: ${{ always() }} uses: ./test-infra/.github/actions/teardown-windows diff --git a/TESTING_AND_CI_DESIGN.md b/TESTING_AND_CI_DESIGN.md new file mode 100644 index 0000000000..a73eafaad1 --- /dev/null +++ b/TESTING_AND_CI_DESIGN.md @@ -0,0 +1,392 @@ +# Torch-TensorRT — Testing & CI Design + +> **Status:** proposal / north-star design. Some pieces already exist on `main` +> (marked **✓ exists**); the rest is the target end-state and a rollout plan. +> +> **One-line goal:** *a developer can run exactly what CI runs, locally, with one +> command — and when something fails, the failure tells them how to reproduce and +> fix it.* + +--- + +## 1. Goals & non-goals + +**Goals** +- **Full coverage** of torch-tensorrt: converters, runtime, lowering, dynamo, torch-compile, torchscript, plugins, models, distributed — across the platforms/CUDA/Python we ship. +- **Ergonomic for developers**: the local experience is the *primary* design target, not an afterthought bolted onto CI. +- **Zero local/CI drift**: the command that runs a tier in CI is the *same* command you run locally. +- **Fast feedback by default, full coverage on demand**: a PR gets a signal in ~15 min; the exhaustive matrix runs when it matters. +- **Failures are never hidden**: nothing gets clobbered, every failure prints its own reproduce command, and the aggregate is machine- and agent-readable. + +**Non-goals** +- Replacing the PyTorch test-infra build plumbing (we reuse it for wheels/runners). +- A bespoke remote-execution cluster (we explicitly assume **no owned cache/RBE infra**). +- 100% of the matrix on every push (that's what nightly + the full lane are for). + +--- + +## 2. Design principles — the "pleasant" contract + +These are the invariants. Every decision below serves one of them. + +1. **One command to run anything.** `just ` — never a 200-char `pytest` incantation. +2. **Local == CI.** Suites are a declarative manifest + one runner; `just` and CI both call `ci run `, so there's nothing to drift. +3. **No rebuild to test.** Running tests never triggers a Bazel rebuild. +4. **Reproduce in one line.** Every CI failure prints `uv run --no-sync pytest … -n0 `. +5. **Build once, test many.** One wheel per (platform, CUDA, Python); every tier reuses it. +6. **Fail loud, aggregate everything.** One consolidated, agent-friendly report; no silent truncation; no fragile third-party glue. +7. **Cheap to retrigger, easy to discover.** Re-run via a PR comment; a menu tells you how. +8. **Pay for the signal you need.** Tiers + lanes + path filters; don't run GPU jobs for a docs typo. +9. **Flakes never mask real failures.** Retries are scoped to known signatures; everything else is quarantined with a tracking issue. + +--- + +## 3. Mental model + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ LAYER 0 — Test logic (platform-agnostic, the single source) │ +│ tests/ci/suites.py SUITES = [Suite(...)] manifest (DATA)│ +│ tests/ci/__main__.py `ci run ` · `ci matrix` one engine │ +│ @pytest.mark.{smoke,critical} lane membership on the test │ +│ tests/py/utils/junit_summary.py aggregate + --agent report ✓ │ +│ tests/py/utils/ci_helpers.sh env + trt_pytest wrapper only │ +│ pyproject.toml markers, optional dep groups │ +└───────────────┬───────────────────────────────┬───────────────────┘ + │ calls "ci run " │ matrix from "ci matrix" + ┌───────▼────────┐ ┌───────▼─────────────────┐ + │ LAYER 1 — LOCAL │ │ LAYER 1 — CI │ + │ justfile ✓ │ │ _build.yml / _test.yml│ + │ uv, jj fix │ │ ci.yml (lanes) │ + └────────────────┘ └───────┬─────────────────┘ + │ + ┌────────▼─────────┐ + │ LAYER 2 — STATUS │ + │ rollup check │ + │ PR comment(--agent)│ + │ /rerun, menu ✓ │ + └──────────────────┘ +``` + +The point: **Layer 0 is the contract — and it is _data + one engine_, not a pile of +shell functions.** A declarative **suite manifest** (`tests/ci/suites.py`) says *what* +each suite is; a single **runner** (`tests/ci/__main__.py`) is the *only* place that knows +*how* to turn a suite into a `pytest` command. `just` and CI are both thin callers of +`ci run `, and CI's job matrix is *generated* by `ci matrix` — so they cannot drift, +and adding a job is a one-line data change (§5.3). + +--- + +## 4. Test taxonomy + +### 4.1 Two orthogonal axes (retiring L0/L1/L2-by-filename) + +A job is a cell in **(subsystem × lane × variant)** — three *independent* axes. The old +`L0/L1/L2` tier numbers fused "what subsystem" with "how deep," and worse, encoded depth +in **filenames** (`runtime/test_000_*` = L0, `runtime/test_001_*` = L1, the rest = L2 via +`-k "not test_000_…"`). That's retired. The axes are now: + +- **Subsystem** *(what — a directory)*: `converters`, `runtime`, `lowering`, `partitioning`, `dynamo-models`, `torch-compile`, `torchscript`, `plugins`, `kernels`, `quantization`, `llm`, `distributed`, `executorch`. +- **Lane** *(how deep — a marker)*: `fast` (= `-m smoke`, every push), `full` (default, the ready-signal lane), `nightly` (everything + perf), plus `python-only` (a build-mode lane: the `PYTHON_ONLY=1` wheel validated against the runtime suite). +- **Variant** *(where — a dimension)*: `standard` / `rtx` — applied centrally by the runner, **not** as `if [ "$USE_TRT_RTX" ]` branches scattered through shell. +- **Platform / channel** *(another dimension)*: `linux-x86_64` / `windows` for tests (SBSA/aarch64 is build-only — no GPU test runners). One reusable per OS family (`uses:` must be literal); the suite *selection* is platform-agnostic. + +| | fast (every push) | full (ready signal) | nightly | +|---|---|---|---| +| converters | smoke subset | all | + fuzz | +| runtime / lowering / partitioning | smoke subset | all | — | +| dynamo-models / torch-compile | — | `-m critical` | all + llm | +| torchscript | api smoke | models | integrations | +| plugins / kernels / quantization | — | — | all (+optional deps) | +| distributed | — | — | multi-GPU | + +Depth lives on the test (a marker), so **moving a test between lanes is editing a +decorator, never renaming a file.** Sharding a big subsystem is a manifest field +(`shards: N`), not a `test_000_*` filename convention. + +### 4.2 Lane membership = markers, uniformly (`pyproject.toml`) +Membership is expressed *one* way — a marker on the test — never filename prefixes or +`-k "not test_000_…"` strings (today three different mechanisms coexist for the same idea). +- `smoke` *(proposed)* — the fast-lane subset of any subsystem. +- `critical` ✓ — the `full`-lane core; nightly runs the complement (`-m "not critical"`). Clean partition, no overlap, no gap. +- `unit` ✓ — the bulk of the suite. +- `flaky` *(proposed)* — quarantined out of gating lanes, tracked by an issue, still run nightly for visibility. + +### 4.3 Optional dependency groups (`-ext`) +`test-ext`, `kernels`, `quantization` (uv groups) + `executorch` (plain package). Suites that need them **skip cleanly when absent** (e.g. `conftest.py` `skip_no_cuda_core`), so a base checkout never hard-fails. `just install-test-ext` pulls them all. + +### 4.4 Directory layout (`tests/py/`) +``` +tests/ + ci/ # ← NEW: the Layer-0 brain — suites.py (manifest) + __main__.py (runner) + py/ + dynamo/{conversion,lowering,runtime,partitioning,models,hlo,llm,executorch,…} + ts/ # torchscript frontend + kernels/ quantization/ # gated on optional deps + distributed/ # multi-GPU + utils/ # ci_helpers.sh (env + trt_pytest wrapper), junit_summary.py +``` + +--- + +## 5. Layer 0 — the suite manifest + one runner + +This replaces the bash `trt_tier_*` library as the source of truth. The bash version +*works*, but config-in-shell conflates data with mechanism and quietly hides errors. Today +it: tiers tests by **filename prefix**; mixes markers / `-k` strings / globs for the same +"depth" concept; drops the L2 suites out of the `trt_pytest` rerun+repro wrapper (so L2 +failures get no repro hint); and — really — points **four** pytest runs at one +`--junitxml` path so three suites' results vanish from the aggregate. Those are all +symptoms of the same root cause, fixed by separating data from engine. + +### 5.1 The manifest — `tests/ci/suites.py` *(data; typed, mypy-validated)* +```python +@dataclass(frozen=True) +class Suite: + name: str # "dynamo-runtime" + paths: tuple[str, ...] # ("dynamo/runtime",) under tests/py + lanes: tuple[Lane, ...] = ("full",) # fast | full | nightly + shards: int = 1 # split a big suite across N CI jobs + dist: str | None = None # "--dist=loadscope" + needs: tuple[str, ...] = () # optional dep groups: ("kernels", "cuda-core") + variants: tuple[Variant, ...] = ("standard", "rtx") + extra_args: tuple[str, ...] = () # ("--ir", "torch_compile"), ("--maxfail", "20") + +SUITES = [ + Suite("dynamo-converters", ("dynamo/conversion",), lanes=("fast", "full"), + dist="--dist=loadscope", shards=4), + Suite("dynamo-runtime", ("dynamo/runtime",), lanes=("fast", "full")), + Suite("dynamo-models", ("dynamo/models",), lanes=("full", "nightly")), + Suite("kernels", ("kernels",), lanes=("nightly",), + needs=("kernels", "cuda-core"), variants=("standard",)), + # … one line per subsystem +] +``` +A typo'd field or path is a **static error**, not a silently-empty glob. (YAML + a pydantic +schema is the friendlier-for-non-coders alternative; we pick Python for the static checking +and because the same module generates the CI matrix.) + +### 5.2 The runner — `tests/ci/__main__.py` *(the ONLY place pytest mechanics live)* +```bash +python -m tests.ci matrix --lane fast --variant standard # → JSON for the GH Actions matrix +python -m tests.ci run dynamo-runtime --shard 2/4 # builds + runs pytest +``` +The runner — and nothing else — knows how to: apply the lane's marker (`fast` → `-m smoke`), +derive a *unique* junit path from `name+shard` (no more collisions), wrap **every** suite in +the `trt_pytest` reruns+repro helper *uniformly*, apply `--dist`/`extra_args`, gate on `needs` +(skip cleanly if a dep group is absent), and resolve variants centrally. Its knobs: + +| Knob | Default | Meaning | +|---|---|---| +| `PYTHON` | `python` | CI: container python; local: `uv run --no-sync python` (no rebuild) | +| `TRT_JOBS` | per-suite | xdist `-n`; **GPU-memory-aware, not `-n auto`** (one GPU OOMs long before it runs out of cores) | +| `TRT_PYTEST_RERUNS` | `1` (CI) / `0` (local) | gated reruns for known flake signatures only | +| `RUNNER_TEST_RESULTS_DIR` | `$TMPDIR/trt_test_results` | where JUnit XMLs land | +| `TMPDIR` | per-user | engine/timing cache; per-user to dodge cross-user permission collisions | + +> **Lesson baked in:** rerun args are a Python list passed to `subprocess` (or, in the +> shrunken shell wrapper, a bash *array*) — **never** an unquoted string. A multi-word +> `--only-rerun "Stream capture invalidated"` expanded unquoted word-splits into phantom +> test paths and silently collects **0 tests** — this is exactly how a real CI run failed. + +### 5.3 Adding a job — one line +- **Before** (bash): write a `trt_tier_*` function in `ci_helpers.sh` + a `just` recipe + a matrix entry in `_test.yml` + a `tests-report` tier-list entry — *4 edits, 3 files, in bash.* +- **After**: add one `Suite(...)` to `SUITES`. `just` exposes it, CI's matrix includes it (it's generated), the report aggregates it — **automatically.** + +### 5.4 `tests/py/utils/ci_helpers.sh` — shrunk, not deleted ✓exists +Keeps only the genuinely-shell bits the runner shells out to: env setup and the +array-safe `trt_pytest` wrapper. No tier definitions, no selection logic. + +### 5.5 `tests/py/utils/junit_summary.py` ✓exists +Reads all JUnit XMLs and emits a **human** report (TTY colors, `NO_COLOR`/`FORCE_COLOR`) and +an **`--agent`** report: Markdown with node id, file, junit path, a copy-paste +`uv run --no-sync pytest -k -n0` repro, message, and capped detail — *paste it +to Claude and it can start fixing.* Exits non-zero on any failure **or empty result set** +(the XMLs are the source of truth for pass/fail). No third-party result actions that crash +on empty input. + +--- + +## 6. Local developer experience + +### 6.1 The golden path +```bash +just test tests/py/dynamo/conversion/test_foo.py # inner loop: one file, fast +just tests-l0 # the smoke tier, exactly as CI runs it +just tests-report l1-ext --agent # run a whole tier past failures → paste-ready report +just lint # all pre-commit hooks (== the linter CI job) +``` +Everything is `uv run --no-sync` underneath — **tests never rebuild torch-tensorrt.** + +### 6.2 Recipes (justfile — a thin caller of the runner) +- `test *args` — raw pytest in the uv env (honors `pyproject` addopts). +- `suite [-- pytest args]` — run one manifest suite, *exactly* as CI runs it (`ci run `). +- `lane ` — run every suite in a lane locally. +- `report [--agent]` — **run a lane past failures, then print one consolidated report** (run + report in one step). +- `summary [--agent]` — re-print the last run's report without re-running. +- `lint` / `lint-changed` — pre-commit over all / changed files. +- `build` *(proposed)* — wrap the clean-rebuild flow; auto-detect ABI staleness after a nightly bump. +- `jobs=N` knob — raise xdist parallelism when your GPU has headroom (`just jobs=8 lane full`). + +During migration the legacy `tests-l0/l1/l2` recipes become thin aliases for the matching lanes, then retire. + +### 6.3 Build ergonomics +- `uv pip install -e . --no-deps --no-build-isolation` for incremental; **clean rebuild after a torch-nightly bump** (libtorch ABI changes → `undefined symbol` otherwise). +- `PYTHON_ONLY=1` for pure-Python iteration (skips Bazel entirely). +- A `build` skill / recipe encodes the decision tree so nobody re-learns it. + +### 6.4 Formatting is automatic, not a chore +- `pre-commit` for the full gate (the checks: mypy, validate-pyproject, uv-lock, …). +- **`jj fix`** wired to the *formatters* (black, ruff, isort, clang-format, buildifier) as stdin→stdout filters, version-pinned via `uvx` to the exact pre-commit revs → format an entire stack in one shot, no per-commit `pre-commit` dance. + +### 6.5 The failure→fix loop +Any failing tier → `just test-summary --agent` → the `analyze-test-report` skill triages (real bug vs torch-API change vs OOM/skip vs flake) and drives each to a fix using the printed repro commands. + +--- + +## 7. CI design + +### 7.1 Lanes — *without a merge queue* + +We **cannot** use GitHub's merge queue, so the full suite is gated by an explicit +"ready" signal instead of a queue: + +| Lane | Trigger | What runs | Required? | +|---|---|---|---| +| **Fast** | every PR push | lint + **1 representative build** (py-latest + newest CUDA, standard) + **L0** | informational | +| **Full** | `ci: full` label · `/ci full` comment · **approval** (`pull_request_review`) | full matrix · L1/L2 · RTX · all platforms | **yes** (`CI / full` rollup) | +| **Main canary** | `push` to `main` | full lane | — (catches trunk breakage → fast revert) | +| **Nightly** | `schedule` | full + `-ext` model/kernels/quant + perf + exhaustive CUDA/Python | — | + +**Gating mechanics.** Branch protection requires `CI / full`. It only reports after +the full lane runs (on label/approval). Pair with **"dismiss stale approvals on new +commits"** so a post-approval push invalidates approval → re-approve → full re-runs on +the new HEAD. + +> **Honest gap.** Without a merge queue we lose the guarantee that the *actually-merged* +> tree (after other PRs land) was tested as a unit — two independently-green PRs can break +> `main` together. Mitigation: (a) require PRs rebased on fresh `main` before the full run, +> (b) the main canary + fast revert. This is the inherent cost of no queue; we accept it. + +### 7.2 Topology — build once, test many ✓(x86_64 today) + +``` +ci.yml (one entry: pull_request | pull_request_review | push | schedule | comment) + ├─ _build.yml matrix{platform, cuda, python, variant} → ONE wheel artifact each + └─ _test.yml download wheel → run tier(s) via ci_helpers.sh → JUnit artifact + └─ rollup job aggregate JUnit → single status/platform + PR comment +``` + +Platform / RTX / python-only stop being **separate workflow files** and become +**matrix inputs**. This collapses today's ~11 near-duplicate entry workflows (incl. the +461-line inline Windows files) into `_build.yml` + `_test.yml` + a thin `ci.yml`. + +### 7.3 Caching — *without owned infra* + +We assume **no remote cache/RBE server**. Use GitHub's own cache, two layers, both wired +into the **vendored** build workflow (it's a local copy, so we can edit its steps): + +1. **sccache with the GitHub Actions backend** (`SCCACHE_GHA_ENABLED=true`) — free, no infra; caches C++ object compiles. Biggest win for Bazel C++ recompiles. +2. **Bazel `--disk_cache=` persisted via `actions/cache@v4`** — key on hashes of `MODULE.bazel` / `.bazelversion` / **torch-nightly + CUDA version** + `restore-keys` for partial hits. + +**Container caveat:** the build runs inside a manylinux container; `actions/cache` runs on +the host. Either **bind-mount** a host dir as the Bazel disk-cache path (no tokens cross +the boundary — preferred), or forward `ACTIONS_CACHE_URL`/`ACTIONS_RUNTIME_TOKEN` into the +container for sccache. + +**Warming model (forks for free):** populate caches on `push` to `main` (trusted, can +write); PRs — **including forks** — restore **read-only**. Forks get no secrets and can't +write the base cache, but they *can* read main's warm cache, so a fork PR still builds +incrementally with zero secret/infra exposure. + +> Caveats: GHA cache is **10 GB/repo, LRU-evicted** — scope it (sccache's compact object +> cache fits better than a raw disk_cache). **Key on the nightly/CUDA version** so a bump +> busts it — a stale cache here is exactly the ABI-mismatch class of bug. Step-up option if +> limits bite: BuildBuddy's free *hosted* tier (just an API key; non-fork PRs + main). + +### 7.4 Status & reporting +- **One rollup check per platform** (`CI / Linux x86_64`, …) summarizes its child jobs → branch protection keys on a *stable* name; reviewers see one green/red + a Markdown table. ✓exists +- JUnit from every job is aggregated by `junit_summary.py` into **one artifact + an auto PR comment** (the `--agent` report). Nothing clobbered, nothing missed. +- `--fail-on-empty` so "0 tests collected" is a failure, not a silent green. + +### 7.5 Ergonomic CI commands +- **`/rerun`** / **`/rerun all`** — re-run failed/cancelled or everything, no new commit (write-access gated). ✓exists +- **PR command menu** — on PR open, a comment lists the commands + local-repro recipes (so contributors discover them). ✓built +- **`/ci full`** *(proposed)* — opt a PR into the full lane (the merge-queue substitute trigger). + +### 7.6 Flake handling +- **Gated reruns for known signatures only** (`--only-rerun cudaErrorStreamCaptureInvalidated`, "Stream capture invalidated"). Never a blanket retry — that masks real failures. +- **Quarantine** `@pytest.mark.flaky` out of gating lanes with a tracking issue; still run nightly for visibility. +- **Cache model weights** as an artifact → kills the corrupted-download flakes (e.g. the mobilenet `unexpected EOF` torchscript failure). + +### 7.7 Concurrency & path filters +- `concurrency: cancel-in-progress` keyed on PR ref — never burn runners on superseded commits. +- **Path filters**: docs-only PR → skip GPU/C++; `.py`-only → skip the C++ test tier. (The single biggest waste-cutter on trivial PRs.) +- Skip CI on draft PRs; fast lane on "ready for review". + +--- + +## 8. Target file layout + +``` +.github/workflows/ + ci.yml # the ONLY build+test entry: lanes by event + _build.yml # reusable: matrix → wheel artifact (all platforms/variants) + _test.yml # reusable: wheel → suite(s) → JUnit; matrix GENERATED by `ci matrix` + nightly.yml # exhaustive matrix + -ext + perf + retrigger-ci.yml # /rerun ✓exists + pr-command-menu.yml # discoverability comment on open ✓built + linter.yml # pre-commit gate + release-*.yml # publish on tags (unchanged) +tests/ci/ + suites.py # ← the manifest (DATA): SUITES = [Suite(...)] + __main__.py # ← the runner/engine: `ci run` · `ci matrix` +tests/py/utils/ + ci_helpers.sh # SHRUNK: env + array-safe trt_pytest wrapper only ✓exists + junit_summary.py # aggregation + --agent report ✓exists +justfile # local entry — thin caller of `ci run`/`ci matrix` ✓exists +pyproject.toml # markers + optional dep groups ✓exists +TESTING_AND_CI_DESIGN.md # this document +``` + +**Deletions this enables:** the ~11 per-platform entry workflows, the parallel +schedule/dispatch build-test set (the source of the old/new *double-run*), and the +inline 461/381-line Windows workflows — all collapse into `_build.yml` + `_test.yml`. + +--- + +## 9. Rollout plan (incremental, each step shippable + verified) + +Use a **byte-equivalence harness** at each step — diff `ci run `'s emitted `pytest` +command against the bash tier it replaces, and render the workflow's generated script — +before flipping it on. Prove we didn't change *what runs*, only *how it's wired*. + +0. **Kill the double-run.** Delete/disable the superseded schedule/dispatch build-test set so a PR runs *one* set of jobs. (Halves PR cost, removes the confusion that surfaced in #4352.) +1. **Stand up the manifest + runner.** Port the `trt_tier_*` functions into `tests/ci/suites.py` + `tests/ci/__main__.py` one suite at a time, each producing a byte-identical command to the bash tier it replaces. Add `smoke` markers to replace the `test_000_*`/`test_001_*` filename tiering; shrink `ci_helpers.sh` to env + wrapper. +2. **Generalize the core** → `_build.yml` + `_test.yml` with a `platform`/`variant` input; `_test.yml`'s matrix is generated by `ci matrix`. Fold RTX + python-only in as variants. +3. **Migrate aarch64 + Windows** onto the same reusables (deletes the inline Windows files). +4. **Lanes** — add `ci.yml` orchestrator + the fast/full split + approval/label/`/ci full` triggers + `concurrency`. +5. **Nightly** — `nightly.yml` for the exhaustive matrix + `-ext`; trim the PR lane to fast. +6. **Caching** — sccache + Bazel disk_cache via GHA cache, warmed on main. +7. **Polish** — path filters, weight-cache, flake quarantine, drop fragile result actions. + +--- + +## 10. Open decisions + +- **Runner availability** for GPU jobs (how many parallel full lanes can we afford?). +- **GHA cache budget** — is 10 GB enough, or do we want BuildBuddy's free hosted tier? +- **Required-check names** for branch protection (must stay stable across the migration). +- **Fork policy** — confirm fork PRs are read-only on cache and never see secrets. +- **`/ci full` vs approval-only** as the full-lane trigger (or both). + +--- + +## 11. What "pleasant" feels like, end-to-end + +> Open a PR → a comment greets you with the command menu → a **fast green check in ~15 min**. +> Iterate on the fast lane. When ready, a reviewer approves (or you comment `/ci full`) → the +> full suite runs once. A failure? **One PR comment** with the exact `uv run --no-sync pytest …` +> for each failing test — paste it to Claude, or run it locally via `just`. Flaky? `/rerun`. +> Formatting? `jj fix`. You never assembled a `pytest` command by hand, never waited on a job +> irrelevant to your change, and never had a real failure hidden behind a flaky one. diff --git a/justfile b/justfile index c0f31ed1f7..54df4ff6e1 100644 --- a/justfile +++ b/justfile @@ -1,4 +1,4 @@ -# Recipes source tests/py/utils/ci_helpers.sh (a bash library), so run them in bash. +# Recipes use bash (shebang recipes + the _ci launcher), so run them in bash. set shell := ["bash", "-cu"] # List all available recipes @@ -11,19 +11,26 @@ default: # PermissionError. Giving each user their own TMPDIR sidesteps that entirely. export TMPDIR := env_var_or_default("TMPDIR", "/tmp/torch_tensorrt_" + env_var_or_default("USER", "local")) -# pytest xdist parallelism for the parallel suites. Defaults to 2: these tiers +# pytest xdist parallelism for the parallel suites. Defaults to 4: these suites # build TensorRT engines, which are GPU-memory-heavy, and a single local GPU # OOMs (CUDA out-of-memory + segfaulting workers) well before it runs out of -# CPU cores — so `-n auto` is the wrong default here. CI runs 8 on dedicated -# GPU runners. Raise it if your GPU has headroom: just jobs=8 l0 +# CPU cores — so `-n auto` is the wrong default here. CI runs more on dedicated +# GPU runners. Raise it if your GPU has headroom: just jobs=8 lane full jobs := "4" -# Launcher + env shared by every tier recipe. The tier definitions themselves -# live in tests/py/utils/ci_helpers.sh — the SAME functions CI calls — so there is a -# single source of truth for what each tier runs. We only set environment -# policy here: PYTHON runs pytest against the already-built .venv (uv --no-sync, -# no rebuild) and TRT_JOBS feeds the parallel suites. -_tier := 'mkdir -p "$TMPDIR" && source tests/py/utils/ci_helpers.sh && export PYTHON="uv run --no-sync python" TRT_JOBS="' + jobs + '" TRT_PYTEST_RERUNS=0 &&' +# Which backend variant to run: standard | rtx. This selects the test SELECTION +# (RTX drops --dist on converters, runs the whole partitioning dir, etc.) — it +# does NOT switch the installed build. To actually run RTX locally you must have +# a torch-tensorrt built with USE_TRT_RTX=1 installed in the .venv. +# just variant=rtx lane fast +variant := "standard" + +# Launcher + env for the suite recipes. Suite definitions live in the manifest +# (tests/ci/suites.py) — the SAME data CI uses — so local and CI cannot drift. +# We only set environment policy here: PYTHON runs pytest against the already-built +# .venv (uv --no-sync, no rebuild), TRT_JOBS feeds xdist, and reruns are off +# locally (the rerunfailures plugin may be absent and you want to SEE flakes). +_ci := 'mkdir -p "$TMPDIR" && PYTHON="uv run --no-sync python" TRT_JOBS="' + jobs + '" TRT_PYTEST_RERUNS=0 uv run --no-sync python -m tests.ci' # ── Testing ─────────────────────────────────────────────────────────────────── @@ -34,148 +41,52 @@ test *args: @mkdir -p "$TMPDIR" uv run --no-sync pytest {{args}} -# ── CI tier reproduction ────────────────────────────────────────────────────── +# ── Suite manifest (tests/ci) — the single source of truth ────────────────────── # -# Run exactly what a CI tier runs, before pushing. Each recipe calls the tier -# function from tests/py/utils/ci_helpers.sh — the same one .github/workflows/ -# _linux-x86_64-core.yml invokes — so local and CI cannot drift. Extra args are -# forwarded to pytest, e.g. `just tests-l0-core -x -k test_foo`. -# (Standard-TensorRT scope; export USE_TRT_RTX=true before running for RTX.) - -# Full L0 smoke tier -tests-l0: tests-l0-converter tests-l0-core tests-l0-py-core tests-l0-torchscript - -tests-l0-converter *args: - {{_tier}} trt_tier_l0_converter {{args}} - -tests-l0-core *args: - {{_tier}} trt_tier_l0_core {{args}} - -tests-l0-py-core *args: - {{_tier}} trt_tier_l0_py_core {{args}} - -tests-l0-torchscript *args: - {{_tier}} trt_tier_l0_torchscript {{args}} - -# Full L1 tier -tests-l1: tests-l1-dynamo-core tests-l1-dynamo-compile tests-l1-torch-compile tests-l1-torchscript - -tests-l1-dynamo-core *args: - {{_tier}} trt_tier_l1_dynamo_core {{args}} - -tests-l1-dynamo-compile *args: - {{_tier}} trt_tier_l1_dynamo_compile {{args}} - -tests-l1-torch-compile *args: - {{_tier}} trt_tier_l1_torch_compile {{args}} - -tests-l1-torchscript *args: - {{_tier}} trt_tier_l1_torchscript {{args}} +# CI and these recipes both call `python -m tests.ci`, so what runs locally is +# exactly what runs in CI. See TESTING_AND_CI_DESIGN.md. + +# Validate the suite manifest (unique names/junit paths, valid setup steps) +doctor: + uv run --no-sync python -m tests.ci doctor + +# List every suite with its tier, lanes, and variants +suites: + uv run --no-sync python -m tests.ci list + +# Run ONE suite exactly as CI runs it (uses the {{variant}} backend). Args after `--`: +# just suite dynamo-runtime -- -k test_foo -x just variant=rtx suite dynamo-converters +suite name *args: + {{_ci}} run {{name}} --variant {{variant}} {{args}} + +# Run every suite in a LANE (fast|full|nightly), continuing past failures: +# just lane fast just variant=rtx lane fast +lane name *args: + {{_ci}} run-lane --lane {{name}} --variant {{variant}} {{args}} + +# Run a lane past failures, then print one consolidated report (--agent for Claude): +# just report fast --agent +report lane *summary_args: + #!/usr/bin/env bash + set -uo pipefail + mkdir -p "$TMPDIR" + results="${RUNNER_TEST_RESULTS_DIR:-$TMPDIR/trt_test_results}" + rm -f "$results"/*.xml 2>/dev/null || true + export PYTHON="uv run --no-sync python" TRT_JOBS="{{jobs}}" TRT_PYTEST_RERUNS=0 + uv run --no-sync python -m tests.ci run-lane --lane {{lane}} --variant {{variant}} || true + uv run --no-sync python tests/py/utils/junit_summary.py "$results" {{summary_args}} -# Pulls every optional test-dep group so the model / kernels / quantization / -# executorch suites run instead of skipping. Uses `uv pip install` (not `uv -# sync`) so it ADDS the deps without rebuilding torch-tensorrt or tearing down -# the env — this also sidesteps the test_ext↔quantization `uv sync` conflict, -# which only applies to lockfile resolution. executorch has no group, so it is -# installed as a plain package. +# Re-print the LAST run's consolidated report without re-running (--agent for Claude) +summary *args: + uv run --no-sync python tests/py/utils/junit_summary.py "${RUNNER_TEST_RESULTS_DIR:-$TMPDIR/trt_test_results}" {{args}} -# Install all optional test deps (test-ext + kernels + quantization + executorch) +# Added without a rebuild via `uv pip install --group` (sidesteps the +# test-ext↔quantization `uv sync` lockfile conflict). Run before `just lane full`. +# Install optional test deps so model/kernels/quantization/executorch suites run install-test-ext: uv pip install --group test-ext --group kernels --group quantization uv pip install pyyaml "executorch>=1.3.1" -# Full L1 tier + test-ext deps, so model-level cases run instead of skipping -tests-l1-ext: install-test-ext tests-l1-dynamo-core tests-l1-dynamo-compile tests-l1-torch-compile tests-l1-torchscript - -# Excludes tests-l2-distributed (needs 2+ GPUs and system MPI). Most L2 suites -# are model-level, so run tests-l2-ext (or `just install-test-ext` first) so -# they don't skip. - -# Full L2 tier (locally runnable suites) -tests-l2: tests-l2-torch-compile tests-l2-dynamo-compile tests-l2-dynamo-core tests-l2-plugin tests-l2-torchscript - -tests-l2-torch-compile *args: - {{_tier}} trt_tier_l2_torch_compile {{args}} - -tests-l2-dynamo-compile *args: - {{_tier}} trt_tier_l2_dynamo_compile {{args}} - -# Also installs executorch (additively) for the executorch/ suite. -tests-l2-dynamo-core *args: - {{_tier}} trt_tier_l2_dynamo_core {{args}} - -# Also installs cuda-python/cuda-core (additively) for the kernels/ QDP suite. -tests-l2-plugin *args: - {{_tier}} trt_tier_l2_plugin {{args}} - -tests-l2-torchscript *args: - {{_tier}} trt_tier_l2_torchscript {{args}} - -# Installs mpich/openmpi via dnf (root-capable container) and runs -# --nproc_per_node=2. Not part of the `tests-l2` aggregate. - -# CI-only: needs 2+ GPUs and system MPI -tests-l2-distributed *args: - {{_tier}} trt_tier_l2_distributed {{args}} - -# Full L2 tier + test-ext deps, so model-level cases run instead of skipping -tests-l2-ext: install-test-ext tests-l2-torch-compile tests-l2-dynamo-compile tests-l2-dynamo-core tests-l2-plugin tests-l2-torchscript - -# ── Run-all + consolidated failure report ───────────────────────────────────── - -# Unlike `just tests-l` (which aborts on the first failing suite), this runs -# every suite and aggregates the JUnit XMLs, so a single consolidated report -# shows all failures — nothing gets clobbered or missed. Append `-ext` to also -# install the test-ext deps first so model-level cases run instead of skipping. -# Pass `--agent` to print the paste-to-Claude Markdown report instead of the -# terminal one — i.e. run + report in one step (e.g. `just tests-report l1-ext -# --agent`). Exits non-zero if anything failed/errored. - -# Run a whole tier (l0|l1|l2[-ext]) past failures, then print one report [--agent] -tests-report level *report_args: - #!/usr/bin/env bash - set -uo pipefail # deliberately no -e: keep going past failures - mkdir -p "$TMPDIR" - # Accept an optional `-ext` suffix: install the test-ext group first. - lvl="{{level}}" - ext=0 - if [[ "$lvl" == *-ext ]]; then ext=1; lvl="${lvl%-ext}"; fi - case "$lvl" in - l0) tiers=(l0_converter l0_core l0_py_core l0_torchscript) ;; - l1) tiers=(l1_dynamo_core l1_dynamo_compile l1_torch_compile l1_torchscript) ;; - l2) tiers=(l2_torch_compile l2_dynamo_compile l2_dynamo_core l2_plugin l2_torchscript) ;; - *) echo "unknown level '{{level}}' (use l0|l1|l2, optionally with -ext)" >&2; exit 2 ;; - esac - if [[ "$ext" == 1 ]]; then - # Same set as `just install-test-ext`: pull every optional test-dep group - # so the model / kernels / quantization / executorch suites run instead of - # skipping. cuda-core resolves via uv here (the plugin tier's vanilla - # `pip install cuda-core` cannot fetch it on all hosts). - echo "==> installing test-ext + kernels + quantization + executorch deps" - uv pip install --group test-ext --group kernels --group quantization \ - || { echo "test-ext group install failed" >&2; exit 1; } - uv pip install pyyaml "executorch>=1.3.1" || true - fi - results="${RUNNER_TEST_RESULTS_DIR:-$TMPDIR/trt_test_results}" - rm -f "$results"/*.xml 2>/dev/null || true # drop stale results from prior runs - source tests/py/utils/ci_helpers.sh - export PYTHON="uv run --no-sync python" TRT_JOBS="{{jobs}}" TRT_PYTEST_RERUNS=0 - for t in "${tiers[@]}"; do - echo "==> trt_tier_$t" - "trt_tier_$t" || echo "::: trt_tier_$t exited non-zero (continuing) :::" - done - # Source of truth is the JUnit XMLs, not exit codes; this sets the final - # status. Extra args (e.g. --agent) are forwarded to the summary. - python3 tests/py/utils/junit_summary.py "$results" {{report_args}} - -# Reads the JUnit XMLs from the previous run (does not re-run). Pass --agent for -# a plain Markdown report to hand to Claude (`just test-summary --agent`). Exits -# non-zero if that run had failures. - -# Print the consolidated failure summary from the last run's JUnit results -test-summary *args: - python3 tests/py/utils/junit_summary.py {{args}} - # ── Linting ─────────────────────────────────────────────────────────────────── # Run all pre-commit hooks across the repo (matches the linter CI job) @@ -203,3 +114,18 @@ docs-serve port="3000": # Build docs then serve them docs-build-serve port="3000": docs python3 -m http.server {{port}} --directory docs + +# ── CI dashboard ────────────────────────────────────────────────────────────── + +# Local web UI for GitHub Actions CI. Per-platform pass/fail, a python×cuda×tier +# grid, failing tests mapped back to source (file:line), a copy-paste local +# repro, and a cross-platform failure rollup. Needs an authenticated `gh` +# (check with `gh auth status`). Defaults to the current branch; override with +# branch= and port=, e.g. just ci branch=nightly | just ci branch=main port=9000 +# +# Uses `uv run --no-sync` on purpose: the dashboard is stdlib-only and has no +# torch-tensorrt dependency, so --no-sync skips the project build entirely and +# just reuses the existing .venv interpreter. Binds 0.0.0.0 so it's reachable +# over Tailscale. +ci branch="" port="8712": + uv run --no-sync tools/ci-dashboard/ci_dashboard.py -b "{{branch}}" -p {{port}} diff --git a/pyproject.toml b/pyproject.toml index 88d9bbc743..b88fc82931 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -143,6 +143,8 @@ addopts = "-n auto --dist=loadfile" markers = [ "unit: a focused unit test (the bulk of the suite)", "critical: a smoke/critical-path case run in the L1 tier (pytest -m critical); the L2 tier runs the complement (-m 'not critical')", + "smoke: the fast-lane subset of a subsystem (pytest -m smoke); run on every push by the `fast` lane", + "flaky: a known-flaky test, quarantined out of the gating lanes and tracked by an issue; still run in nightly", ] norecursedirs = [ "bazel-*", @@ -156,6 +158,7 @@ norecursedirs = [ "tools", "modules", "utils", + "ci", ] [tool.uv] diff --git a/setup.py b/setup.py index 91651c7e15..3b170e16d5 100644 --- a/setup.py +++ b/setup.py @@ -270,6 +270,16 @@ def build_libtorchtrt_cxx11_abi( else: cmd.append("--platforms=//toolchains:ci_rhel_x86_64_linux") + # Persist bazel's action cache across builds when a cache dir is provided. + # The C++/CUDA compile is the serial long pole; with a warm --disk_cache an + # unchanged (or partially-changed) tree reuses cached action outputs, turning + # a from-scratch rebuild into seconds. CI sets BAZEL_DISK_CACHE and restores/ + # saves the dir via actions/cache (coarse key); no-op locally unless opted in. + disk_cache = os.environ.get("BAZEL_DISK_CACHE") + if disk_cache: + cmd.append(f"--disk_cache={disk_cache}") + print(f"Using bazel --disk_cache={disk_cache}") + env = os.environ.copy() if "TORCH_PATH" not in env: stable_torch_path = resolve_torch_path() diff --git a/tests/ci/__init__.py b/tests/ci/__init__.py new file mode 100644 index 0000000000..20cec47fd1 --- /dev/null +++ b/tests/ci/__init__.py @@ -0,0 +1,16 @@ +"""Torch-TensorRT test-suite manifest + runner (the Layer-0 "brain"). + +This package is the single source of truth for *what each CI/local test job runs*. +It replaces the ``trt_tier_*`` bash functions in ``tests/py/utils/ci_helpers.sh``. + + * ``suites.py`` — the manifest: ``SUITES = [Suite(...), ...]`` (pure data). + * ``runner.py`` — the one engine that turns a ``Suite`` into a ``pytest`` command. + * ``__main__.py`` — the CLI: ``python -m tests.ci {list,show,run,matrix}``. + +Both ``just`` (local) and CI call ``python -m tests.ci run ``; CI's job +matrix is generated by ``python -m tests.ci matrix``. There is nothing to drift. +""" + +from .suites import SUITES, Lane, Suite, Tier, Variant + +__all__ = ["SUITES", "Suite", "Lane", "Tier", "Variant"] diff --git a/tests/ci/__main__.py b/tests/ci/__main__.py new file mode 100644 index 0000000000..362e46d788 --- /dev/null +++ b/tests/ci/__main__.py @@ -0,0 +1,208 @@ +"""CLI for the test-suite manifest: python -m tests.ci {list,show,run,matrix,doctor} + +list all suites, tiers, lanes, variants +show a suite's resolved command per variant +run [opts] [-- ...] run one suite (the call CI + just both make) +matrix [--lane|--tier] JSON matrix `include` for GitHub Actions +doctor validate the manifest (CI lints this) +""" + +from __future__ import annotations + +import argparse +import json +import sys + +from .runner import REPO_ROOT, describe, junit_path, matrix, run_suite, select +from .suites import SUITES, by_name + + +def _cmd_list(_: argparse.Namespace) -> int: + width = max(len(s.name) for s in SUITES) + print( + f"{'SUITE'.ljust(width)} TIER LANES VARIANTS PLATFORMS" + ) + for s in SUITES: + print( + f"{s.name.ljust(width)} {s.tier:<4} " + f"{','.join(s.lanes):<21} {','.join(s.variants):<15} {','.join(s.platforms)}" + ) + print( + f"\n{len(SUITES)} suites. " + f"Run one: python -m tests.ci run (or `just suite `)" + ) + return 0 + + +def _cmd_show(args: argparse.Namespace) -> int: + s = by_name(args.name) + print(f"# {s.name} (tier={s.tier}, lanes={','.join(s.lanes)})") + for var in s.variants: + print(f"\n## variant: {var} junit: {junit_path(s).name}") + print(describe(s, var)) + return 0 + + +def _cmd_run(args: argparse.Namespace) -> int: + s = by_name(args.name) + variants = [args.variant] if args.variant else list(s.variants) + if args.variant and args.variant not in s.variants: + print( + f"::warning::{s.name} does not run on variant {args.variant!r}; " + f"it runs on {s.variants}", + file=sys.stderr, + ) + return 0 + rc = 0 + for var in variants: + rc = run_suite(s, var, dry_run=args.dry_run, extra=args.pytest_args) or rc + return rc + + +def _read_changed(args: argparse.Namespace) -> list[str] | None: + """Changed-file list for path-based selection, from ``--changed-from`` (a + file path, or ``-`` for stdin; one repo-relative path per line). Returns + ``None`` when not supplied → no narrowing (run everything the filters select).""" + src = getattr(args, "changed_from", None) + if not src: + return None + if src == "-": + text = sys.stdin.read() + else: + with open(src, encoding="utf-8") as fh: + text = fh.read() + return [ln.strip() for ln in text.splitlines() if ln.strip()] + + +def _cmd_run_lane(args: argparse.Namespace) -> int: + """Run every suite in a lane/tier, continuing past failures (so one consolidated + report sees them all). Returns non-zero if any suite failed.""" + jobs = select( + lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform, + changed=_read_changed(args), + ) + if not jobs: + print("::warning::no suites match the given filters", file=sys.stderr) + return 0 + rc = 0 + for s, var in jobs: + rc = run_suite(s, var, dry_run=args.dry_run) or rc + return rc + + +def _cmd_matrix(args: argparse.Namespace) -> int: + include = matrix( + lane=args.lane, tier=args.tier, variant=args.variant, platform=args.platform, + changed=_read_changed(args), + ) + if not include: + print("::warning::matrix is empty for the given filters", file=sys.stderr) + print(json.dumps({"include": include})) + return 0 + + +def _cmd_doctor(_: argparse.Namespace) -> int: + """Static checks CI can gate on: unique names, unique junit paths, valid setup + steps, declared cwd dirs exist, every suite is reachable by some lane.""" + problems: list[str] = [] + names = [s.name for s in SUITES] + dupes = {n for n in names if names.count(n) > 1} + if dupes: + problems.append(f"duplicate suite names: {sorted(dupes)}") + + junits = [junit_path(s).name for s in SUITES] + jdupes = {j for j in junits if junits.count(j) > 1} + if jdupes: + problems.append(f"colliding junit paths: {sorted(jdupes)}") + + valid_setup = {"hub", "executorch", "cuda-core", "mpi"} + for s in SUITES: + for step in s.setup: + if step not in valid_setup: + problems.append(f"{s.name}: unknown setup step {step!r}") + if not s.lanes: + problems.append(f"{s.name}: belongs to no lane") + if not s.variants: + problems.append(f"{s.name}: runs on no variant") + if not s.platforms: + problems.append(f"{s.name}: runs on no platform") + cwd = REPO_ROOT / s.cwd + if not cwd.is_dir(): + problems.append(f"{s.name}: cwd {s.cwd} does not exist") + for var in s.variants: + if var not in (s.overrides.keys() | {"standard", "rtx"}): + problems.append(f"{s.name}: bad variant {var!r}") + + # Every suite should be exercised by some lane and some tier path. + if problems: + for p in problems: + print(f"✗ {p}", file=sys.stderr) + print(f"\n{len(problems)} manifest problem(s).", file=sys.stderr) + return 1 + print( + f"✓ manifest OK — {len(SUITES)} suites, " + f"{len(set(junits))} unique junit paths, no collisions." + ) + return 0 + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(prog="python -m tests.ci", description=__doc__) + sub = p.add_subparsers(dest="cmd", required=True) + + sub.add_parser("list", help="list all suites").set_defaults(fn=_cmd_list) + + sp = sub.add_parser("show", help="show a suite's resolved command") + sp.add_argument("name") + sp.set_defaults(fn=_cmd_show) + + sp = sub.add_parser("run", help="run one suite") + sp.add_argument("name") + sp.add_argument("--variant", choices=("standard", "rtx")) + sp.add_argument( + "--dry-run", action="store_true", help="print the command, don't run" + ) + sp.add_argument( + "pytest_args", + nargs="*", + help="extra args forwarded to pytest " "(use `-- -k foo`)", + ) + sp.set_defaults(fn=_cmd_run) + + sp = sub.add_parser( + "run-lane", help="run every suite in a lane/tier, past failures" + ) + g = sp.add_mutually_exclusive_group() + g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only")) + g.add_argument("--tier", choices=("l0", "l1", "l2")) + sp.add_argument("--variant", choices=("standard", "rtx")) + sp.add_argument("--platform", choices=("linux-x86_64", "windows")) + sp.add_argument("--dry-run", action="store_true") + sp.add_argument( + "--changed-from", metavar="FILE", + help="narrow to suites affected by these changed files (a file, or '-' " + "for stdin; one repo-relative path per line). Broad changes → no narrowing.", + ) + sp.set_defaults(fn=_cmd_run_lane) + + sp = sub.add_parser("matrix", help="emit a GitHub Actions matrix as JSON") + g = sp.add_mutually_exclusive_group() + g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only")) + g.add_argument("--tier", choices=("l0", "l1", "l2")) + sp.add_argument("--variant", choices=("standard", "rtx")) + sp.add_argument("--platform", choices=("linux-x86_64", "windows")) + sp.add_argument( + "--changed-from", metavar="FILE", + help="narrow to suites affected by these changed files (a file, or '-' " + "for stdin; one repo-relative path per line). Broad changes → no narrowing.", + ) + sp.set_defaults(fn=_cmd_matrix) + + sub.add_parser("doctor", help="validate the manifest").set_defaults(fn=_cmd_doctor) + + args = p.parse_args(argv) + return args.fn(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/ci/runner.py b/tests/ci/runner.py new file mode 100644 index 0000000000..98980a79e5 --- /dev/null +++ b/tests/ci/runner.py @@ -0,0 +1,313 @@ +"""The one engine that turns a ``Suite`` into a ``pytest`` command and runs it. + +This is the *only* place that knows pytest mechanics (xdist workers, junit paths, +the flake-rerun wrapper, ``--dist``, optional-dep setup, variant resolution). The +manifest (``suites.py``) stays pure data. +""" + +from __future__ import annotations + +import glob +import os +import shlex +import subprocess +import sys +from pathlib import Path + +from .suites import SUITES, Suite, Variant, by_name + +# Repo root: tests/ci/runner.py -> parents[2]. Honor TRT_REPO_ROOT like the bash. +REPO_ROOT = Path( + os.environ.get("TRT_REPO_ROOT", str(Path(__file__).resolve().parents[2])) +) + +# Known transient cudagraph/TRT-driver flake signatures. Expand ONLY with +# concrete evidence — a broad regex hides real bugs. +_RERUN_ARGS = [ + "--reruns", + "1", + "--reruns-delay", + "5", + "--only-rerun", + "cudaErrorStreamCaptureInvalidated", + "--only-rerun", + "Stream capture invalidated", +] + + +def _launcher() -> list[str]: + """The python/pytest launcher. CI leaves PYTHON unset (-> container python); + the justfile sets PYTHON='uv run --no-sync python' to use the built venv.""" + return shlex.split(os.environ.get("PYTHON", "python")) + + +def _results_dir() -> Path: + d = os.environ.get("RUNNER_TEST_RESULTS_DIR") + if not d: + tmp = os.environ.get("TMPDIR", "/tmp") + d = str(Path(tmp) / "trt_test_results") + Path(d).mkdir(parents=True, exist_ok=True) + return Path(d) + + +def junit_path(suite: Suite) -> Path: + """Unique per suite (no more four-runs-one-file collisions).""" + return _results_dir() / f"{suite.name}.xml" + + +def _nproc(jobs: str | None) -> list[str]: + """``-n`` token. TRT_JOBS overrides the suite default; jobs=None -> serial.""" + if jobs is None: + return [] + return ["-n", os.environ.get("TRT_JOBS") or jobs] + + +def _reruns_enabled(reruns: bool) -> bool: + return reruns and os.environ.get("TRT_PYTEST_RERUNS", "1") != "0" + + +def _expand_paths(cwd: Path, paths: tuple[str, ...]) -> list[str]: + """Shell-style glob expansion (the bash relied on the shell for this). + + A pattern with ``*`` is expanded relative to ``cwd`` and sorted; a pattern + that matches nothing is kept literal (matches bash nullglob-off behavior, so + pytest reports the missing path rather than silently collecting 0 tests). + """ + out: list[str] = [] + for p in paths: + if "*" in p: + matches = sorted( + str(Path(m).relative_to(cwd)) for m in glob.glob(str(cwd / p)) + ) + out.extend(matches or [p]) + else: + out.append(p) + return out + + +def build_pytest_args(suite: Suite, variant: Variant) -> list[str]: + """The pytest args (everything after ``-m pytest``) for this suite+variant.""" + v = suite.for_variant(variant) + cwd = REPO_ROOT / v["cwd"] + args: list[str] = [] + if _reruns_enabled(v["reruns"]): + args += _RERUN_ARGS + if v["markers"]: + args += ["-m", v["markers"]] + args += ["-ra"] + args += _nproc(v["jobs"]) + args += ["--junitxml", str(junit_path(suite))] + if v["dist"]: + args += [v["dist"]] + if v["maxfail"] is not None: + args += [f"--maxfail={v['maxfail']}"] + if v["ir"]: + args += ["--ir", v["ir"]] + if v["keyword"]: + args += ["-k", v["keyword"]] + if v["verbose"]: + args += ["-v"] + args += _expand_paths(cwd, tuple(v["paths"])) + return args + + +def _setup_commands(step: str) -> list[tuple[list[str], Path]]: + """(argv, cwd) pairs for a named setup step.""" + launcher = _launcher() + if step == "hub": + return [(launcher + ["hub.py"], REPO_ROOT / "tests/modules")] + if step == "executorch": + return [ + ( + launcher + ["-m", "pip", "install", "pyyaml", "executorch>=1.3.1"], + REPO_ROOT, + ) + ] + if step == "cuda-core": + return [ + (launcher + ["-m", "pip", "install", "cuda-python", "cuda-core"], REPO_ROOT) + ] + if step == "mpi": + return [ + ( + [ + "dnf", + "install", + "-y", + "mpich", + "mpich-devel", + "openmpi", + "openmpi-devel", + ], + REPO_ROOT, + ) + ] + raise KeyError(f"unknown setup step {step!r} in a suite definition") + + +def describe(suite: Suite, variant: Variant) -> str: + """The full command line, for --dry-run / show (quoting-safe display).""" + v = suite.for_variant(variant) + pre = [] + for step in v["setup"]: + for argv, cwd in _setup_commands(step): + pre.append(f" (cd {cwd.relative_to(REPO_ROOT)} && {shlex.join(argv)})") + cmd = shlex.join(_launcher() + ["-m", "pytest"] + build_pytest_args(suite, variant)) + lines = pre + [f" (cd {v['cwd']} && {cmd})"] + for f in v["follow"]: + lines.append(f" (cd {v['cwd']} && {shlex.join(_launcher() + list(f))})") + return "\n".join(lines) + + +def run_suite( + suite: Suite, + variant: Variant, + *, + dry_run: bool = False, + extra: list[str] | None = None, +) -> int: + """Run setup steps, the pytest command, then any follow commands. Returns the + process exit code (non-zero on first failure), mirroring the bash tiers.""" + v = suite.for_variant(variant) + extra = extra or [] + env = {**os.environ, **{k: str(val) for k, val in v["env"].items()}} + cwd = REPO_ROOT / v["cwd"] + pytest_cmd = ( + _launcher() + ["-m", "pytest"] + build_pytest_args(suite, variant) + extra + ) + + if dry_run: + print(describe(suite, variant)) + if extra: + print(f" # + extra pytest args: {shlex.join(extra)}") + return 0 + + for step in v["setup"]: + for argv, scwd in _setup_commands(step): + print(f"==> setup[{step}]: {shlex.join(argv)}", flush=True) + rc = subprocess.run(argv, cwd=scwd, env=env).returncode + if rc != 0: + print(f"::warning::setup step {step!r} exited {rc}", flush=True) + + print(f"==> {suite.name} [{variant}]: {shlex.join(pytest_cmd)}", flush=True) + rc = subprocess.run(pytest_cmd, cwd=cwd, env=env).returncode + if rc != 0: + repro = shlex.join( + ["uv", "run", "--no-sync", "pytest"] + + build_pytest_args(suite, variant) + + extra + ) + print( + f"::warning::{suite.name} failed. Reproduce: cd {v['cwd']} && {repro}", + flush=True, + ) + return rc + + for f in v["follow"]: + fcmd = _launcher() + list(f) + print(f"==> {suite.name} follow: {shlex.join(fcmd)}", flush=True) + frc = subprocess.run(fcmd, cwd=cwd, env=env).returncode + if frc != 0: + return frc + return 0 + + +# Path prefixes whose changes cannot affect any GPU test outcome. A PR touching +# ONLY these can skip the whole test matrix. +_NO_TEST_IMPACT = ( + "docs/", "docsrc/", "tools/", "notebooks/", "examples/", ".devcontainer/", + "README", "CHANGELOG", "LICENSE", ".gitignore", ".pre-commit", ".flake8", +) + + +def _owned_dirs(s: Suite) -> set[str]: + """Repo-relative directories a suite owns — the directory of each path target + (glob tails dropped, file targets mapped to their dir). Coarse on purpose: + change-based selection must OVER-select (never miss a suite a change breaks).""" + dirs: set[str] = set() + for var in s.variants: + fv = s.for_variant(var) + cwd = fv["cwd"].rstrip("/") + for p in fv["paths"] or (".",): + t = p.split("*")[0].strip("/") # drop any glob tail + full = f"{cwd}/{t}" if t and t != "." else cwd + base = full.rsplit("/", 1)[-1] + dirs.add(full.rsplit("/", 1)[0] if "." in base else full) + return dirs + + +def affected_suites(changed: list[str]) -> set[str] | None: + """Suite names a `changed` file set can affect. + + Returns ``None`` when the change set can affect ANYTHING — any file that is + neither a no-test-impact path nor owned by a specific suite (i.e. library + source, build files, a shared conftest/harness). The caller then runs the + full selection (safe default). An empty set means "nothing to test" (e.g. a + docs-only PR). This never narrows away a suite a change could break. + """ + owned = {s.name: _owned_dirs(s) for s in SUITES} + hit: set[str] = set() + for f in changed: + f = f.strip() + if not f: + continue + if any(f.startswith(p) for p in _NO_TEST_IMPACT): + continue # no test impact + matched = { + name + for name, dirs in owned.items() + if any(f == d or f.startswith(d + "/") for d in dirs) + } + if matched: + hit |= matched + else: + return None # source / build / shared-test change → cannot narrow + return hit + + +def select( + *, + lane: str | None = None, + tier: str | None = None, + variant: str | None = None, + platform: str | None = None, + names: list[str] | None = None, + changed: list[str] | None = None, +) -> list[tuple[Suite, Variant]]: + """All (suite, variant) jobs matching the filters. No filter on an axis = all. + + ``changed`` (a list of repo-relative changed paths) narrows to only the + affected suites — but ONLY when every change is confined to test dirs; any + source/build/shared change keeps the full pool (see ``affected_suites``). + """ + jobs: list[tuple[Suite, Variant]] = [] + pool = [by_name(n) for n in names] if names else list(SUITES) + if changed is not None: + aff = affected_suites(changed) + if aff is not None: # None → a broad change → do not narrow + pool = [s for s in pool if s.name in aff] + for s in pool: + if lane is not None and lane not in s.lanes: + continue + if tier is not None and s.tier != tier: + continue + if platform is not None and platform not in s.platforms: + continue + for var in s.variants: + if variant is not None and var != variant: + continue + jobs.append((s, var)) + return jobs + + +def matrix(**filters: str | None) -> list[dict[str, str]]: + """GitHub-Actions matrix ``include`` entries for the selected jobs.""" + return [ + { + "suite": s.name, + "variant": var, + "tier": s.tier, + "cwd": s.for_variant(var)["cwd"], + } + for s, var in select(**filters) + ] diff --git a/tests/ci/suites.py b/tests/ci/suites.py new file mode 100644 index 0000000000..800803b7e7 --- /dev/null +++ b/tests/ci/suites.py @@ -0,0 +1,356 @@ +"""The test-suite manifest — pure data describing every test job. + +A *suite* is one ``pytest`` invocation against one subsystem. A CI/local *job* +is a (suite x variant) pair, optionally sharded across N runners. Suites are +grouped two ways: + + * ``tier`` — the legacy L0/L1/L2 grouping, kept so the migration can be + coverage-equivalent to today's ``ci_helpers.sh`` (``ci matrix --tier l0``). + * ``lanes`` — the target grouping (``fast`` / ``full`` / ``nightly``) the + redesign moves to (``ci matrix --lane fast``). Depth within a subsystem is + expressed by a marker on the test, not a filename prefix. + +Deliberate differences from the bash tiers (improvements, not regressions): + * ``hlo/`` ran in BOTH l0_core (standard) and l1_dynamo_core — wasteful. It is + one ``dynamo-hlo`` suite here, run once. + * l2_plugin re-ran the whole ``conversion/`` dir (already covered by + ``dynamo-converters``) and pointed four ``--junitxml`` runs at ONE path, so + three suites' results vanished from the aggregate. Each suite here owns a + unique junit name (derived from ``name`` + shard), and the redundant + ``conversion/`` re-run is dropped. + * L2 suites used raw ``pytest`` (no rerun wrapper / no repro hint); every suite + now runs through the same wrapper uniformly (gated by ``TRT_PYTEST_RERUNS``). + +Validate the manifest: ``python -m tests.ci doctor`` +Inspect a command: ``python -m tests.ci run --dry-run`` +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +Tier = Literal["l0", "l1", "l2"] +# python-only validates the PYTHON_ONLY=1 wheel (no C++ runtime) against the +# dynamo runtime suite. It's its own lane because it pairs a distinct BUILD mode +# with a focused test set, orthogonal to fast/full/nightly depth. +Lane = Literal["fast", "full", "nightly", "python-only"] +Variant = Literal["standard", "rtx"] +# Test channels. SBSA (linux-aarch64) is build-only — no GPU test runners — so it +# is a build channel handled at the workflow level, not a suite platform here. +Platform = Literal["linux-x86_64", "windows"] + +ALL_VARIANTS: tuple[Variant, ...] = ("standard", "rtx") +ALL_PLATFORMS: tuple[Platform, ...] = ("linux-x86_64", "windows") + +# xdist worker counts (`-n`). GPU tests are MEMORY-bound, not CPU-bound: each +# worker builds its own TensorRT engine, so N workers ≈ N× peak GPU memory. +# `-n auto` (= CPU count, up to ~48 on big runners) is a footgun here — it OOMs +# the model/runtime suites on real 24 GB runners, and an OOM'd engine build fails +# `assert cuda_engine` and then *reruns*, doubling wall-clock. So workers are +# tiered by worst-case engine size, not by core count: +_LIGHT = "8" # small single-op / graph tests (conversion, partitioning, hlo) +_HEAVY = "4" # full compile+execute suites (lowering, runtime) — OOM'd at -n 8 +_MODEL = "2" # real models / LLMs (ResNet, BERT, Llama) — biggest engines + + +@dataclass(frozen=True) +class Suite: + """One ``pytest`` invocation against one subsystem. + + Fields map 1:1 onto the command the runner builds. ``overrides`` lets a + single suite differ per variant (e.g. RTX selects a different path set) + without splitting it into two entries with two names. + """ + + name: str + tier: Tier + lanes: tuple[Lane, ...] + cwd: str = "tests/py/dynamo" # relative to repo root + paths: tuple[str, ...] = () # pytest positionals (rel to cwd); globs ok + markers: str | None = None # -m EXPR + keyword: str | None = None # -k EXPR + dist: str | None = None # --dist=loadscope + maxfail: int | None = None # --maxfail=N + ir: str | None = None # --ir torch_compile + jobs: str | None = None # xdist default: None=serial, "8"/"auto"/"4" + reruns: bool = True # wrap in the flake-rerun helper + verbose: bool = False # -v + variants: tuple[Variant, ...] = ALL_VARIANTS + platforms: tuple[Platform, ...] = ALL_PLATFORMS # channels this suite runs on + setup: tuple[str, ...] = () # named pre-steps: hub|executorch|cuda-core|mpi + follow: tuple[tuple[str, ...], ...] = () # extra argv to run AFTER pytest + env: dict[str, str] = field(default_factory=dict) + overrides: dict[str, dict[str, Any]] = field(default_factory=dict) # per-variant + + def for_variant(self, variant: Variant) -> dict[str, Any]: + """This suite's effective fields for ``variant`` (applies overrides).""" + base = { + f: getattr(self, f) + for f in ( + "cwd", + "paths", + "markers", + "keyword", + "dist", + "maxfail", + "ir", + "jobs", + "reruns", + "verbose", + "setup", + "follow", + "env", + ) + } + base.update(self.overrides.get(variant, {})) + return base + + +# ── L0 — smoke / fast lane ──────────────────────────────────────────────────── +_L0: list[Suite] = [ + Suite( + "dynamo-converters", + tier="l0", + lanes=("fast", "full"), + paths=("conversion/",), + dist="--dist=loadscope", + maxfail=20, + jobs="8", + # RTX does not shard converters with loadscope. + overrides={"rtx": {"dist": None}}, + ), + Suite( + "dynamo-runtime-smoke", + tier="l0", + lanes=("fast", "full"), + paths=("runtime/test_000_*",), + jobs=_HEAVY, + ), + Suite( + "dynamo-partitioning-smoke", + tier="l0", + lanes=("fast", "full"), + paths=("partitioning/test_000_*",), + jobs="8", + # RTX runs the whole partitioning suite (no smoke subset split). + overrides={"rtx": {"paths": ("partitioning/",)}}, + ), + Suite( + "dynamo-lowering", + tier="l0", + lanes=("fast", "full"), + paths=("lowering/",), + jobs=_HEAVY, + ), + Suite( + "py-core", + tier="l0", + lanes=("fast", "full"), + cwd="tests/py/core", + paths=(".",), + jobs="8", + ), + Suite( + "ts-api", + tier="l0", + lanes=("fast", "full"), + cwd="tests/py/ts", + paths=("api/",), + setup=("hub",), + variants=("standard",), + ), +] + +# ── L1 — critical-path / full lane ──────────────────────────────────────────── +_L1: list[Suite] = [ + Suite( + "dynamo-runtime", + tier="l1", + lanes=("full",), + paths=("runtime/test_001_*",), + jobs=_HEAVY, + ), + Suite( + "dynamo-partitioning", + tier="l1", + lanes=("full",), + paths=("partitioning/test_001_*",), + jobs="8", + variants=("standard",), + ), + Suite( + # Was run in BOTH l0_core (std) and l1_dynamo_core (both) — deduped to once. + "dynamo-hlo", + tier="l1", + lanes=("full",), + paths=("hlo/",), + jobs="8", + ), + Suite( + "dynamo-models-critical", + tier="l1", + lanes=("full",), + paths=("models/",), + markers="critical", + ), + Suite( + "torch-compile-backend", + tier="l1", + lanes=("full",), + paths=("backend/",), + ), + Suite( + "torch-compile-models-critical", + tier="l1", + lanes=("full",), + paths=("models/test_models.py", "models/test_dyn_models.py"), + markers="critical", + ir="torch_compile", + ), + Suite( + "ts-models", + tier="l1", + lanes=("full",), + cwd="tests/py/ts", + paths=("models/",), + setup=("hub",), + variants=("standard",), + ), +] + +# ── L2 — exhaustive / full + nightly ────────────────────────────────────────── +_L2: list[Suite] = [ + Suite( + "torch-compile-models", + tier="l2", + lanes=("full", "nightly"), + paths=("models/test_models.py", "models/test_dyn_models.py"), + markers="not critical", + ir="torch_compile", + jobs=_MODEL, + ), + Suite( + "dynamo-models", + tier="l2", + lanes=("full", "nightly"), + paths=("models/",), + markers="not critical", + jobs=_MODEL, + ), + Suite( + "dynamo-llm", + tier="l2", + lanes=("nightly",), + paths=("llm/",), + jobs=_MODEL, + ), + Suite( + "dynamo-runtime-full", + tier="l2", + lanes=("full", "nightly"), + paths=("runtime/",), + keyword="not test_000_ and not test_001_", + jobs=_HEAVY, + ), + Suite( + "executorch", + tier="l2", + lanes=("nightly",), + paths=("executorch/",), + setup=("executorch",), + jobs="auto", + variants=("standard",), + platforms=("linux-x86_64",), + ), + Suite( + # Standard: the automatic-plugin trio. RTX: the whole automatic_plugin dir. + # (The redundant conversion/ re-run from the old l2_plugin is dropped.) + "plugins-automatic", + tier="l2", + lanes=("nightly",), + jobs="auto", + paths=( + "automatic_plugin/test_automatic_plugin.py", + "automatic_plugin/test_automatic_plugin_with_attrs.py", + "automatic_plugin/test_flashinfer_rmsnorm.py", + ), + overrides={"rtx": {"paths": ("automatic_plugin/",)}}, + ), + Suite( + "kernels", + tier="l2", + lanes=("nightly",), + cwd="tests/py/kernels", + paths=(".",), + setup=("cuda-core",), + jobs="auto", + variants=("standard",), + platforms=("linux-x86_64",), + ), + Suite( + "ts-integrations", + tier="l2", + lanes=("nightly",), + cwd="tests/py/ts", + paths=("integrations/",), + setup=("hub",), + jobs="auto", + variants=("standard",), + ), + Suite( + "distributed", + tier="l2", + lanes=("nightly",), + paths=( + "distributed/test_nccl_ops.py", + "distributed/test_native_nccl.py", + "distributed/test_export_save_load.py", + ), + jobs="auto", + verbose=True, + reruns=False, + variants=("standard",), + platforms=("linux-x86_64",), + setup=("mpi",), + env={"USE_HOST_DEPS": "1", "CI_BUILD": "1", "USE_TRTLLM_PLUGINS": "1"}, + follow=( + ( + "-m", + "torch_tensorrt.distributed.run", + "--nproc_per_node=2", + "distributed/test_native_nccl.py", + "--multirank", + ), + ( + "-m", + "torch_tensorrt.distributed.run", + "--nproc_per_node=2", + "distributed/test_export_save_load.py", + "--multirank", + ), + ), + ), +] + +# ── python-only — validates the PYTHON_ONLY=1 wheel against the runtime suite ── +_PYTHON_ONLY: list[Suite] = [ + Suite( + "python-only-runtime", + tier="l1", + lanes=("python-only",), + paths=("runtime/",), + jobs=_HEAVY, + # Runs for BOTH backends: the PYTHON_ONLY=1 wheel is validated against + # standard TensorRT and TensorRT-RTX (variants default to both). + ), +] + +SUITES: tuple[Suite, ...] = tuple(_L0 + _L1 + _L2 + _PYTHON_ONLY) + + +def by_name(name: str) -> Suite: + for s in SUITES: + if s.name == name: + return s + raise KeyError(f"no suite named {name!r}; try `python -m tests.ci list`") diff --git a/tools/ci-dashboard/.gitignore b/tools/ci-dashboard/.gitignore new file mode 100644 index 0000000000..7a60b85e14 --- /dev/null +++ b/tools/ci-dashboard/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/tools/ci-dashboard/README.md b/tools/ci-dashboard/README.md new file mode 100644 index 0000000000..c4a2e764d6 --- /dev/null +++ b/tools/ci-dashboard/README.md @@ -0,0 +1,113 @@ +# CI dashboard + +A local web UI for Torch-TensorRT's GitHub Actions CI. It answers the questions +the raw Actions tab makes painful: **which platforms are green, which are red, +which test broke, and where that test lives in the tree.** + +![what it shows](#) + +## Run it + +```bash +just ci # current branch +just ci branch=nightly # a specific branch +just ci branch=main port=9000 # pick a port + +# or directly (no build — stdlib only): +uv run --no-sync tools/ci-dashboard/ci_dashboard.py -b nightly +python3 tools/ci-dashboard/ci_dashboard.py -b nightly # also fine +``` + +It opens `http://127.0.0.1:8712/` locally. It also **binds `0.0.0.0`**, so it's +reachable over your tailnet/LAN — the startup log prints the Tailscale + LAN URLs +(e.g. `http://100.x.y.z:8712/`). To keep it local-only, pass `--host 127.0.0.1`. + +### Requirements + +- An **authenticated `gh`** — the only data source. Check with `gh auth status`; + log in with `gh auth login`. +- Python 3.9+ (**stdlib only** — no `pip install`, and **no torch-tensorrt build**). + `just ci` uses `uv run --no-sync`, which reuses the existing `.venv` interpreter + and skips the project build; plain `python3` works too since nothing outside the + stdlib is imported. `htmx` is vendored under `static/`, so it works offline. + +## What you get + +- **Platform board** — one card per workflow (Linux x86_64, aarch64, Windows, + the RTX and python-only variants, jetpack…), sorted worst-first, with a live + status badge. A summary strip up top counts failing / running / queued / + passing at a glance. +- **Drill in** — open a platform to see its jobs as a **python × cuda × suite** + grid. Green/red cells; each suite is labelled with the pytest paths it runs + (read from the `tests/ci` manifest — the same data CI runs). +- **Click a red cell** — a drawer shows every failing test with its error, the + **source file:line it maps to** (local path + a GitHub link pinned to the run's + commit), and a **copy-paste command to reproduce it locally** — the exact + `python -m tests.ci run ` command CI used, narrowed to the failing tests. +- **Relevant logs, captured** — the drawer then pulls that job's log and extracts + just the **pytest FAILURES block per failing test** (traceback + captured + stdout/stderr), so you read the actual failure without scrolling a 1 MB log. + A **raw log ↗** link opens the full plain-text log (grep/save it); a + build/env failure with no test annotations falls back to the end of the log. +- **Failures across platforms** — a rollup that dedupes a failing test across + every platform it breaks on ("fails on 3 platforms → likely this code"), so a + systemic break stands out from a one-off. +- **"Didn't run" ≠ "failed"** — skipped/gated-off tiers (common on PR branches), + jobs blocked by a failed dependency, and cancelled/never-started jobs are shown + as a distinct, dashed/muted **didn't run** state — never colored red and never + counted as failing. The summary strip tallies them separately, and the + cross-platform rollup only ever counts jobs that actually *ran and failed*. +- **Live** — status badges refresh in the background (~every 25s) without + collapsing whatever you've expanded. `↻ Refresh` (or `r`) forces a full reload; + `/` focuses the branch box; the *failing only* toggle hides the green cards. + +## PR commands + +Comment these on a PR to control its CI without pushing a new commit (write +access required; handled by `.github/workflows/retrigger-ci.yml`). They're also +listed in the dashboard header under **PR commands** with copy buttons. + +| comment | effect | +|---|---| +| `/rerun` | re-run only the failed / cancelled jobs on the current commit | +| `/rerun all` | re-run every job from scratch on the current commit | +| `/cancel` | cancel stale “zombie” runs still in-flight on **old** commits of the PR | +| `/cancel all` | cancel **every** in-flight run for the PR (full stop) | +| `/test [backend]` | dispatch a fresh run at an exact lane — `lane` ∈ `fast\|full\|nightly`, `backend` ∈ `standard\|rtx\|both` (default `both`). e.g. `/test full rtx`, `/test nightly` | + +Typical unstick: `/cancel` to clear zombies, then `/rerun all` for a clean wave. +Run everything on demand: `/test full` (or `/test nightly` for llm/kernels/distributed). +(These take effect from `main`, since `issue_comment` workflows always run there.) + +## How it works + +The server (`ci_dashboard.py`) is a thin, caching proxy over `gh`; all rendering +is server-side HTML fragments and `htmx` wires up the interactions (lazy-load a +platform on open, out-of-band badge polling, the failure drawer). + +- **Runs / jobs**: `gh run list` + `gh api …/actions/runs/{id}/jobs`. Job names + encode the matrix — `core / L2 dynamo distributed tests / + L2-dynamo-distributed-tests--3.12-cu130` — which we parse into + `{tier, python, cuda, kind}`. +- **Why a red cell → a test → a file**: the test jobs surface each pytest + failure as a GitHub **check-run annotation** (via `pytest-results-action`), so + the failing test name + traceback is one cheap API call + (`…/check-runs/{id}/annotations`) — no wheel or junit download. We then + `git grep` the test symbol in `tests/` to resolve it to `file:line`. +- **Suite → paths / reproduce command** come straight from the `tests/ci` + manifest (`tests/ci/suites.py`) via `info_for()` — the CI names each test job + `-`, which is exactly a manifest suite, so there is nothing to + keep in sync. A small `TIER_MAP` remains only as a fallback so historical, + pre-migration runs (tier-named) still resolve; it can be dropped once those age + out. + +## Limitations + +- Results are only as granular as the CI annotations. A **build/env/setup** + failure (not a test assertion) has no pytest annotation — the drawer says so + and links you to the raw job log. +- Expanded platform grids don't auto-refresh (only the top-level badges do). Open + a stale platform again, or hit `↻ Refresh`, to re-pull its jobs. +- The cross-platform rollup fans out `gh` calls for every failing test job; on a + branch that's failing everywhere the first scan takes a few seconds (results + are cached; `↻ rescan` forces a refresh). diff --git a/tools/ci-dashboard/ci_dashboard.py b/tools/ci-dashboard/ci_dashboard.py new file mode 100755 index 0000000000..cebfe3ff57 --- /dev/null +++ b/tools/ci-dashboard/ci_dashboard.py @@ -0,0 +1,1026 @@ +#!/usr/bin/env python3 +"""Torch-TensorRT CI dashboard — a local, pleasant view of GitHub Actions. + +Run: uv run tools/ci-dashboard/ci_dashboard.py # current branch + uv run tools/ci-dashboard/ci_dashboard.py -b nightly # a branch + python3 tools/ci-dashboard/ci_dashboard.py --port 8712 + +No pip dependencies. Data comes from the `gh` CLI (must be authenticated: +`gh auth status`). The server renders HTML fragments; htmx (vendored under +./static) drives lazy-loading, live status polling, and the failure drawer. + +Why annotations, not artifacts: the test jobs surface every pytest failure as a +GitHub check-run *annotation* (via pytest-results-action), so the failing test +name + traceback is one cheap API call away — no wheel/junit download needed. + +Job names encode the whole matrix, e.g. + test / dynamo-converters-standard / dynamo-converters-standard--3.12-cu130 +We parse that into {group, python, cuda, kind}; the group is a `-` +from the tests/ci manifest (tests/ci/suites.py — the SAME data CI runs), so a red +cell maps to the suite's pytest paths and its exact `python -m tests.ci run` +reproduce command. Pre-migration runs (tier-named) fall back to the legacy map. +""" +from __future__ import annotations + +import argparse +import html +import json +import os +import re +import socket +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from functools import lru_cache +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, quote, urlparse + +HERE = os.path.dirname(os.path.abspath(__file__)) +STATIC = os.path.join(HERE, "static") +REPO_ROOT = os.path.abspath(os.path.join(HERE, "..", "..")) + +# ── tests/ci manifest (single source of truth) ─────────────────────────────── +# The dashboard resolves a red CI cell back to "what ran + how to reproduce" from +# the SAME manifest CI and the `just` recipes use (tests/ci/suites.py). CI names +# each test job `-` (tests/ci → linux-test.yml), so the group the +# dashboard parses out of a job IS a manifest suite — no duplicated path lists to +# drift. Imported best-effort: a checkout predating the manifest falls back to the +# legacy TIER_MAP below. +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) +try: + from tests.ci import SUITES as _SUITES # noqa: E402 +except Exception: # pragma: no cover — manifest absent on pre-migration branches + _SUITES = () + +# ── Legacy tier → source mapping (fallback) ────────────────────────────────── +# Kept only so historical, pre-migration runs (named "L2 dynamo core tests", not +# "-") still resolve. New runs go through the manifest above. +# Keys are normalized job group names (see _norm()); `fn` is the old shell tier. +TIER_MAP = { + "l0 dynamo converter tests": dict( + fn="trt_tier_l0_converter", paths=["tests/py/dynamo/conversion/"]), + "l0 dynamo core tests": dict( + fn="trt_tier_l0_core", + paths=["tests/py/dynamo/runtime/test_000_*", "tests/py/dynamo/partitioning/test_000_*", + "tests/py/dynamo/lowering/", "tests/py/dynamo/hlo/"]), + "l0 core python tests": dict(fn="trt_tier_l0_py_core", paths=["tests/py/core/"]), + "l0 torchscript tests": dict( + fn="trt_tier_l0_torchscript", paths=["tests/py/ts/api/", "tests/modules/hub.py"]), + "l1 dynamo core tests": dict( + fn="trt_tier_l1_dynamo_core", + paths=["tests/py/dynamo/runtime/test_001_*", "tests/py/dynamo/partitioning/test_001_*", + "tests/py/dynamo/hlo/"]), + "l1 dynamo compile tests": dict( + fn="trt_tier_l1_dynamo_compile", paths=["tests/py/dynamo/models/"]), + "l1 torch compile tests": dict( + fn="trt_tier_l1_torch_compile", + paths=["tests/py/dynamo/backend/", "tests/py/dynamo/models/test_models.py", + "tests/py/dynamo/models/test_dyn_models.py"]), + "l1 torch script tests": dict(fn="trt_tier_l1_torchscript", paths=["tests/py/ts/models/"]), + "l2 torch compile tests": dict( + fn="trt_tier_l2_torch_compile", + paths=["tests/py/dynamo/models/test_models.py", "tests/py/dynamo/models/test_dyn_models.py"]), + "l2 dynamo compile tests": dict( + fn="trt_tier_l2_dynamo_compile", paths=["tests/py/dynamo/models/", "tests/py/dynamo/llm/"]), + "l2 dynamo core tests": dict( + fn="trt_tier_l2_dynamo_core", paths=["tests/py/dynamo/runtime/", "tests/py/dynamo/executorch/"]), + "l2 dynamo plugin tests": dict( + fn="trt_tier_l2_plugin", + paths=["tests/py/dynamo/conversion/", "tests/py/dynamo/automatic_plugin/", "tests/py/kernels/"]), + "l2 torch script tests": dict(fn="trt_tier_l2_torchscript", paths=["tests/py/ts/integrations/"]), + "l2 dynamo distributed tests": dict( + fn="trt_tier_l2_distributed", paths=["tests/py/dynamo/distributed/"]), +} + + +def _norm(s: str) -> str: + """Lowercase and collapse separators so 'L2-dynamo-core-tests' and + 'L2 dynamo core tests' hash to the same tier key.""" + return re.sub(r"[^a-z0-9]+", " ", s.lower()).strip() + + +# ── manifest-backed resolution (group → what ran + how to reproduce) ────────── +def _suite_group_index(): + """Map a normalized CI job group to ``(suite, variant)``. + + CI names each test job ``-`` so the parsed group is exactly a + manifest suite. Index both the ``-`` and bare ```` + forms; per-variant ``paths`` overrides (e.g. RTX) are applied at lookup.""" + idx = {} + for s in _SUITES: + for v in s.variants: + idx.setdefault(_norm(f"{s.name}-{v}"), (s, v)) + idx.setdefault(_norm(s.name), (s, s.variants[0])) + return idx + + +_SUITE_INDEX = _suite_group_index() +# Longest keys first so 'dynamo runtime smoke' wins over 'dynamo runtime'. +_SUITE_KEYS = sorted(_SUITE_INDEX, key=len, reverse=True) + + +def _suite_repro(suite, variant, kexpr=""): + """The exact command CI ran for this suite (what `just suite` invokes).""" + var = f" --variant {variant}" if variant and variant != "standard" else "" + ka = f' -- -k "{kexpr}"' if kexpr else "" + return ( + f"TRT_PYTEST_RERUNS=0 uv run --no-sync python -m tests.ci " + f"run {suite.name}{var}{ka}" + ) + + +def info_for(group: str): + """Resolve a CI job group → ``{paths, repro, label}`` from the tests/ci + manifest, falling back to the legacy ``TIER_MAP`` for pre-migration runs. + + ``paths`` are repo-relative (``cwd`` joined with the suite's pytest + positionals) so a red cell still points at code. ``repro(kexpr)`` returns the + local reproduce command, narrowed to the failing tests when ``kexpr`` given. + """ + g = _norm(group) + hit = _SUITE_INDEX.get(g) or next( + (_SUITE_INDEX[k] for k in _SUITE_KEYS if k in g), None + ) + if hit: + s, v = hit + fv = s.for_variant(v) + cwd = fv["cwd"].rstrip("/") + paths = [cwd if p in (".", "./") else f"{cwd}/{p}" for p in fv["paths"]] + paths = paths or [cwd] + return dict( + paths=paths, + repro=lambda kexpr="", _s=s, _v=v: _suite_repro(_s, _v, kexpr), + label=f"{s.name} · {v}", + ) + t = TIER_MAP.get(g) + if t: + return dict( + paths=t["paths"], + repro=lambda kexpr="", _t=t: ( + 'source tests/py/utils/ci_helpers.sh && PYTHON="uv run ' + f'--no-sync python" TRT_PYTEST_RERUNS=0 {_t["fn"]}' + + (f' -k "{kexpr}"' if kexpr else "") + ), + label=None, + ) + return None + + +# ── gh plumbing (cached) ───────────────────────────────────────────────────── +class Cache: + """Tiny TTL cache. Completed runs/jobs are immutable so we cache them long; + anything still in flight gets a short TTL so the UI stays live.""" + + def __init__(self): + self._d: dict[str, tuple[float, object]] = {} + self._lock = threading.Lock() + + def get(self, key): + with self._lock: + hit = self._d.get(key) + if not hit: + return None + expires, val = hit + return None if time.time() > expires else val + + def put(self, key, val, ttl): + with self._lock: + self._d[key] = (time.time() + ttl, val) + + def clear(self): + with self._lock: + self._d.clear() + + +CACHE = Cache() + + +def gh(args, parse=True, timeout=90): + proc = subprocess.run(["gh", *args], capture_output=True, text=True, + timeout=timeout, cwd=REPO_ROOT) + if proc.returncode != 0: + raise RuntimeError((proc.stderr or proc.stdout or "gh failed").strip()) + return json.loads(proc.stdout) if parse else proc.stdout + + +@lru_cache(maxsize=1) +def repo_slug(): + try: + return gh(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"], + parse=False).strip() + except Exception: + return "pytorch/TensorRT" + + +_DEFAULT_BRANCH = None + + +def default_branch(): + if _DEFAULT_BRANCH: + return _DEFAULT_BRANCH + try: + b = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, text=True, cwd=REPO_ROOT).stdout.strip() + return b if b and b != "HEAD" else "main" + except Exception: + return "main" + + +# ── data layer ─────────────────────────────────────────────────────────────── +def get_runs(branch, limit=40, refresh=False): + key = f"runs:{branch}:{limit}" + if not refresh and (c := CACHE.get(key)) is not None: + return c + fields = ("databaseId,workflowName,displayTitle,headBranch,headSha,status," + "conclusion,event,createdAt,updatedAt,url,number") + runs = gh(["run", "list", "--branch", branch, "--limit", str(limit), "--json", fields]) + newest = {} + for r in runs: + r["id"] = r.get("databaseId") # normalize (gh run list uses databaseId) + wf = r.get("workflowName") or "?" + if wf not in newest or r["createdAt"] > newest[wf]["createdAt"]: + newest[wf] = r + result = list(newest.values()) + live = any(r["status"] != "completed" for r in result) + CACHE.put(key, result, ttl=15 if live else 120) + return result + + +def _parse_job(j): + raw = j.get("name", "") + parts = [p.strip() for p in raw.split(" / ")] + last = parts[-1] + py = re.search(r"(? 1 and bool(re.search(r"--|build-wheel|3\.\d+", last)) + group_parts = parts[:-1] if is_matrix else parts[:] + if group_parts and group_parts[0] in ("core", "build"): + rest = group_parts[1:] + group_parts = rest if rest else (parts[:-1] if is_matrix else parts) + group = " / ".join(group_parts) if group_parts else raw + gl = _norm(group) + if "build-wheel" in last or gl.startswith("build "): + kind = "build" + elif re.match(r"l[012]\b", gl): + kind = "test" + elif "matrix" in gl or "generate" in gl or group == "filter-matrix": + kind = "setup" + elif raw.startswith("CI /") or "aggregate" in gl or "collect results" in gl or "rollup" in gl: + kind = "rollup" + else: + kind = "other" + return dict( + id=j.get("databaseId") or j.get("id"), raw=raw, group=group, kind=kind, + python=py.group(0) if py else None, cuda=cu.group(0) if cu else None, + status=j.get("status"), conclusion=j.get("conclusion"), + url=j.get("html_url") or j.get("url"), + failedSteps=[s["name"] for s in j.get("steps", []) if s.get("conclusion") == "failure"], + ) + + +def get_jobs(run_id, refresh=False): + key = f"jobs:{run_id}" + if not refresh and (c := CACHE.get(key)) is not None: + return c + data = gh(["api", f"repos/{repo_slug()}/actions/runs/{run_id}/jobs", + "--paginate", "-q", ".jobs[]"], parse=False) + jobs = [json.loads(line) for line in data.splitlines() if line.strip()] + parsed = [_parse_job(j) for j in jobs] + live = any(j["status"] != "completed" for j in parsed) + CACHE.put(key, parsed, ttl=15 if live else 300) + return parsed + + +NOISE = re.compile( + r"Process completed with exit code|Node\.js \d+ is deprecated|" + r"is deprecated\. Please switch|conda\.cli|defaults' channel|auto_activate|" + r"might have been added implicitly", re.I) +TESTLINE = re.compile(r"^(?P[\w./-]+(?:\.[\w\[\]-]+)+)\s*$") + + +@lru_cache(maxsize=4096) +def grep_test(symbol): + """Resolve a failing-test symbol (Class.method / module.func) to (file, line) + via `git grep`. Returns None if not found.""" + leaf = re.split(r"[.:]", symbol.strip())[-1] + leaf = re.sub(r"\[.*$", "", leaf) + if not re.match(r"^[A-Za-z_]\w+$", leaf): + return None + pat = f"def {leaf}\\b" if leaf.startswith("test") else f"class {leaf}\\b" + try: + out = subprocess.run(["git", "grep", "-n", "-E", pat, "--", "tests/"], + capture_output=True, text=True, cwd=REPO_ROOT, timeout=15).stdout + except Exception: + return None + for line in out.splitlines(): + path, lineno, _ = line.split(":", 2) + return (path, int(lineno)) + return None + + +def get_failures(job_id, refresh=False): + key = f"fail:{job_id}" + if not refresh and (c := CACHE.get(key)) is not None: + return c + try: + anns = gh(["api", f"repos/{repo_slug()}/check-runs/{job_id}/annotations"]) + except Exception as e: + return dict(error=str(e), failures=[]) + seen, failures = set(), [] + for a in anns: + if a.get("annotation_level") != "failure": + continue + msg = (a.get("message") or "").strip() + if not msg or NOISE.search(msg.splitlines()[0]): + continue + head = msg.splitlines()[0].strip() + m = TESTLINE.match(head) + test = m.group("test") if m else None + dedupe = test or head + if dedupe in seen: + continue + seen.add(dedupe) + loc = grep_test(test) if test else None + failures.append(dict(test=test, message=msg[:1400], + file=loc[0] if loc else None, line=loc[1] if loc else None)) + result = dict(failures=failures) + CACHE.put(key, result, ttl=600) + return result + + +# ── job logs (fetch + per-test extraction) ────────────────────────────────── +_TS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ") # GH line prefix +_SEP = re.compile(r"^_{4,}\s+(.+?)\s+_{4,}\s*$") # pytest FAILURES sep +_SECT = re.compile(r"^={4,}") # ==== section ==== + + +def get_job_log(job_id, refresh=False): + """Raw plain-text log for one job (immutable once complete → cached 1h).""" + key = f"log:{job_id}" + if not refresh and (c := CACHE.get(key)) is not None: + return c + try: + raw = gh(["api", f"repos/{repo_slug()}/actions/jobs/{job_id}/logs"], + parse=False, timeout=60) + except Exception: + raw = None + CACHE.put(key, raw, ttl=3600 if raw else 20) + return raw + + +def _clean_lines(raw): + return [_TS.sub("", ln) for ln in raw.splitlines()] + + +def extract_test_log(lines, test, max_lines=160): + """Pull the pytest FAILURES block for `test`. Falls back to a context window + around the test's last mention (covers custom harnesses w/o a FAILURES sep). + Returns the excerpt string, or None.""" + leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None + seps = [(i, m.group(1).strip()) for i, ln in enumerate(lines) + if (m := _SEP.match(ln))] + start = None + if test: + # Prefer an exact title match — the leaf method name (e.g. test_trt_compile) + # is shared across many classes, so fuzzy matching would grab the wrong block. + for i, title in seps: + if title == test or title.split("[")[0] == test: + start = i + break + if start is None and leaf: # fall back when the annotation gave only a leaf + for i, title in seps: + base = title.split("[")[0] + if base == leaf or base.endswith("." + leaf): + start = i + break + if start is not None: + nexts = [i for i, _ in seps if i > start] + [len(lines)] + end = min(nexts) + for j in range(start + 1, end): + if _SECT.match(lines[j]): + end = j + break + block = lines[start:end] + else: + needle = leaf or test + hit = next((i for i in range(len(lines) - 1, -1, -1) if needle and needle in lines[i]), None) + if hit is None: + return None + block = lines[max(0, hit - 4): hit + 44] + if len(block) > max_lines: + block = block[:max_lines] + [f"… (+{len(block) - max_lines} more lines — open the raw log)"] + return "\n".join(block).strip() + + +def summary_reasons(lines, test): + """The concise `FAILED …/ERROR … - ` lines from pytest's summary.""" + leaf = re.split(r"[.:]", test)[-1].split("[")[0] if test else None + return [ln for ln in lines + if re.match(r"^(FAILED|ERROR) ", ln) and (not leaf or leaf in ln)] + + +def render_joblog(job_id, url): + log = get_job_log(job_id) + ghlink = f'job on GitHub ↗' if url else "" + rawlink = f'raw log ↗' + head = f'
relevant logs{rawlink} · {ghlink}
' + if not log: + return head + ('
Log not available yet — the job may still be ' + f'running, or GitHub expired it. {ghlink}
') + lines = _clean_lines(log) + fails = get_failures(job_id).get("failures", []) + blocks = [] + for f in fails: + excerpt = extract_test_log(lines, f["test"]) + reasons = summary_reasons(lines, f["test"]) + title = e(f["test"] or "failure") + reason_html = (f'
{e(reasons[0][:300])}
' if reasons else "") + if excerpt: + blocks.append(f'
{title}' + f'{reason_html}
{e(excerpt)}
') + else: + blocks.append(f'
{title}' + f'{reason_html}
No matching block in the log — ' + f'see the raw log.
') + if not fails: # build/env/setup failure: no test annotations → show the tail + tail = "\n".join(lines[-180:]).strip() + blocks.append('
end of job log' + f'
{e(tail)}
') + return head + "".join(blocks) + + +# ── status helpers ─────────────────────────────────────────────────────────── +def classify(status, conclusion): + """→ (css class, label). "Didn't run" states (skipped/cancelled/never-started) + are kept DISTINCT from "fail" — a gated-off or blocked tier is not a failure, + and a cancelled job didn't produce a verdict. Only `failure`/`timed_out` + (actually executed and failed) are `fail`.""" + if status and status != "completed": + if status in ("in_progress", "running"): + return "run", "running" + return "queue", status.replace("_", " ") # queued / waiting / pending / requested + c = conclusion or "" + return { + "success": ("pass", "passed"), + "failure": ("fail", "failed"), + "timed_out": ("fail", "timed out"), + # ── didn't run (by design / gating / blocked by a failed dependency) ── + "skipped": ("skip", "didn’t run"), + "neutral": ("skip", "neutral"), + "stale": ("skip", "stale"), + # ── didn't finish / never started (distinct again from a test failure) ── + "cancelled": ("cancel", "cancelled"), + "startup_failure": ("cancel", "didn’t start"), + "action_required": ("queue", "action req"), + }.get(c, ("skip", c or "no result")) + + +# Sort worst-first: real failures, then in-flight, then didn't-finish, then green, +# then the didn't-run noise last. "skip" (gated-off) is deliberately below "pass". +RANK = {"fail": 0, "run": 1, "queue": 2, "cancel": 3, "pass": 4, "skip": 5} +DIDNT_RUN = ("skip", "cancel") + + +# ── failure categorization ─────────────────────────────────────────────────── +# Triage-at-a-glance: the judgment a human makes reading each traceback — is this +# infra (OOM/resource → retry/parallelism, not a code bug), an env/import gap, a +# converter gap, or a real regression (numerical/assertion)? First pattern wins, +# so order is most-specific → most-generic. `kind` drives the tag colour: +# infra = "probably not your bug" · env/gap = setup/feature hole · bug = real. +# NOTE: an OOM often surfaces AS an AssertionError ("assert cuda_engine"); the OOM +# signatures below are listed first on purpose so they win over the generic assert. +_FAIL_CATS = [ + ("oom", "GPU / resource", "infra", re.compile( + r"OutOfMemory|out of memory|createCaskHardwareInfo|\bCask\b|" + r"build_serialized_network returned None|assert cuda_engine|cuda_engine\b|" + r"catchCudaError|cudaError|cuInit|defaultAllocator|no CUDA GPUs|" + r"CUDA error|cudaMalloc|device-side assert", re.I)), + ("timeout", "timeout", "infra", re.compile( + r"timed out|timeout|deadline exceeded|took too long", re.I)), + ("import", "import / collection", "env", re.compile( + r"ModuleNotFoundError|ImportError|No module named|cannot import name|" + r"error collecting|errors during collection|failed to import", re.I)), + ("unsupported", "unsupported op", "env", re.compile( + r"no converter|not support|unsupported|Could not find any implementation|" + r"UnsupportedOperator|no implementation for", re.I)), + ("numerical", "numerical / accuracy", "bug", re.compile( + r"not close|Mismatched elements|cosine|tolerance|allclose|" + r"Max absolute difference|relative difference|accuracy|atol|rtol", re.I)), + ("assertion", "assertion", "bug", re.compile( + r"\bAssertionError\b|^\s*assert\s", re.I | re.M)), + ("typeerr", "type / shape", "bug", re.compile( + r"\b(TypeError|AttributeError|KeyError|IndexError|ValueError)\b|" + r"shape mismatch|size mismatch|dtype", re.I)), + ("runtime", "runtime error", "bug", re.compile( + r"\b(RuntimeError|NotImplementedError)\b", re.I)), +] + + +# Roll-up labels for the per-run category tally (kind → human summary word). +_KIND_LABEL = { + "bug": "real bug", + "env": "env / gap", + "infra": "infra / OOM", + "unknown": "uncategorized", +} + + +def categorize_failure(message): + """Classify a failure's error text → {key, label, kind}. `kind` ∈ + {infra, env, bug, unknown} and drives the tag colour.""" + text = message or "" + for key, label, kind, rx in _FAIL_CATS: + if rx.search(text): + return dict(key=key, label=label, kind=kind) + return dict(key="other", label="error", kind="unknown") + + +def _cat_tag(cat): + return (f'{e(cat["label"])}') + + +def _config_pattern(occ, all_pys, all_cudas): + """Which configuration a failure correlates with — the strongest diagnostic + signal after the category. Only asserts "X only" when X is a strict subset of + what actually ran (so we never claim a pattern that isn't real).""" + fpys = {o["py"] for o in occ if o.get("py")} + fcudas = {o["cuda"] for o in occ if o.get("cuda")} + fplats = {o["plat"] for o in occ} + bits = [] + if fplats and all("RTX" in p or "rtx" in p for p in fplats): + bits.append("RTX only") + if fplats and all("py-only" in p for p in fplats): + bits.append("python-only") + if len(all_cudas) > 1 and len(fcudas) == 1 and fcudas < all_cudas: + bits.append(f"{next(iter(fcudas))} only") + if len(all_pys) > 1 and len(fpys) == 1 and fpys < all_pys: + bits.append(f"py{next(iter(fpys))} only") + return bits + + +def rel_time(iso): + if not iso: + return "" + try: + t = datetime.fromisoformat(iso.replace("Z", "+00:00")) + except Exception: + return iso + s = (datetime.now(timezone.utc) - t).total_seconds() + if s < 60: + return "just now" + for unit, n in (("d", 86400), ("h", 3600), ("m", 60)): + if s >= n: + return f"{int(s // n)}{unit} ago" + return "just now" + + +def pretty_platform(name): + n = (name.replace("Python-only build and test ", "py-only ") + .replace("RTX - Build and test ", "RTX ") + .replace("RTX - Python-only build and test ", "RTX py-only ") + .replace("Build and test ", "").replace(" wheels", "") + .replace(" for Jetpack", " · jetpack")) + return n.strip() + + +def e(s): + return html.escape(str(s if s is not None else "")) + + +def qp(**kw): + """Build a URL-encoded, HTML-attribute-safe query string from kwargs, so + values with spaces/&/ (platform + tier names, job URLs) survive intact.""" + return e("&".join(f"{k}={quote(str(v if v is not None else ''), safe='')}" + for k, v in kw.items())) + + +# ── HTML fragment renderers ────────────────────────────────────────────────── +def _summary_html(runs, oob=False): + """The top counts strip. Shared by the board and the live poller so the two + never drift. 'didn't run' folds skipped + cancelled together and is counted + separately from 'failing'.""" + counts = {"pass": 0, "fail": 0, "run": 0, "queue": 0, "skip": 0, "cancel": 0} + for r in runs: + cls, _ = classify(r["status"], r["conclusion"]) + counts[cls] = counts.get(cls, 0) + 1 + didnt_run = counts["skip"] + counts["cancel"] + head = runs[0] if runs else {} + sha = (head.get("headSha") or "")[:9] + commit = (f'
{e(sha)} · {e(head.get("displayTitle", ""))[:80]}' + f' · {e(rel_time(head.get("createdAt")))}
') + + def stat(cls, n, word): + return (f'' + f'{n} {word}') if n else "" + oob_attr = ' hx-swap-oob="true"' if oob else '' + return (f'
' + f'{stat("fail", counts["fail"], "failing")}' + f'{stat("run", counts["run"], "running")}' + f'{stat("queue", counts["queue"], "queued")}' + f'{stat("pass", counts["pass"], "passing")}' + f'{stat("skip", didnt_run, "didn’t run")}' + f'{commit}
') + + +def render_board(branch, refresh=False): + try: + runs = get_runs(branch, refresh=refresh) + except Exception as ex: + return f'
Could not load {e(branch)}:

{e(ex)}
' + if not runs: + return f'
No CI runs found for {e(branch)}.
' + + for r in runs: + r["_cls"], r["_label"] = classify(r["status"], r["conclusion"]) + runs.sort(key=lambda r: (RANK.get(r["_cls"], 9), pretty_platform(r["workflowName"]).lower())) + summary = _summary_html(runs) + + cards = [] + for r in runs: + rid, cls, label = r["id"], r["_cls"], r["_label"] + plat = pretty_platform(r["workflowName"]) + sub = f'{e(r.get("event",""))} · #{r.get("number","")} · {e(rel_time(r.get("createdAt")))}' + q = qp(run=rid, sha=r.get("headSha", ""), platform=plat, + refresh="1" if refresh else "0") + cards.append(f''' +
+ + + {e(plat)}
{sub}
 + {e(label)} + ▸ + +
loading jobs…
+
''') + + poller = (f'
') + agg = (f'

Failures across platforms

' + f'
' + f'
scanning failing tests…
') + board = f'

Platforms

{"".join(cards)}
' + return summary + agg + board + poller + + +def render_status(branch): + """OOB-only fragment: refresh the summary counts + every card badge in place + without disturbing expanded platform grids.""" + try: + runs = get_runs(branch, refresh=True) + except Exception: + return "" + spans = [] + for r in runs: + cls, label = classify(r["status"], r["conclusion"]) + spans.append(f'{e(label)}') + return _summary_html(runs, oob=True) + "".join(spans) + + +KIND_ORDER = {"test": 0, "build": 1, "setup": 2, "rollup": 3, "other": 4} + + +def render_platform(run_id, sha, platform, refresh=False): + try: + jobs = get_jobs(run_id, refresh=refresh) + except Exception as ex: + return f'
Could not load jobs: {e(ex)}
' + if not jobs: + return '
No jobs.
' + + # group by (kind, group) + groups = {} + for j in jobs: + groups.setdefault((KIND_ORDER.get(j["kind"], 9), j["group"]), []).append(j) + + out = [] + for (_, gname), gjobs in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1])): + info = info_for(gname) + tierpath = f'{e(", ".join(info["paths"]))}' if info else "" + cells, rows = [], [] + for j in sorted(gjobs, key=lambda j: (j["python"] or "", j["cuda"] or "")): + cls, label = classify(j["status"], j["conclusion"]) + failable = cls == "fail" and j["kind"] in ("test", "build") + if j["python"] and j["cuda"]: + inner = (f'{e(j["python"])}·{e(j["cuda"])}' + f'{e(label)}') + if failable: + q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, + py=j["python"], cuda=j["cuda"], url=j["url"]) + cells.append(f'
{inner}
') + else: + cells.append(f'
{inner}
') + else: + name = j["raw"].split(" / ")[-1] or j["group"] + link = f' log ↗' if j["url"] else "" + extra = "" + if failable: + q = qp(job=j["id"], sha=sha, platform=platform, tier=gname, url=j["url"]) + extra = (f' · details') + rows.append(f'
' + f'{e(name)}' + f'{e(label)}{link}{extra}
') + body = "" + if cells: + body += f'
{"".join(cells)}
' + if rows: + body += f'
{"".join(rows)}
' + out.append(f'

{e(gname)} {tierpath}

{body}
') + return "".join(out) + + +def render_failures(job_id, sha, platform, tier, py, cuda, url): + data = get_failures(job_id) + joblink = (f' · job log ↗') if url else "" + oob = (f'{e(platform)} — {e(tier)}' + f'{e(py)} {e(cuda)}{joblink}') + if data.get("error"): + return oob + f'
Could not load annotations: {e(data["error"])}
' + fails = data["failures"] + info = info_for(tier) + body = [] + if not fails: + loglink = f'Open the job log ↗' if url else "" + body.append('
No pytest failure annotations — this is likely a ' + f'build / environment / setup failure rather than a test assertion.

{loglink}
') + for f in fails: + loc = "" + if f["file"]: + gh_url = f'https://github.com/{repo_slug()}/blob/{sha}/{f["file"]}#L{f["line"]}' + loc = (f'
↳ ' + f'{e(f["file"])}:{f["line"]} ↗
') + cat = categorize_failure(f["message"]) + body.append(f'
{e(f["test"] or "failure")}' + f' {_cat_tag(cat)}
' + f'{loc}
{e(f["message"])}
') + if info: + leaves = [] + for f in fails: + if f["test"]: + leaf = re.split(r"[.:]", f["test"])[-1] + leaf = re.sub(r"\[.*$", "", leaf) + if leaf and leaf not in leaves: + leaves.append(leaf) + kexpr = " or ".join(leaves[:6]) + cmd = info["repro"](kexpr) + body.append(f'
' + f'
reproduce locally
{e(cmd)}
') + # Lazy: fetch the job log + extract the relevant block(s) once the drawer is in. + body.append(f'
' + f'
' + f'fetching relevant logs…
') + return oob + "".join(body) + + +def render_aggregate(branch, refresh=False): + """Cross-platform failure rollup: dedupe a failing test across every platform + it breaks on, so 'fails on N platforms' points straight at the likely code.""" + try: + runs = get_runs(branch, refresh=refresh) + except Exception as ex: + return f'
Could not scan: {e(ex)}
' + + def jobs_of(r): + try: + return [(r, j) for j in get_jobs(r["id"], refresh=refresh)] + except Exception: + return [] + + with ThreadPoolExecutor(max_workers=8) as ex: + alljobs = [x for sub in ex.map(jobs_of, runs) for x in sub] + failed = [(r, j) for r, j in alljobs + if classify(j["status"], j["conclusion"])[0] == "fail" and j["kind"] == "test"] + if not failed: + healthy = all(classify(r["status"], r["conclusion"])[0] != "fail" for r in runs) + msg = ("No failing test jobs 🎉" if healthy + else "No test-level failures found (failures are build/setup — see the platform grids).") + return f'
{msg}
' + + def fails_of(rj): + r, j = rj + return (r, j, get_failures(j["id"], refresh=refresh).get("failures", [])) + with ThreadPoolExecutor(max_workers=8) as ex: + results = list(ex.map(fails_of, failed)) + + agg = {} + for r, j, fails in results: + plat = pretty_platform(r["workflowName"]) + if not fails: # failed job but no parseable test → bucket under the tier + key = f"[{j['group']}]" + a = agg.setdefault(key, dict(test=key, file=None, line=None, occ=[])) + a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg="")) + continue + for f in fails: + key = f["test"] or f["message"].splitlines()[0] + a = agg.setdefault(key, dict(test=f["test"] or key, file=f["file"], line=f["line"], occ=[])) + a["file"] = a["file"] or f["file"] + a["line"] = a["line"] or f["line"] + a["occ"].append(dict(plat=plat, py=j["python"], cuda=j["cuda"], url=j["url"], msg=f["message"])) + + # Config universe (what actually ran) so "X only" hints are real subsets. + test_jobs = [j for _, j in alljobs if j["kind"] == "test"] + all_pys = {j["python"] for j in test_jobs if j["python"]} + all_cudas = {j["cuda"] for j in test_jobs if j["cuda"]} + + # Categorize once per row (from the sample traceback) so we can tally + tag. + for a in agg.values(): + a["sample"] = next((o["msg"] for o in a["occ"] if o["msg"]), "") + a["cat"] = categorize_failure(a["sample"] or a["test"]) + + # Real regressions (bug) first, then env/gap, infra last; ties → most cells. + _KIND_RANK = {"bug": 0, "env": 1, "unknown": 2, "infra": 3} + rows = sorted( + agg.values(), + key=lambda a: (_KIND_RANK.get(a["cat"]["kind"], 2), + -len({o["plat"] for o in a["occ"]}), -len(a["occ"]), a["test"])) + out = [] + for a in rows: + plats = sorted({o["plat"] for o in a["occ"]}) + ncells = len(a["occ"]) + loc = "" + if a["file"]: + gh_url = f'https://github.com/{repo_slug()}/blob/{runs[0].get("headSha","")}/{a["file"]}#L{a["line"]}' + loc = f'
↳ {e(a["file"])}:{a["line"]} ↗
' + chips = "".join(f'{e(p)}' for p in plats) + pat = _config_pattern(a["occ"], all_pys, all_cudas) + patchips = "".join(f'{e(p)}' for p in pat) + out.append(f'''
+ +
{e(a["test"])} {_cat_tag(a["cat"])}{patchips}
{loc}
+ {len(plats)} platform{"s" if len(plats)!=1 else ""} · {ncells} cell{"s" if ncells!=1 else ""} + +
{chips}
{f"
{e(a['sample'])}
" if a["sample"] else ""}
+
''') + # Category tally — an instant read on the run's character (infra flakes vs regressions). + tally = {} + for a in rows: + tally.setdefault(a["cat"]["kind"], dict(n=0, label=a["cat"]["label"]))["n"] += 1 + order = ["bug", "env", "unknown", "infra"] + tstr = " · ".join( + f'{tally[k]["n"]} {e(_KIND_LABEL[k])}' + for k in order if k in tally) + header = (f'
{len(rows)} distinct failing test' + f'{"s" if len(rows)!=1 else ""} across ' + f'{len({o["plat"] for a in rows for o in a["occ"]})} platforms' + f'{tstr}' + f'
') + return header + f'
{"".join(out)}
' + + +# ── HTTP server ────────────────────────────────────────────────────────────── +class Handler(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _send(self, code, body, ctype="text/html; charset=utf-8"): + if isinstance(body, (dict, list)): + body, ctype = json.dumps(body), "application/json" + body = body.encode() if isinstance(body, str) else body + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + u = urlparse(self.path) + q = {k: v[0] for k, v in parse_qs(u.query).items()} + refresh = q.get("refresh") == "1" + try: + p = u.path + if p in ("/", "/index.html"): + return self._index() + if p.startswith("/static/"): + return self._static(p[len("/static/"):]) + if p == "/ui/board": + return self._send(200, render_board(q.get("branch") or default_branch(), refresh)) + if p == "/ui/status": + return self._send(200, render_status(q.get("branch") or default_branch())) + if p == "/ui/platform": + return self._send(200, render_platform(q["run"], q.get("sha", ""), + q.get("platform", ""), refresh)) + if p == "/ui/failures": + return self._send(200, render_failures( + q["job"], q.get("sha", ""), q.get("platform", ""), q.get("tier", ""), + q.get("py", ""), q.get("cuda", ""), q.get("url", ""))) + if p == "/ui/aggregate": + return self._send(200, render_aggregate(q.get("branch") or default_branch(), refresh)) + if p == "/ui/joblog": + if q.get("raw") == "1": + log = get_job_log(q["job"], refresh) + return self._send(200 if log else 404, + log or "log not available", "text/plain; charset=utf-8") + return self._send(200, render_joblog(q["job"], q.get("url", ""))) + return self._send(404, "not found", "text/plain") + except KeyError as ex: + self._send(400, f"missing param {ex}", "text/plain") + except Exception as ex: + self._send(500, f'
error: {e(ex)}
') + + def _index(self): + with open(os.path.join(STATIC, "index.html"), encoding="utf-8") as f: + html_s = f.read() + html_s = html_s.replace("{{BRANCH}}", e(default_branch())).replace("{{REPO}}", e(repo_slug())) + self._send(200, html_s) + + def _static(self, rel): + rel = rel.split("?")[0].lstrip("/") + path = os.path.normpath(os.path.join(STATIC, rel)) + if not path.startswith(STATIC) or not os.path.isfile(path): + return self._send(404, "not found", "text/plain") + ctype = {"html": "text/html", "js": "text/javascript", + "css": "text/css"}.get(rel.rsplit(".", 1)[-1], "application/octet-stream") + with open(path, "rb") as f: + self._send(200, f.read(), ctype + "; charset=utf-8") + + +def preflight(): + try: + subprocess.run(["gh", "auth", "status"], capture_output=True, check=True) + except Exception: + sys.exit("error: `gh` is not authenticated. Run `gh auth login` first.") + + +def tailscale_ip(): + """Best-effort Tailscale IPv4 (100.64.0.0/10). Uses the CLI if present, + otherwise sniffs local interfaces. Returns None if not on a tailnet.""" + try: + out = subprocess.run(["tailscale", "ip", "-4"], capture_output=True, + text=True, timeout=3).stdout.strip().splitlines() + if out and out[0]: + return out[0].strip() + except Exception: + pass + try: + for res in socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET): + ip = res[4][0] + if ip.startswith("100.") and 64 <= int(ip.split(".")[1]) <= 127: + return ip + except Exception: + pass + return None + + +def lan_ip(): + """Primary outbound-facing LAN IP (no packets sent).""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + ip = s.getsockname()[0] + s.close() + return ip + except Exception: + return None + + +def main(): + global _DEFAULT_BRANCH + ap = argparse.ArgumentParser(description="Local Torch-TensorRT CI dashboard") + ap.add_argument("-b", "--branch", default=None, help="default branch to show") + ap.add_argument("-p", "--port", type=int, default=8712) + ap.add_argument("--host", default="0.0.0.0", + help="bind address (default 0.0.0.0 — reachable over LAN/Tailscale; " + "use 127.0.0.1 to keep it local-only)") + ap.add_argument("--no-open", action="store_true") + args = ap.parse_args() + preflight() + if args.branch: + _DEFAULT_BRANCH = args.branch + srv = ThreadingHTTPServer((args.host, args.port), Handler) + local = f"http://127.0.0.1:{args.port}/" + print(f"CI dashboard (repo {repo_slug()}, branch {default_branch()})") + print(f" local {local}") + if args.host in ("0.0.0.0", "::"): + ts = tailscale_ip() + if ts: + print(f" tailscale http://{ts}:{args.port}/") + lan = lan_ip() + if lan and lan != ts: + print(f" lan http://{lan}:{args.port}/") + print(" (bound to all interfaces — anyone who can reach this host can view CI status)") + print("Ctrl-C to stop.", flush=True) + if not args.no_open: + try: + import webbrowser + webbrowser.open(local) # always open the loopback URL locally + except Exception: + pass + try: + srv.serve_forever() + except KeyboardInterrupt: + print("\nbye") + + +if __name__ == "__main__": + main() diff --git a/tools/ci-dashboard/static/htmx.min.js b/tools/ci-dashboard/static/htmx.min.js new file mode 100644 index 0000000000..59937d712d --- /dev/null +++ b/tools/ci-dashboard/static/htmx.min.js @@ -0,0 +1 @@ +var htmx=function(){"use strict";const Q={onLoad:null,process:null,on:null,off:null,trigger:null,ajax:null,find:null,findAll:null,closest:null,values:function(e,t){const n=cn(e,t||"post");return n.values},remove:null,addClass:null,removeClass:null,toggleClass:null,takeClass:null,swap:null,defineExtension:null,removeExtension:null,logAll:null,logNone:null,logger:null,config:{historyEnabled:true,historyCacheSize:10,refreshOnHistoryMiss:false,defaultSwapStyle:"innerHTML",defaultSwapDelay:0,defaultSettleDelay:20,includeIndicatorStyles:true,indicatorClass:"htmx-indicator",requestClass:"htmx-request",addedClass:"htmx-added",settlingClass:"htmx-settling",swappingClass:"htmx-swapping",allowEval:true,allowScriptTags:true,inlineScriptNonce:"",inlineStyleNonce:"",attributesToSettle:["class","style","width","height"],withCredentials:false,timeout:0,wsReconnectDelay:"full-jitter",wsBinaryType:"blob",disableSelector:"[hx-disable], [data-hx-disable]",scrollBehavior:"instant",defaultFocusScroll:false,getCacheBusterParam:false,globalViewTransitions:false,methodsThatUseUrlParams:["get","delete"],selfRequestsOnly:true,ignoreTitle:false,scrollIntoViewOnBoost:true,triggerSpecsCache:null,disableInheritance:false,responseHandling:[{code:"204",swap:false},{code:"[23]..",swap:true},{code:"[45]..",swap:false,error:true}],allowNestedOobSwaps:true},parseInterval:null,_:null,version:"2.0.4"};Q.onLoad=j;Q.process=kt;Q.on=ye;Q.off=be;Q.trigger=he;Q.ajax=Rn;Q.find=u;Q.findAll=x;Q.closest=g;Q.remove=z;Q.addClass=K;Q.removeClass=G;Q.toggleClass=W;Q.takeClass=Z;Q.swap=$e;Q.defineExtension=Fn;Q.removeExtension=Bn;Q.logAll=V;Q.logNone=_;Q.parseInterval=d;Q._=e;const n={addTriggerHandler:St,bodyContains:le,canAccessLocalStorage:B,findThisElement:Se,filterValues:hn,swap:$e,hasAttribute:s,getAttributeValue:te,getClosestAttributeValue:re,getClosestMatch:o,getExpressionVars:En,getHeaders:fn,getInputValues:cn,getInternalData:ie,getSwapSpecification:gn,getTriggerSpecs:st,getTarget:Ee,makeFragment:P,mergeObjects:ce,makeSettleInfo:xn,oobSwap:He,querySelectorExt:ae,settleImmediately:Kt,shouldCancel:ht,triggerEvent:he,triggerErrorEvent:fe,withExtensions:Ft};const r=["get","post","put","delete","patch"];const H=r.map(function(e){return"[hx-"+e+"], [data-hx-"+e+"]"}).join(", ");function d(e){if(e==undefined){return undefined}let t=NaN;if(e.slice(-2)=="ms"){t=parseFloat(e.slice(0,-2))}else if(e.slice(-1)=="s"){t=parseFloat(e.slice(0,-1))*1e3}else if(e.slice(-1)=="m"){t=parseFloat(e.slice(0,-1))*1e3*60}else{t=parseFloat(e)}return isNaN(t)?undefined:t}function ee(e,t){return e instanceof Element&&e.getAttribute(t)}function s(e,t){return!!e.hasAttribute&&(e.hasAttribute(t)||e.hasAttribute("data-"+t))}function te(e,t){return ee(e,t)||ee(e,"data-"+t)}function c(e){const t=e.parentElement;if(!t&&e.parentNode instanceof ShadowRoot)return e.parentNode;return t}function ne(){return document}function m(e,t){return e.getRootNode?e.getRootNode({composed:t}):ne()}function o(e,t){while(e&&!t(e)){e=c(e)}return e||null}function i(e,t,n){const r=te(t,n);const o=te(t,"hx-disinherit");var i=te(t,"hx-inherit");if(e!==t){if(Q.config.disableInheritance){if(i&&(i==="*"||i.split(" ").indexOf(n)>=0)){return r}else{return null}}if(o&&(o==="*"||o.split(" ").indexOf(n)>=0)){return"unset"}}return r}function re(t,n){let r=null;o(t,function(e){return!!(r=i(t,ue(e),n))});if(r!=="unset"){return r}}function h(e,t){const n=e instanceof Element&&(e.matches||e.matchesSelector||e.msMatchesSelector||e.mozMatchesSelector||e.webkitMatchesSelector||e.oMatchesSelector);return!!n&&n.call(e,t)}function T(e){const t=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i;const n=t.exec(e);if(n){return n[1].toLowerCase()}else{return""}}function q(e){const t=new DOMParser;return t.parseFromString(e,"text/html")}function L(e,t){while(t.childNodes.length>0){e.append(t.childNodes[0])}}function A(e){const t=ne().createElement("script");se(e.attributes,function(e){t.setAttribute(e.name,e.value)});t.textContent=e.textContent;t.async=false;if(Q.config.inlineScriptNonce){t.nonce=Q.config.inlineScriptNonce}return t}function N(e){return e.matches("script")&&(e.type==="text/javascript"||e.type==="module"||e.type==="")}function I(e){Array.from(e.querySelectorAll("script")).forEach(e=>{if(N(e)){const t=A(e);const n=e.parentNode;try{n.insertBefore(t,e)}catch(e){O(e)}finally{e.remove()}}})}function P(e){const t=e.replace(/]*)?>[\s\S]*?<\/head>/i,"");const n=T(t);let r;if(n==="html"){r=new DocumentFragment;const i=q(e);L(r,i.body);r.title=i.title}else if(n==="body"){r=new DocumentFragment;const i=q(t);L(r,i.body);r.title=i.title}else{const i=q('");r=i.querySelector("template").content;r.title=i.title;var o=r.querySelector("title");if(o&&o.parentNode===r){o.remove();r.title=o.innerText}}if(r){if(Q.config.allowScriptTags){I(r)}else{r.querySelectorAll("script").forEach(e=>e.remove())}}return r}function oe(e){if(e){e()}}function t(e,t){return Object.prototype.toString.call(e)==="[object "+t+"]"}function k(e){return typeof e==="function"}function D(e){return t(e,"Object")}function ie(e){const t="htmx-internal-data";let n=e[t];if(!n){n=e[t]={}}return n}function M(t){const n=[];if(t){for(let e=0;e=0}function le(e){return e.getRootNode({composed:true})===document}function F(e){return e.trim().split(/\s+/)}function ce(e,t){for(const n in t){if(t.hasOwnProperty(n)){e[n]=t[n]}}return e}function S(e){try{return JSON.parse(e)}catch(e){O(e);return null}}function B(){const e="htmx:localStorageTest";try{localStorage.setItem(e,e);localStorage.removeItem(e);return true}catch(e){return false}}function U(t){try{const e=new URL(t);if(e){t=e.pathname+e.search}if(!/^\/$/.test(t)){t=t.replace(/\/+$/,"")}return t}catch(e){return t}}function e(e){return vn(ne().body,function(){return eval(e)})}function j(t){const e=Q.on("htmx:load",function(e){t(e.detail.elt)});return e}function V(){Q.logger=function(e,t,n){if(console){console.log(t,e,n)}}}function _(){Q.logger=null}function u(e,t){if(typeof e!=="string"){return e.querySelector(t)}else{return u(ne(),e)}}function x(e,t){if(typeof e!=="string"){return e.querySelectorAll(t)}else{return x(ne(),e)}}function E(){return window}function z(e,t){e=y(e);if(t){E().setTimeout(function(){z(e);e=null},t)}else{c(e).removeChild(e)}}function ue(e){return e instanceof Element?e:null}function $(e){return e instanceof HTMLElement?e:null}function J(e){return typeof e==="string"?e:null}function f(e){return e instanceof Element||e instanceof Document||e instanceof DocumentFragment?e:null}function K(e,t,n){e=ue(y(e));if(!e){return}if(n){E().setTimeout(function(){K(e,t);e=null},n)}else{e.classList&&e.classList.add(t)}}function G(e,t,n){let r=ue(y(e));if(!r){return}if(n){E().setTimeout(function(){G(r,t);r=null},n)}else{if(r.classList){r.classList.remove(t);if(r.classList.length===0){r.removeAttribute("class")}}}}function W(e,t){e=y(e);e.classList.toggle(t)}function Z(e,t){e=y(e);se(e.parentElement.children,function(e){G(e,t)});K(ue(e),t)}function g(e,t){e=ue(y(e));if(e&&e.closest){return e.closest(t)}else{do{if(e==null||h(e,t)){return e}}while(e=e&&ue(c(e)));return null}}function l(e,t){return e.substring(0,t.length)===t}function Y(e,t){return e.substring(e.length-t.length)===t}function ge(e){const t=e.trim();if(l(t,"<")&&Y(t,"/>")){return t.substring(1,t.length-2)}else{return t}}function p(t,r,n){if(r.indexOf("global ")===0){return p(t,r.slice(7),true)}t=y(t);const o=[];{let t=0;let n=0;for(let e=0;e"){t--}}if(n0){const r=ge(o.shift());let e;if(r.indexOf("closest ")===0){e=g(ue(t),ge(r.substr(8)))}else if(r.indexOf("find ")===0){e=u(f(t),ge(r.substr(5)))}else if(r==="next"||r==="nextElementSibling"){e=ue(t).nextElementSibling}else if(r.indexOf("next ")===0){e=pe(t,ge(r.substr(5)),!!n)}else if(r==="previous"||r==="previousElementSibling"){e=ue(t).previousElementSibling}else if(r.indexOf("previous ")===0){e=me(t,ge(r.substr(9)),!!n)}else if(r==="document"){e=document}else if(r==="window"){e=window}else if(r==="body"){e=document.body}else if(r==="root"){e=m(t,!!n)}else if(r==="host"){e=t.getRootNode().host}else{s.push(r)}if(e){i.push(e)}}if(s.length>0){const e=s.join(",");const c=f(m(t,!!n));i.push(...M(c.querySelectorAll(e)))}return i}var pe=function(t,e,n){const r=f(m(t,n)).querySelectorAll(e);for(let e=0;e=0;e--){const o=r[e];if(o.compareDocumentPosition(t)===Node.DOCUMENT_POSITION_FOLLOWING){return o}}};function ae(e,t){if(typeof e!=="string"){return p(e,t)[0]}else{return p(ne().body,e)[0]}}function y(e,t){if(typeof e==="string"){return u(f(t)||document,e)}else{return e}}function xe(e,t,n,r){if(k(t)){return{target:ne().body,event:J(e),listener:t,options:n}}else{return{target:y(e),event:J(t),listener:n,options:r}}}function ye(t,n,r,o){Vn(function(){const e=xe(t,n,r,o);e.target.addEventListener(e.event,e.listener,e.options)});const e=k(n);return e?n:r}function be(t,n,r){Vn(function(){const e=xe(t,n,r);e.target.removeEventListener(e.event,e.listener)});return k(n)?n:r}const ve=ne().createElement("output");function we(e,t){const n=re(e,t);if(n){if(n==="this"){return[Se(e,t)]}else{const r=p(e,n);if(r.length===0){O('The selector "'+n+'" on '+t+" returned no matches!");return[ve]}else{return r}}}}function Se(e,t){return ue(o(e,function(e){return te(ue(e),t)!=null}))}function Ee(e){const t=re(e,"hx-target");if(t){if(t==="this"){return Se(e,"hx-target")}else{return ae(e,t)}}else{const n=ie(e);if(n.boosted){return ne().body}else{return e}}}function Ce(t){const n=Q.config.attributesToSettle;for(let e=0;e0){s=e.substring(0,e.indexOf(":"));n=e.substring(e.indexOf(":")+1)}else{s=e}o.removeAttribute("hx-swap-oob");o.removeAttribute("data-hx-swap-oob");const r=p(t,n,false);if(r){se(r,function(e){let t;const n=o.cloneNode(true);t=ne().createDocumentFragment();t.appendChild(n);if(!Re(s,e)){t=f(n)}const r={shouldSwap:true,target:e,fragment:t};if(!he(e,"htmx:oobBeforeSwap",r))return;e=r.target;if(r.shouldSwap){qe(t);_e(s,e,e,t,i);Te()}se(i.elts,function(e){he(e,"htmx:oobAfterSwap",r)})});o.parentNode.removeChild(o)}else{o.parentNode.removeChild(o);fe(ne().body,"htmx:oobErrorNoTarget",{content:o})}return e}function Te(){const e=u("#--htmx-preserve-pantry--");if(e){for(const t of[...e.children]){const n=u("#"+t.id);n.parentNode.moveBefore(t,n);n.remove()}e.remove()}}function qe(e){se(x(e,"[hx-preserve], [data-hx-preserve]"),function(e){const t=te(e,"id");const n=ne().getElementById(t);if(n!=null){if(e.moveBefore){let e=u("#--htmx-preserve-pantry--");if(e==null){ne().body.insertAdjacentHTML("afterend","
");e=u("#--htmx-preserve-pantry--")}e.moveBefore(n,null)}else{e.parentNode.replaceChild(n,e)}}})}function Le(l,e,c){se(e.querySelectorAll("[id]"),function(t){const n=ee(t,"id");if(n&&n.length>0){const r=n.replace("'","\\'");const o=t.tagName.replace(":","\\:");const e=f(l);const i=e&&e.querySelector(o+"[id='"+r+"']");if(i&&i!==e){const s=t.cloneNode();Oe(t,i);c.tasks.push(function(){Oe(t,s)})}}})}function Ae(e){return function(){G(e,Q.config.addedClass);kt(ue(e));Ne(f(e));he(e,"htmx:load")}}function Ne(e){const t="[autofocus]";const n=$(h(e,t)?e:e.querySelector(t));if(n!=null){n.focus()}}function a(e,t,n,r){Le(e,n,r);while(n.childNodes.length>0){const o=n.firstChild;K(ue(o),Q.config.addedClass);e.insertBefore(o,t);if(o.nodeType!==Node.TEXT_NODE&&o.nodeType!==Node.COMMENT_NODE){r.tasks.push(Ae(o))}}}function Ie(e,t){let n=0;while(n0}function $e(e,t,r,o){if(!o){o={}}e=y(e);const i=o.contextElement?m(o.contextElement,false):ne();const n=document.activeElement;let s={};try{s={elt:n,start:n?n.selectionStart:null,end:n?n.selectionEnd:null}}catch(e){}const l=xn(e);if(r.swapStyle==="textContent"){e.textContent=t}else{let n=P(t);l.title=n.title;if(o.selectOOB){const u=o.selectOOB.split(",");for(let t=0;t0){E().setTimeout(c,r.settleDelay)}else{c()}}function Je(e,t,n){const r=e.getResponseHeader(t);if(r.indexOf("{")===0){const o=S(r);for(const i in o){if(o.hasOwnProperty(i)){let e=o[i];if(D(e)){n=e.target!==undefined?e.target:n}else{e={value:e}}he(n,i,e)}}}else{const s=r.split(",");for(let e=0;e0){const s=o[0];if(s==="]"){e--;if(e===0){if(n===null){t=t+"true"}o.shift();t+=")})";try{const l=vn(r,function(){return Function(t)()},function(){return true});l.source=t;return l}catch(e){fe(ne().body,"htmx:syntax:error",{error:e,source:t});return null}}}else if(s==="["){e++}if(tt(s,n,i)){t+="(("+i+"."+s+") ? ("+i+"."+s+") : (window."+s+"))"}else{t=t+s}n=o.shift()}}}function C(e,t){let n="";while(e.length>0&&!t.test(e[0])){n+=e.shift()}return n}function rt(e){let t;if(e.length>0&&Ye.test(e[0])){e.shift();t=C(e,Qe).trim();e.shift()}else{t=C(e,v)}return t}const ot="input, textarea, select";function it(e,t,n){const r=[];const o=et(t);do{C(o,w);const l=o.length;const c=C(o,/[,\[\s]/);if(c!==""){if(c==="every"){const u={trigger:"every"};C(o,w);u.pollInterval=d(C(o,/[,\[\s]/));C(o,w);var i=nt(e,o,"event");if(i){u.eventFilter=i}r.push(u)}else{const a={trigger:c};var i=nt(e,o,"event");if(i){a.eventFilter=i}C(o,w);while(o.length>0&&o[0]!==","){const f=o.shift();if(f==="changed"){a.changed=true}else if(f==="once"){a.once=true}else if(f==="consume"){a.consume=true}else if(f==="delay"&&o[0]===":"){o.shift();a.delay=d(C(o,v))}else if(f==="from"&&o[0]===":"){o.shift();if(Ye.test(o[0])){var s=rt(o)}else{var s=C(o,v);if(s==="closest"||s==="find"||s==="next"||s==="previous"){o.shift();const h=rt(o);if(h.length>0){s+=" "+h}}}a.from=s}else if(f==="target"&&o[0]===":"){o.shift();a.target=rt(o)}else if(f==="throttle"&&o[0]===":"){o.shift();a.throttle=d(C(o,v))}else if(f==="queue"&&o[0]===":"){o.shift();a.queue=C(o,v)}else if(f==="root"&&o[0]===":"){o.shift();a[f]=rt(o)}else if(f==="threshold"&&o[0]===":"){o.shift();a[f]=C(o,v)}else{fe(e,"htmx:syntax:error",{token:o.shift()})}C(o,w)}r.push(a)}}if(o.length===l){fe(e,"htmx:syntax:error",{token:o.shift()})}C(o,w)}while(o[0]===","&&o.shift());if(n){n[t]=r}return r}function st(e){const t=te(e,"hx-trigger");let n=[];if(t){const r=Q.config.triggerSpecsCache;n=r&&r[t]||it(e,t,r)}if(n.length>0){return n}else if(h(e,"form")){return[{trigger:"submit"}]}else if(h(e,'input[type="button"], input[type="submit"]')){return[{trigger:"click"}]}else if(h(e,ot)){return[{trigger:"change"}]}else{return[{trigger:"click"}]}}function lt(e){ie(e).cancelled=true}function ct(e,t,n){const r=ie(e);r.timeout=E().setTimeout(function(){if(le(e)&&r.cancelled!==true){if(!gt(n,e,Mt("hx:poll:trigger",{triggerSpec:n,target:e}))){t(e)}ct(e,t,n)}},n.pollInterval)}function ut(e){return location.hostname===e.hostname&&ee(e,"href")&&ee(e,"href").indexOf("#")!==0}function at(e){return g(e,Q.config.disableSelector)}function ft(t,n,e){if(t instanceof HTMLAnchorElement&&ut(t)&&(t.target===""||t.target==="_self")||t.tagName==="FORM"&&String(ee(t,"method")).toLowerCase()!=="dialog"){n.boosted=true;let r,o;if(t.tagName==="A"){r="get";o=ee(t,"href")}else{const i=ee(t,"method");r=i?i.toLowerCase():"get";o=ee(t,"action");if(o==null||o===""){o=ne().location.href}if(r==="get"&&o.includes("?")){o=o.replace(/\?[^#]+/,"")}}e.forEach(function(e){pt(t,function(e,t){const n=ue(e);if(at(n)){b(n);return}de(r,o,n,t)},n,e,true)})}}function ht(e,t){const n=ue(t);if(!n){return false}if(e.type==="submit"||e.type==="click"){if(n.tagName==="FORM"){return true}if(h(n,'input[type="submit"], button')&&(h(n,"[form]")||g(n,"form")!==null)){return true}if(n instanceof HTMLAnchorElement&&n.href&&(n.getAttribute("href")==="#"||n.getAttribute("href").indexOf("#")!==0)){return true}}return false}function dt(e,t){return ie(e).boosted&&e instanceof HTMLAnchorElement&&t.type==="click"&&(t.ctrlKey||t.metaKey)}function gt(e,t,n){const r=e.eventFilter;if(r){try{return r.call(t,n)!==true}catch(e){const o=r.source;fe(ne().body,"htmx:eventFilter:error",{error:e,source:o});return true}}return false}function pt(l,c,e,u,a){const f=ie(l);let t;if(u.from){t=p(l,u.from)}else{t=[l]}if(u.changed){if(!("lastValue"in f)){f.lastValue=new WeakMap}t.forEach(function(e){if(!f.lastValue.has(u)){f.lastValue.set(u,new WeakMap)}f.lastValue.get(u).set(e,e.value)})}se(t,function(i){const s=function(e){if(!le(l)){i.removeEventListener(u.trigger,s);return}if(dt(l,e)){return}if(a||ht(e,l)){e.preventDefault()}if(gt(u,l,e)){return}const t=ie(e);t.triggerSpec=u;if(t.handledFor==null){t.handledFor=[]}if(t.handledFor.indexOf(l)<0){t.handledFor.push(l);if(u.consume){e.stopPropagation()}if(u.target&&e.target){if(!h(ue(e.target),u.target)){return}}if(u.once){if(f.triggeredOnce){return}else{f.triggeredOnce=true}}if(u.changed){const n=event.target;const r=n.value;const o=f.lastValue.get(u);if(o.has(n)&&o.get(n)===r){return}o.set(n,r)}if(f.delayed){clearTimeout(f.delayed)}if(f.throttle){return}if(u.throttle>0){if(!f.throttle){he(l,"htmx:trigger");c(l,e);f.throttle=E().setTimeout(function(){f.throttle=null},u.throttle)}}else if(u.delay>0){f.delayed=E().setTimeout(function(){he(l,"htmx:trigger");c(l,e)},u.delay)}else{he(l,"htmx:trigger");c(l,e)}}};if(e.listenerInfos==null){e.listenerInfos=[]}e.listenerInfos.push({trigger:u.trigger,listener:s,on:i});i.addEventListener(u.trigger,s)})}let mt=false;let xt=null;function yt(){if(!xt){xt=function(){mt=true};window.addEventListener("scroll",xt);window.addEventListener("resize",xt);setInterval(function(){if(mt){mt=false;se(ne().querySelectorAll("[hx-trigger*='revealed'],[data-hx-trigger*='revealed']"),function(e){bt(e)})}},200)}}function bt(e){if(!s(e,"data-hx-revealed")&&X(e)){e.setAttribute("data-hx-revealed","true");const t=ie(e);if(t.initHash){he(e,"revealed")}else{e.addEventListener("htmx:afterProcessNode",function(){he(e,"revealed")},{once:true})}}}function vt(e,t,n,r){const o=function(){if(!n.loaded){n.loaded=true;he(e,"htmx:trigger");t(e)}};if(r>0){E().setTimeout(o,r)}else{o()}}function wt(t,n,e){let i=false;se(r,function(r){if(s(t,"hx-"+r)){const o=te(t,"hx-"+r);i=true;n.path=o;n.verb=r;e.forEach(function(e){St(t,e,n,function(e,t){const n=ue(e);if(g(n,Q.config.disableSelector)){b(n);return}de(r,o,n,t)})})}});return i}function St(r,e,t,n){if(e.trigger==="revealed"){yt();pt(r,n,t,e);bt(ue(r))}else if(e.trigger==="intersect"){const o={};if(e.root){o.root=ae(r,e.root)}if(e.threshold){o.threshold=parseFloat(e.threshold)}const i=new IntersectionObserver(function(t){for(let e=0;e0){t.polling=true;ct(ue(r),n,e)}else{pt(r,n,t,e)}}function Et(e){const t=ue(e);if(!t){return false}const n=t.attributes;for(let e=0;e", "+e).join(""));return o}else{return[]}}function Tt(e){const t=g(ue(e.target),"button, input[type='submit']");const n=Lt(e);if(n){n.lastButtonClicked=t}}function qt(e){const t=Lt(e);if(t){t.lastButtonClicked=null}}function Lt(e){const t=g(ue(e.target),"button, input[type='submit']");if(!t){return}const n=y("#"+ee(t,"form"),t.getRootNode())||g(t,"form");if(!n){return}return ie(n)}function At(e){e.addEventListener("click",Tt);e.addEventListener("focusin",Tt);e.addEventListener("focusout",qt)}function Nt(t,e,n){const r=ie(t);if(!Array.isArray(r.onHandlers)){r.onHandlers=[]}let o;const i=function(e){vn(t,function(){if(at(t)){return}if(!o){o=new Function("event",n)}o.call(t,e)})};t.addEventListener(e,i);r.onHandlers.push({event:e,listener:i})}function It(t){ke(t);for(let e=0;eQ.config.historyCacheSize){i.shift()}while(i.length>0){try{localStorage.setItem("htmx-history-cache",JSON.stringify(i));break}catch(e){fe(ne().body,"htmx:historyCacheError",{cause:e,cache:i});i.shift()}}}function Vt(t){if(!B()){return null}t=U(t);const n=S(localStorage.getItem("htmx-history-cache"))||[];for(let e=0;e=200&&this.status<400){he(ne().body,"htmx:historyCacheMissLoad",i);const e=P(this.response);const t=e.querySelector("[hx-history-elt],[data-hx-history-elt]")||e;const n=Ut();const r=xn(n);kn(e.title);qe(e);Ve(n,t,r);Te();Kt(r.tasks);Bt=o;he(ne().body,"htmx:historyRestore",{path:o,cacheMiss:true,serverResponse:this.response})}else{fe(ne().body,"htmx:historyCacheMissLoadError",i)}};e.send()}function Wt(e){zt();e=e||location.pathname+location.search;const t=Vt(e);if(t){const n=P(t.content);const r=Ut();const o=xn(r);kn(t.title);qe(n);Ve(r,n,o);Te();Kt(o.tasks);E().setTimeout(function(){window.scrollTo(0,t.scroll)},0);Bt=e;he(ne().body,"htmx:historyRestore",{path:e,item:t})}else{if(Q.config.refreshOnHistoryMiss){window.location.reload(true)}else{Gt(e)}}}function Zt(e){let t=we(e,"hx-indicator");if(t==null){t=[e]}se(t,function(e){const t=ie(e);t.requestCount=(t.requestCount||0)+1;e.classList.add.call(e.classList,Q.config.requestClass)});return t}function Yt(e){let t=we(e,"hx-disabled-elt");if(t==null){t=[]}se(t,function(e){const t=ie(e);t.requestCount=(t.requestCount||0)+1;e.setAttribute("disabled","");e.setAttribute("data-disabled-by-htmx","")});return t}function Qt(e,t){se(e.concat(t),function(e){const t=ie(e);t.requestCount=(t.requestCount||1)-1});se(e,function(e){const t=ie(e);if(t.requestCount===0){e.classList.remove.call(e.classList,Q.config.requestClass)}});se(t,function(e){const t=ie(e);if(t.requestCount===0){e.removeAttribute("disabled");e.removeAttribute("data-disabled-by-htmx")}})}function en(t,n){for(let e=0;en.indexOf(e)<0)}else{e=e.filter(e=>e!==n)}r.delete(t);se(e,e=>r.append(t,e))}}function on(t,n,r,o,i){if(o==null||en(t,o)){return}else{t.push(o)}if(tn(o)){const s=ee(o,"name");let e=o.value;if(o instanceof HTMLSelectElement&&o.multiple){e=M(o.querySelectorAll("option:checked")).map(function(e){return e.value})}if(o instanceof HTMLInputElement&&o.files){e=M(o.files)}nn(s,e,n);if(i){sn(o,r)}}if(o instanceof HTMLFormElement){se(o.elements,function(e){if(t.indexOf(e)>=0){rn(e.name,e.value,n)}else{t.push(e)}if(i){sn(e,r)}});new FormData(o).forEach(function(e,t){if(e instanceof File&&e.name===""){return}nn(t,e,n)})}}function sn(e,t){const n=e;if(n.willValidate){he(n,"htmx:validation:validate");if(!n.checkValidity()){t.push({elt:n,message:n.validationMessage,validity:n.validity});he(n,"htmx:validation:failed",{message:n.validationMessage,validity:n.validity})}}}function ln(n,e){for(const t of e.keys()){n.delete(t)}e.forEach(function(e,t){n.append(t,e)});return n}function cn(e,t){const n=[];const r=new FormData;const o=new FormData;const i=[];const s=ie(e);if(s.lastButtonClicked&&!le(s.lastButtonClicked)){s.lastButtonClicked=null}let l=e instanceof HTMLFormElement&&e.noValidate!==true||te(e,"hx-validate")==="true";if(s.lastButtonClicked){l=l&&s.lastButtonClicked.formNoValidate!==true}if(t!=="get"){on(n,o,i,g(e,"form"),l)}on(n,r,i,e,l);if(s.lastButtonClicked||e.tagName==="BUTTON"||e.tagName==="INPUT"&&ee(e,"type")==="submit"){const u=s.lastButtonClicked||e;const a=ee(u,"name");nn(a,u.value,o)}const c=we(e,"hx-include");se(c,function(e){on(n,r,i,ue(e),l);if(!h(e,"form")){se(f(e).querySelectorAll(ot),function(e){on(n,r,i,e,l)})}});ln(r,o);return{errors:i,formData:r,values:An(r)}}function un(e,t,n){if(e!==""){e+="&"}if(String(n)==="[object Object]"){n=JSON.stringify(n)}const r=encodeURIComponent(n);e+=encodeURIComponent(t)+"="+r;return e}function an(e){e=qn(e);let n="";e.forEach(function(e,t){n=un(n,t,e)});return n}function fn(e,t,n){const r={"HX-Request":"true","HX-Trigger":ee(e,"id"),"HX-Trigger-Name":ee(e,"name"),"HX-Target":te(t,"id"),"HX-Current-URL":ne().location.href};bn(e,"hx-headers",false,r);if(n!==undefined){r["HX-Prompt"]=n}if(ie(e).boosted){r["HX-Boosted"]="true"}return r}function hn(n,e){const t=re(e,"hx-params");if(t){if(t==="none"){return new FormData}else if(t==="*"){return n}else if(t.indexOf("not ")===0){se(t.slice(4).split(","),function(e){e=e.trim();n.delete(e)});return n}else{const r=new FormData;se(t.split(","),function(t){t=t.trim();if(n.has(t)){n.getAll(t).forEach(function(e){r.append(t,e)})}});return r}}else{return n}}function dn(e){return!!ee(e,"href")&&ee(e,"href").indexOf("#")>=0}function gn(e,t){const n=t||re(e,"hx-swap");const r={swapStyle:ie(e).boosted?"innerHTML":Q.config.defaultSwapStyle,swapDelay:Q.config.defaultSwapDelay,settleDelay:Q.config.defaultSettleDelay};if(Q.config.scrollIntoViewOnBoost&&ie(e).boosted&&!dn(e)){r.show="top"}if(n){const s=F(n);if(s.length>0){for(let e=0;e0?o.join(":"):null;r.scroll=u;r.scrollTarget=i}else if(l.indexOf("show:")===0){const a=l.slice(5);var o=a.split(":");const f=o.pop();var i=o.length>0?o.join(":"):null;r.show=f;r.showTarget=i}else if(l.indexOf("focus-scroll:")===0){const h=l.slice("focus-scroll:".length);r.focusScroll=h=="true"}else if(e==0){r.swapStyle=l}else{O("Unknown modifier in hx-swap: "+l)}}}}return r}function pn(e){return re(e,"hx-encoding")==="multipart/form-data"||h(e,"form")&&ee(e,"enctype")==="multipart/form-data"}function mn(t,n,r){let o=null;Ft(n,function(e){if(o==null){o=e.encodeParameters(t,r,n)}});if(o!=null){return o}else{if(pn(n)){return ln(new FormData,qn(r))}else{return an(r)}}}function xn(e){return{tasks:[],elts:[e]}}function yn(e,t){const n=e[0];const r=e[e.length-1];if(t.scroll){var o=null;if(t.scrollTarget){o=ue(ae(n,t.scrollTarget))}if(t.scroll==="top"&&(n||o)){o=o||n;o.scrollTop=0}if(t.scroll==="bottom"&&(r||o)){o=o||r;o.scrollTop=o.scrollHeight}}if(t.show){var o=null;if(t.showTarget){let e=t.showTarget;if(t.showTarget==="window"){e="body"}o=ue(ae(n,e))}if(t.show==="top"&&(n||o)){o=o||n;o.scrollIntoView({block:"start",behavior:Q.config.scrollBehavior})}if(t.show==="bottom"&&(r||o)){o=o||r;o.scrollIntoView({block:"end",behavior:Q.config.scrollBehavior})}}}function bn(r,e,o,i){if(i==null){i={}}if(r==null){return i}const s=te(r,e);if(s){let e=s.trim();let t=o;if(e==="unset"){return null}if(e.indexOf("javascript:")===0){e=e.slice(11);t=true}else if(e.indexOf("js:")===0){e=e.slice(3);t=true}if(e.indexOf("{")!==0){e="{"+e+"}"}let n;if(t){n=vn(r,function(){return Function("return ("+e+")")()},{})}else{n=S(e)}for(const l in n){if(n.hasOwnProperty(l)){if(i[l]==null){i[l]=n[l]}}}}return bn(ue(c(r)),e,o,i)}function vn(e,t,n){if(Q.config.allowEval){return t()}else{fe(e,"htmx:evalDisallowedError");return n}}function wn(e,t){return bn(e,"hx-vars",true,t)}function Sn(e,t){return bn(e,"hx-vals",false,t)}function En(e){return ce(wn(e),Sn(e))}function Cn(t,n,r){if(r!==null){try{t.setRequestHeader(n,r)}catch(e){t.setRequestHeader(n,encodeURIComponent(r));t.setRequestHeader(n+"-URI-AutoEncoded","true")}}}function On(t){if(t.responseURL&&typeof URL!=="undefined"){try{const e=new URL(t.responseURL);return e.pathname+e.search}catch(e){fe(ne().body,"htmx:badResponseUrl",{url:t.responseURL})}}}function R(e,t){return t.test(e.getAllResponseHeaders())}function Rn(t,n,r){t=t.toLowerCase();if(r){if(r instanceof Element||typeof r==="string"){return de(t,n,null,null,{targetOverride:y(r)||ve,returnPromise:true})}else{let e=y(r.target);if(r.target&&!e||r.source&&!e&&!y(r.source)){e=ve}return de(t,n,y(r.source),r.event,{handler:r.handler,headers:r.headers,values:r.values,targetOverride:e,swapOverride:r.swap,select:r.select,returnPromise:true})}}else{return de(t,n,null,null,{returnPromise:true})}}function Hn(e){const t=[];while(e){t.push(e);e=e.parentElement}return t}function Tn(e,t,n){let r;let o;if(typeof URL==="function"){o=new URL(t,document.location.href);const i=document.location.origin;r=i===o.origin}else{o=t;r=l(t,document.location.origin)}if(Q.config.selfRequestsOnly){if(!r){return false}}return he(e,"htmx:validateUrl",ce({url:o,sameHost:r},n))}function qn(e){if(e instanceof FormData)return e;const t=new FormData;for(const n in e){if(e.hasOwnProperty(n)){if(e[n]&&typeof e[n].forEach==="function"){e[n].forEach(function(e){t.append(n,e)})}else if(typeof e[n]==="object"&&!(e[n]instanceof Blob)){t.append(n,JSON.stringify(e[n]))}else{t.append(n,e[n])}}}return t}function Ln(r,o,e){return new Proxy(e,{get:function(t,e){if(typeof e==="number")return t[e];if(e==="length")return t.length;if(e==="push"){return function(e){t.push(e);r.append(o,e)}}if(typeof t[e]==="function"){return function(){t[e].apply(t,arguments);r.delete(o);t.forEach(function(e){r.append(o,e)})}}if(t[e]&&t[e].length===1){return t[e][0]}else{return t[e]}},set:function(e,t,n){e[t]=n;r.delete(o);e.forEach(function(e){r.append(o,e)});return true}})}function An(o){return new Proxy(o,{get:function(e,t){if(typeof t==="symbol"){const r=Reflect.get(e,t);if(typeof r==="function"){return function(){return r.apply(o,arguments)}}else{return r}}if(t==="toJSON"){return()=>Object.fromEntries(o)}if(t in e){if(typeof e[t]==="function"){return function(){return o[t].apply(o,arguments)}}else{return e[t]}}const n=o.getAll(t);if(n.length===0){return undefined}else if(n.length===1){return n[0]}else{return Ln(e,t,n)}},set:function(t,n,e){if(typeof n!=="string"){return false}t.delete(n);if(e&&typeof e.forEach==="function"){e.forEach(function(e){t.append(n,e)})}else if(typeof e==="object"&&!(e instanceof Blob)){t.append(n,JSON.stringify(e))}else{t.append(n,e)}return true},deleteProperty:function(e,t){if(typeof t==="string"){e.delete(t)}return true},ownKeys:function(e){return Reflect.ownKeys(Object.fromEntries(e))},getOwnPropertyDescriptor:function(e,t){return Reflect.getOwnPropertyDescriptor(Object.fromEntries(e),t)}})}function de(t,n,r,o,i,D){let s=null;let l=null;i=i!=null?i:{};if(i.returnPromise&&typeof Promise!=="undefined"){var e=new Promise(function(e,t){s=e;l=t})}if(r==null){r=ne().body}const M=i.handler||Dn;const X=i.select||null;if(!le(r)){oe(s);return e}const c=i.targetOverride||ue(Ee(r));if(c==null||c==ve){fe(r,"htmx:targetError",{target:te(r,"hx-target")});oe(l);return e}let u=ie(r);const a=u.lastButtonClicked;if(a){const L=ee(a,"formaction");if(L!=null){n=L}const A=ee(a,"formmethod");if(A!=null){if(A.toLowerCase()!=="dialog"){t=A}}}const f=re(r,"hx-confirm");if(D===undefined){const K=function(e){return de(t,n,r,o,i,!!e)};const G={target:c,elt:r,path:n,verb:t,triggeringEvent:o,etc:i,issueRequest:K,question:f};if(he(r,"htmx:confirm",G)===false){oe(s);return e}}let h=r;let d=re(r,"hx-sync");let g=null;let F=false;if(d){const N=d.split(":");const I=N[0].trim();if(I==="this"){h=Se(r,"hx-sync")}else{h=ue(ae(r,I))}d=(N[1]||"drop").trim();u=ie(h);if(d==="drop"&&u.xhr&&u.abortable!==true){oe(s);return e}else if(d==="abort"){if(u.xhr){oe(s);return e}else{F=true}}else if(d==="replace"){he(h,"htmx:abort")}else if(d.indexOf("queue")===0){const W=d.split(" ");g=(W[1]||"last").trim()}}if(u.xhr){if(u.abortable){he(h,"htmx:abort")}else{if(g==null){if(o){const P=ie(o);if(P&&P.triggerSpec&&P.triggerSpec.queue){g=P.triggerSpec.queue}}if(g==null){g="last"}}if(u.queuedRequests==null){u.queuedRequests=[]}if(g==="first"&&u.queuedRequests.length===0){u.queuedRequests.push(function(){de(t,n,r,o,i)})}else if(g==="all"){u.queuedRequests.push(function(){de(t,n,r,o,i)})}else if(g==="last"){u.queuedRequests=[];u.queuedRequests.push(function(){de(t,n,r,o,i)})}oe(s);return e}}const p=new XMLHttpRequest;u.xhr=p;u.abortable=F;const m=function(){u.xhr=null;u.abortable=false;if(u.queuedRequests!=null&&u.queuedRequests.length>0){const e=u.queuedRequests.shift();e()}};const B=re(r,"hx-prompt");if(B){var x=prompt(B);if(x===null||!he(r,"htmx:prompt",{prompt:x,target:c})){oe(s);m();return e}}if(f&&!D){if(!confirm(f)){oe(s);m();return e}}let y=fn(r,c,x);if(t!=="get"&&!pn(r)){y["Content-Type"]="application/x-www-form-urlencoded"}if(i.headers){y=ce(y,i.headers)}const U=cn(r,t);let b=U.errors;const j=U.formData;if(i.values){ln(j,qn(i.values))}const V=qn(En(r));const v=ln(j,V);let w=hn(v,r);if(Q.config.getCacheBusterParam&&t==="get"){w.set("org.htmx.cache-buster",ee(c,"id")||"true")}if(n==null||n===""){n=ne().location.href}const S=bn(r,"hx-request");const _=ie(r).boosted;let E=Q.config.methodsThatUseUrlParams.indexOf(t)>=0;const C={boosted:_,useUrlParams:E,formData:w,parameters:An(w),unfilteredFormData:v,unfilteredParameters:An(v),headers:y,target:c,verb:t,errors:b,withCredentials:i.credentials||S.credentials||Q.config.withCredentials,timeout:i.timeout||S.timeout||Q.config.timeout,path:n,triggeringEvent:o};if(!he(r,"htmx:configRequest",C)){oe(s);m();return e}n=C.path;t=C.verb;y=C.headers;w=qn(C.parameters);b=C.errors;E=C.useUrlParams;if(b&&b.length>0){he(r,"htmx:validation:halted",C);oe(s);m();return e}const z=n.split("#");const $=z[0];const O=z[1];let R=n;if(E){R=$;const Z=!w.keys().next().done;if(Z){if(R.indexOf("?")<0){R+="?"}else{R+="&"}R+=an(w);if(O){R+="#"+O}}}if(!Tn(r,R,C)){fe(r,"htmx:invalidPath",C);oe(l);return e}p.open(t.toUpperCase(),R,true);p.overrideMimeType("text/html");p.withCredentials=C.withCredentials;p.timeout=C.timeout;if(S.noHeaders){}else{for(const k in y){if(y.hasOwnProperty(k)){const Y=y[k];Cn(p,k,Y)}}}const H={xhr:p,target:c,requestConfig:C,etc:i,boosted:_,select:X,pathInfo:{requestPath:n,finalRequestPath:R,responsePath:null,anchor:O}};p.onload=function(){try{const t=Hn(r);H.pathInfo.responsePath=On(p);M(r,H);if(H.keepIndicators!==true){Qt(T,q)}he(r,"htmx:afterRequest",H);he(r,"htmx:afterOnLoad",H);if(!le(r)){let e=null;while(t.length>0&&e==null){const n=t.shift();if(le(n)){e=n}}if(e){he(e,"htmx:afterRequest",H);he(e,"htmx:afterOnLoad",H)}}oe(s);m()}catch(e){fe(r,"htmx:onLoadError",ce({error:e},H));throw e}};p.onerror=function(){Qt(T,q);fe(r,"htmx:afterRequest",H);fe(r,"htmx:sendError",H);oe(l);m()};p.onabort=function(){Qt(T,q);fe(r,"htmx:afterRequest",H);fe(r,"htmx:sendAbort",H);oe(l);m()};p.ontimeout=function(){Qt(T,q);fe(r,"htmx:afterRequest",H);fe(r,"htmx:timeout",H);oe(l);m()};if(!he(r,"htmx:beforeRequest",H)){oe(s);m();return e}var T=Zt(r);var q=Yt(r);se(["loadstart","loadend","progress","abort"],function(t){se([p,p.upload],function(e){e.addEventListener(t,function(e){he(r,"htmx:xhr:"+t,{lengthComputable:e.lengthComputable,loaded:e.loaded,total:e.total})})})});he(r,"htmx:beforeSend",H);const J=E?null:mn(p,r,w);p.send(J);return e}function Nn(e,t){const n=t.xhr;let r=null;let o=null;if(R(n,/HX-Push:/i)){r=n.getResponseHeader("HX-Push");o="push"}else if(R(n,/HX-Push-Url:/i)){r=n.getResponseHeader("HX-Push-Url");o="push"}else if(R(n,/HX-Replace-Url:/i)){r=n.getResponseHeader("HX-Replace-Url");o="replace"}if(r){if(r==="false"){return{}}else{return{type:o,path:r}}}const i=t.pathInfo.finalRequestPath;const s=t.pathInfo.responsePath;const l=re(e,"hx-push-url");const c=re(e,"hx-replace-url");const u=ie(e).boosted;let a=null;let f=null;if(l){a="push";f=l}else if(c){a="replace";f=c}else if(u){a="push";f=s||i}if(f){if(f==="false"){return{}}if(f==="true"){f=s||i}if(t.pathInfo.anchor&&f.indexOf("#")===-1){f=f+"#"+t.pathInfo.anchor}return{type:a,path:f}}else{return{}}}function In(e,t){var n=new RegExp(e.code);return n.test(t.toString(10))}function Pn(e){for(var t=0;t0){E().setTimeout(e,x.swapDelay)}else{e()}}if(f){fe(o,"htmx:responseError",ce({error:"Response Status Error Code "+s.status+" from "+i.pathInfo.requestPath},i))}}const Mn={};function Xn(){return{init:function(e){return null},getSelectors:function(){return null},onEvent:function(e,t){return true},transformResponse:function(e,t,n){return e},isInlineSwap:function(e){return false},handleSwap:function(e,t,n,r){return false},encodeParameters:function(e,t,n){return null}}}function Fn(e,t){if(t.init){t.init(n)}Mn[e]=ce(Xn(),t)}function Bn(e){delete Mn[e]}function Un(e,n,r){if(n==undefined){n=[]}if(e==undefined){return n}if(r==undefined){r=[]}const t=te(e,"hx-ext");if(t){se(t.split(","),function(e){e=e.replace(/ /g,"");if(e.slice(0,7)=="ignore:"){r.push(e.slice(7));return}if(r.indexOf(e)<0){const t=Mn[e];if(t&&n.indexOf(t)<0){n.push(t)}}})}return Un(ue(c(e)),n,r)}var jn=false;ne().addEventListener("DOMContentLoaded",function(){jn=true});function Vn(e){if(jn||ne().readyState==="complete"){e()}else{ne().addEventListener("DOMContentLoaded",e)}}function _n(){if(Q.config.includeIndicatorStyles!==false){const e=Q.config.inlineStyleNonce?` nonce="${Q.config.inlineStyleNonce}"`:"";ne().head.insertAdjacentHTML("beforeend"," ."+Q.config.indicatorClass+"{opacity:0} ."+Q.config.requestClass+" ."+Q.config.indicatorClass+"{opacity:1; transition: opacity 200ms ease-in;} ."+Q.config.requestClass+"."+Q.config.indicatorClass+"{opacity:1; transition: opacity 200ms ease-in;} ")}}function zn(){const e=ne().querySelector('meta[name="htmx-config"]');if(e){return S(e.content)}else{return null}}function $n(){const e=zn();if(e){Q.config=ce(Q.config,e)}}Vn(function(){$n();_n();let e=ne().body;kt(e);const t=ne().querySelectorAll("[hx-trigger='restored'],[data-hx-trigger='restored']");e.addEventListener("htmx:abort",function(e){const t=e.target;const n=ie(t);if(n&&n.xhr){n.xhr.abort()}});const n=window.onpopstate?window.onpopstate.bind(window):null;window.onpopstate=function(e){if(e.state&&e.state.htmx){Wt();se(t,function(e){he(e,"htmx:restored",{document:ne(),triggerEvent:he})})}else{if(n){n(e)}}};E().setTimeout(function(){he(e,"htmx:load",{});e=null},0)});return Q}(); \ No newline at end of file diff --git a/tools/ci-dashboard/static/index.html b/tools/ci-dashboard/static/index.html new file mode 100644 index 0000000000..89ac3b5d1d --- /dev/null +++ b/tools/ci-dashboard/static/index.html @@ -0,0 +1,91 @@ + + + + + + Torch-TensorRT CI · {{BRANCH}} + + + + +
+

Torch-TensorRT CI

+
+ + + + +
+
+ +
+
+ passed + failed + running + queued + didn’t run + · open a platform for its python×cuda×tier grid · click a red cell for failing tests + code +
+
+ PR commands — comment these on a PR to control its CI (write access) +
+
/rerunre-run only the failed / cancelled jobs (current commit)
+
/rerun allre-run every job from scratch (current commit)
+
/cancelcancel stale “zombie” runs still in-flight on old commits
+
/cancel allcancel every in-flight run for the PR (full stop)
+
/test fullrun the full suite (all tiers, both engines)
+
/test full rtxfull suite, RTX engine only (also standard)
+
/test nightlyeverything incl. llm / kernels / distributed
+
+
+
+
loading {{BRANCH}}…
+
+
+ +
+ + +
+ + + + diff --git a/tools/ci-dashboard/static/style.css b/tools/ci-dashboard/static/style.css new file mode 100644 index 0000000000..25146987eb --- /dev/null +++ b/tools/ci-dashboard/static/style.css @@ -0,0 +1,250 @@ +:root { + --bg: #f6f7f9; --surface: #fff; --surface-2: #f0f2f5; --border: #e2e6ea; + --text: #1c2024; --muted: #5b6570; --accent: #0969da; + --pass: #1a7f37; --pass-bg: #e6f4ea; --pass-line: #b7dcc0; + --fail: #cf222e; --fail-bg: #ffebe9; --fail-line: #f3c0c2; + --run: #9a6700; --run-bg: #fff8e6; --run-line: #e6d9a8; + --queue: #6e7781; --queue-bg: #eef0f2; --queue-line: #dbe0e4; + --skip: #7d8590; --skip-bg: #eef1f4; --skip-line: #d3d9df; + --cancel: #8250df; --cancel-bg: #f4f0fb; --cancel-line: #e1d5f5; + --radius: 10px; --shadow: 0 1px 2px rgba(0,0,0,.06), 0 3px 12px rgba(0,0,0,.04); + --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; +} +@media (prefers-color-scheme: dark) { + :root { + --bg: #0d1117; --surface: #161b22; --surface-2: #1c2128; --border: #30363d; + --text: #e6edf3; --muted: #8b949e; --accent: #58a6ff; + --pass: #3fb950; --pass-bg: #12261a; --pass-line: #23552f; + --fail: #f85149; --fail-bg: #2c1518; --fail-line: #5c2a2a; + --run: #d29922; --run-bg: #2a230f; --run-line: #574a1e; + --queue: #8b949e; --queue-bg: #1c2128; --queue-line: #363c44; + --skip: #8b949e; --skip-bg: #1a1f26; --skip-line: #363c44; + --cancel: #a371f7; --cancel-bg: #201a2c; --cancel-line: #3d3357; + --shadow: 0 1px 2px rgba(0,0,0,.4), 0 3px 14px rgba(0,0,0,.3); + } +} +* { box-sizing: border-box; } +body { + margin: 0; background: var(--bg); color: var(--text); + font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; +} +a { color: var(--accent); text-decoration: none; } +a:hover { text-decoration: underline; } +button, input { font: inherit; color: inherit; } + +/* header */ +header { + position: sticky; top: 0; z-index: 20; background: var(--surface); + border-bottom: 1px solid var(--border); padding: 10px 18px; + display: flex; align-items: center; gap: 14px; flex-wrap: wrap; +} +header h1 { font-size: 15px; margin: 0; font-weight: 650; letter-spacing: .2px; } +header h1 .dim { color: var(--muted); font-weight: 400; } +.controls { display: flex; align-items: center; gap: 8px; margin-left: auto; flex-wrap: wrap; } +.controls input[type=text] { + background: var(--bg); border: 1px solid var(--border); border-radius: 8px; + padding: 6px 10px; width: 190px; +} +.controls input[type=text]:focus { outline: 2px solid var(--accent); border-color: transparent; } +.btn { + background: var(--surface-2); border: 1px solid var(--border); border-radius: 8px; + padding: 6px 12px; cursor: pointer; display: inline-flex; align-items: center; gap: 6px; +} +.btn:hover { border-color: var(--accent); } +.btn.primary { background: var(--accent); color: #fff; border-color: transparent; } +.btn.primary:hover { filter: brightness(1.08); } +.toggle { display: inline-flex; align-items: center; gap: 6px; color: var(--muted); font-size: 13px; cursor: pointer; } + +main { max-width: 1280px; margin: 0 auto; padding: 18px; } + +/* summary strip */ +.summary { display: flex; gap: 10px; flex-wrap: wrap; margin-bottom: 8px; align-items: center; } +.stat { display: inline-flex; align-items: center; gap: 7px; padding: 5px 12px; border-radius: 999px; + border: 1px solid var(--border); background: var(--surface); font-weight: 600; } +.stat .n { font-variant-numeric: tabular-nums; } +.stat.pass { color: var(--pass); border-color: var(--pass-line); background: var(--pass-bg); } +.stat.fail { color: var(--fail); border-color: var(--fail-line); background: var(--fail-bg); } +.stat.run { color: var(--run); border-color: var(--run-line); background: var(--run-bg); } +.stat.skip { color: var(--skip); border-color: var(--skip-line); background: var(--skip-bg); border-style: dashed; } +.commit { color: var(--muted); font-size: 13px; margin-left: auto; } +.commit code { font-family: var(--mono); color: var(--text); } + +/* dot / pill */ +.dot { width: 9px; height: 9px; border-radius: 50%; display: inline-block; flex: none; } +.dot.pass { background: var(--pass); } +.dot.fail { background: var(--fail); } +.dot.run { background: var(--run); animation: pulse 1.4s ease-in-out infinite; } +.dot.queue{ background: var(--queue); } +.dot.skip { background: transparent; box-shadow: inset 0 0 0 1.5px var(--skip); } +.dot.cancel { background: var(--cancel); } +@keyframes pulse { 0%,100% { opacity: 1; } 50% { opacity: .35; } } + +/* section headings */ +h2.sec { font-size: 12px; text-transform: uppercase; letter-spacing: .08em; color: var(--muted); + margin: 22px 0 10px; font-weight: 700; } + +/* aggregate failures panel */ +.agg { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); + box-shadow: var(--shadow); overflow: hidden; margin-bottom: 8px; } +.agg-row-d { border-top: 1px solid var(--border); } +.agg-row-d:first-child { border-top: none; } +.agg-row-d > summary { list-style: none; } +.agg-row-d > summary::-webkit-details-marker { display: none; } +.agg-row { display: grid; grid-template-columns: 1fr auto; gap: 12px; align-items: center; + padding: 11px 14px; cursor: pointer; } +.agg-row:hover { background: var(--surface-2); } +.agg-head-row { display: flex; align-items: center; gap: 10px; color: var(--muted); + font-size: 13px; margin: 2px 0 8px; } +.agg-loading { color: var(--muted); padding: 12px 2px; } +.agg-ok { color: var(--pass); font-weight: 600; padding: 12px 2px; } +.btn.small { padding: 3px 9px; font-size: 12px; margin-left: auto; } +#content.filter-fail .card:not(.fail) { display: none; } +.agg-test { font-family: var(--mono); font-size: 13px; word-break: break-all; } +.agg-file { color: var(--muted); font-size: 12px; margin-top: 2px; } +.agg-file code { font-family: var(--mono); } +.agg-count { display: inline-flex; align-items: center; gap: 6px; white-space: nowrap; + color: var(--fail); font-weight: 650; } +.agg-detail { padding: 10px 14px 12px 14px; border-top: 1px dashed var(--border); + background: var(--surface-2); font-size: 13px; } +.agg-detail pre { background: var(--bg); border: 1px solid var(--border); border-radius: 8px; + padding: 10px; overflow: auto; font-family: var(--mono); font-size: 12px; margin: 10px 0; max-height: 240px; } +.chips { display: flex; gap: 6px; flex-wrap: wrap; margin-top: 8px; } +.chip { font-size: 11px; padding: 2px 8px; border-radius: 999px; border: 1px solid var(--border); + background: var(--surface); color: var(--muted); } + +/* failure category tags — colour encodes triage kind: + bug=real regression (red) · infra=OOM/resource (purple) · env=setup/feature gap (amber) */ +.cat { display: inline-block; font-size: 10.5px; font-weight: 650; letter-spacing: .02em; + padding: 1px 7px; border-radius: 999px; border: 1px solid transparent; + vertical-align: middle; white-space: nowrap; } +.cat.bug { color: var(--fail); background: var(--fail-bg); border-color: var(--fail-line); } +.cat.infra { color: var(--cancel); background: var(--cancel-bg); border-color: var(--cancel-line); } +.cat.env { color: var(--run); background: var(--run-bg); border-color: var(--run-line); } +.cat.unknown { color: var(--skip); background: var(--skip-bg); border-color: var(--skip-line); } +/* config-correlation hint (e.g. "RTX only", "cu132 only") */ +.cfgpat { display: inline-block; font-size: 10.5px; font-weight: 600; padding: 1px 7px; + border-radius: 999px; margin-left: 4px; vertical-align: middle; white-space: nowrap; + color: var(--accent); background: transparent; border: 1px dashed var(--accent); } +.agg-tally { display: inline-flex; gap: 6px; flex-wrap: wrap; align-items: center; } +.agg-tally .cat { font-weight: 700; } + +/* platform board */ +.board { display: grid; grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); gap: 12px; } +.card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); + box-shadow: var(--shadow); overflow: hidden; } +.card.fail { border-color: var(--fail-line); } +.card-head { display: flex; align-items: center; gap: 10px; padding: 12px 14px; cursor: pointer; } +.card-head:hover { background: var(--surface-2); } +.card-title { font-weight: 620; flex: 1; min-width: 0; } +.card-title .sub { color: var(--muted); font-weight: 400; font-size: 12px; margin-top: 1px; } +.card-badge { font-size: 12px; font-weight: 650; padding: 3px 9px; border-radius: 999px; white-space: nowrap; } +.card-badge.pass { color: var(--pass); background: var(--pass-bg); border: 1px solid var(--pass-line); } +.card-badge.fail { color: var(--fail); background: var(--fail-bg); border: 1px solid var(--fail-line); } +.card-badge.run { color: var(--run); background: var(--run-bg); border: 1px solid var(--run-line); } +.card-badge.queue{ color: var(--queue);background: var(--queue-bg);border: 1px solid var(--queue-line); } +.card-badge.skip { color: var(--skip); background: var(--skip-bg); border: 1px dashed var(--skip-line); } +.card-badge.cancel { color: var(--cancel); background: var(--cancel-bg); border: 1px solid var(--cancel-line); } +.progress { height: 4px; background: var(--surface-2); display: flex; } +.progress > span { height: 100%; } +.progress .p-pass { background: var(--pass); } +.progress .p-fail { background: var(--fail); } +.progress .p-run { background: var(--run); } +.card-body { padding: 6px 14px 14px; border-top: 1px solid var(--border); } +.card > summary { list-style: none; } +.card > summary::-webkit-details-marker { display: none; } +.caret { color: var(--muted); transition: transform .15s; } +.card[open] .caret { transform: rotate(90deg); } + +/* job groups + matrix */ +.jobgroup { margin-top: 12px; } +.jobgroup h3 { font-size: 12px; font-weight: 650; margin: 0 0 6px; display: flex; align-items: center; gap: 8px; } +.jobgroup h3 .tierpath { color: var(--muted); font-weight: 400; font-family: var(--mono); font-size: 11px; } +.matrix { display: flex; gap: 6px; flex-wrap: wrap; } +.cell { min-width: 84px; border: 1px solid var(--border); border-radius: 8px; padding: 6px 9px; + display: flex; flex-direction: column; gap: 3px; cursor: default; background: var(--surface); } +.cell.pass { background: var(--pass-bg); border-color: var(--pass-line); } +.cell.fail { background: var(--fail-bg); border-color: var(--fail-line); cursor: pointer; } +.cell.fail:hover { outline: 2px solid var(--fail); } +.cell.run { background: var(--run-bg); border-color: var(--run-line); } +.cell.queue{ background: var(--queue-bg); } +/* "didn't run" — dashed + muted so it reads as inactive, never as pass or fail */ +.cell.skip { background: var(--surface); border-style: dashed; border-color: var(--skip-line); color: var(--muted); } +.cell.skip .k, .cell.cancel .k { color: var(--muted); font-weight: 500; } +.cell.cancel { background: var(--cancel-bg); border-style: dashed; border-color: var(--cancel-line); } +.cell .k { font-size: 11px; font-weight: 650; font-variant-numeric: tabular-nums; display: flex; gap: 5px; align-items: center; } +.cell .v { font-size: 10px; color: var(--muted); font-family: var(--mono); } +.rows { display: flex; flex-direction: column; gap: 4px; } +.jobrow { display: flex; align-items: center; gap: 8px; padding: 4px 6px; border-radius: 6px; font-size: 13px; } +.jobrow:hover { background: var(--surface-2); } +.jobrow .name { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.jobrow.fail .name { color: var(--fail); } +.jobstate { font-size: 11px; padding: 1px 8px; border-radius: 999px; white-space: nowrap; color: var(--muted); border: 1px solid var(--border); } +.jobstate.fail { color: var(--fail); border-color: var(--fail-line); background: var(--fail-bg); } +.jobstate.pass { color: var(--pass); border-color: var(--pass-line); background: var(--pass-bg); } +.jobstate.skip { border-style: dashed; } +.jobstate.cancel { color: var(--cancel); border-color: var(--cancel-line); border-style: dashed; } + +/* drawer */ +.drawer-bg { position: fixed; inset: 0; background: rgba(0,0,0,.35); z-index: 40; display: none; } +.drawer-bg.open { display: block; } +.drawer { position: fixed; top: 0; right: 0; height: 100%; width: min(560px, 94vw); z-index: 41; + background: var(--surface); border-left: 1px solid var(--border); box-shadow: -8px 0 30px rgba(0,0,0,.2); + transform: translateX(100%); transition: transform .2s ease; display: flex; flex-direction: column; } +.drawer.open { transform: translateX(0); } +.drawer-head { padding: 14px 16px; border-bottom: 1px solid var(--border); display: flex; align-items: flex-start; gap: 10px; } +.drawer-head .t { flex: 1; } +.drawer-head .t .title { font-weight: 650; } +.drawer-head .t .sub { color: var(--muted); font-size: 12px; margin-top: 2px; } +.drawer-head .x { cursor: pointer; border: none; background: none; color: var(--muted); font-size: 20px; line-height: 1; } +.drawer-body { padding: 14px 16px; overflow: auto; } +.fail-item { border: 1px solid var(--border); border-radius: 8px; padding: 10px 12px; margin-bottom: 10px; background: var(--surface); } +.fail-item .ftest { font-family: var(--mono); font-size: 13px; font-weight: 600; word-break: break-all; } +.fail-item .floc { font-size: 12px; margin-top: 4px; } +.fail-item .floc code { font-family: var(--mono); } +.fail-item pre { background: var(--bg); border: 1px solid var(--border); border-radius: 6px; + padding: 9px; font-family: var(--mono); font-size: 12px; overflow: auto; max-height: 260px; margin: 8px 0 0; white-space: pre-wrap; } +.repro { background: var(--surface-2); border: 1px solid var(--border); border-radius: 8px; padding: 10px 12px; margin-top: 10px; } +.repro .lbl { font-size: 11px; text-transform: uppercase; letter-spacing: .06em; color: var(--muted); margin-bottom: 6px; } +.repro code { font-family: var(--mono); font-size: 12px; word-break: break-all; display: block; } +.copy { float: right; font-size: 11px; cursor: pointer; color: var(--accent); background: none; border: none; } + +/* relevant logs (in drawer) */ +.logsec { margin-top: 14px; border-top: 1px solid var(--border); padding-top: 12px; } +.logsec-head { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--muted); + margin-bottom: 8px; flex-wrap: wrap; } +.logsec-head .lbl { text-transform: uppercase; letter-spacing: .06em; font-weight: 600; + color: var(--text); margin-right: auto; } +.logblock { border: 1px solid var(--border); border-radius: 8px; margin-bottom: 8px; overflow: hidden; } +.logblock > summary { cursor: pointer; padding: 8px 12px; font-family: var(--mono); font-size: 12px; + font-weight: 600; background: var(--surface-2); word-break: break-all; list-style: none; } +.logblock > summary::-webkit-details-marker { display: none; } +.logblock > summary::before { content: "▸ "; color: var(--muted); } +.logblock[open] > summary::before { content: "▾ "; } +.logblock .logreason { padding: 8px 12px 0; font-family: var(--mono); font-size: 12px; color: var(--fail); word-break: break-word; } +.logblock pre { margin: 8px 12px 12px; background: var(--bg); border: 1px solid var(--border); + border-radius: 6px; padding: 10px; font-family: var(--mono); font-size: 12px; line-height: 1.45; + overflow: auto; max-height: 420px; white-space: pre; } + +.muted { color: var(--muted); } +.spinner { display: inline-block; width: 13px; height: 13px; border: 2px solid var(--border); + border-top-color: var(--accent); border-radius: 50%; animation: spin .7s linear infinite; } +@keyframes spin { to { transform: rotate(360deg); } } +.empty-state { text-align: center; color: var(--muted); padding: 60px 20px; } +.toast { position: fixed; bottom: 20px; left: 50%; transform: translateX(-50%); background: var(--text); + color: var(--bg); padding: 8px 16px; border-radius: 8px; font-size: 13px; z-index: 60; opacity: 0; + transition: opacity .2s; pointer-events: none; } +.toast.show { opacity: 1; } +.legend { display: flex; gap: 14px; color: var(--muted); font-size: 12px; margin: 4px 0 0; flex-wrap: wrap; } +.legend span { display: inline-flex; align-items: center; gap: 5px; } + +/* PR command reference (collapsible) */ +.cmds { margin: 8px 0 0; font-size: 12px; } +.cmds > summary { cursor: pointer; color: var(--muted); user-select: none; padding: 2px 0; } +.cmds > summary:hover { color: var(--text); } +.cmd-list { display: flex; flex-direction: column; gap: 6px; margin: 8px 0 2px; padding: 10px 12px; + background: var(--surface-2); border: 1px solid var(--border); border-radius: 8px; } +.cmd { display: flex; align-items: center; gap: 10px; } +.cmd code { font-family: var(--mono); font-size: 12px; color: var(--text); background: var(--surface); + border: 1px solid var(--border); border-radius: 6px; padding: 2px 8px; white-space: nowrap; min-width: 82px; } +.cmd .copy { float: none; } +.cmd .lbl { color: var(--muted); }