diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 3c8df49637..daeb4ea4f8 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -79,150 +79,6 @@ jobs: - name: Generate coverage report (85%) # Coverage threshold should only increase over time — never decrease it! run: COVERAGE_FAIL_UNDER=85 make coverage-report - integration-test: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: '3.12' - - name: Install UV - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: true - - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - - name: Install - run: make install - - name: Run integration tests with coverage - run: COVERAGE=1 make test-integration - - name: Show debug logs - if: ${{ failure() }} - run: docker compose -f dev/docker-compose-integration.yml logs - - name: Upload coverage data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: coverage-integration - path: .coverage* - include-hidden-files: true - - integration-test-s3: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: '3.12' - - name: Install UV - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: true - - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - - name: Install - run: make install - - name: Run s3 integration tests with coverage - run: COVERAGE=1 make test-s3 - - name: Show debug logs - if: ${{ failure() }} - run: docker compose -f dev/docker-compose.yml logs - - name: Upload coverage data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: coverage-s3 - path: .coverage* - include-hidden-files: true - - integration-test-adls: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: '3.12' - - name: Install UV - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: true - - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - - name: Install - run: make install - - name: Run adls integration tests with coverage - run: COVERAGE=1 make test-adls - - name: Show debug logs - if: ${{ failure() }} - run: docker compose -f dev/docker-compose-azurite.yml logs - - name: Upload coverage data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: coverage-adls - path: .coverage* - include-hidden-files: true - - integration-test-gcs: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: '3.12' - - name: Install UV - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: true - - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - - name: Install - run: make install - - name: Run gcs integration tests with coverage - run: COVERAGE=1 make test-gcs - - name: Show debug logs - if: ${{ failure() }} - run: docker compose -f dev/docker-compose-gcs-server.yml logs - - name: Upload coverage data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: coverage-gcs - path: .coverage* - include-hidden-files: true - - integration-coverage-report: - runs-on: ubuntu-latest - needs: [integration-test, integration-test-s3, integration-test-adls, integration-test-gcs] - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: '3.12' - - name: Install UV - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: true - - name: Install dependencies - run: uv sync --group dev - - name: Download all coverage artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: coverage-* - merge-multiple: true - - name: Generate coverage report (75%) # Coverage threshold should only increase over time — never decrease it! - run: COVERAGE_FAIL_UNDER=75 make coverage-report - cibw-dev-env-smoke-test: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/python-integration.yml b/.github/workflows/python-integration.yml new file mode 100644 index 0000000000..4c9ecc6eaa --- /dev/null +++ b/.github/workflows/python-integration.yml @@ -0,0 +1,185 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "Python Integration" + +on: + push: + branches: + - 'main' + pull_request: + paths: # Only run integration tests when Python (or the code they exercise) changes. + - '**/*.py' + - 'pyproject.toml' + - 'uv.lock' + - 'Makefile' + - 'dev/**' # docker-compose files and fixtures used by the integration suites. + - '.github/workflows/python-integration.yml' # this file itself. + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + integration-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.12' + - name: Install UV + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: true + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos + - name: Install + run: make install + - name: Run integration tests with coverage + run: COVERAGE=1 make test-integration + - name: Show debug logs + if: ${{ failure() }} + run: docker compose -f dev/docker-compose-integration.yml logs + - name: Upload coverage data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-integration + path: .coverage* + include-hidden-files: true + + integration-test-s3: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.12' + - name: Install UV + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: true + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos + - name: Install + run: make install + - name: Run s3 integration tests with coverage + run: COVERAGE=1 make test-s3 + - name: Show debug logs + if: ${{ failure() }} + run: docker compose -f dev/docker-compose.yml logs + - name: Upload coverage data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-s3 + path: .coverage* + include-hidden-files: true + + integration-test-adls: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.12' + - name: Install UV + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: true + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos + - name: Install + run: make install + - name: Run adls integration tests with coverage + run: COVERAGE=1 make test-adls + - name: Show debug logs + if: ${{ failure() }} + run: docker compose -f dev/docker-compose-azurite.yml logs + - name: Upload coverage data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-adls + path: .coverage* + include-hidden-files: true + + integration-test-gcs: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.12' + - name: Install UV + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: true + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos + - name: Install + run: make install + - name: Run gcs integration tests with coverage + run: COVERAGE=1 make test-gcs + - name: Show debug logs + if: ${{ failure() }} + run: docker compose -f dev/docker-compose-gcs-server.yml logs + - name: Upload coverage data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-gcs + path: .coverage* + include-hidden-files: true + + integration-coverage-report: + runs-on: ubuntu-latest + needs: [integration-test, integration-test-s3, integration-test-adls, integration-test-gcs] + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.12' + - name: Install UV + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: true + - name: Install dependencies + run: uv sync --group dev + - name: Download all coverage artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: coverage-* + merge-multiple: true + - name: Generate coverage report (75%) # Coverage threshold should only increase over time — never decrease it! + run: COVERAGE_FAIL_UNDER=75 make coverage-report