main.yml

name: Py 3.10, 3.11, 3.12 | Windows Mac Linux

on:
  pull_request:
    branches:
      - main
      - develop
  push:
    branches:
      - main
      - develop

# Global configuration for all jobs. Can be overridden at the job level if needed.
env:
  # Conda channels: Set conda forge as only channel and disable defaults channel
  CONDA_MINICONDA_VERSION: latest
  CONDA_AUTO_UPDATE_CONDA: true
  CONDA_CHANNELS: conda-forge
  CONDA_REMOVE_DEFAULTS: true
  CONDA_USE_ONLY_TAR_BZ2: false
  CONDA_ENVIRONMENT_FILE: environment-dev.yml
  CONDA_ENVIRONMENT_NAME: mhkit-dev-env
  # Coveralls: Set to false to ignore upload failures during outages.
  #   Check status: https://status.coveralls.io/
  COVERALLS_FAIL_ON_ERROR: true

jobs:
  # Set the operating system matrix for the following jobs/tests
  # conda-forge-build
  # pip-build
  # hindcast-calls
  set-os:
    runs-on: ubuntu-latest
    outputs:
      matrix_os: ${{ steps.set-matrix.outputs.matrix_os }}
    steps:
      - id: set-matrix
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "develop" ]]; then
            echo "matrix_os=[ \"ubuntu-latest\"]" >> $GITHUB_OUTPUT
          else
            echo "matrix_os=[\"windows-latest\", \"ubuntu-latest\", \"macos-latest\"]" >> $GITHUB_OUTPUT
          fi


  # This job decides if the hindcast test suite should run.
  check-changes:
    runs-on: ubuntu-latest
    outputs:
      wave_io_hindcast_changed: ${{ steps.changes.outputs.wave_io_hindcast }}
      should-run-hindcast: ${{ steps.hindcast-logic.outputs.should-run-hindcast }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Check for changes in wave/io/hindcast
        id: changes
        uses: dorny/paths-filter@v4
        with:
          filters: |
            wave_io_hindcast:
              - 'mhkit/wave/io/hindcast/**'
              - 'mhkit/tests/wave/io/hindcast/**'

      - id: hindcast-logic
        run: |
          if [[ "${{ github.event.pull_request.base.ref }}" == "main" || "${{ steps.changes.outputs.wave_io_hindcast }}" == "true" ]]; then
            echo "should-run-hindcast=true" >> "$GITHUB_OUTPUT"
          else
            echo "should-run-hindcast=false" >> "$GITHUB_OUTPUT"
          fi

  prepare-nonhindcast-cache:
    runs-on: ubuntu-latest
    env:
      PYTHON_VER: 3.11
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniconda-version: ${{ env.CONDA_MINICONDA_VERSION }}
          auto-update-conda: ${{ env.CONDA_AUTO_UPDATE_CONDA }}
          environment-file: ${{ env.CONDA_ENVIRONMENT_FILE }}
          activate-environment: ${{ env.CONDA_ENVIRONMENT_NAME }}
          python-version: ${{ matrix.python-version }}
          use-only-tar-bz2: ${{ env.CONDA_USE_ONLY_TAR_BZ2 }}
          channels: ${{ env.CONDA_CHANNELS }}
          conda-remove-defaults: ${{ env.CONDA_REMOVE_DEFAULTS }}

      - name: Install testing dependencies
        shell: bash -l {0}
        run: |
          conda install -y pytest coverage coveralls

      - name: Install mhkit
        shell: bash -l {0}
        run: |
          pip install -e ".[all,dev]" --no-deps

      - name: Prepare non-hindcast API data
        shell: bash -l {0}
        run: |
          pytest mhkit/tests/river/test_io_usgs.py
          pytest mhkit/tests/tidal/test_io.py
          pytest mhkit/tests/wave/io/test_cdip.py

      - name: Upload data as artifact
        uses: actions/upload-artifact@v6
        with:
          name: data
          path: ~/.cache/mhkit

  prepare-wave-hindcast-cache:
    needs: [check-changes]
    runs-on: ubuntu-latest
    env:
      PYTHON_VER: 3.11
    if: (needs.check-changes.outputs.should-run-hindcast == 'true')
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniconda-version: ${{ env.CONDA_MINICONDA_VERSION }}
          auto-update-conda: ${{ env.CONDA_AUTO_UPDATE_CONDA }}
          environment-file: ${{ env.CONDA_ENVIRONMENT_FILE }}
          activate-environment: ${{ env.CONDA_ENVIRONMENT_NAME }}
          python-version: ${{ matrix.python-version }}
          use-only-tar-bz2: ${{ env.CONDA_USE_ONLY_TAR_BZ2 }}
          channels: ${{ env.CONDA_CHANNELS }}
          conda-remove-defaults: ${{ env.CONDA_REMOVE_DEFAULTS }}

      - name: Install testing dependencies
        shell: bash -l {0}
        run: |
          conda install -y pytest coverage coveralls

      - name: Install mhkit
        shell: bash -l {0}
        run: |
          pip install -e ".[all,dev]" --no-deps

      - name: Prepare Wave Hindcast data
        shell: bash -l {0}
        run: |
          pytest mhkit/tests/wave/io/hindcast/test_hindcast.py

      - name: Upload Wave Hindcast data as artifact
        uses: actions/upload-artifact@v6
        with:
          name: wave-hindcast-data
          path: ~/.cache/mhkit

  prepare-wind-hindcast-cache:
    needs: [check-changes, prepare-wave-hindcast-cache]
    runs-on: ubuntu-latest
    env:
      PYTHON_VER: 3.11
    if: (needs.check-changes.outputs.should-run-hindcast == 'true')
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniconda-version: ${{ env.CONDA_MINICONDA_VERSION }}
          auto-update-conda: ${{ env.CONDA_AUTO_UPDATE_CONDA }}
          environment-file: ${{ env.CONDA_ENVIRONMENT_FILE }}
          activate-environment: ${{ env.CONDA_ENVIRONMENT_NAME }}
          python-version: ${{ matrix.python-version }}
          use-only-tar-bz2: ${{ env.CONDA_USE_ONLY_TAR_BZ2 }}
          channels: ${{ env.CONDA_CHANNELS }}
          conda-remove-defaults: ${{ env.CONDA_REMOVE_DEFAULTS }}

      - name: Install testing dependencies
        shell: bash -l {0}
        run: |
          conda install -y pytest coverage coveralls

      - name: Install mhkit
        shell: bash -l {0}
        run: |
          pip install -e ".[all,dev]" --no-deps

      - name: Prepare Wind Hindcast data
        shell: bash -l {0}
        run: |
          pytest mhkit/tests/wave/io/hindcast/test_wind_toolkit.py

      - name: Upload Wind Hindcast data as artifact
        uses: actions/upload-artifact@v6
        with:
          name: wind-hindcast-data
          path: ~/.cache/mhkit

  conda-forge-build:
    name: conda-forge-${{ matrix.os }}/${{ matrix.python-version }}
    needs: [set-os, prepare-nonhindcast-cache]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: ${{fromJson(needs.set-os.outputs.matrix_os)}}
        python-version: ['3.10', '3.11', '3.12']
    env:
      PYTHON_VER: ${{ matrix.python-version }}

    steps:
      - uses: actions/checkout@v6

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniconda-version: ${{ env.CONDA_MINICONDA_VERSION }}
          auto-update-conda: ${{ env.CONDA_AUTO_UPDATE_CONDA }}
          environment-file: ${{ env.CONDA_ENVIRONMENT_FILE }}
          activate-environment: ${{ env.CONDA_ENVIRONMENT_NAME }}
          python-version: ${{ matrix.python-version }}
          use-only-tar-bz2: ${{ env.CONDA_USE_ONLY_TAR_BZ2 }}
          channels: ${{ env.CONDA_CHANNELS }}
          conda-remove-defaults: ${{ env.CONDA_REMOVE_DEFAULTS }}

      - name: Install testing dependencies
        shell: bash -l {0}
        run: |
          conda install -y pytest coverage coveralls

      - name: Install mhkit
        shell: bash -l {0}
        run: |
          pip install -e ".[all,dev]" --no-deps

      - name: Download data from artifact
        uses: actions/download-artifact@v4
        with:
          name: data
          path: ~/.cache/mhkit

      - name: Run pytest & generate coverage report
        shell: bash -l {0}
        run: |
          coverage run --rcfile=.github/workflows/.coveragerc --source=./mhkit/ -m pytest -c .github/workflows/pytest.ini
          coverage lcov

      - name: Upload coverage data to coveralls.io
        uses: coverallsapp/github-action@v2
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          flag-name: conda-forge-${{ runner.os }}-py${{ matrix.python-version }}
          parallel: true
          path-to-lcov: ./coverage.lcov
          fail-on-error: ${{ env.COVERALLS_FAIL_ON_ERROR }}

  pip-build:
    name: pip-${{ matrix.os }}/${{ matrix.python-version }}
    needs: [set-os, prepare-nonhindcast-cache]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: ${{fromJson(needs.set-os.outputs.matrix_os)}}
        python-version: ['3.10', '3.11', '3.12']

    steps:
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Setup Python Virtual Environment
        shell: bash
        # The second command puts the venv bin directory on the PATH for
        # subsequent steps in the job, which ensures that the correct Python
        # version and installed packages are used.
        run: |
          python -m venv venv
          echo "$PWD/venv/bin" >> $GITHUB_PATH

      - name: Set up Git repository
        uses: actions/checkout@v6

      - name: Download data from artifact
        uses: actions/download-artifact@v4
        with:
          name: data
          path: ~/.cache/mhkit

      - name: Install system dependencies
        if: runner.os == 'Linux'
        run: |
          # Update apt-get cache
          sudo apt-get update
          sudo apt-get install -y libhdf5-dev libnetcdf-dev

      - name: Update and install packages
        shell: bash
        run: |
          python -m pip install --upgrade pip wheel
          pip install -e ".[all,dev]"

      - name: Reinstall h5py and netCDF4 with system libraries
        if: runner.os == 'Linux'
        shell: bash
        run: |
          pip install --force-reinstall --no-binary=:all: h5py netCDF4

      - name: Install setuptools for Python 3.12
        if: matrix.python-version == '3.12'
        shell: bash
        run: |
          pip install setuptools

      - name: Run pytest & generate coverage report
        shell: bash
        run: |
          coverage run --rcfile=.github/workflows/.coveragerc --source=./mhkit/ -m pytest -c .github/workflows/pytest.ini
          coverage lcov

      - name: Upload coverage data to coveralls.io
        uses: coverallsapp/github-action@v2
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          flag-name: pip-${{ runner.os }}-py${{ matrix.python-version }}
          parallel: true
          path-to-lcov: ./coverage.lcov
          fail-on-error: ${{ env.COVERALLS_FAIL_ON_ERROR }}

  hindcast-calls:
    name: hindcast-${{ matrix.os }}/${{ matrix.python-version }}
    needs:
      [
        check-changes,
        prepare-wave-hindcast-cache,
        prepare-wind-hindcast-cache,
        set-os,
      ]
    if: (needs.check-changes.outputs.should-run-hindcast == 'true')

    runs-on: ${{ matrix.os }}
    strategy:
      max-parallel: 1
      fail-fast: false
      matrix:
        os: ${{fromJson(needs.set-os.outputs.matrix_os)}}
        python-version: ['3.10', '3.11', '3.12']

    steps:
      - uses: actions/checkout@v6

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniconda-version: ${{ env.CONDA_MINICONDA_VERSION }}
          auto-update-conda: ${{ env.CONDA_AUTO_UPDATE_CONDA }}
          environment-file: ${{ env.CONDA_ENVIRONMENT_FILE }}
          activate-environment: ${{ env.CONDA_ENVIRONMENT_NAME }}
          python-version: ${{ matrix.python-version }}
          use-only-tar-bz2: ${{ env.CONDA_USE_ONLY_TAR_BZ2 }}
          channels: ${{ env.CONDA_CHANNELS }}
          conda-remove-defaults: ${{ env.CONDA_REMOVE_DEFAULTS }}

      - name: Install testing dependencies
        shell: bash -l {0}
        run: |
          conda install -y pytest coverage coveralls

      - name: Install mhkit
        shell: bash -l {0}
        run: |
          pip install -e ".[all,dev]" --no-deps

      - name: Download Wave Hindcast data from artifact
        uses: actions/download-artifact@v4
        with:
          name: wave-hindcast-data
          path: ~/.cache/mhkit/wave-hindcast

      - name: Download Wind Hindcast data from artifact
        uses: actions/download-artifact@v4
        with:
          name: wind-hindcast-data
          path: ~/.cache/mhkit/wind-hindcast

      - name: Consolidate hindcast data
        run: |
          mkdir -p ~/.cache/mhkit/hindcast
          mv ~/.cache/mhkit/wave-hindcast/hindcast/* ~/.cache/mhkit/hindcast/
          mv ~/.cache/mhkit/wind-hindcast/hindcast/* ~/.cache/mhkit/hindcast/
        shell: bash

      - name: Run hindcast pytest
        shell: bash -l {0}
        run: |
          coverage run --rcfile=.github/workflows/.coveragehindcastrc -m pytest -c .github/workflows/pytest-hindcast.ini
          coverage lcov

      - name: Upload coverage data to coveralls.io
        uses: coverallsapp/github-action@v2
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          flag-name: hindcast-${{ runner.os }}-py${{ matrix.python-version }}
          parallel: true
          path-to-lcov: ./coverage.lcov
          fail-on-error: ${{ env.COVERALLS_FAIL_ON_ERROR }}

  test-wheel-packaging:
    name: Test Built Wheel
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-python@v6
        with:
          python-version: '3.11'

      - name: Install build dependencies
        run: python -m pip install build --user

      - name: Build wheel
        run: python -m build --wheel --outdir dist/ .

      - name: Install MHKiT from built wheel
        run: |
          # Install from build wheel in dist/ directory, not from PyPI
           # -f, --find-links <url>      If a URL or path to an html file, then parse for links to archives such as sdist (.tar.gz) or wheel (.whl) files.
             # If a local path or file:// URL that's a directory, then look for archives in the directory listing.
          pip install 'mhkit[all]' --find-links dist/

      - name: Test mhkit submodule imports
        run: |
          python -c "
          import mhkit
          print(f'Version: {mhkit.__version__}')

          # Test all submodules can be imported
          modules = ['wave', 'river', 'tidal', 'dolfyn', 'power', 'loads', 'mooring', 'acoustics', 'qc', 'utils']
          for mod in modules:
              exec(f'from mhkit import {mod}')
              print(f'Successfully imported mhkit.{mod}')

          print('All submodules imported successfully!')
          "

      - name: Verify mhkit package structure
        run: |
          python -c "
          import mhkit
          import os

          mhkit_path = os.path.dirname(mhkit.__file__)
          expected_dirs = ['wave', 'river', 'tidal', 'dolfyn', 'power', 'loads', 'mooring', 'acoustics', 'qc', 'utils']

          for d in expected_dirs:
              dir_path = os.path.join(mhkit_path, d)
              assert os.path.isdir(dir_path), f'Missing submodule directory: {d}'
              init_file = os.path.join(dir_path, '__init__.py')
              assert os.path.isfile(init_file), f'Missing __init__.py in {d}'

          print('Package structure verified!')
          "

  test-optional-pip-dependencies:
    needs: [set-os, prepare-nonhindcast-cache]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        module:
          [wave, tidal, river, dolfyn, power, loads, mooring, acoustics, utils]
        python-version: ['3.12']

    steps:
      - uses: actions/checkout@v6

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Download non-hindcast data
        uses: actions/download-artifact@v4
        with:
          name: data
          path: ~/.cache/mhkit

      - name: Install system dependencies
        if: matrix.module != 'river' || matrix.module != 'power' || matrix.module != 'utils' || matrix.module != 'loads'
        run: |
          # Update apt-get cache
          sudo apt-get update
          sudo apt-get install -y libhdf5-dev libnetcdf-dev

      - name: Install MHKiT with optional dependency
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[${{ matrix.module }}]"
          pip install pytest

      - name: Reinstall h5py and netCDF4 with system libraries
        if: matrix.module != 'river' || matrix.module != 'power' || matrix.module != 'utils' || matrix.module != 'loads'
        run: "pip install --force-reinstall --no-binary=:all: h5py netCDF4"

      - name: Run tests for ${{ matrix.module }}
        env:
          MPLBACKEND: Agg
        run: |
          python -m pytest -c .github/workflows/pytest.ini mhkit/tests/${{ matrix.module }}/

  notebook-matrix:
    runs-on: ubuntu-latest
    needs:
      [
        check-changes,
        prepare-nonhindcast-cache,
        prepare-wave-hindcast-cache,
        prepare-wind-hindcast-cache,
      ]
    if: |
      always() &&
      (
        (
          needs.prepare-nonhindcast-cache.result == 'success' &&
          needs.prepare-wave-hindcast-cache.result == 'skipped' &&
          needs.prepare-wind-hindcast-cache.result == 'skipped' &&
          needs.check-changes.outputs.should-run-hindcast == 'false'
        ) ||
        (
          needs.prepare-nonhindcast-cache.result == 'success' &&
          needs.prepare-wave-hindcast-cache.result == 'success' &&
          needs.prepare-wind-hindcast-cache.result == 'success' &&
          needs.check-changes.outputs.should-run-hindcast == 'true' 
        )
      )
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.11'

      - name: Generate matrix
        id: set-matrix
        run: |
          matrix_json=$(python .github/workflows/generate_notebook_matrix.py)
          echo "$matrix_json" >> $GITHUB_OUTPUT

  test-notebooks:
    needs:
      [
        notebook-matrix,
        check-changes,
        prepare-nonhindcast-cache,
        prepare-wave-hindcast-cache,
        prepare-wind-hindcast-cache,
      ]
    strategy:
      matrix: ${{ fromJson(needs.notebook-matrix.outputs.matrix) }}
    if: |
      always()
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniconda-version: ${{ env.CONDA_MINICONDA_VERSION }}
          auto-update-conda: ${{ env.CONDA_AUTO_UPDATE_CONDA }}
          environment-file: ${{ env.CONDA_ENVIRONMENT_FILE }}
          activate-environment: ${{ env.CONDA_ENVIRONMENT_NAME }}
          python-version: ${{ matrix.python-version }}
          use-only-tar-bz2: ${{ env.CONDA_USE_ONLY_TAR_BZ2 }}
          channels: ${{ env.CONDA_CHANNELS }}
          conda-remove-defaults: ${{ env.CONDA_REMOVE_DEFAULTS }}

      - name: Install notebook testing dependencies
        shell: bash -l {0}
        run: |
          conda install -y pytest coverage coveralls nbval jupyter utm folium

      - name: Install mhkit
        shell: bash -l {0}
        run: |
          pip install -e ".[all,dev]" --no-deps

      - name: Download non-hindcast data
        uses: actions/download-artifact@v4
        with:
          name: data
          path: ~/.cache/mhkit

      - name: Download Wave Hindcast data (if available)
        if: (needs.check-changes.outputs.should-run-hindcast == 'true')
        uses: actions/download-artifact@v4
        with:
          name: wave-hindcast-data
          path: ~/.cache/mhkit/wave-hindcast

      - name: Download Wind Hindcast data (if available)
        if: (needs.check-changes.outputs.should-run-hindcast == 'true')
        uses: actions/download-artifact@v4
        with:
          name: wind-hindcast-data
          path: ~/.cache/mhkit/wind-hindcast

      - name: Consolidate hindcast data
        if: (needs.check-changes.outputs.should-run-hindcast == 'true')
        run: |
          mkdir -p ~/.cache/mhkit/hindcast
          mv ~/.cache/mhkit/wave-hindcast/hindcast/* ~/.cache/mhkit/hindcast/
          mv ~/.cache/mhkit/wind-hindcast/hindcast/* ~/.cache/mhkit/hindcast/
        shell: bash

      - name: Copy .hscfg file to examples directory
        shell: bash -l {0}
        run: |
          cp .hscfg examples/

      - name: Run notebook
        shell: bash -l {0}
        run: |
          if [[ "${{ matrix.notebook }}" == "examples/metocean_example.ipynb" || "${{ matrix.notebook }}" == "examples/WPTO_hindcast_example.ipynb" ]]; then
            if [[ "${{ needs.check-changes.outputs.should-run-hindcast }}" == 'true' ]]; then
              jupyter nbconvert --to notebook --execute --inplace --ExecutePreprocessor.timeout=${{ matrix.timeout }} "${{ matrix.notebook }}"
            else
              echo "Skipping ${{ matrix.notebook }}"
            fi
          else
            jupyter nbconvert --to notebook --execute --inplace --ExecutePreprocessor.timeout=${{ matrix.timeout }} "${{ matrix.notebook }}"
          fi

  coveralls:
    name: Indicate completion to coveralls.io
    needs:
      [
        prepare-wave-hindcast-cache,
        prepare-wind-hindcast-cache,
        conda-forge-build,
        pip-build,
        hindcast-calls,
      ]
    if: |
      always() &&
      (
        (
          needs.conda-forge-build.result == 'success' &&
          needs.pip-build.result == 'success' &&
          needs.prepare-wave-hindcast-cache.result == 'skipped' &&
          needs.prepare-wind-hindcast-cache.result == 'skipped' &&
          needs.hindcast-calls.result == 'skipped'
        ) ||
        (
          needs.conda-forge-build.result == 'success' &&
          needs.pip-build.result == 'success' &&
          needs.prepare-wave-hindcast-cache.result == 'success' &&
          needs.prepare-wind-hindcast-cache.result == 'success' &&
          needs.hindcast-calls.result == 'success'
        )
      )
    runs-on: ubuntu-latest
    steps:
      - name: Coveralls Finished
        uses: coverallsapp/github-action@v2
        with:
          parallel-finished: true
          github-token: ${{ secrets.GITHUB_TOKEN }}
          fail-on-error: ${{ env.COVERALLS_FAIL_ON_ERROR }}