diff --git a/.github/scripts/shorten-conda-env-windows.sh b/.github/scripts/shorten-conda-env-windows.sh new file mode 100644 index 0000000000..0ff1d2d2d5 --- /dev/null +++ b/.github/scripts/shorten-conda-env-windows.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ -z "${CONDA_ENV:-}" ]]; then + echo "::error::CONDA_ENV is not set" + exit 1 +fi + +if [[ -z "${GITHUB_ENV:-}" ]]; then + echo "::error::GITHUB_ENV is not set" + exit 1 +fi + +if ! command -v cygpath >/dev/null 2>&1; then + echo "::error::cygpath is required to shorten Windows paths" + exit 1 +fi + +to_windows_path() { + local path="${1//\\//}" + + if [[ "${path}" =~ ^[A-Za-z]:/ ]]; then + printf "%s\n" "${path//\//\\}" + else + cygpath -w "${path}" + fi +} + +to_bash_path() { + local path="${1//\\//}" + + if [[ "${path}" =~ ^([A-Za-z]):(/.*)?$ ]]; then + printf "/%s%s\n" "${BASH_REMATCH[1],,}" "${BASH_REMATCH[2]}" + else + printf "%s\n" "${path}" + fi +} + +find_unused_drive() { + local drive + + for drive in T S R Q P O N M L K J I H G F E D; do + if [[ ! -e "/${drive,,}" ]]; then + printf "%s:\n" "${drive}" + return 0 + fi + done + + return 1 +} + +conda_env="${CONDA_ENV%/}" +conda_env_bash="$(to_bash_path "${conda_env}")" +conda_env_name="$(basename "${conda_env_bash}")" +conda_env_parent="$(dirname "${conda_env_bash}")" +conda_env_parent_win="$(to_windows_path "${conda_env_parent}")" + +if [[ -n "${SHORT_CONDA_DRIVE:-}" ]]; then + short_drive="${SHORT_CONDA_DRIVE%:}:" +elif ! short_drive="$(find_unused_drive)"; then + echo "::error::Could not find an unused drive letter for the Conda env path" + exit 1 +fi + +if [[ ! "${short_drive}" =~ ^[A-Za-z]:$ ]]; then + echo "::error::SHORT_CONDA_DRIVE must be a Windows drive letter like T:" + exit 1 +fi + +MSYS2_ARG_CONV_EXCL="*" MSYS2_ENV_CONV_EXCL="CONDA_ENV_PARENT_WIN" SHORT_CONDA_DRIVE="${short_drive}" CONDA_ENV_PARENT_WIN="${conda_env_parent_win}" powershell.exe -NoProfile -ExecutionPolicy Bypass -Command ' +$ErrorActionPreference = "Stop" +$drive = $env:SHORT_CONDA_DRIVE +$target = $env:CONDA_ENV_PARENT_WIN +$target = $target -replace "^[\\/]+(?=[A-Za-z]:[\\/])", "" +Write-Host "Mapping Conda env parent path: $target" +if (-not (Test-Path -LiteralPath $target -PathType Container)) { + throw "Conda env parent path not found: $target" +} +& subst.exe $drive /D 2>$null +& subst.exe $drive $target +if ($LASTEXITCODE -ne 0) { + throw "subst failed to map $drive to $target" +} +' + +short_conda_env="${short_drive}/${conda_env_name}" +short_conda_run="conda run --no-capture-output -p ${short_conda_env}" + +{ + echo "CONDA_ENV=${short_conda_env}" + echo "CONDA_RUN=${short_conda_run}" +} >> "${GITHUB_ENV}" + +build_env_file="${BUILD_ENV_FILE:-}" +if [[ -n "${build_env_file}" ]]; then + build_env_file="$(to_bash_path "${build_env_file}")" +fi + +if [[ -n "${build_env_file}" && -f "${build_env_file}" ]]; then + { + printf "export CONDA_ENV=%q\n" "${short_conda_env}" + printf "export CONDA_RUN=%q\n" "${short_conda_run}" + } >> "${build_env_file}" +fi + +echo "Mapped ${conda_env_parent_win} to ${short_drive}" +echo "Using ${short_conda_env} as the Conda environment prefix" diff --git a/.github/workflows/build-tensorrt-windows.yml b/.github/workflows/build-tensorrt-windows.yml index b9be0b0530..3c69427a22 100644 --- a/.github/workflows/build-tensorrt-windows.yml +++ b/.github/workflows/build-tensorrt-windows.yml @@ -131,6 +131,9 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} cuda-version: ${{ env.CU_VERSION }} arch: ${{ env.ARCH }} + - name: Shorten Conda environment path + run: | + bash "${REPOSITORY}/.github/scripts/shorten-conda-env-windows.sh" - name: Install XPU support package if: ${{ matrix.gpu_arch_type == 'xpu' }} run: | diff --git a/.github/workflows/build_windows.yml b/.github/workflows/build_windows.yml index ba5b6ac1a9..5187f6e2a6 100644 --- a/.github/workflows/build_windows.yml +++ b/.github/workflows/build_windows.yml @@ -167,6 +167,10 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} cuda-version: ${{ env.CU_VERSION }} arch: ${{ inputs.architecture }} + - name: Shorten Conda environment path + if: inputs.architecture == 'x64' + run: | + bash "${REPOSITORY}/.github/scripts/shorten-conda-env-windows.sh" - name: Install XPU support package if: ${{ matrix.gpu_arch_type == 'xpu' }} env: diff --git a/.github/workflows/windows-test.yml b/.github/workflows/windows-test.yml index ad17387e7c..0d919ca3c3 100644 --- a/.github/workflows/windows-test.yml +++ b/.github/workflows/windows-test.yml @@ -102,6 +102,10 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} cuda-version: ${{ env.CU_VERSION }} arch: ${{ env.ARCH }} + - name: Shorten Conda environment path + if: inputs.architecture == 'x64' + run: | + bash "${REPOSITORY}/.github/scripts/shorten-conda-env-windows.sh" - name: Run Pre-Script with Caching if: ${{ inputs.pre-script != '' }} uses: ./test-infra/.github/actions/run-script-with-cache diff --git a/packaging/pre_build_script_windows.sh b/packaging/pre_build_script_windows.sh index 0b52e24064..9698ba07ce 100644 --- a/packaging/pre_build_script_windows.sh +++ b/packaging/pre_build_script_windows.sh @@ -1,6 +1,6 @@ set -x -pip install -U numpy packaging pyyaml setuptools wheel fmt +python -m pip install -U numpy packaging pyyaml setuptools wheel fmt choco install bazelisk -y @@ -20,12 +20,13 @@ fi TORCH=$(grep "^torch>" py/requirements.txt) INDEX_URL=https://download.pytorch.org/whl/${CHANNEL}/${CU_VERSION} -# Install all the dependencies required for Torch-TensorRT -pip uninstall -y torch torchvision -pip install --force-reinstall --pre ${TORCH} --index-url ${INDEX_URL} +# The workflow installs torch before this script runs. Avoid uninstalling and +# force-reinstalling it here: with the shortened Windows conda prefix, pip can +# rediscover a half-removed torch dist-info through the original C:\ path. +python -m pip install --pre "${TORCH}" --index-url "${INDEX_URL}" || exit 1 export CUDA_HOME="$(echo ${CUDA_PATH} | sed -e 's#\\#\/#g')" -export TORCH_INSTALL_PATH="$(python -c "import torch, os; print(os.path.dirname(torch.__file__))" | sed -e 's#\\#\/#g')" +export TORCH_INSTALL_PATH="$(python -c "import torch, os; print(os.path.dirname(torch.__file__).replace('\\\\', '/'))")" || exit 1 # tried with conda install -c conda-forge fmt -y, but build still failed in windows with the following error: # C:\actions-runner\_work\_temp\conda_environment_18042354682\lib\site-packages\torch\include\torch/csrc/utils/python_arg_parser.h(42): fatal error C1083: Cannot open include file: 'fmt/format.h': No such file or directory