diff --git a/.github/workflows/build_test_cbsinit.yml b/.github/workflows/build_test_cbsinit.yml index 2cfa3ddd..89c63561 100644 --- a/.github/workflows/build_test_cbsinit.yml +++ b/.github/workflows/build_test_cbsinit.yml @@ -10,7 +10,8 @@ jobs: matrix: os: ['windows-2022'] download_official_python_msi: ["true"] - remove_python_pycs: ["true"] + remove_python_pycs: ["false"] + install_with_pymanager: ["true"] cbsinit_repo: ['https://github.com/cloudbase/cloudbase-init'] cbsinit_branch: ['master'] python_version: ['3.14_4'] @@ -34,10 +35,11 @@ jobs: -CloudbaseInitRepoBranch ${{ matrix.cbsinit_branch }} ^ -InstallOfficialPythonMsi:$${{ matrix.download_official_python_msi }} ^ -OfficialPythonMsiChecksum "C10234D0D9BD89F6F6DD55BAE28EDE0F97EE0DF4" ^ - -RemovePythonPycs:$${{ matrix.remove_python_pycs }} + -RemovePythonPycs:$${{ matrix.remove_python_pycs }} ^ + -InstallOfficialPythonUsingPyManager:$${{ matrix.install_with_pymanager }} - uses: actions/upload-artifact@v7 with: - name: "CloudbaseInit_platform-${{ matrix.platform }}_build-env-os-${{ matrix.os }}_download-official-msi-${{ matrix.download_official_python_msi }}_remove-pycs-${{ matrix.remove_python_pycs }}_cbs-init-branch-${{ matrix.cbsinit_branch }}_MSI" + name: "CloudbaseInit_platform-${{ matrix.platform }}_build-env-os-${{ matrix.os }}_download-official-msi-${{ matrix.download_official_python_msi }}_remove-pycs-${{ matrix.remove_python_pycs }}_install-with-pymanager${{ matrix.install_with_pymanager }}_cbs-init-branch-${{ matrix.cbsinit_branch }}_MSI" path: 'CloudbaseInitSetup/bin/release/${{ matrix.platform }}/CloudbaseInitSetup.msi' - name: Download external dependencies shell: powershell @@ -99,6 +101,7 @@ jobs: } } & $pythonPath -m pip install pip-audit + & $pythonPath -m pip install venv $pipAuditLogs = & $pipAuditPath Write-Output "$pipAuditLogs" if ($LASTEXITCODE) { diff --git a/BuildAutomation/BuildCloudbaseInitSetup.ps1 b/BuildAutomation/BuildCloudbaseInitSetup.ps1 index 1ad9a7c1..870cb795 100644 --- a/BuildAutomation/BuildCloudbaseInitSetup.ps1 +++ b/BuildAutomation/BuildCloudbaseInitSetup.ps1 @@ -13,7 +13,8 @@ Param( [string]$VCVars = "2019", [switch]$InstallOfficialPythonMsi = $false, [string]$OfficialPythonMsiChecksum = "C10234D0D9BD89F6F6DD55BAE28EDE0F97EE0DF4", - [switch]$RemovePythonPycs = $false + [switch]$RemovePythonPycs = $false, + [switch]$InstallOfficialPythonUsingPyManager = $false ) $ErrorActionPreference = "Stop" @@ -80,16 +81,31 @@ try $python_template_dir = join-path $cloudbaseInitInstallerDir "Python$($pythonversion.replace('.', ''))_${platform}_Template" + # TODO(avladu): stick to just only one way to download Python, preferably the pymanager once it gets ironed out if ($InstallOfficialPythonMsi) { - if (!$OfficialPythonMsiChecksum) { - throw "Please set a OfficialPythonMsiChecksum parameter value." + if ($InstallOfficialPythonUsingPyManager) { + Remove-Item -Recurse -Force $python_template_dir -ErrorAction SilentlyContinue + DownloadInstall-PythonUsingPyManager $platform $python_template_dir $pythonversion + } else { + if (!$OfficialPythonMsiChecksum) { + throw "Please set a OfficialPythonMsiChecksum parameter value." + } + Remove-Item -Recurse -Force $python_template_dir -ErrorAction SilentlyContinue + DownloadInstall-PythonMsi $platform $python_template_dir $pythonversion $OfficialPythonMsiChecksum } - Remove-Item -Recurse -Force $python_template_dir -ErrorAction SilentlyContinue - DownloadInstall-PythonMsi $platform $python_template_dir $pythonversion $OfficialPythonMsiChecksum } CheckCopyDir $python_template_dir $python_dir + $pythonGetPipUrl = "https://bootstrap.pypa.io/get-pip.py" + $pythonGetPipPath = Join-Path (Resolve-Path "${python_dir}/..").Path "/get-pip.py" + ExecRetry { DownloadFile $pythonGetPipUrl $pythonGetPipPath } + + & python.exe "${pythonGetPipPath}" + if ($LASTEXITCODE) { + throw "Failed to install pip in directory: ${python_dir}" + } + # Make sure that we don't have temp files from a previous build $python_build_path = "$ENV:LOCALAPPDATA\Temp\pip_build_$ENV:USERNAME" if (Test-Path $python_build_path) { @@ -97,6 +113,11 @@ try } ExecRetry { PipInstall "pip" -update $true } + # When using embed Python, pbr needs to be reinstalled so that the project builds + # Warning received: UserWarning: Unknown distribution option: 'pbr' + # pbr is already installed but most likely this is a problem with knowing where it is, + # and reinstalling fixing the issue + ExecRetry { PipInstall "pbr" -update $true } ExecRetry { PipInstall "wheel" -update $true } ExecRetry { PipInstall "setuptools" -update $true } diff --git a/BuildAutomation/BuildUtils.ps1 b/BuildAutomation/BuildUtils.ps1 index 604f52e2..405b3ec2 100644 --- a/BuildAutomation/BuildUtils.ps1 +++ b/BuildAutomation/BuildUtils.ps1 @@ -460,3 +460,77 @@ function DownloadInstall-PythonMsi($platform, $python_template_dir, $pythonVersi } } + +function DownloadInstall-PythonUsingPyManager($platform, $python_template_dir, $pythonVersion) { + $pythonManagerUrl = "https://www.python.org/ftp/python/pymanager/python-manager-26.1.msix" + $pythonManagerPath = Join-Path (Resolve-Path "${python_template_dir}/..").Path "/python-manager.exe" + + $pythonManagerPackage = Get-AppPackage -Name PythonSoftwareFoundation.PythonManager -ErrorAction SilentlyContinue + + if (!$pythonManagerPackage) { + ExecRetry { DownloadFile $pythonManagerUrl $pythonManagerPath } + Add-AppPackage -Path $pythonManagerPath + } + + $platformSuffix = "" + if ($platform -eq "x86") { + $platformSuffix = "-32" + } + if ($platform -eq "arm64") { + $platformSuffix = "-arm64" + } + + if (Test-Path $python_template_dir) { + throw "$python_template_dir folder already exists" + } + + $pythonVersionEscaped = "PythonEmbed\" + $pythonVersion.replace("_",".") + $platformSuffix + pymanager.exe install --target=$python_template_dir --force --update $pythonVersionEscaped + if ($LASTEXITCODE) { + throw "Failed to install python in directory: ${python_template_dir}" + } + + if (!(Test-Path $python_template_dir)) { + throw "$python_template_dir has not been created" + } + + & "$python_template_dir/python.exe" --version + if ($LASTEXITCODE) { + throw "Failed to run python in directory: ${python_template_dir}" + } + + Out-File -Append -InputObject "Lib\site-packages" -Encoding ascii $python_template_dir\python*._pth + Out-File -Append -InputObject "libs" -Encoding ascii $python_template_dir\python*._pth + + $python_template_dir_full = $python_template_dir + "_full" + $pythonVersionEscaped = $pythonVersion.replace("_",".") + $platformSuffix + pymanager.exe install --target=$python_template_dir_full --force --update $pythonVersionEscaped + if ($LASTEXITCODE) { + throw "Failed to install python in directory: ${python_template_dir_full}" + } + + Move-Item $python_template_dir_full/include $python_template_dir/ + Move-Item $python_template_dir_full/libs $python_template_dir/ + + mkdir $python_template_dir\Lib\site-packages + + Out-File -Append -InputObject "win32" -Encoding ascii "$python_template_dir\Lib\site-packages\pywin32.pth" + Out-File -Append -InputObject "win32\lib" -Encoding ascii "$python_template_dir\Lib\site-packages\pywin32.pth" + Out-File -Append -InputObject "Pythonwin" -Encoding ascii "$python_template_dir\Lib\site-packages\pywin32.pth" + Out-File -Append -InputObject "import pywin32_bootstrap" -Encoding ascii "$python_template_dir\Lib\site-packages\pywin32.pth" + + Out-File -Append -InputObject "Lib\site-packages\win32" -Encoding ascii $python_template_dir\python*._pth + Out-File -Append -InputObject "Lib\site-packages\win32\lib" -Encoding ascii $python_template_dir\python*._pth + Out-File -Append -InputObject "Lib\site-packages\Pythonwin" -Encoding ascii $python_template_dir\python*._pth + + Remove-Item -Force -Recurse "$python_template_dir_full" -ErrorAction SilentlyContinue + + Remove-Item -Force -Recurse "$python_template_dir/DLLs/_tkinter.pyd" -ErrorAction SilentlyContinue + Remove-Item -Force -Recurse "$python_template_dir/DLLs/tcl*.dll" -ErrorAction SilentlyContinue + Remove-Item -Force -Recurse "$python_template_dir/DLLs/tk*.dll" -ErrorAction SilentlyContinue + Remove-Item -Force -Recurse "$python_template_dir/Doc" -ErrorAction SilentlyContinue + Remove-Item -Force -Recurse "$python_template_dir/Lib/tkinter" -ErrorAction SilentlyContinue + Remove-Item -Force -Recurse "$python_template_dir/Lib/turtle.py" -ErrorAction SilentlyContinue + Remove-Item -Force -Recurse "$python_template_dir/Lib/turtledemo" -ErrorAction SilentlyContinue + Remove-Item -Force -Recurse "$python_template_dir/tcl" -ErrorAction SilentlyContinue +}