From 0e328f256de502c3bed5fd43abf5af223230f66b Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Thu, 23 Jul 2026 02:50:50 +0000 Subject: [PATCH 1/4] perf(http-client-csharp): parallelize SDK regeneration Preinstall shared tooling once, regenerate matching SDK projects with bounded parallelism, and report per-project timing. Refs #11361 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6626962a-f09e-496d-8a15-d2d9fd96d09d --- .../eng/scripts/Submit-AzureSdkForNetPr.ps1 | 145 ++++++++++++++---- 1 file changed, 115 insertions(+), 30 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 index 3e2b94ff1a2..ad71c0ed47b 100755 --- a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 +++ b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 @@ -125,14 +125,14 @@ This PR updates the UnbrandedGeneratorVersion property in eng/centralpackagemana - Ran npm install to update package-lock.json - Ran eng/packages/http-client-csharp/eng/scripts/Generate.ps1 to regenerate test projects - Generated emitter-package.json artifacts using tsp-client -- Regenerated SDK libraries using the unbranded emitter via dotnet msbuild /t:GenerateCode +- Regenerated SDK libraries using the unbranded emitter via dotnet build /t:GenerateCode $(if ($RegenerateAzureLibraries) { @" ### Additional changes (Azure data plane regeneration) - Built and packaged Azure emitter locally from eng/packages/http-client-csharp - Updated Azure emitter package artifacts in eng/ -- Regenerated Azure data plane SDK libraries via dotnet msbuild /t:GenerateCode +- Regenerated Azure data plane SDK libraries via dotnet build /t:GenerateCode "@ }) $(if ($RegenerateMgmtLibraries) { @@ -141,7 +141,7 @@ $(if ($RegenerateMgmtLibraries) { ### Additional changes (mgmt regeneration) - Built and packaged management plane emitter locally from eng/packages/http-client-csharp-mgmt - Updated mgmt emitter package artifacts in eng/ -- Regenerated mgmt SDK libraries via dotnet msbuild /t:GenerateCode +- Regenerated mgmt SDK libraries via dotnet build /t:GenerateCode "@ }) @@ -580,9 +580,9 @@ try { } } - # Discover service directories with tsp-location.yaml referencing any of the matched emitter patterns + # Discover SDK projects with tsp-location.yaml referencing any of the matched emitter patterns $tspLocations = Get-ChildItem -Path (Join-Path $tempDir "sdk") -Filter "tsp-location.yaml" -Recurse - $serviceDirectories = @() + $sdkProjects = @() foreach ($tspLocation in $tspLocations) { $content = Get-Content $tspLocation.FullName -Raw $matched = $false @@ -594,41 +594,126 @@ try { } if ($matched) { $relativePath = $tspLocation.DirectoryName -replace ".*[\\/]sdk[\\/]", "" - $serviceDirectory = $relativePath -replace "[\\/].*", "" - if ($serviceDirectories -notcontains $serviceDirectory) { - $serviceDirectories += $serviceDirectory + $pathSegments = $relativePath -split "[\\/]" + $srcPath = Join-Path $tspLocation.DirectoryName "src" + $projectFiles = @(Get-ChildItem -Path $srcPath -Filter "*.csproj" -File -ErrorAction SilentlyContinue) + if ($projectFiles.Count -ne 1) { + Register-StepFailure "Expected one SDK project under $srcPath but found $($projectFiles.Count). Skipping $relativePath." + continue + } + + $sdkProjects += [pscustomobject]@{ + Service = $pathSegments[0] + Library = $pathSegments[-1] + ProjectPath = $projectFiles[0].FullName } } } - if ($serviceDirectories.Count -eq 0) { + if ($sdkProjects.Count -eq 0) { Write-Host "No SDK libraries found matching emitter patterns. Skipping SDK regeneration." } else { - $serviceProj = Join-Path $tempDir "eng/service.proj" - - # Service directories whose libraries share a single code generator plugin that each - # library's generation builds into a common output folder. These services are regenerated - # serially since they share a common plugin project that shouldn't be built in parallel. - $serialCodeGenServiceDirectories = @("ai") + $sdkProjects = @($sdkProjects | Sort-Object -Property ProjectPath -Unique) + $setupStopwatch = [System.Diagnostics.Stopwatch]::StartNew() + $regenerationSetupSucceeded = $true + $previousErrorAction = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + $tspClientDir = Join-Path $tempDir "eng/common/tsp-client" + Write-Host "##[section]Pre-installing tsp-client..." + Invoke "npm ci --prefix `"$tspClientDir`"" $tempDir + if ($LASTEXITCODE -ne 0) { + Register-StepFailure "Failed to pre-install tsp-client. Skipping SDK regeneration." + $regenerationSetupSucceeded = $false + } - foreach ($serviceDirectory in $serviceDirectories) { - Write-Host "Regenerating code for service directory: $serviceDirectory" - $previousErrorAction = $ErrorActionPreference - $ErrorActionPreference = "Continue" - try { - $generateCommand = "dotnet msbuild $serviceProj /restore /t:GenerateCode /p:Trace=true /p:ServiceDirectory=$serviceDirectory" - if ($serialCodeGenServiceDirectories -contains $serviceDirectory) { - $generateCommand += " /m:1 /p:BuildInParallel=false" - } - Invoke $generateCommand $tempDir + if ($regenerationSetupSucceeded) { + $codeGenerationTargetPath = Join-Path $tempDir "eng/CodeGeneration.targets" + Write-Host "##[section]Pre-building the shared client plugin..." + Invoke "dotnet build `"$codeGenerationTargetPath`" /t:BuildPlugin /p:TypeSpecInput=temp" $tempDir if ($LASTEXITCODE -ne 0) { - Register-StepFailure "Code generation failed for $serviceDirectory with exit code $LASTEXITCODE. Continuing with next service directory." + Register-StepFailure "Failed to pre-build the shared client plugin. Skipping SDK regeneration." + $regenerationSetupSucceeded = $false } - } catch { - Register-StepFailure "Code generation failed for $serviceDirectory`: $($_.Exception.Message). Continuing with next service directory." - } finally { - $ErrorActionPreference = $previousErrorAction } + } catch { + Register-StepFailure "SDK regeneration setup failed: $($_.Exception.Message). Skipping SDK regeneration." + $regenerationSetupSucceeded = $false + } finally { + $ErrorActionPreference = $previousErrorAction + } + + $setupStopwatch.Stop() + + if ($regenerationSetupSucceeded) { + $cpuCores = [Environment]::ProcessorCount + $throttleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) + $completed = [System.Collections.Concurrent.ConcurrentBag[int]]::new() + $totalCount = $sdkProjects.Count + $regenerationStopwatch = [System.Diagnostics.Stopwatch]::StartNew() + + Write-Host "##[section]Regenerating $totalCount SDK projects with $throttleLimit concurrent jobs..." + $results = $sdkProjects | ForEach-Object -ThrottleLimit $throttleLimit -Parallel { + $sdkProject = $_ + $completedBag = $using:completed + $total = $using:totalCount + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + + try { + $output = & dotnet build $sdkProject.ProjectPath /t:GenerateCode /p:Trace=true /p:SkipTspClientInstall=true /p:SkipBuildPlugin=true 2>&1 + $exitCode = $LASTEXITCODE + $timingOutput = @($output | Where-Object { "$_" -match "Total Elapsed time" }) + + $completedBag.Add(1) + $currentCount = $completedBag.Count + $status = if ($exitCode -eq 0) { "passed" } else { "failed" } + Write-Host "[$currentCount/$total] $($sdkProject.Library) $status in $($stopwatch.Elapsed)" + foreach ($timingLine in $timingOutput) { + Write-Host " $timingLine" + } + + [pscustomobject]@{ + Service = $sdkProject.Service + Library = $sdkProject.Library + Success = $exitCode -eq 0 + ExitCode = $exitCode + Elapsed = $stopwatch.Elapsed + Output = if ($exitCode -eq 0) { "" } else { $output -join [Environment]::NewLine } + } + } catch { + $completedBag.Add(1) + $currentCount = $completedBag.Count + Write-Host "[$currentCount/$total] $($sdkProject.Library) failed in $($stopwatch.Elapsed)" + [pscustomobject]@{ + Service = $sdkProject.Service + Library = $sdkProject.Library + Success = $false + ExitCode = -1 + Elapsed = $stopwatch.Elapsed + Output = $_.Exception.ToString() + } + } finally { + $stopwatch.Stop() + } + } + + $regenerationStopwatch.Stop() + $failedResults = @($results | Where-Object { -not $_.Success }) + foreach ($result in $failedResults) { + Write-Host "##[error]$($result.Library) generation output:`n$($result.Output)" + Register-StepFailure "Code generation failed for $($result.Library) in service $($result.Service) with exit code $($result.ExitCode)." + } + + Write-Host "##[section]SDK regeneration timing summary" + Write-Host "Setup: $($setupStopwatch.Elapsed)" + Write-Host "Project regeneration: $($regenerationStopwatch.Elapsed)" + Write-Host "Succeeded: $($results.Count - $failedResults.Count)" + Write-Host "Failed: $($failedResults.Count)" + Write-Host "Slowest projects:" + $results | + Sort-Object -Property Elapsed -Descending | + Select-Object -First 20 | + ForEach-Object { Write-Host " $($_.Library): $($_.Elapsed)" } } } } From d668c183538db04eaa574f00974805cb6437214d Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Thu, 23 Jul 2026 03:19:53 +0000 Subject: [PATCH 2/4] perf(http-client-csharp): shard SDK regeneration jobs Split full Azure SDK regeneration across 18 pipeline jobs, pin every shard to one azure-sdk-for-net commit, and aggregate their binary patches before creating the final PR. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6626962a-f09e-496d-8a15-d2d9fd96d09d --- .../eng/pipeline/publish.yml | 211 +++++++++++------- .../steps/prepare-azure-sdk-for-net.yml | 63 ++++++ .../eng/scripts/Submit-AzureSdkForNetPr.ps1 | 125 ++++++++++- 3 files changed, 303 insertions(+), 96 deletions(-) create mode 100644 packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml diff --git a/packages/http-client-csharp/eng/pipeline/publish.yml b/packages/http-client-csharp/eng/pipeline/publish.yml index 8e480d9943f..f8c15258dd6 100644 --- a/packages/http-client-csharp/eng/pipeline/publish.yml +++ b/packages/http-client-csharp/eng/pipeline/publish.yml @@ -24,6 +24,11 @@ parameters: type: boolean default: false + - name: SdkRegenerationJobCount + displayName: Number of parallel Azure SDK regeneration jobs + type: number + default: 18 + - name: RunTests displayName: Run unit tests type: boolean @@ -115,18 +120,87 @@ extends: image: $(LINUXVMIMAGE) os: linux jobs: + - ${{ if or(parameters.RegenerateAzureLibraries, parameters.RegenerateMgmtLibraries) }}: + - job: PrepareSdkRegeneration + displayName: Prepare SDK regeneration + steps: + - checkout: none + - pwsh: | + $remote = "https://github.com/Azure/azure-sdk-for-net.git" + $commit = (git ls-remote $remote refs/heads/main).Split()[0] + if ($commit -notmatch "^[0-9a-f]{40}$") { + throw "Failed to resolve Azure SDK main commit from $remote" + } + Write-Host "Pinned Azure SDK commit: $commit" + Write-Host "##vso[task.setvariable variable=AzureSdkCommit;isOutput=true]$commit" + name: resolve + displayName: Pin Azure SDK commit + + - job: GenerateSdkChanges + displayName: Generate Azure SDK changes + dependsOn: PrepareSdkRegeneration + timeoutInMinutes: 90 + strategy: + parallel: ${{ parameters.SdkRegenerationJobCount }} + variables: + AzureSdkCommit: $[ dependencies.PrepareSdkRegeneration.outputs['resolve.AzureSdkCommit'] ] + steps: + - template: /packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml + parameters: + PackageVersion: $(PackageVersion) + UseTypeSpecNext: ${{ parameters.UseTypeSpecNext }} + + - task: PowerShell@2 + displayName: Regenerate SDK shard + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 + workingDirectory: $(Build.SourcesDirectory)/packages/http-client-csharp + arguments: > + -PackageVersion '$(PackageVersion)' + -TypeSpecCommitUrl '$(Build.Repository.Uri)/commit/$(Build.SourceVersion)' + -TypeSpecSourcePackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' + -AzureSdkCommit '$(AzureSdkCommit)' + -ShardIndex '$(System.JobPositionInPhase)' + -ShardCount '$(System.TotalJobsInPhase)' + -SdkPatchOutputPath '$(Build.ArtifactStagingDirectory)/sdk-changes.patch' + ${{ replace(replace('True', eq(variables['Build.SourceBranchName'], 'main'), ''), 'True', '-Internal') }} + ${{ replace(replace('True', eq(parameters.RegenerateAzureLibraries, false), ''), 'True', '-RegenerateAzureLibraries') }} + ${{ replace(replace('True', eq(parameters.RegenerateMgmtLibraries, false), ''), 'True', '-RegenerateMgmtLibraries') }} + -BuildArtifactsPath '$(Pipeline.Workspace)/build_artifacts_csharp/packages' + -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} + -BuildReason '$(Build.Reason)' + + - task: 1ES.PublishPipelineArtifact@1 + displayName: Publish SDK shard patch + inputs: + path: $(Build.ArtifactStagingDirectory)/sdk-changes.patch + artifact: sdk_regen_$(System.JobPositionInPhase) + + - pwsh: | + Remove-Item "$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc" -Force -ErrorAction SilentlyContinue + displayName: Cleanup .npmrc + condition: always() + - job: CreatePR timeoutInMinutes: 90 + ${{ if or(parameters.RegenerateAzureLibraries, parameters.RegenerateMgmtLibraries) }}: + dependsOn: + - PrepareSdkRegeneration + - GenerateSdkChanges variables: + ${{ if or(parameters.RegenerateAzureLibraries, parameters.RegenerateMgmtLibraries) }}: + AzureSdkCommit: $[ dependencies.PrepareSdkRegeneration.outputs['resolve.AzureSdkCommit'] ] # Redirect temp and cache directories to Agent.TempDirectory (a separate, larger partition) # to avoid running out of disk space on the root partition during generation TMPDIR: $(Agent.TempDirectory) NUGET_PACKAGES: $(Agent.TempDirectory)/nuget npm_config_cache: $(Agent.TempDirectory)/npm-cache steps: - - checkout: self - - - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + - template: /packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml + parameters: + PackageVersion: $(PackageVersion) + UseTypeSpecNext: ${{ parameters.UseTypeSpecNext }} - pwsh: | # Determine the TypeSpec PR URL @@ -147,68 +221,6 @@ extends: Write-Host "##vso[task.setvariable variable=PipelineRunUrl]$pipelineRunUrl" displayName: Set variables for PR creation - - task: UseNode@1 - displayName: "Install Node.js" - inputs: - version: "22.x" - - - task: NuGetAuthenticate@1 - - - pwsh: | - Write-Host "Creating .npmrc file $(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc for registry https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/" - $parentFolder = Split-Path -Path '$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc' -Parent - - if (!(Test-Path $parentFolder)) { - Write-Host "Creating folder $parentFolder" - New-Item -Path $parentFolder -ItemType Directory | Out-Null - } - - $content = "registry=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/`n`nalways-auth=true" - $content | Out-File '$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc' - - Write-Host "##[section].npmrc file created successfully. Contents:" - Get-Content '$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc' | ForEach-Object { Write-Host " $_" } - displayName: "Create .npmrc" - - - task: npmAuthenticate@0 - displayName: Authenticate npm for Azure Artifacts - inputs: - workingFile: $(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc - - - download: current - artifact: build_artifacts_csharp - displayName: Download pipeline artifacts - - - pwsh: | - npm install -g @azure-tools/typespec-client-generator-cli@latest - displayName: Install tsp-client - - - pwsh: | - npm install -g semver - displayName: Install semver - - - task: PowerShell@2 - displayName: Update package.json with injected dependencies - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Update-PackageJson.ps1 - workingDirectory: $(Build.SourcesDirectory)/packages/http-client-csharp - arguments: > - -PackageVersion '$(PackageVersion)' - -PackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' - - - pwsh: | - Write-Host "##vso[task.setvariable variable=npm_config_force]true" - Write-Host "##vso[task.setvariable variable=TSPCLIENT_FORCE_INSTALL]true" - Write-Host "Set npm --force and TSPCLIENT_FORCE_INSTALL for TypeSpec Next" - displayName: Configure npm for TypeSpec Next - condition: ${{ parameters.UseTypeSpecNext }} - - - task: UseDotNet@2 - inputs: - useGlobalJson: true - workingDirectory: $(Build.SourcesDirectory)/packages/http-client-csharp - - pwsh: | Write-Host "##[section]Checking npm configuration before PR submission..." Write-Host "Current npm registry:" @@ -227,30 +239,61 @@ extends: displayName: "Log npm configuration" condition: eq(${{ parameters.CreateAzureSdkForNetPR }}, true) + - ${{ if or(parameters.RegenerateAzureLibraries, parameters.RegenerateMgmtLibraries) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download SDK regeneration patches + inputs: + buildType: current + itemPattern: "**/*.patch" + targetPath: $(Pipeline.Workspace)/sdk_regen + - template: /eng/common/pipelines/templates/steps/login-to-github.yml parameters: TokenOwners: - Azure - - task: AzureCLI@2 - displayName: Generate emitter-package.json files & create PR in azure-sdk-for-net - inputs: - azureSubscription: "AzureSDKEngKeyVault Secrets" - scriptType: pscore - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 - arguments: > - -PackageVersion '$(PackageVersion)' - -TypeSpecCommitUrl '$(TypeSpecCommitUrl)' - -AuthToken '$(GH_TOKEN)' - -TypeSpecSourcePackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' - -PipelineRunUrl '$(PipelineRunUrl)' - ${{ replace(replace('True', eq(variables['Build.SourceBranchName'], 'main'), ''), 'True', '-Internal') }} - ${{ replace(replace('True', eq(parameters.RegenerateAzureLibraries, false), ''), 'True', '-RegenerateAzureLibraries') }} - ${{ replace(replace('True', eq(parameters.RegenerateMgmtLibraries, false), ''), 'True', '-RegenerateMgmtLibraries') }} - -BuildArtifactsPath '$(Pipeline.Workspace)/build_artifacts_csharp/packages' - -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} - -BuildReason '$(Build.Reason)' + - ${{ if or(parameters.RegenerateAzureLibraries, parameters.RegenerateMgmtLibraries) }}: + - task: AzureCLI@2 + displayName: Aggregate SDK changes & create PR in azure-sdk-for-net + inputs: + azureSubscription: "AzureSDKEngKeyVault Secrets" + scriptType: pscore + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 + arguments: > + -PackageVersion '$(PackageVersion)' + -TypeSpecCommitUrl '$(TypeSpecCommitUrl)' + -AuthToken '$(GH_TOKEN)' + -TypeSpecSourcePackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' + -PipelineRunUrl '$(PipelineRunUrl)' + -AzureSdkCommit '$(AzureSdkCommit)' + -SdkPatchDirectory '$(Pipeline.Workspace)/sdk_regen' + -SkipSdkRegeneration + ${{ replace(replace('True', eq(variables['Build.SourceBranchName'], 'main'), ''), 'True', '-Internal') }} + ${{ replace(replace('True', eq(parameters.RegenerateAzureLibraries, false), ''), 'True', '-RegenerateAzureLibraries') }} + ${{ replace(replace('True', eq(parameters.RegenerateMgmtLibraries, false), ''), 'True', '-RegenerateMgmtLibraries') }} + -BuildArtifactsPath '$(Pipeline.Workspace)/build_artifacts_csharp/packages' + -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} + -BuildReason '$(Build.Reason)' + + - ${{ else }}: + - task: AzureCLI@2 + displayName: Generate emitter-package.json files & create PR in azure-sdk-for-net + inputs: + azureSubscription: "AzureSDKEngKeyVault Secrets" + scriptType: pscore + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 + arguments: > + -PackageVersion '$(PackageVersion)' + -TypeSpecCommitUrl '$(TypeSpecCommitUrl)' + -AuthToken '$(GH_TOKEN)' + -TypeSpecSourcePackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' + -PipelineRunUrl '$(PipelineRunUrl)' + ${{ replace(replace('True', eq(variables['Build.SourceBranchName'], 'main'), ''), 'True', '-Internal') }} + -BuildArtifactsPath '$(Pipeline.Workspace)/build_artifacts_csharp/packages' + -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} + -BuildReason '$(Build.Reason)' - pwsh: | $npmrcPath = "$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc" diff --git a/packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml b/packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml new file mode 100644 index 00000000000..5e14e4ae66b --- /dev/null +++ b/packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml @@ -0,0 +1,63 @@ +parameters: + - name: PackageVersion + type: string + - name: UseTypeSpecNext + type: boolean + default: false + +steps: + - checkout: self + + - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + + - task: UseNode@1 + displayName: "Install Node.js" + inputs: + version: "22.x" + + - task: NuGetAuthenticate@1 + + - pwsh: | + $npmrcPath = "$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc" + Write-Host "Creating $npmrcPath for the Azure SDK public npm feed" + @( + "registry=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/" + "always-auth=true" + ) | Set-Content $npmrcPath + displayName: "Create C# emitter .npmrc" + + - task: npmAuthenticate@0 + displayName: Authenticate npm for Azure Artifacts + inputs: + workingFile: $(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc + + - download: current + artifact: build_artifacts_csharp + displayName: Download pipeline artifacts + + - pwsh: npm install -g @azure-tools/typespec-client-generator-cli@latest + displayName: Install tsp-client + + - pwsh: npm install -g semver + displayName: Install semver + + - task: PowerShell@2 + displayName: Update package.json with injected dependencies + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Update-PackageJson.ps1 + workingDirectory: $(Build.SourcesDirectory)/packages/http-client-csharp + arguments: > + -PackageVersion '${{ parameters.PackageVersion }}' + -PackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' + + - ${{ if parameters.UseTypeSpecNext }}: + - pwsh: | + Write-Host "##vso[task.setvariable variable=npm_config_force]true" + Write-Host "##vso[task.setvariable variable=TSPCLIENT_FORCE_INSTALL]true" + displayName: Configure npm for TypeSpec Next + + - task: UseDotNet@2 + inputs: + useGlobalJson: true + workingDirectory: $(Build.SourcesDirectory)/packages/http-client-csharp diff --git a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 index ad71c0ed47b..7635ebba2a8 100755 --- a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 +++ b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 @@ -1,4 +1,5 @@ #!/usr/bin/env pwsh +# cspell:ignore pscustomobject <# .DESCRIPTION @@ -23,6 +24,18 @@ Path to the build artifacts directory containing the published .tgz and .nupkg f The URL of the pipeline run that triggered this PR. When provided, it is included in the PR description for traceability. .PARAMETER BuildReason The reason the pipeline was triggered (for example, 'Manual', 'Schedule', or 'IndividualCI'). When set to 'Manual', step failures fail the pipeline instead of being downgraded to warnings and opening a PR. +.PARAMETER AzureSdkCommit +The azure-sdk-for-net commit to regenerate. Pipeline shards must use the same commit so their patches can be combined. +.PARAMETER ShardIndex +The zero-based index of this regeneration shard. +.PARAMETER ShardCount +The total number of regeneration shards. +.PARAMETER SdkPatchOutputPath +When provided, writes SDK changes to this patch and exits without creating a pull request. +.PARAMETER SdkPatchDirectory +When provided, applies all SDK patch files under this directory before creating the pull request. +.PARAMETER SkipSdkRegeneration +Skips SDK generation. Used by the final aggregation job, which applies patches produced by generation shards. #> [CmdletBinding(SupportsShouldProcess = $true)] param( @@ -32,7 +45,7 @@ param( [Parameter(Mandatory = $true)] [string]$TypeSpecCommitUrl, - [Parameter(Mandatory = $true)] + [Parameter(Mandatory = $false)] [string]$AuthToken, [Parameter(Mandatory = $false)] @@ -60,9 +73,33 @@ param( [switch]$UseTypeSpecNext, [Parameter(Mandatory = $false)] - [string]$BuildReason + [string]$BuildReason, + + [Parameter(Mandatory = $false)] + [string]$AzureSdkCommit, + + [Parameter(Mandatory = $false)] + [ValidateRange(0, [int]::MaxValue)] + [int]$ShardIndex = 0, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, [int]::MaxValue)] + [int]$ShardCount = 1, + + [Parameter(Mandatory = $false)] + [string]$SdkPatchOutputPath, + + [Parameter(Mandatory = $false)] + [string]$SdkPatchDirectory, + + [Parameter(Mandatory = $false)] + [switch]$SkipSdkRegeneration ) +if ($ShardIndex -ge $ShardCount) { + throw "ShardIndex ($ShardIndex) must be less than ShardCount ($ShardCount)." +} + # When the pipeline is triggered manually, failures should fail the pipeline with an # error instead of being downgraded to warnings and still opening a PR, which gives a # false positive to reviewers. For automated (scheduled/CI) runs, keep the existing @@ -81,7 +118,9 @@ function Register-StepFailure { $script:StepFailures.Add($Message) Write-Warning $Message - Write-Host "##vso[task.complete result=SucceededWithIssues;]" + if (-not $SdkPatchOutputPath) { + Write-Host "##vso[task.complete result=SucceededWithIssues;]" + } } # Import the Generation module to use the Invoke helper function @@ -173,7 +212,9 @@ try { # Set the authentication token for gh CLI early so that scripts invoked # during the build (e.g. Emitter_Version_Dashboard.ps1) can call the # GitHub API to resolve commit hashes in shallow clones. - $env:GH_TOKEN = $AuthToken + if ($AuthToken) { + $env:GH_TOKEN = $AuthToken + } # Configure git user for commits in this repository git config user.name "azure-sdk" @@ -199,17 +240,18 @@ try { throw "Failed to set sparse checkout patterns" } - # Fetch only the main branch with depth 1 - Write-Host "Fetching $BaseBranch branch with sparse checkout..." - git fetch --depth 1 origin $BaseBranch + # Fetch the pinned commit when sharding so every patch has the same base. + $sourceRef = if ($AzureSdkCommit) { $AzureSdkCommit } else { $BaseBranch } + Write-Host "Fetching $sourceRef with sparse checkout..." + git fetch --depth 1 origin $sourceRef if ($LASTEXITCODE -ne 0) { throw "Failed to fetch repository" } # Checkout the fetched branch - git checkout $BaseBranch + git checkout --detach FETCH_HEAD if ($LASTEXITCODE -ne 0) { - throw "Failed to checkout $BaseBranch" + throw "Failed to checkout $sourceRef" } # Create a new branch @@ -369,7 +411,7 @@ try { } # Run Generate.ps1 from the package root - if ($shouldRunGenerate -eq $true) + if ($shouldRunGenerate -eq $true -and -not $SdkPatchOutputPath) { Write-Host "Running eng/packages/http-client-csharp/eng/scripts/Generate.ps1..." $previousErrorAction = $ErrorActionPreference @@ -453,7 +495,7 @@ try { } # Regenerate all SDK libraries that use the unbranded emitter - if ($installSucceeded) { + if ($installSucceeded -and -not $SkipSdkRegeneration) { Write-Host "Expanding sparse checkout to include sdk directory for SDK regeneration..." git sparse-checkout add sdk if ($LASTEXITCODE -ne 0) { @@ -614,6 +656,18 @@ try { Write-Host "No SDK libraries found matching emitter patterns. Skipping SDK regeneration." } else { $sdkProjects = @($sdkProjects | Sort-Object -Property ProjectPath -Unique) + if ($ShardCount -gt 1) { + $allProjects = $sdkProjects + $sdkProjects = @( + for ($i = 0; $i -lt $allProjects.Count; $i++) { + if ($i % $ShardCount -eq $ShardIndex) { + $allProjects[$i] + } + } + ) + Write-Host "Shard $ShardIndex/$ShardCount selected $($sdkProjects.Count) of $($allProjects.Count) SDK projects." + } + $setupStopwatch = [System.Diagnostics.Stopwatch]::StartNew() $regenerationSetupSucceeded = $true $previousErrorAction = $ErrorActionPreference @@ -647,7 +701,7 @@ try { if ($regenerationSetupSucceeded) { $cpuCores = [Environment]::ProcessorCount - $throttleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) + $throttleLimit = if ($ShardCount -gt 1) { 1 } else { [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) } $completed = [System.Collections.Concurrent.ConcurrentBag[int]]::new() $totalCount = $sdkProjects.Count $regenerationStopwatch = [System.Diagnostics.Stopwatch]::StartNew() @@ -719,6 +773,53 @@ try { } } + if ($SdkPatchOutputPath) { + if ($script:StepFailures.Count -gt 0) { + $failureSummary = ($script:StepFailures | ForEach-Object { "- $_" }) -join [Environment]::NewLine + throw "Regeneration shard failed; no patch will be published:$([Environment]::NewLine)$failureSummary" + } + + $patchDirectory = Split-Path -Path $SdkPatchOutputPath -Parent + New-Item -ItemType Directory -Path $patchDirectory -Force | Out-Null + git add --all sdk + if ($LASTEXITCODE -ne 0) { + throw "Failed to stage SDK changes for shard $ShardIndex/$ShardCount" + } + git diff --cached --binary --output="$SdkPatchOutputPath" -- sdk + if ($LASTEXITCODE -ne 0) { + throw "Failed to write SDK patch to $SdkPatchOutputPath" + } + Write-Host "Wrote SDK patch for shard $ShardIndex/$ShardCount to $SdkPatchOutputPath" + return + } + + if ($SdkPatchDirectory) { + Write-Host "Expanding sparse checkout to apply SDK regeneration patches..." + git sparse-checkout add sdk + if ($LASTEXITCODE -ne 0) { + throw "Failed to expand sparse checkout before applying SDK patches." + } + + $sdkPatches = @(Get-ChildItem -Path $SdkPatchDirectory -Filter "*.patch" -File -Recurse | Sort-Object -Property FullName) + if ($sdkPatches.Count -eq 0) { + throw "No SDK patches found under $SdkPatchDirectory" + } + + Write-Host "##[section]Applying $($sdkPatches.Count) SDK regeneration patches..." + foreach ($patch in $sdkPatches) { + if ($patch.Length -eq 0) { + Write-Host "Skipping empty patch $($patch.FullName)" + continue + } + + git apply --binary $patch.FullName + if ($LASTEXITCODE -ne 0) { + throw "Failed to apply SDK patch $($patch.FullName)" + } + Write-Host "Applied $($patch.Name)" + } + } + # Regenerate the emitter version dashboard Write-Host "Regenerating emitter version dashboard..." $dashboardScript = Join-Path $tempDir "doc/GeneratorVersions/Emitter_Version_Dashboard.ps1" From 09d17b16509108c159e7821bf8370deb52acdbf3 Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Thu, 23 Jul 2026 03:34:24 +0000 Subject: [PATCH 3/4] revert(http-client-csharp): defer SDK regeneration sharding Keep SDK regeneration on one pipeline agent and defer job-level fan-out to a separate change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6626962a-f09e-496d-8a15-d2d9fd96d09d --- .../eng/pipeline/publish.yml | 211 +++++++----------- .../steps/prepare-azure-sdk-for-net.yml | 63 ------ .../eng/scripts/Submit-AzureSdkForNetPr.ps1 | 125 +---------- 3 files changed, 96 insertions(+), 303 deletions(-) delete mode 100644 packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml diff --git a/packages/http-client-csharp/eng/pipeline/publish.yml b/packages/http-client-csharp/eng/pipeline/publish.yml index f8c15258dd6..8e480d9943f 100644 --- a/packages/http-client-csharp/eng/pipeline/publish.yml +++ b/packages/http-client-csharp/eng/pipeline/publish.yml @@ -24,11 +24,6 @@ parameters: type: boolean default: false - - name: SdkRegenerationJobCount - displayName: Number of parallel Azure SDK regeneration jobs - type: number - default: 18 - - name: RunTests displayName: Run unit tests type: boolean @@ -120,87 +115,18 @@ extends: image: $(LINUXVMIMAGE) os: linux jobs: - - ${{ if or(parameters.RegenerateAzureLibraries, parameters.RegenerateMgmtLibraries) }}: - - job: PrepareSdkRegeneration - displayName: Prepare SDK regeneration - steps: - - checkout: none - - pwsh: | - $remote = "https://github.com/Azure/azure-sdk-for-net.git" - $commit = (git ls-remote $remote refs/heads/main).Split()[0] - if ($commit -notmatch "^[0-9a-f]{40}$") { - throw "Failed to resolve Azure SDK main commit from $remote" - } - Write-Host "Pinned Azure SDK commit: $commit" - Write-Host "##vso[task.setvariable variable=AzureSdkCommit;isOutput=true]$commit" - name: resolve - displayName: Pin Azure SDK commit - - - job: GenerateSdkChanges - displayName: Generate Azure SDK changes - dependsOn: PrepareSdkRegeneration - timeoutInMinutes: 90 - strategy: - parallel: ${{ parameters.SdkRegenerationJobCount }} - variables: - AzureSdkCommit: $[ dependencies.PrepareSdkRegeneration.outputs['resolve.AzureSdkCommit'] ] - steps: - - template: /packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml - parameters: - PackageVersion: $(PackageVersion) - UseTypeSpecNext: ${{ parameters.UseTypeSpecNext }} - - - task: PowerShell@2 - displayName: Regenerate SDK shard - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 - workingDirectory: $(Build.SourcesDirectory)/packages/http-client-csharp - arguments: > - -PackageVersion '$(PackageVersion)' - -TypeSpecCommitUrl '$(Build.Repository.Uri)/commit/$(Build.SourceVersion)' - -TypeSpecSourcePackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' - -AzureSdkCommit '$(AzureSdkCommit)' - -ShardIndex '$(System.JobPositionInPhase)' - -ShardCount '$(System.TotalJobsInPhase)' - -SdkPatchOutputPath '$(Build.ArtifactStagingDirectory)/sdk-changes.patch' - ${{ replace(replace('True', eq(variables['Build.SourceBranchName'], 'main'), ''), 'True', '-Internal') }} - ${{ replace(replace('True', eq(parameters.RegenerateAzureLibraries, false), ''), 'True', '-RegenerateAzureLibraries') }} - ${{ replace(replace('True', eq(parameters.RegenerateMgmtLibraries, false), ''), 'True', '-RegenerateMgmtLibraries') }} - -BuildArtifactsPath '$(Pipeline.Workspace)/build_artifacts_csharp/packages' - -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} - -BuildReason '$(Build.Reason)' - - - task: 1ES.PublishPipelineArtifact@1 - displayName: Publish SDK shard patch - inputs: - path: $(Build.ArtifactStagingDirectory)/sdk-changes.patch - artifact: sdk_regen_$(System.JobPositionInPhase) - - - pwsh: | - Remove-Item "$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc" -Force -ErrorAction SilentlyContinue - displayName: Cleanup .npmrc - condition: always() - - job: CreatePR timeoutInMinutes: 90 - ${{ if or(parameters.RegenerateAzureLibraries, parameters.RegenerateMgmtLibraries) }}: - dependsOn: - - PrepareSdkRegeneration - - GenerateSdkChanges variables: - ${{ if or(parameters.RegenerateAzureLibraries, parameters.RegenerateMgmtLibraries) }}: - AzureSdkCommit: $[ dependencies.PrepareSdkRegeneration.outputs['resolve.AzureSdkCommit'] ] # Redirect temp and cache directories to Agent.TempDirectory (a separate, larger partition) # to avoid running out of disk space on the root partition during generation TMPDIR: $(Agent.TempDirectory) NUGET_PACKAGES: $(Agent.TempDirectory)/nuget npm_config_cache: $(Agent.TempDirectory)/npm-cache steps: - - template: /packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml - parameters: - PackageVersion: $(PackageVersion) - UseTypeSpecNext: ${{ parameters.UseTypeSpecNext }} + - checkout: self + + - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml - pwsh: | # Determine the TypeSpec PR URL @@ -221,6 +147,68 @@ extends: Write-Host "##vso[task.setvariable variable=PipelineRunUrl]$pipelineRunUrl" displayName: Set variables for PR creation + - task: UseNode@1 + displayName: "Install Node.js" + inputs: + version: "22.x" + + - task: NuGetAuthenticate@1 + + - pwsh: | + Write-Host "Creating .npmrc file $(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc for registry https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/" + $parentFolder = Split-Path -Path '$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc' -Parent + + if (!(Test-Path $parentFolder)) { + Write-Host "Creating folder $parentFolder" + New-Item -Path $parentFolder -ItemType Directory | Out-Null + } + + $content = "registry=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/`n`nalways-auth=true" + $content | Out-File '$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc' + + Write-Host "##[section].npmrc file created successfully. Contents:" + Get-Content '$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc' | ForEach-Object { Write-Host " $_" } + displayName: "Create .npmrc" + + - task: npmAuthenticate@0 + displayName: Authenticate npm for Azure Artifacts + inputs: + workingFile: $(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc + + - download: current + artifact: build_artifacts_csharp + displayName: Download pipeline artifacts + + - pwsh: | + npm install -g @azure-tools/typespec-client-generator-cli@latest + displayName: Install tsp-client + + - pwsh: | + npm install -g semver + displayName: Install semver + + - task: PowerShell@2 + displayName: Update package.json with injected dependencies + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Update-PackageJson.ps1 + workingDirectory: $(Build.SourcesDirectory)/packages/http-client-csharp + arguments: > + -PackageVersion '$(PackageVersion)' + -PackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' + + - pwsh: | + Write-Host "##vso[task.setvariable variable=npm_config_force]true" + Write-Host "##vso[task.setvariable variable=TSPCLIENT_FORCE_INSTALL]true" + Write-Host "Set npm --force and TSPCLIENT_FORCE_INSTALL for TypeSpec Next" + displayName: Configure npm for TypeSpec Next + condition: ${{ parameters.UseTypeSpecNext }} + + - task: UseDotNet@2 + inputs: + useGlobalJson: true + workingDirectory: $(Build.SourcesDirectory)/packages/http-client-csharp + - pwsh: | Write-Host "##[section]Checking npm configuration before PR submission..." Write-Host "Current npm registry:" @@ -239,61 +227,30 @@ extends: displayName: "Log npm configuration" condition: eq(${{ parameters.CreateAzureSdkForNetPR }}, true) - - ${{ if or(parameters.RegenerateAzureLibraries, parameters.RegenerateMgmtLibraries) }}: - - task: DownloadPipelineArtifact@2 - displayName: Download SDK regeneration patches - inputs: - buildType: current - itemPattern: "**/*.patch" - targetPath: $(Pipeline.Workspace)/sdk_regen - - template: /eng/common/pipelines/templates/steps/login-to-github.yml parameters: TokenOwners: - Azure - - ${{ if or(parameters.RegenerateAzureLibraries, parameters.RegenerateMgmtLibraries) }}: - - task: AzureCLI@2 - displayName: Aggregate SDK changes & create PR in azure-sdk-for-net - inputs: - azureSubscription: "AzureSDKEngKeyVault Secrets" - scriptType: pscore - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 - arguments: > - -PackageVersion '$(PackageVersion)' - -TypeSpecCommitUrl '$(TypeSpecCommitUrl)' - -AuthToken '$(GH_TOKEN)' - -TypeSpecSourcePackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' - -PipelineRunUrl '$(PipelineRunUrl)' - -AzureSdkCommit '$(AzureSdkCommit)' - -SdkPatchDirectory '$(Pipeline.Workspace)/sdk_regen' - -SkipSdkRegeneration - ${{ replace(replace('True', eq(variables['Build.SourceBranchName'], 'main'), ''), 'True', '-Internal') }} - ${{ replace(replace('True', eq(parameters.RegenerateAzureLibraries, false), ''), 'True', '-RegenerateAzureLibraries') }} - ${{ replace(replace('True', eq(parameters.RegenerateMgmtLibraries, false), ''), 'True', '-RegenerateMgmtLibraries') }} - -BuildArtifactsPath '$(Pipeline.Workspace)/build_artifacts_csharp/packages' - -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} - -BuildReason '$(Build.Reason)' - - - ${{ else }}: - - task: AzureCLI@2 - displayName: Generate emitter-package.json files & create PR in azure-sdk-for-net - inputs: - azureSubscription: "AzureSDKEngKeyVault Secrets" - scriptType: pscore - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 - arguments: > - -PackageVersion '$(PackageVersion)' - -TypeSpecCommitUrl '$(TypeSpecCommitUrl)' - -AuthToken '$(GH_TOKEN)' - -TypeSpecSourcePackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' - -PipelineRunUrl '$(PipelineRunUrl)' - ${{ replace(replace('True', eq(variables['Build.SourceBranchName'], 'main'), ''), 'True', '-Internal') }} - -BuildArtifactsPath '$(Pipeline.Workspace)/build_artifacts_csharp/packages' - -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} - -BuildReason '$(Build.Reason)' + - task: AzureCLI@2 + displayName: Generate emitter-package.json files & create PR in azure-sdk-for-net + inputs: + azureSubscription: "AzureSDKEngKeyVault Secrets" + scriptType: pscore + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 + arguments: > + -PackageVersion '$(PackageVersion)' + -TypeSpecCommitUrl '$(TypeSpecCommitUrl)' + -AuthToken '$(GH_TOKEN)' + -TypeSpecSourcePackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' + -PipelineRunUrl '$(PipelineRunUrl)' + ${{ replace(replace('True', eq(variables['Build.SourceBranchName'], 'main'), ''), 'True', '-Internal') }} + ${{ replace(replace('True', eq(parameters.RegenerateAzureLibraries, false), ''), 'True', '-RegenerateAzureLibraries') }} + ${{ replace(replace('True', eq(parameters.RegenerateMgmtLibraries, false), ''), 'True', '-RegenerateMgmtLibraries') }} + -BuildArtifactsPath '$(Pipeline.Workspace)/build_artifacts_csharp/packages' + -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} + -BuildReason '$(Build.Reason)' - pwsh: | $npmrcPath = "$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc" diff --git a/packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml b/packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml deleted file mode 100644 index 5e14e4ae66b..00000000000 --- a/packages/http-client-csharp/eng/pipeline/steps/prepare-azure-sdk-for-net.yml +++ /dev/null @@ -1,63 +0,0 @@ -parameters: - - name: PackageVersion - type: string - - name: UseTypeSpecNext - type: boolean - default: false - -steps: - - checkout: self - - - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml - - - task: UseNode@1 - displayName: "Install Node.js" - inputs: - version: "22.x" - - - task: NuGetAuthenticate@1 - - - pwsh: | - $npmrcPath = "$(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc" - Write-Host "Creating $npmrcPath for the Azure SDK public npm feed" - @( - "registry=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/" - "always-auth=true" - ) | Set-Content $npmrcPath - displayName: "Create C# emitter .npmrc" - - - task: npmAuthenticate@0 - displayName: Authenticate npm for Azure Artifacts - inputs: - workingFile: $(Build.SourcesDirectory)/packages/http-client-csharp/.npmrc - - - download: current - artifact: build_artifacts_csharp - displayName: Download pipeline artifacts - - - pwsh: npm install -g @azure-tools/typespec-client-generator-cli@latest - displayName: Install tsp-client - - - pwsh: npm install -g semver - displayName: Install semver - - - task: PowerShell@2 - displayName: Update package.json with injected dependencies - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/packages/http-client-csharp/eng/scripts/Update-PackageJson.ps1 - workingDirectory: $(Build.SourcesDirectory)/packages/http-client-csharp - arguments: > - -PackageVersion '${{ parameters.PackageVersion }}' - -PackageJsonPath '$(Build.SourcesDirectory)/packages/http-client-csharp/package.json' - - - ${{ if parameters.UseTypeSpecNext }}: - - pwsh: | - Write-Host "##vso[task.setvariable variable=npm_config_force]true" - Write-Host "##vso[task.setvariable variable=TSPCLIENT_FORCE_INSTALL]true" - displayName: Configure npm for TypeSpec Next - - - task: UseDotNet@2 - inputs: - useGlobalJson: true - workingDirectory: $(Build.SourcesDirectory)/packages/http-client-csharp diff --git a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 index 7635ebba2a8..ad71c0ed47b 100755 --- a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 +++ b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 @@ -1,5 +1,4 @@ #!/usr/bin/env pwsh -# cspell:ignore pscustomobject <# .DESCRIPTION @@ -24,18 +23,6 @@ Path to the build artifacts directory containing the published .tgz and .nupkg f The URL of the pipeline run that triggered this PR. When provided, it is included in the PR description for traceability. .PARAMETER BuildReason The reason the pipeline was triggered (for example, 'Manual', 'Schedule', or 'IndividualCI'). When set to 'Manual', step failures fail the pipeline instead of being downgraded to warnings and opening a PR. -.PARAMETER AzureSdkCommit -The azure-sdk-for-net commit to regenerate. Pipeline shards must use the same commit so their patches can be combined. -.PARAMETER ShardIndex -The zero-based index of this regeneration shard. -.PARAMETER ShardCount -The total number of regeneration shards. -.PARAMETER SdkPatchOutputPath -When provided, writes SDK changes to this patch and exits without creating a pull request. -.PARAMETER SdkPatchDirectory -When provided, applies all SDK patch files under this directory before creating the pull request. -.PARAMETER SkipSdkRegeneration -Skips SDK generation. Used by the final aggregation job, which applies patches produced by generation shards. #> [CmdletBinding(SupportsShouldProcess = $true)] param( @@ -45,7 +32,7 @@ param( [Parameter(Mandatory = $true)] [string]$TypeSpecCommitUrl, - [Parameter(Mandatory = $false)] + [Parameter(Mandatory = $true)] [string]$AuthToken, [Parameter(Mandatory = $false)] @@ -73,33 +60,9 @@ param( [switch]$UseTypeSpecNext, [Parameter(Mandatory = $false)] - [string]$BuildReason, - - [Parameter(Mandatory = $false)] - [string]$AzureSdkCommit, - - [Parameter(Mandatory = $false)] - [ValidateRange(0, [int]::MaxValue)] - [int]$ShardIndex = 0, - - [Parameter(Mandatory = $false)] - [ValidateRange(1, [int]::MaxValue)] - [int]$ShardCount = 1, - - [Parameter(Mandatory = $false)] - [string]$SdkPatchOutputPath, - - [Parameter(Mandatory = $false)] - [string]$SdkPatchDirectory, - - [Parameter(Mandatory = $false)] - [switch]$SkipSdkRegeneration + [string]$BuildReason ) -if ($ShardIndex -ge $ShardCount) { - throw "ShardIndex ($ShardIndex) must be less than ShardCount ($ShardCount)." -} - # When the pipeline is triggered manually, failures should fail the pipeline with an # error instead of being downgraded to warnings and still opening a PR, which gives a # false positive to reviewers. For automated (scheduled/CI) runs, keep the existing @@ -118,9 +81,7 @@ function Register-StepFailure { $script:StepFailures.Add($Message) Write-Warning $Message - if (-not $SdkPatchOutputPath) { - Write-Host "##vso[task.complete result=SucceededWithIssues;]" - } + Write-Host "##vso[task.complete result=SucceededWithIssues;]" } # Import the Generation module to use the Invoke helper function @@ -212,9 +173,7 @@ try { # Set the authentication token for gh CLI early so that scripts invoked # during the build (e.g. Emitter_Version_Dashboard.ps1) can call the # GitHub API to resolve commit hashes in shallow clones. - if ($AuthToken) { - $env:GH_TOKEN = $AuthToken - } + $env:GH_TOKEN = $AuthToken # Configure git user for commits in this repository git config user.name "azure-sdk" @@ -240,18 +199,17 @@ try { throw "Failed to set sparse checkout patterns" } - # Fetch the pinned commit when sharding so every patch has the same base. - $sourceRef = if ($AzureSdkCommit) { $AzureSdkCommit } else { $BaseBranch } - Write-Host "Fetching $sourceRef with sparse checkout..." - git fetch --depth 1 origin $sourceRef + # Fetch only the main branch with depth 1 + Write-Host "Fetching $BaseBranch branch with sparse checkout..." + git fetch --depth 1 origin $BaseBranch if ($LASTEXITCODE -ne 0) { throw "Failed to fetch repository" } # Checkout the fetched branch - git checkout --detach FETCH_HEAD + git checkout $BaseBranch if ($LASTEXITCODE -ne 0) { - throw "Failed to checkout $sourceRef" + throw "Failed to checkout $BaseBranch" } # Create a new branch @@ -411,7 +369,7 @@ try { } # Run Generate.ps1 from the package root - if ($shouldRunGenerate -eq $true -and -not $SdkPatchOutputPath) + if ($shouldRunGenerate -eq $true) { Write-Host "Running eng/packages/http-client-csharp/eng/scripts/Generate.ps1..." $previousErrorAction = $ErrorActionPreference @@ -495,7 +453,7 @@ try { } # Regenerate all SDK libraries that use the unbranded emitter - if ($installSucceeded -and -not $SkipSdkRegeneration) { + if ($installSucceeded) { Write-Host "Expanding sparse checkout to include sdk directory for SDK regeneration..." git sparse-checkout add sdk if ($LASTEXITCODE -ne 0) { @@ -656,18 +614,6 @@ try { Write-Host "No SDK libraries found matching emitter patterns. Skipping SDK regeneration." } else { $sdkProjects = @($sdkProjects | Sort-Object -Property ProjectPath -Unique) - if ($ShardCount -gt 1) { - $allProjects = $sdkProjects - $sdkProjects = @( - for ($i = 0; $i -lt $allProjects.Count; $i++) { - if ($i % $ShardCount -eq $ShardIndex) { - $allProjects[$i] - } - } - ) - Write-Host "Shard $ShardIndex/$ShardCount selected $($sdkProjects.Count) of $($allProjects.Count) SDK projects." - } - $setupStopwatch = [System.Diagnostics.Stopwatch]::StartNew() $regenerationSetupSucceeded = $true $previousErrorAction = $ErrorActionPreference @@ -701,7 +647,7 @@ try { if ($regenerationSetupSucceeded) { $cpuCores = [Environment]::ProcessorCount - $throttleLimit = if ($ShardCount -gt 1) { 1 } else { [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) } + $throttleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) $completed = [System.Collections.Concurrent.ConcurrentBag[int]]::new() $totalCount = $sdkProjects.Count $regenerationStopwatch = [System.Diagnostics.Stopwatch]::StartNew() @@ -773,53 +719,6 @@ try { } } - if ($SdkPatchOutputPath) { - if ($script:StepFailures.Count -gt 0) { - $failureSummary = ($script:StepFailures | ForEach-Object { "- $_" }) -join [Environment]::NewLine - throw "Regeneration shard failed; no patch will be published:$([Environment]::NewLine)$failureSummary" - } - - $patchDirectory = Split-Path -Path $SdkPatchOutputPath -Parent - New-Item -ItemType Directory -Path $patchDirectory -Force | Out-Null - git add --all sdk - if ($LASTEXITCODE -ne 0) { - throw "Failed to stage SDK changes for shard $ShardIndex/$ShardCount" - } - git diff --cached --binary --output="$SdkPatchOutputPath" -- sdk - if ($LASTEXITCODE -ne 0) { - throw "Failed to write SDK patch to $SdkPatchOutputPath" - } - Write-Host "Wrote SDK patch for shard $ShardIndex/$ShardCount to $SdkPatchOutputPath" - return - } - - if ($SdkPatchDirectory) { - Write-Host "Expanding sparse checkout to apply SDK regeneration patches..." - git sparse-checkout add sdk - if ($LASTEXITCODE -ne 0) { - throw "Failed to expand sparse checkout before applying SDK patches." - } - - $sdkPatches = @(Get-ChildItem -Path $SdkPatchDirectory -Filter "*.patch" -File -Recurse | Sort-Object -Property FullName) - if ($sdkPatches.Count -eq 0) { - throw "No SDK patches found under $SdkPatchDirectory" - } - - Write-Host "##[section]Applying $($sdkPatches.Count) SDK regeneration patches..." - foreach ($patch in $sdkPatches) { - if ($patch.Length -eq 0) { - Write-Host "Skipping empty patch $($patch.FullName)" - continue - } - - git apply --binary $patch.FullName - if ($LASTEXITCODE -ne 0) { - throw "Failed to apply SDK patch $($patch.FullName)" - } - Write-Host "Applied $($patch.Name)" - } - } - # Regenerate the emitter version dashboard Write-Host "Regenerating emitter version dashboard..." $dashboardScript = Join-Path $tempDir "doc/GeneratorVersions/Emitter_Version_Dashboard.ps1" From af22ddb848421df2e0b4d1db39d5395312404e5d Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Thu, 23 Jul 2026 06:30:37 +0000 Subject: [PATCH 4/4] perf(http-client-csharp): use available regen concurrency Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ec786e41-5618-4414-9552-1c28e2362114 --- .../http-client-csharp/eng/scripts/Generation.psm1 | 13 +++++++++++++ .../http-client-csharp/eng/scripts/RegenPreview.ps1 | 6 +++--- .../eng/scripts/Submit-AzureSdkForNetPr.ps1 | 12 ++++++------ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/Generation.psm1 b/packages/http-client-csharp/eng/scripts/Generation.psm1 index 35291b6fcaf..992269b3279 100644 --- a/packages/http-client-csharp/eng/scripts/Generation.psm1 +++ b/packages/http-client-csharp/eng/scripts/Generation.psm1 @@ -24,6 +24,18 @@ function Invoke($command, $executePath=$packageRoot) } } +function Get-ParallelThrottleLimit { + param ( + [Parameter(Mandatory = $true)] + [ValidateRange(1, [int]::MaxValue)] + [int]$ProcessorCount, + [ValidateRange(1, [int]::MaxValue)] + [int]$Maximum = 8 + ) + + return [Math]::Min($Maximum, $ProcessorCount) +} + function Get-TspCommand { param ( [string]$specFile, @@ -191,6 +203,7 @@ function Set-LaunchSettings { } Export-ModuleMember -Function "Invoke" +Export-ModuleMember -Function "Get-ParallelThrottleLimit" Export-ModuleMember -Function "Get-TspCommand" Export-ModuleMember -Function "Refresh-Build" Export-ModuleMember -Function "Compare-Paths" diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 index 30a83af5d45..8122623fec5 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 @@ -957,7 +957,7 @@ try { $failedCount = 0 } else { - # Determine parallel execution throttle limit: (CPU cores - 2), min 1, max 8 + # Determine parallel execution throttle limit based on available logical processors, capped at 8. $cpuCores = if ($IsWindows -or $PSVersionTable.PSVersion.Major -lt 6) { (Get-CimInstance -ClassName Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum } elseif ($IsMacOS) { @@ -966,7 +966,7 @@ try { [int](nproc) } - $throttleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) + $throttleLimit = Get-ParallelThrottleLimit -ProcessorCount $cpuCores Write-Host "Using $throttleLimit concurrent jobs (detected $cpuCores logical processors)" -ForegroundColor Gray Write-Host "" @@ -1033,7 +1033,7 @@ try { } else { Push-Location $buildPath try { - $output = & dotnet build /t:GenerateCode /p:SkipTspClientInstall=true /p:SkipBuildPlugin=true 2>&1 + $output = & dotnet msbuild /t:GenerateCode /p:SkipTspClientInstall=true /p:SkipBuildPlugin=true 2>&1 $exitCode = $LASTEXITCODE if ($exitCode -ne 0) { diff --git a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 index ad71c0ed47b..1fb4a5c6456 100755 --- a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 +++ b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 @@ -125,14 +125,14 @@ This PR updates the UnbrandedGeneratorVersion property in eng/centralpackagemana - Ran npm install to update package-lock.json - Ran eng/packages/http-client-csharp/eng/scripts/Generate.ps1 to regenerate test projects - Generated emitter-package.json artifacts using tsp-client -- Regenerated SDK libraries using the unbranded emitter via dotnet build /t:GenerateCode +- Regenerated SDK libraries using the unbranded emitter via dotnet msbuild /t:GenerateCode $(if ($RegenerateAzureLibraries) { @" ### Additional changes (Azure data plane regeneration) - Built and packaged Azure emitter locally from eng/packages/http-client-csharp - Updated Azure emitter package artifacts in eng/ -- Regenerated Azure data plane SDK libraries via dotnet build /t:GenerateCode +- Regenerated Azure data plane SDK libraries via dotnet msbuild /t:GenerateCode "@ }) $(if ($RegenerateMgmtLibraries) { @@ -141,7 +141,7 @@ $(if ($RegenerateMgmtLibraries) { ### Additional changes (mgmt regeneration) - Built and packaged management plane emitter locally from eng/packages/http-client-csharp-mgmt - Updated mgmt emitter package artifacts in eng/ -- Regenerated mgmt SDK libraries via dotnet build /t:GenerateCode +- Regenerated mgmt SDK libraries via dotnet msbuild /t:GenerateCode "@ }) @@ -647,12 +647,12 @@ try { if ($regenerationSetupSucceeded) { $cpuCores = [Environment]::ProcessorCount - $throttleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) + $throttleLimit = Get-ParallelThrottleLimit -ProcessorCount $cpuCores $completed = [System.Collections.Concurrent.ConcurrentBag[int]]::new() $totalCount = $sdkProjects.Count $regenerationStopwatch = [System.Diagnostics.Stopwatch]::StartNew() - Write-Host "##[section]Regenerating $totalCount SDK projects with $throttleLimit concurrent jobs..." + Write-Host "##[section]Regenerating $totalCount SDK projects with $throttleLimit concurrent jobs (detected $cpuCores logical processors)..." $results = $sdkProjects | ForEach-Object -ThrottleLimit $throttleLimit -Parallel { $sdkProject = $_ $completedBag = $using:completed @@ -660,7 +660,7 @@ try { $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() try { - $output = & dotnet build $sdkProject.ProjectPath /t:GenerateCode /p:Trace=true /p:SkipTspClientInstall=true /p:SkipBuildPlugin=true 2>&1 + $output = & dotnet msbuild $sdkProject.ProjectPath /t:GenerateCode /p:Trace=true /p:SkipTspClientInstall=true /p:SkipBuildPlugin=true 2>&1 $exitCode = $LASTEXITCODE $timingOutput = @($output | Where-Object { "$_" -match "Total Elapsed time" })