Enable VS integration tests on CI

azure-pipelines-PR.yml

@@ -436,22 +436,28 @@ stages:
               coreclr_release:
                 _configuration: Release
                 _testKind: testCoreclr
+                prepareMachineArg: ''
                 transparentCompiler: # Empty display name part.
               vs_release:
                 _configuration: Release
                 _testKind: testVs
                 setupVsHive: true
                 FSharp_CacheEvictionImmediate: true
+                prepareMachineArg: ''
                 transparentCompiler:
               transparent_compiler_release:
                 _configuration: Release
                 _testKind: testCoreclr
                 TEST_TRANSPARENT_COMPILER: 1 # Pipeline variable will map to env var.
+                prepareMachineArg: ''
                 transparentCompiler: TransparentCompiler
-              # inttests_release:
-              #   _configuration: Release
-              #   _testKind: testIntegration
-              #   setupVsHive: true
+              inttests_release:
+                _configuration: Release
+                _testKind: testIntegration
+                setupVsHive: true
+                # Enables on-exit devenv cleanup; the integration leg also runs VS on an
+                # interactive desktop session (Setup-IntegrationTestRun in eng\Build.ps1).
+                prepareMachineArg: -prepareMachine
           steps:
           - checkout: self
             clean: true
@@ -460,14 +466,26 @@ stages:
             displayName: Setup VS Hive
             condition: eq(variables.setupVsHive, 'true')
 
-          - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind)
+          - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) $(prepareMachineArg)
             env:
               DOTNET_DbgEnableMiniDump: 1
               DOTNET_DbgMiniDumpType: 2 # 1=mini, 2=heap, 3=triage, 4=full. Heap dumps include managed object data for debugging.
               DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\$(_configuration)\$(Build.BuildId)-%e-%p-%t.dmp
               NativeToolsOnMachine: true
             displayName: Build and Test $(_testKind) $(transparentCompiler)
 
+          # Publish the integration-test startup screenshot to prove devenv ran on a live
+          # interactive desktop session (captured by Setup-IntegrationTestRun).
+          - task: PublishBuildArtifacts@1
+            displayName: Publish Integration Test Screenshot
+            condition: and(always(), eq(variables._testKind, 'testIntegration'))
+            continueOnError: true
+            inputs:
+                PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\$(_configuration)\StartingBuild.png'
+                ArtifactName: 'Windows $(_configuration) $(_testKind) screenshot'
+                ArtifactType: Container
+                parallel: true
+
           - task: PublishTestResults@2
             displayName: Publish Test Results
             inputs:

eng/Build.ps1

@@ -402,6 +402,90 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
     Exec-Console $dotnetExe $test_args
 }
 
+# Runs a test assembly via the xUnit v2 console runner, bypassing `dotnet test` (and therefore
+# the repo-wide Microsoft.Testing.Platform gate declared in global.json). Used only for projects
+# whose harness cannot run under MTP -- today that is FSharp.Editor.IntegrationTests, which
+# depends on Microsoft.VisualStudio.Extensibility.Testing.Xunit (VS-hive launcher, xUnit-v2-locked).
+# The runner is restored via a <PackageDownload Include="xunit.runner.console" .../> on the
+# test project, so it is present in the NuGet global packages folder after a normal slnx restore.
+function TestUsingXUnitConsole([string] $testProject, [string] $targetFramework) {
+
+    $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject)
+    $assemblyPath = "$ArtifactsDir\bin\$projectName\$configuration\$targetFramework\$projectName.dll"
+    if (-not (Test-Path $assemblyPath)) {
+        throw "Test assembly not found at $assemblyPath. Was $projectName built before -testIntegration was invoked?"
+    }
+
+    # Get-PackageVersion (eng\build-utils.ps1) reads <XunitRunnerConsoleV2Version> from eng\Versions.props,
+    # and Get-PackageDir resolves the NuGet cache path. Defensive Trim() covers accidental whitespace in the value.
+    $runnerVersion = (Get-PackageVersion "XunitRunnerConsoleV2").Trim()
+    $xunitConsole = Join-Path (Get-PackageDir "xunit.runner.console" $runnerVersion) "tools\net472\xunit.console.exe"
+    if (-not (Test-Path $xunitConsole)) {
+        throw "xunit.console.exe not found at $xunitConsole. Ensure restore of $projectName ran first (PackageDownload of xunit.runner.console v$runnerVersion)."
+    }
+
+    $testResultsDir = "$ArtifactsDir\TestResults\$configuration"
+    Create-Directory $testResultsDir
+    $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" }
+    $resultsXml = Join-Path $testResultsDir "$projectName.$targetFramework.$jobName.xml"
+
+    # -parallel none / -noshadow mirror the project's xunit.runner.json (parallelizeTestCollections=false, shadowCopy=false).
+    $xunit_args = """$assemblyPath"" -xml ""$resultsXml"" -parallel none -noshadow -nologo"
+
+    Write-Host("$xunitConsole $xunit_args")
+
+    Exec-Console $xunitConsole $xunit_args
+}
+
+# Sets up the CI machine for running our VS integration tests, mirroring dotnet/roslyn's
+# Setup-IntegrationTestRun (eng\build.ps1). The VS editor's view-driven diagnostic producers
+# (e.g. the error-squiggle tagger) only run reliably with a realized, visible desktop, so we
+# ensure devenv runs on an interactive session: probe by taking a screenshot, and if that fails
+# reconnect the session to the physical console via tscon, then re-capture (fail-fast otherwise).
+function Setup-IntegrationTestRun() {
+    # Ensure leftover devenv instances are cleaned up on exit (honored when -ci -prepareMachine).
+    $script:processesToStopOnExit += "devenv"
+
+    $screenshotPath = (Join-Path $LogDir "StartingBuild.png")
+    try {
+        Capture-Screenshot $screenshotPath
+    }
+    catch {
+        Write-Host "Screenshot failed; attempting to connect to the console"
+
+        # Keep the session open so we have a UI to interact with
+        $quserItems = ((quser $env:USERNAME | Select-Object -Skip 1) -split '\s+')
+        $sessionid = $quserItems[2]
+        if ($sessionid -eq 'Disc') {
+            # When the session isn't connected, the third value is 'Disc' instead of the ID
+            $sessionid = $quserItems[1]
+        }
+
+        if ($quserItems[1] -eq 'console') {
+            Write-Host "Disconnecting from console before attempting reconnection"
+            try {
+                tsdiscon
+            } catch {
+                # ignore
+            }
+
+            # Disconnection is asynchronous, so wait a few seconds for it to complete
+            Start-Sleep -Seconds 3
+            query user
+        }
+
+        Write-Host "tscon $sessionid /dest:console"
+        tscon $sessionid /dest:console
+
+        # Connection is asynchronous, so wait a few seconds for it to complete
+        Start-Sleep 3
+        query user
+
+        # Make sure we can capture a screenshot. An exception at this point will fail-fast the build.
+        Capture-Screenshot $screenshotPath
+    }
+}
+
 function Prepare-TempDir() {
     Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.props") $TempDir
     Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.targets") $TempDir
@@ -529,6 +613,7 @@ try {
     Process-Arguments
 
     . (Join-Path $PSScriptRoot "build-utils.ps1")
+    . (Join-Path $PSScriptRoot "build-utils-win.ps1")
 
     Update-Arguments
 
@@ -544,6 +629,15 @@ try {
     if ($ci) {
         Prepare-TempDir
         EnablePreviewSdks
+
+        if ($testIntegration -and -not $noVisualStudio) {
+            # Minimize all windows to avoid interference during integration test runs, then ensure
+            # devenv has an interactive desktop session (see Setup-IntegrationTestRun).
+            $shell = New-Object -ComObject "Shell.Application"
+            $shell.MinimizeAll()
+
+            Setup-IntegrationTestRun
+        }
     }
 
     $buildTool = InitializeBuildTool
@@ -665,8 +759,8 @@ try {
         TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\UnitTests\VisualFSharp.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework
     }
 
-    if ($testIntegration) {
-        TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework
+    if ($testIntegration -and -not $noVisualStudio) {
+        TestUsingXUnitConsole -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework
     }
 
     if ($testAOT) {

eng/SetupVSHive.ps1

@@ -12,6 +12,32 @@ $vsRegEdit = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'VSRegEdit
 $hive = "RoslynDev"
 &$vsRegEdit set "$vsDir" $hive HKCU "Roslyn\Internal\OnOff\Features" OOP64Bit dword 0
 
+# VS/UI hardening for headless integration test runs, ported from dotnet/roslyn eng\build.ps1.
+# These reduce interference (toasts, dialogs, roaming) that can disrupt the editor and the
+# error list / light bulb during code-action tests.
+
+# Disable roaming settings to avoid interference from the online user profile
+&$vsRegEdit set "$vsDir" $hive HKCU "ApplicationPrivateSettings\Microsoft\VisualStudio" RoamingEnabled string "1*System.Boolean*False"
+
+# Disable IntelliCode line completions to avoid interference with completion testing
+&$vsRegEdit set "$vsDir" $hive HKCU "ApplicationPrivateSettings\Microsoft\VisualStudio\IntelliCode" wholeLineCompletions string "0*System.Int32*2"
+
+# Disable IntelliCode RepositoryAttachedModels since it requires authentication which can fail in CI
+&$vsRegEdit set "$vsDir" $hive HKCU "ApplicationPrivateSettings\Microsoft\VisualStudio\IntelliCode" repositoryAttachedModels string "0*System.Int32*2"
+
+# Disable background download UI to avoid toasts
+&$vsRegEdit set "$vsDir" $hive HKCU "FeatureFlags\Setup\BackgroundDownload" Value dword 0
+
+# Disable text spell checker to avoid spurious warnings in the error list
+&$vsRegEdit set "$vsDir" $hive HKCU "FeatureFlags\Editor\EnableSpellChecker" Value dword 0
+
+# Disable text editor error reporting because it pops up a dialog. We want to fail fast or
+# fail silently and continue testing, never block on a modal dialog.
+&$vsRegEdit set "$vsDir" $hive HKCU "Text Editor" "Report Exceptions" dword 0
+
+# Disable targeted notifications (does not work via vsregedit, so set the registry directly)
+reg add hkcu\Software\Microsoft\VisualStudio\RemoteSettings /f /t REG_DWORD /v TurnOffSwitch /d 1
+
 Write-Host "-- VS Info --"
 $isolationIni = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'devenv.isolation.ini'
 Get-Content $isolationIni | Write-Host

eng/Versions.props

@@ -130,7 +130,7 @@
     <MicrosoftVisualStudioPlatformVSEditorVersion>$(VisualStudioEditorPackagesVersion)</MicrosoftVisualStudioPlatformVSEditorVersion>
     <MicrosoftVisualStudioTextUIWpfVersion>$(VisualStudioEditorPackagesVersion)</MicrosoftVisualStudioTextUIWpfVersion>
     <NuGetVisualStudioVersion>17.14.0</NuGetVisualStudioVersion>
-    <MicrosoftVisualStudioExtensibilityTestingVersion>0.1.800-beta</MicrosoftVisualStudioExtensibilityTestingVersion>
+    <MicrosoftVisualStudioExtensibilityTestingVersion>5.3.0-2.26055.8</MicrosoftVisualStudioExtensibilityTestingVersion>
     <MicrosoftVisualStudioExtensibilityTestingSourceGeneratorVersion>$(MicrosoftVisualStudioExtensibilityTestingVersion)</MicrosoftVisualStudioExtensibilityTestingSourceGeneratorVersion>
 
     <!-- Visual Studio Threading packages -->
@@ -161,6 +161,8 @@
     <NewtonsoftJsonVersion>13.0.4</NewtonsoftJsonVersion>
     <XunitVersion>3.2.2</XunitVersion>
     <XunitRunnerConsoleVersion>3.2.2</XunitRunnerConsoleVersion>
+    <!-- xUnit v2 runner, only consumed by FSharp.Editor.IntegrationTests (VS-hive harness is xUnit-v2-locked). -->
+    <XunitRunnerConsoleV2Version>2.9.3</XunitRunnerConsoleV2Version>
     <XunitXmlTestLoggerVersion>8.0.0</XunitXmlTestLoggerVersion>
     <!-- -->
   </PropertyGroup>

eng/build-utils-win.ps1

@@ -0,0 +1,34 @@
+
+# Collection of Windows-only PowerShell build utility functions used by our CI scripts.
+# Ported from dotnet/roslyn eng\build-utils-win.ps1 to support running the VS integration
+# tests on an interactive desktop session (screenshot probe + tscon reconnect).
+
+Set-StrictMode -version 2.0
+$ErrorActionPreference = "Stop"
+
+Add-Type -AssemblyName 'System.Drawing'
+Add-Type -AssemblyName 'System.Windows.Forms'
+function Capture-Screenshot($path) {
+  $width = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Width
+  $height = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Height
+
+  $bitmap = New-Object System.Drawing.Bitmap $width, $height
+  try {
+    $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+    try {
+      $graphics.CopyFromScreen( `
+        [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.X, `
+        [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Y, `
+        0, `
+        0, `
+        $bitmap.Size, `
+        [System.Drawing.CopyPixelOperation]::SourceCopy)
+    } finally {
+      $graphics.Dispose()
+    }
+
+    $bitmap.Save($path, [System.Drawing.Imaging.ImageFormat]::Png)
+  } finally {
+    $bitmap.Dispose()
+  }
+}

src/Compiler/Service/ServiceAnalysis.fs

@@ -304,19 +304,49 @@ module UnusedOpens =
         async {
             use! _holder = Cancellable.UseToken()
 
+            System.Diagnostics.Trace.TraceInformation(
+                "[UnusedOpens] getUnusedOpens called. OpenDeclarations.Length={0}",
+                checkFileResults.OpenDeclarations.Length
+            )
+
             if checkFileResults.OpenDeclarations.Length = 0 then
+                System.Diagnostics.Trace.TraceInformation("[UnusedOpens] No open declarations, returning empty")
                 return []
             else
                 let! ct = Async.CancellationToken
-                let symbolUses = checkFileResults.GetAllUsesOfAllSymbolsInFile(ct)
-                let symbolUses = filterSymbolUses getSourceLineStr symbolUses
-                let symbolUses = splitSymbolUses symbolUses
+                let allSymbolUses = checkFileResults.GetAllUsesOfAllSymbolsInFile(ct)
+
+                let filteredSymbolUses = filterSymbolUses getSourceLineStr allSymbolUses
+                let symbolUses1, symbolUses2 = splitSymbolUses filteredSymbolUses
                 let openStatements = getOpenStatements checkFileResults.OpenDeclarations
 
+                System.Diagnostics.Trace.TraceInformation(
+                    "[UnusedOpens] filteredSymbolUses={0}, symbolUses1={1}, symbolUses2={2}, openStatements={3}",
+                    filteredSymbolUses.Length,
+                    symbolUses1.Length,
+                    symbolUses2.Length,
+                    openStatements.Length
+                )
+
+                for os in openStatements do
+                    System.Diagnostics.Trace.TraceInformation(
+                        "[UnusedOpens]   openStatement range={0}, openedGroups={1}",
+                        os.Range,
+                        os.OpenedGroups.Length
+                    )
+
                 if openStatements.Length = 0 then
+                    System.Diagnostics.Trace.TraceInformation("[UnusedOpens] No open statements, returning empty")
                     return []
                 else
-                    return! filterOpenStatements symbolUses openStatements
+                    let! result = filterOpenStatements (symbolUses1, symbolUses2) openStatements
+
+                    System.Diagnostics.Trace.TraceInformation("[UnusedOpens] Result: {0} unused opens", result.Length)
+
+                    for r in result do
+                        System.Diagnostics.Trace.TraceInformation("[UnusedOpens]   unused: {0}", r)
+
+                    return result
         }
 
 module SimplifyNames =