From 0c8f3e5ed6e5ef8708810a02d5a4cfb335b9ea30 Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Sat, 16 May 2026 13:44:03 -0700 Subject: [PATCH 1/6] Add agent host user message telemetry Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../platform/agentHost/node/agentHostMain.ts | 13 +++- .../agentHost/node/agentHostServerMain.ts | 6 +- .../node/agentHostTelemetryService.ts | 64 ++++++++++++++++ .../platform/agentHost/node/agentService.ts | 4 + .../agentHost/node/agentSideEffects.ts | 34 ++++++++- .../test/node/agentSideEffects.test.ts | 75 ++++++++++++++++++- 6 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 src/vs/platform/agentHost/node/agentHostTelemetryService.ts diff --git a/src/vs/platform/agentHost/node/agentHostMain.ts b/src/vs/platform/agentHost/node/agentHostMain.ts index 8bcfefc7b4629..2dc1125a8d858 100644 --- a/src/vs/platform/agentHost/node/agentHostMain.ts +++ b/src/vs/platform/agentHost/node/agentHostMain.ts @@ -59,15 +59,20 @@ import { AgentPluginManager } from './agentPluginManager.js'; import { AgentHostGitService, IAgentHostGitService } from './agentHostGitService.js'; import { registerPendingEditContentProvider } from './copilot/pendingEditContentStore.js'; import { join } from '../../../base/common/path.js'; +import { createAgentHostTelemetryService } from './agentHostTelemetryService.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; // Entry point for the agent host utility process. // Sets up IPC, logging, and registers agent providers (Copilot). // When VSCODE_AGENT_HOST_PORT or VSCODE_AGENT_HOST_SOCKET_PATH env vars // are set, also starts a WebSocket server for external clients. -startAgentHost(); +void startAgentHost().catch(err => { + console.error(err); + process.exit(1); +}); -function startAgentHost(): void { +async function startAgentHost(): Promise { // Setup RPC - supports both Electron utility process and Node child process let server: ChildProcessServer | UtilityProcessServer; if (isUtilityProcess(process)) { @@ -100,6 +105,7 @@ function startAgentHost(): void { // Session data service const sessionDataService = new SessionDataService(URI.file(environmentService.userDataPath), fileService, logService); const rootConfigResource = joinPath(environmentService.appSettingsHome, 'globalStorage', 'agent-host-config.json'); + const telemetryService = await createAgentHostTelemetryService({ environmentService, productService, fileService, loggerService, logService, disposables }); // Create the real service implementation that lives in this process let agentService: AgentService; @@ -113,6 +119,7 @@ function startAgentHost(): void { diServices.set(IFileService, fileService); diServices.set(ISessionDataService, sessionDataService); diServices.set(IProductService, productService); + diServices.set(ITelemetryService, telemetryService); instantiationService = new InstantiationService(diServices); const gitService = instantiationService.createInstance(AgentHostGitService); diServices.set(IAgentHostGitService, gitService); @@ -124,7 +131,7 @@ function startAgentHost(): void { diServices.set(IClaudeAgentSdkService, claudeAgentSdkService); const agentHostOTelService = disposables.add(instantiationService.createInstance(AgentHostOTelService)); diServices.set(IAgentHostOTelService, agentHostOTelService); - agentService = new AgentService(logService, fileService, sessionDataService, productService, gitService, rootConfigResource); + agentService = new AgentService(logService, fileService, sessionDataService, productService, gitService, rootConfigResource, telemetryService); const pluginManager = new AgentPluginManager(URI.file(environmentService.userDataPath), fileService, logService); diServices.set(IAgentPluginManager, pluginManager); const diffComputeService = disposables.add(new NodeWorkerDiffComputeService(logService)); diff --git a/src/vs/platform/agentHost/node/agentHostServerMain.ts b/src/vs/platform/agentHost/node/agentHostServerMain.ts index aa3ceba14d706..5f8f2c8814fe6 100644 --- a/src/vs/platform/agentHost/node/agentHostServerMain.ts +++ b/src/vs/platform/agentHost/node/agentHostServerMain.ts @@ -60,6 +60,8 @@ import { AgentPluginManager } from './agentPluginManager.js'; import { IAgentPluginManager } from '../common/agentPluginManager.js'; import { registerPendingEditContentProvider } from './copilot/pendingEditContentStore.js'; import { AgentHostGitService, IAgentHostGitService } from './agentHostGitService.js'; +import { createAgentHostTelemetryService } from './agentHostTelemetryService.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; /** Log to stderr so messages appear in the terminal alongside the process. */ function log(msg: string): void { @@ -186,17 +188,19 @@ async function main(): Promise { // `createInstance` (it needs IFileService + INativeEnvironmentService). // The git service is shared by AgentService (for diff computation + // showBlob) and the production agent registration path. + const telemetryService = await createAgentHostTelemetryService({ environmentService, productService, fileService, loggerService, logService, disposables, disableTelemetry: options.quiet }); const diServices = new ServiceCollection(); diServices.set(IProductService, productService); diServices.set(INativeEnvironmentService, environmentService); diServices.set(ILogService, logService); diServices.set(IFileService, fileService); diServices.set(ISessionDataService, sessionDataService); + diServices.set(ITelemetryService, telemetryService); const instantiationService = new InstantiationService(diServices); const gitService = instantiationService.createInstance(AgentHostGitService); // Create the agent service (owns AgentHostStateManager + AgentSideEffects internally) - const agentService = new AgentService(logService, fileService, sessionDataService, productService, gitService, rootConfigResource); + const agentService = new AgentService(logService, fileService, sessionDataService, productService, gitService, rootConfigResource, telemetryService); disposables.add(agentService); // Register agents diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryService.ts b/src/vs/platform/agentHost/node/agentHostTelemetryService.ts new file mode 100644 index 0000000000000..924633c31d8d1 --- /dev/null +++ b/src/vs/platform/agentHost/node/agentHostTelemetryService.ts @@ -0,0 +1,64 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { hostname, release } from 'os'; +import { toDisposable, type DisposableStore } from '../../../base/common/lifecycle.js'; +import { joinPath } from '../../../base/common/resources.js'; +import { getDevDeviceId, getMachineId, getSqmMachineId } from '../../../base/node/id.js'; +import { ConfigurationService } from '../../configuration/common/configurationService.js'; +import { INativeEnvironmentService } from '../../environment/common/environment.js'; +import { IFileService } from '../../files/common/files.js'; +import { ILogService, ILoggerService } from '../../log/common/log.js'; +import { NullPolicyService } from '../../policy/common/policy.js'; +import { IProductService } from '../../product/common/productService.js'; +import { OneDataSystemAppender } from '../../telemetry/node/1dsAppender.js'; +import { resolveCommonProperties } from '../../telemetry/common/commonProperties.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; +import { TelemetryLogAppender } from '../../telemetry/common/telemetryLogAppender.js'; +import { TelemetryService } from '../../telemetry/common/telemetryService.js'; +import { getPiiPathsFromEnvironment, isInternalTelemetry, isLoggingOnly, NullTelemetryService, supportsTelemetry, type ITelemetryAppender } from '../../telemetry/common/telemetryUtils.js'; + +export interface IAgentHostTelemetryServiceOptions { + readonly environmentService: INativeEnvironmentService; + readonly productService: IProductService; + readonly fileService: IFileService; + readonly loggerService: ILoggerService | undefined; + readonly logService: ILogService; + readonly disposables: DisposableStore; + readonly disableTelemetry?: boolean; +} + +export async function createAgentHostTelemetryService(options: IAgentHostTelemetryServiceOptions): Promise { + const { environmentService, productService, fileService, loggerService, logService, disposables } = options; + if (options.disableTelemetry || !loggerService || !supportsTelemetry(productService, environmentService)) { + return NullTelemetryService; + } + + const configurationService = disposables.add(new ConfigurationService(joinPath(environmentService.appSettingsHome, 'settings.json'), fileService, new NullPolicyService(), logService)); + await configurationService.initialize(); + + const appenders: ITelemetryAppender[] = [ + disposables.add(new TelemetryLogAppender('', false, loggerService, environmentService, productService)), + ]; + const internalTelemetry = isInternalTelemetry(productService, configurationService); + if (!isLoggingOnly(productService, environmentService) && productService.aiConfig?.ariaKey) { + const collectorAppender = new OneDataSystemAppender(undefined, internalTelemetry, 'monacoworkbench', null, productService.aiConfig.ariaKey); + disposables.add(toDisposable(() => { void collectorAppender.flush(); })); + appenders.push(collectorAppender); + } + + const [machineId, sqmId, devDeviceId] = await Promise.all([ + getMachineId(error => logService.error(error)), + getSqmMachineId(error => logService.error(error)), + getDevDeviceId(error => logService.error(error)), + ]); + + return disposables.add(new TelemetryService({ + appenders, + sendErrorTelemetry: true, + commonProperties: resolveCommonProperties(release(), hostname(), process.arch, productService.commit, productService.version, machineId, sqmId, devDeviceId, internalTelemetry, productService.date), + piiPaths: getPiiPathsFromEnvironment(environmentService), + }, configurationService, productService)); +} diff --git a/src/vs/platform/agentHost/node/agentService.ts b/src/vs/platform/agentHost/node/agentService.ts index b7dc4d4abd02f..c824b2af4588d 100644 --- a/src/vs/platform/agentHost/node/agentService.ts +++ b/src/vs/platform/agentHost/node/agentService.ts @@ -40,6 +40,8 @@ import { AgentHostFileCompletionProvider } from './agentHostFileCompletionProvid import { AgentHostSkillCompletionProvider } from './agentHostSkillCompletionProvider.js'; import { AgentHostWorkspaceFiles } from './agentHostWorkspaceFiles.js'; import { toAgentClientUri } from '../common/agentClientUri.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; +import { NullTelemetryService } from '../../telemetry/common/telemetryUtils.js'; /** * Grace period before an empty, unsubscribed session is garbage-collected @@ -131,6 +133,7 @@ export class AgentService extends Disposable implements IAgentService { private readonly _productService: IProductService, private readonly _gitService: IAgentHostGitService, private readonly _rootConfigResource?: URI, + private readonly _telemetryService: ITelemetryService = NullTelemetryService, ) { super(); this._logService.info('AgentService initialized'); @@ -148,6 +151,7 @@ export class AgentService extends Disposable implements IAgentService { [IProductService, this._productService], [IAgentConfigurationService, configurationService], [IAgentHostGitService, this._gitService], + [ITelemetryService, this._telemetryService], ); const instantiationService = this._register(new InstantiationService(services, /*strict*/ true)); diff --git a/src/vs/platform/agentHost/node/agentSideEffects.ts b/src/vs/platform/agentHost/node/agentSideEffects.ts index e5b35815c8677..607840d69be2a 100644 --- a/src/vs/platform/agentHost/node/agentSideEffects.ts +++ b/src/vs/platform/agentHost/node/agentSideEffects.ts @@ -15,7 +15,7 @@ import { ILogService } from '../../log/common/log.js'; import { AgentSignal, IAgent, IAgentToolPendingConfirmationSignal } from '../common/agentService.js'; import { IDiffComputeService } from '../common/diffComputeService.js'; import { ISessionDatabase, ISessionDataService } from '../common/sessionDataService.js'; -import type { AgentInfo } from '../common/state/protocol/state.js'; +import type { AgentInfo, MessageAttachment } from '../common/state/protocol/state.js'; import { ActionType, isSessionAction, StateAction, type SessionToolCallCompleteAction } from '../common/state/sessionActions.js'; import { buildSubagentSessionUri, @@ -35,6 +35,25 @@ import { AgentHostStateManager } from './agentHostStateManager.js'; import { NodeWorkerDiffComputeService } from './diffComputeService.js'; import { computeSessionDiffs, type IIncrementalDiffOptions } from './sessionDiffAggregator.js'; import { SessionPermissionManager } from './sessionPermissions.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; + +type AgentHostUserMessageSentSource = 'direct' | 'queued'; + +type AgentHostUserMessageSentEvent = { + provider: string; + source: AgentHostUserMessageSentSource; + hasAttachments: boolean; + attachmentCount: number; +}; + +type AgentHostUserMessageSentClassification = { + provider: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The provider handling the agent host session.' }; + source: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user message was sent directly or from the queued-message flow.' }; + hasAttachments: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the user message included attachments.' }; + attachmentCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of attachments included with the user message.' }; + owner: 'roblourens'; + comment: 'Tracks user messages sent from the agent host process to an agent provider.'; +}; /** * Options for constructing an {@link AgentSideEffects} instance. @@ -110,6 +129,7 @@ export class AgentSideEffects extends Disposable { @IInstantiationService instantiationService: IInstantiationService, @ILogService private readonly _logService: ILogService, @IAgentHostGitService private readonly _gitService: IAgentHostGitService, + @ITelemetryService private readonly _telemetryService: ITelemetryService, ) { super(); this._diffComputeService = this._register(new NodeWorkerDiffComputeService(this._logService)); @@ -712,6 +732,7 @@ export class AgentSideEffects extends Disposable { return; } const attachments = action.userMessage.attachments; + this._logUserMessageSent(agent.id, 'direct', attachments); agent.sendMessage(URI.parse(action.session), action.userMessage.text, attachments, action.turnId).catch(err => { const errCode = (err as { code?: number })?.code; this._logService.error(`[AgentSideEffects] sendMessage failed for session=${action.session}: code=${errCode}, message=${err instanceof Error ? err.message : String(err)}, type=${err?.constructor?.name}`, err); @@ -943,6 +964,7 @@ export class AgentSideEffects extends Disposable { return; } const attachments = msg.userMessage.attachments; + this._logUserMessageSent(agent.id, 'queued', attachments); agent.sendMessage(URI.parse(session), msg.userMessage.text, attachments, turnId).catch(err => { this._logService.error('[AgentSideEffects] sendMessage failed (queued)', err); this._stateManager.dispatchServerAction({ @@ -954,6 +976,16 @@ export class AgentSideEffects extends Disposable { }); } + private _logUserMessageSent(provider: string, source: AgentHostUserMessageSentSource, attachments: readonly MessageAttachment[] | undefined): void { + const attachmentCount = attachments?.length ?? 0; + this._telemetryService.publicLog2('agentHostUserMessageSent', { + provider, + source, + hasAttachments: attachmentCount > 0, + attachmentCount, + }); + } + // ---- Session diff computation ---------------------------------------------- /** diff --git a/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts b/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts index 11423965c64a0..b1213710ed2d9 100644 --- a/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts +++ b/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts @@ -24,6 +24,8 @@ import { CustomizationStatus } from '../../common/state/protocol/state.js'; import { ActionType, ActionEnvelope, SessionAction } from '../../common/state/sessionActions.js'; import { buildSubagentSessionUri, MessageAttachmentKind, PendingMessageKind, ResponsePartKind, SessionStatus, ToolCallConfirmationReason, ToolCallStatus, ToolResultContentType } from '../../common/state/sessionState.js'; import { IProductService } from '../../../product/common/productService.js'; +import { ITelemetryService, TelemetryLevel } from '../../../telemetry/common/telemetry.js'; +import { NullTelemetryService } from '../../../telemetry/common/telemetryUtils.js'; import { AgentConfigurationService, IAgentConfigurationService } from '../../node/agentConfigurationService.js'; import { IAgentHostGitService } from '../../node/agentHostGitService.js'; import { AgentService } from '../../node/agentService.js'; @@ -40,17 +42,39 @@ import { MockAgent } from './mockAgent.js'; * scope that satisfies its {@link IAgentConfigurationService} / * {@link ILogService} / {@link IAgentHostGitService} dependencies. */ -function createTestSideEffects(disposables: DisposableStore, stateManager: AgentHostStateManager, options: IAgentSideEffectsOptions, gitService?: IAgentHostGitService): AgentSideEffects { +function createTestSideEffects(disposables: DisposableStore, stateManager: AgentHostStateManager, options: IAgentSideEffectsOptions, gitService?: IAgentHostGitService, telemetryService: ITelemetryService = NullTelemetryService): AgentSideEffects { const logService = new NullLogService(); const configService = disposables.add(new AgentConfigurationService(stateManager, logService)); const instantiationService = disposables.add(new InstantiationService(new ServiceCollection( [ILogService, logService], [IAgentConfigurationService, configService], [IAgentHostGitService, gitService ?? createNoopGitService()], + [ITelemetryService, telemetryService], ), /*strict*/ true)); return disposables.add(instantiationService.createInstance(AgentSideEffects, stateManager, options)); } +class TestTelemetryService implements ITelemetryService { + declare readonly _serviceBrand: undefined; + readonly telemetryLevel = TelemetryLevel.USAGE; + readonly sessionId = 'test-session'; + readonly machineId = 'test-machine'; + readonly sqmId = 'test-sqm'; + readonly devDeviceId = 'test-dev-device'; + readonly firstSessionDate = 'test-first-session-date'; + readonly sendErrorTelemetry = false; + readonly events: { eventName: string; data: unknown }[] = []; + + publicLog(): void { } + publicLog2(eventName: string, data?: unknown): void { + this.events.push({ eventName, data }); + } + publicLogError(): void { } + publicLogError2(): void { } + setExperimentProperty(): void { } + setCommonProperty(): void { } +} + suite('AgentSideEffects', () => { const disposables = new DisposableStore(); @@ -59,6 +83,7 @@ suite('AgentSideEffects', () => { let agent: MockAgent; let sideEffects: AgentSideEffects; let agentList: ReturnType>; + let telemetryService: TestTelemetryService; const sessionUri = AgentSession.uri('mock', 'session-1'); @@ -97,12 +122,13 @@ suite('AgentSideEffects', () => { disposables.add(toDisposable(() => agent.dispose())); stateManager = disposables.add(new AgentHostStateManager(new NullLogService())); agentList = observableValue('agents', [agent]); + telemetryService = new TestTelemetryService(); sideEffects = createTestSideEffects(disposables, stateManager, { getAgent: () => agent, agents: agentList, sessionDataService: createNullSessionDataService(), onTurnComplete: () => { }, - }); + }, undefined, telemetryService); }); teardown(() => { @@ -130,6 +156,27 @@ suite('AgentSideEffects', () => { assert.deepStrictEqual(agent.sendMessageCalls, [{ session: URI.parse(sessionUri.toString()), prompt: 'hello world', attachments: undefined }]); }); + test('logs telemetry when sending a direct user message', () => { + setupSession(); + const fileUri = URI.file('/workspace/direct.ts'); + sideEffects.handleAction({ + type: ActionType.SessionTurnStarted, + session: sessionUri.toString(), + turnId: 'turn-1', + userMessage: { text: 'hello world', attachments: [{ type: MessageAttachmentKind.Resource, uri: fileUri.toString(), label: 'direct.ts', displayKind: 'document' }] }, + }); + + assert.deepStrictEqual(telemetryService.events, [{ + eventName: 'agentHostUserMessageSent', + data: { + provider: 'mock', + source: 'direct', + hasAttachments: true, + attachmentCount: 1, + }, + }]); + }); + test('parses protocol attachment URI strings before passing them to the agent', () => { setupSession(); const fileUri = URI.file('/workspace/test.ts'); @@ -577,6 +624,30 @@ suite('AgentSideEffects', () => { }]); }); + test('logs telemetry when sending a queued user message', () => { + setupSession(); + + const action = { + type: ActionType.SessionPendingMessageSet as const, + session: sessionUri.toString(), + kind: PendingMessageKind.Queued, + id: 'q-telemetry', + userMessage: { text: 'queued message' }, + }; + stateManager.dispatchClientAction(action, { clientId: 'test', clientSeq: 1 }); + sideEffects.handleAction(action); + + assert.deepStrictEqual(telemetryService.events, [{ + eventName: 'agentHostUserMessageSent', + data: { + provider: 'mock', + source: 'queued', + hasAttachments: false, + attachmentCount: 0, + }, + }]); + }); + test('syncs on SessionPendingMessageRemoved', () => { setupSession(); From c4b9277690072db9700ae55e9380132536300b47 Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Sat, 16 May 2026 15:52:57 -0700 Subject: [PATCH 2/6] Propagate agent host telemetry level Use root config to propagate the client telemetry level into the Agent Host process and clamp Agent Host telemetry to the most restrictive level. Also forwards parent process telemetry disablement into spawned Agent Host processes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../browser/remoteAgentHostProtocolClient.ts | 22 ++++ .../agentHost/common/agentHostSchema.ts | 38 ++++++ .../electron-browser/localAgentHostService.ts | 25 +++- .../electron-main/electronAgentHostStarter.ts | 13 +- .../agentHost/node/agentHostStateManager.ts | 4 +- .../node/agentHostTelemetryService.ts | 118 +++++++++++++++++- .../platform/agentHost/node/agentService.ts | 2 + .../agentHost/node/agentSideEffects.ts | 2 + .../agentHost/node/nodeAgentHostStarter.ts | 15 ++- .../remoteAgentHostProtocolClient.test.ts | 40 ++++-- .../node/agentHostTelemetryService.test.ts | 91 ++++++++++++++ .../test/node/agentSideEffects.test.ts | 22 +++- 12 files changed, 363 insertions(+), 29 deletions(-) create mode 100644 src/vs/platform/agentHost/test/node/agentHostTelemetryService.test.ts diff --git a/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts b/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts index bf769f49702ff..3990228ff7cea 100644 --- a/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts +++ b/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts @@ -17,6 +17,7 @@ import { URI } from '../../../base/common/uri.js'; import { generateUuid } from '../../../base/common/uuid.js'; import { ILogService } from '../../log/common/log.js'; import { FileSystemProviderErrorCode, IFileService, toFileSystemProviderErrorCode } from '../../files/common/files.js'; +import { IConfigurationService } from '../../configuration/common/configuration.js'; import { AgentSession, IAgentConnection, IAgentCreateSessionConfig, IAgentResolveSessionConfigParams, IAgentSessionConfigCompletionsParams, IAgentSessionMetadata, AuthenticateParams, AuthenticateResult } from '../common/agentService.js'; import { AgentSubscriptionManager, type IAgentSubscription } from '../common/state/agentSubscription.js'; import { agentHostAuthority, fromAgentHostUri, toAgentHostUri } from '../common/agentHostUri.js'; @@ -32,6 +33,9 @@ import { AhpErrorCodes } from '../common/state/protocol/errors.js'; import { ContentEncoding, ResourceRequestParams, type CompletionsParams, type CompletionsResult, type CreateTerminalParams, type ResolveSessionConfigResult, type SessionConfigCompletionsResult } from '../common/state/protocol/commands.js'; import { decodeBase64, encodeBase64, VSBuffer } from '../../../base/common/buffer.js'; import { ILoadEstimator, LoadEstimator } from '../../../base/parts/ipc/common/ipc.net.js'; +import { TELEMETRY_CRASH_REPORTER_SETTING_ID, TELEMETRY_OLD_SETTING_ID, TELEMETRY_SETTING_ID } from '../../telemetry/common/telemetry.js'; +import { getTelemetryLevel } from '../../telemetry/common/telemetryUtils.js'; +import { AgentHostTelemetryLevelConfigKey, telemetryLevelToAgentHostConfigValue } from '../common/agentHostSchema.js'; const AHP_CLIENT_CONNECTION_CLOSED = -32000; @@ -242,6 +246,7 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC @ILogService private readonly _logService: ILogService, @IFileService private readonly _fileService: IFileService, @IAgentHostPermissionService private readonly _permissionService: IAgentHostPermissionService, + @IConfigurationService private readonly _configurationService: IConfigurationService, ) { super(); this._address = address; @@ -269,6 +274,15 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC this._subscriptionManager.receiveEnvelope(envelope); })); + this._register(this._configurationService.onDidChangeConfiguration(e => { + if (e.affectsConfiguration(TELEMETRY_SETTING_ID) || e.affectsConfiguration(TELEMETRY_OLD_SETTING_ID) || e.affectsConfiguration(TELEMETRY_CRASH_REPORTER_SETTING_ID)) { + if (this._state.kind !== AgentHostClientState.Connected) { + return; + } + this._updateTelemetryLevel(); + } + })); + // Detect silently-dead transports — see {@link _watchdogTick}. this._watchdog.cancelAndSet(() => this._watchdogTick(), WATCHDOG_CHECK_INTERVAL_MS); } @@ -351,6 +365,7 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC } this._completionTriggerCharacters = result.completionTriggerCharacters ?? []; + this._updateTelemetryLevel(); this._transitionTo({ kind: AgentHostClientState.Connected }); } @@ -1111,6 +1126,13 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC return this._dispatchRequest(method, params); } + private _updateTelemetryLevel(): void { + this.dispatchAction({ + type: ActionType.RootConfigChanged, + config: { [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(getTelemetryLevel(this._configurationService)) }, + }, this._clientId, 0); + } + /** * Common path for outgoing JSON-RPC requests: gate on any in-flight * reconnect (unless explicitly bypassed for the `reconnect` RPC itself), diff --git a/src/vs/platform/agentHost/common/agentHostSchema.ts b/src/vs/platform/agentHost/common/agentHostSchema.ts index 1136c5fd147c6..9b36c902f4eb0 100644 --- a/src/vs/platform/agentHost/common/agentHostSchema.ts +++ b/src/vs/platform/agentHost/common/agentHostSchema.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { localize } from '../../../nls.js'; +import { TelemetryConfiguration, TelemetryLevel } from '../../telemetry/common/telemetry.js'; import { SessionConfigKey } from './sessionConfigKeys.js'; import type { SessionConfigPropertySchema, SessionConfigSchema } from './state/protocol/commands.js'; import { JsonRpcErrorCodes, ProtocolError } from './state/sessionProtocol.js'; @@ -349,6 +350,43 @@ export const platformSessionSchema = createSchema({ * auto-approval. See `SessionPermissionManager` for the evaluation * rules. */ +export const AgentHostTelemetryLevelConfigKey = 'telemetryLevel'; + +export function telemetryLevelToAgentHostConfigValue(telemetryLevel: TelemetryLevel): TelemetryConfiguration { + switch (telemetryLevel) { + case TelemetryLevel.NONE: + return TelemetryConfiguration.OFF; + case TelemetryLevel.CRASH: + return TelemetryConfiguration.CRASH; + case TelemetryLevel.ERROR: + return TelemetryConfiguration.ERROR; + case TelemetryLevel.USAGE: + return TelemetryConfiguration.ON; + } +} + +export function agentHostConfigValueToTelemetryLevel(value: unknown): TelemetryLevel | undefined { + switch (value) { + case TelemetryConfiguration.OFF: + return TelemetryLevel.NONE; + case TelemetryConfiguration.CRASH: + return TelemetryLevel.CRASH; + case TelemetryConfiguration.ERROR: + return TelemetryLevel.ERROR; + case TelemetryConfiguration.ON: + return TelemetryLevel.USAGE; + default: + return undefined; + } +} + export const platformRootSchema = createSchema({ [SessionConfigKey.Permissions]: permissionsProperty, + [AgentHostTelemetryLevelConfigKey]: schemaProperty({ + type: 'string', + title: localize('agentHost.config.telemetryLevel.title', "Telemetry Level"), + description: localize('agentHost.config.telemetryLevel.description', "Most restrictive telemetry level requested by connected clients."), + enum: [TelemetryConfiguration.ON, TelemetryConfiguration.ERROR, TelemetryConfiguration.CRASH, TelemetryConfiguration.OFF], + default: TelemetryConfiguration.ON, + }), }); diff --git a/src/vs/platform/agentHost/electron-browser/localAgentHostService.ts b/src/vs/platform/agentHost/electron-browser/localAgentHostService.ts index 9f86d14de1508..347625c486262 100644 --- a/src/vs/platform/agentHost/electron-browser/localAgentHostService.ts +++ b/src/vs/platform/agentHost/electron-browser/localAgentHostService.ts @@ -20,13 +20,16 @@ import { AhpJsonlLogger, getAhpLogByteLength } from '../common/ahpJsonlLogger.js import { wrapAgentServiceWithAhpLogging } from './localAhpJsonlLogging.js'; import { AgentSubscriptionManager, type IAgentSubscription } from '../common/state/agentSubscription.js'; import type { CompletionsParams, CompletionsResult, CreateTerminalParams, ResolveSessionConfigResult, SessionConfigCompletionsResult } from '../common/state/protocol/commands.js'; -import type { ActionEnvelope, INotification, IRootConfigChangedAction, SessionAction, TerminalAction } from '../common/state/sessionActions.js'; +import { ActionType, type ActionEnvelope, type INotification, type IRootConfigChangedAction, type SessionAction, type TerminalAction } from '../common/state/sessionActions.js'; import type { ResourceCopyParams, ResourceCopyResult, ResourceDeleteParams, ResourceDeleteResult, ResourceListResult, ResourceMoveParams, ResourceMoveResult, ResourceReadResult, ResourceWriteParams, ResourceWriteResult, IStateSnapshot } from '../common/state/sessionProtocol.js'; import { StateComponents, ROOT_STATE_URI, type RootState } from '../common/state/sessionState.js'; import { revive } from '../../../base/common/marshalling.js'; import { URI } from '../../../base/common/uri.js'; import { IFileService } from '../../files/common/files.js'; import { AGENT_HOST_CLIENT_RESOURCE_CHANNEL, AgentHostClientResourceChannel } from '../common/agentHostClientResourceChannel.js'; +import { TELEMETRY_CRASH_REPORTER_SETTING_ID, TELEMETRY_OLD_SETTING_ID, TELEMETRY_SETTING_ID } from '../../telemetry/common/telemetry.js'; +import { getTelemetryLevel } from '../../telemetry/common/telemetryUtils.js'; +import { AgentHostTelemetryLevelConfigKey, telemetryLevelToAgentHostConfigValue } from '../common/agentHostSchema.js'; /** * Renderer-side implementation of {@link IAgentHostService} that connects @@ -76,7 +79,7 @@ export class LocalAgentHostServiceClient extends Disposable implements IAgentHos constructor( @ILogService private readonly _logService: ILogService, - @IConfigurationService configurationService: IConfigurationService, + @IConfigurationService private readonly _configurationService: IConfigurationService, @IFileService private readonly _fileService: IFileService, @IEnvironmentService environmentService: IEnvironmentService, @IInstantiationService instantiationService: IInstantiationService, @@ -92,7 +95,7 @@ export class LocalAgentHostServiceClient extends Disposable implements IAgentHos // Optionally wrap the proxy with a logging layer that synthesizes JSON-RPC // frames for every request/response/notification on the in-process MessagePort // channel, mirroring the AHP transport JSONL logs produced by remote agent hosts. - this._ahpLogger = configurationService.getValue(AgentHostAhpJsonlLoggingSettingId) + this._ahpLogger = this._configurationService.getValue(AgentHostAhpJsonlLoggingSettingId) ? this._register(instantiationService.createInstance(AhpJsonlLogger, { logsHome: environmentService.logsHome, connectionId: this.clientId, @@ -113,7 +116,13 @@ export class LocalAgentHostServiceClient extends Disposable implements IAgentHos resource => this.unsubscribe(resource), )); - if (configurationService.getValue(AgentHostEnabledSettingId)) { + this._register(this._configurationService.onDidChangeConfiguration(e => { + if (e.affectsConfiguration(TELEMETRY_SETTING_ID) || e.affectsConfiguration(TELEMETRY_OLD_SETTING_ID) || e.affectsConfiguration(TELEMETRY_CRASH_REPORTER_SETTING_ID)) { + this._updateTelemetryLevel(); + } + })); + + if (this._configurationService.getValue(AgentHostEnabledSettingId)) { this._connect(); } } @@ -133,6 +142,7 @@ export class LocalAgentHostServiceClient extends Disposable implements IAgentHos // AgentHostClientFileSystemProvider that calls back through this channel. client.registerChannel(AGENT_HOST_CLIENT_RESOURCE_CHANNEL, new AgentHostClientResourceChannel(this._fileService, this._ahpLogger)); this._clientEventually.complete(client); + this._updateTelemetryLevel(); store.add(this._proxy.onDidAction(e => { const revived = revive(e) as ActionEnvelope; @@ -161,6 +171,13 @@ export class LocalAgentHostServiceClient extends Disposable implements IAgentHos }); } + private _updateTelemetryLevel(): void { + this.dispatchAction({ + type: ActionType.RootConfigChanged, + config: { [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(getTelemetryLevel(this._configurationService)) }, + }, this.clientId, 0); + } + // ---- IAgentService forwarding (no await needed, delayed channel handles queuing) ---- authenticate(params: AuthenticateParams): Promise { diff --git a/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts b/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts index de12b6329ca37..db790842723e6 100644 --- a/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts +++ b/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts @@ -85,15 +85,20 @@ export class ElectronAgentHostStarter extends Disposable implements IAgentHostSt dbSpanExporterEnabled: this._configurationService.getValue(AgentHostOTelDbSpanExporterEnabledSettingId), }, process.env); + const args = [ + '--logsPath', this._environmentMainService.logsHome.with({ scheme: Schemas.file }).fsPath, + '--user-data-dir', this._environmentMainService.userDataPath, + ]; + if (this._environmentMainService.disableTelemetry) { + args.push('--disable-telemetry'); + } + this.utilityProcess.start({ type: 'agentHost', name: 'agent-host', entryPoint: 'vs/platform/agentHost/node/agentHostMain', execArgv, - args: [ - '--logsPath', this._environmentMainService.logsHome.with({ scheme: Schemas.file }).fsPath, - '--user-data-dir', this._environmentMainService.userDataPath, - ], + args, env: { ...deepClone(process.env), ...shellEnv, diff --git a/src/vs/platform/agentHost/node/agentHostStateManager.ts b/src/vs/platform/agentHost/node/agentHostStateManager.ts index a6705f0f56959..3dc7e1f58bdbe 100644 --- a/src/vs/platform/agentHost/node/agentHostStateManager.ts +++ b/src/vs/platform/agentHost/node/agentHostStateManager.ts @@ -8,11 +8,12 @@ import { Emitter, Event } from '../../../base/common/event.js'; import { Disposable } from '../../../base/common/lifecycle.js'; import { equals } from '../../../base/common/objects.js'; import { ILogService } from '../../log/common/log.js'; +import { TelemetryLevel } from '../../telemetry/common/telemetry.js'; import { ActionType, NotificationType, ActionEnvelope, ActionOrigin, INotification, IRootConfigChangedAction, SessionAction, RootAction, StateAction, TerminalAction, isRootAction, isSessionAction } from '../common/state/sessionActions.js'; import type { IStateSnapshot } from '../common/state/sessionProtocol.js'; import { rootReducer, sessionReducer } from '../common/state/sessionReducers.js'; import { createRootState, createSessionState, SessionLifecycle, type RootState, type SessionMeta, type SessionState, type SessionSummary, type Turn, type URI, ROOT_STATE_URI } from '../common/state/sessionState.js'; -import { IPermissionsValue, platformRootSchema } from '../common/agentHostSchema.js'; +import { AgentHostTelemetryLevelConfigKey, IPermissionsValue, platformRootSchema, telemetryLevelToAgentHostConfigValue } from '../common/agentHostSchema.js'; import { SessionConfigKey } from '../common/sessionConfigKeys.js'; /** @@ -67,6 +68,7 @@ export class AgentHostStateManager extends Disposable { schema: platformRootSchema.toProtocol(), values: platformRootSchema.validateOrDefault({}, { [SessionConfigKey.Permissions]: { allow: [], deny: [] } satisfies IPermissionsValue, + [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(TelemetryLevel.USAGE), }), }, }; diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryService.ts b/src/vs/platform/agentHost/node/agentHostTelemetryService.ts index 924633c31d8d1..295d0d14f9648 100644 --- a/src/vs/platform/agentHost/node/agentHostTelemetryService.ts +++ b/src/vs/platform/agentHost/node/agentHostTelemetryService.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { hostname, release } from 'os'; -import { toDisposable, type DisposableStore } from '../../../base/common/lifecycle.js'; +import { Disposable, isDisposable, toDisposable, type DisposableStore } from '../../../base/common/lifecycle.js'; import { joinPath } from '../../../base/common/resources.js'; import { getDevDeviceId, getMachineId, getSqmMachineId } from '../../../base/node/id.js'; import { ConfigurationService } from '../../configuration/common/configurationService.js'; @@ -15,10 +15,12 @@ import { NullPolicyService } from '../../policy/common/policy.js'; import { IProductService } from '../../product/common/productService.js'; import { OneDataSystemAppender } from '../../telemetry/node/1dsAppender.js'; import { resolveCommonProperties } from '../../telemetry/common/commonProperties.js'; -import { ITelemetryService } from '../../telemetry/common/telemetry.js'; +import { ClassifiedEvent, IGDPRProperty, OmitMetadata, StrictPropertyCheck } from '../../telemetry/common/gdprTypings.js'; +import { ITelemetryData, ITelemetryService, TelemetryLevel } from '../../telemetry/common/telemetry.js'; import { TelemetryLogAppender } from '../../telemetry/common/telemetryLogAppender.js'; import { TelemetryService } from '../../telemetry/common/telemetryService.js'; import { getPiiPathsFromEnvironment, isInternalTelemetry, isLoggingOnly, NullTelemetryService, supportsTelemetry, type ITelemetryAppender } from '../../telemetry/common/telemetryUtils.js'; +import { AgentHostTelemetryLevelConfigKey, agentHostConfigValueToTelemetryLevel } from '../common/agentHostSchema.js'; export interface IAgentHostTelemetryServiceOptions { readonly environmentService: INativeEnvironmentService; @@ -30,10 +32,112 @@ export interface IAgentHostTelemetryServiceOptions { readonly disableTelemetry?: boolean; } -export async function createAgentHostTelemetryService(options: IAgentHostTelemetryServiceOptions): Promise { +export interface IAgentHostTelemetryService extends ITelemetryService { + updateTelemetryLevel(telemetryLevel: TelemetryLevel): void; +} + +export class AgentHostTelemetryService extends Disposable implements IAgentHostTelemetryService { + declare readonly _serviceBrand: undefined; + + private _telemetryLevel = TelemetryLevel.USAGE; + + constructor(private readonly _delegate: ITelemetryService) { + super(); + if (isDisposable(_delegate)) { + this._register(_delegate); + } + } + + get telemetryLevel(): TelemetryLevel { + return Math.min(this._delegate.telemetryLevel, this._telemetryLevel); + } + + get sendErrorTelemetry(): boolean { + return this.telemetryLevel >= TelemetryLevel.ERROR && this._delegate.sendErrorTelemetry; + } + + get sessionId(): string { + return this._delegate.sessionId; + } + + get machineId(): string { + return this._delegate.machineId; + } + + get sqmId(): string { + return this._delegate.sqmId; + } + + get devDeviceId(): string { + return this._delegate.devDeviceId; + } + + get firstSessionDate(): string { + return this._delegate.firstSessionDate; + } + + get msftInternal(): boolean | undefined { + return this._delegate.msftInternal; + } + + publicLog(eventName: string, data?: ITelemetryData): void { + if (this.telemetryLevel < TelemetryLevel.USAGE) { + return; + } + this._delegate.publicLog(eventName, data); + } + + publicLogError(eventName: string, data?: ITelemetryData): void { + if (this.telemetryLevel < TelemetryLevel.ERROR) { + return; + } + this._delegate.publicLogError(eventName, data); + } + + publicLog2> = never, T extends IGDPRProperty = never>(eventName: string, data?: StrictPropertyCheck): void { + if (this.telemetryLevel < TelemetryLevel.USAGE) { + return; + } + this._delegate.publicLog2(eventName, data); + } + + publicLogError2> = never, T extends IGDPRProperty = never>(eventName: string, data?: StrictPropertyCheck): void { + if (this.telemetryLevel < TelemetryLevel.ERROR) { + return; + } + this._delegate.publicLogError2(eventName, data); + } + + setExperimentProperty(name: string, value: string): void { + this._delegate.setExperimentProperty(name, value); + } + + setCommonProperty(name: string, value: string): void { + this._delegate.setCommonProperty(name, value); + } + + updateTelemetryLevel(telemetryLevel: TelemetryLevel): void { + this._telemetryLevel = Math.min(this._telemetryLevel, telemetryLevel); + } +} + +export function updateAgentHostTelemetryLevelFromConfig(telemetryService: ITelemetryService, config: Record | undefined): void { + const telemetryLevel = config?.[AgentHostTelemetryLevelConfigKey]; + const telemetryLevelValue = agentHostConfigValueToTelemetryLevel(telemetryLevel); + if (!isAgentHostTelemetryService(telemetryService) || telemetryLevelValue === undefined) { + return; + } + telemetryService.updateTelemetryLevel(telemetryLevelValue); +} + +function isAgentHostTelemetryService(telemetryService: ITelemetryService): telemetryService is IAgentHostTelemetryService { + return typeof (telemetryService as IAgentHostTelemetryService).updateTelemetryLevel === 'function'; +} + +export async function createAgentHostTelemetryService(options: IAgentHostTelemetryServiceOptions): Promise { const { environmentService, productService, fileService, loggerService, logService, disposables } = options; if (options.disableTelemetry || !loggerService || !supportsTelemetry(productService, environmentService)) { - return NullTelemetryService; + return disposables.add(new AgentHostTelemetryService(NullTelemetryService)); } const configurationService = disposables.add(new ConfigurationService(joinPath(environmentService.appSettingsHome, 'settings.json'), fileService, new NullPolicyService(), logService)); @@ -55,10 +159,12 @@ export async function createAgentHostTelemetryService(options: IAgentHostTelemet getDevDeviceId(error => logService.error(error)), ]); - return disposables.add(new TelemetryService({ + const telemetryService = new TelemetryService({ appenders, sendErrorTelemetry: true, commonProperties: resolveCommonProperties(release(), hostname(), process.arch, productService.commit, productService.version, machineId, sqmId, devDeviceId, internalTelemetry, productService.date), piiPaths: getPiiPathsFromEnvironment(environmentService), - }, configurationService, productService)); + }, configurationService, productService); + + return disposables.add(new AgentHostTelemetryService(telemetryService)); } diff --git a/src/vs/platform/agentHost/node/agentService.ts b/src/vs/platform/agentHost/node/agentService.ts index c824b2af4588d..53c463e653191 100644 --- a/src/vs/platform/agentHost/node/agentService.ts +++ b/src/vs/platform/agentHost/node/agentService.ts @@ -42,6 +42,7 @@ import { AgentHostWorkspaceFiles } from './agentHostWorkspaceFiles.js'; import { toAgentClientUri } from '../common/agentClientUri.js'; import { ITelemetryService } from '../../telemetry/common/telemetry.js'; import { NullTelemetryService } from '../../telemetry/common/telemetryUtils.js'; +import { updateAgentHostTelemetryLevelFromConfig } from './agentHostTelemetryService.js'; /** * Grace period before an empty, unsubscribed session is garbage-collected @@ -146,6 +147,7 @@ export class AgentService extends Disposable implements IAgentService { // via DI rather than being plumbed plain-class references. const configurationService: IAgentConfigurationService = this._register(new AgentConfigurationService(this._stateManager, this._logService, this._rootConfigResource)); this._configurationService = configurationService; + updateAgentHostTelemetryLevelFromConfig(this._telemetryService, this._stateManager.rootState.config?.values); const services = new ServiceCollection( [ILogService, this._logService], [IProductService, this._productService], diff --git a/src/vs/platform/agentHost/node/agentSideEffects.ts b/src/vs/platform/agentHost/node/agentSideEffects.ts index 607840d69be2a..ced6f54862207 100644 --- a/src/vs/platform/agentHost/node/agentSideEffects.ts +++ b/src/vs/platform/agentHost/node/agentSideEffects.ts @@ -36,6 +36,7 @@ import { NodeWorkerDiffComputeService } from './diffComputeService.js'; import { computeSessionDiffs, type IIncrementalDiffOptions } from './sessionDiffAggregator.js'; import { SessionPermissionManager } from './sessionPermissions.js'; import { ITelemetryService } from '../../telemetry/common/telemetry.js'; +import { updateAgentHostTelemetryLevelFromConfig } from './agentHostTelemetryService.js'; type AgentHostUserMessageSentSource = 'direct' | 'queued'; @@ -827,6 +828,7 @@ export class AgentSideEffects extends Disposable { break; } case ActionType.RootConfigChanged: { + updateAgentHostTelemetryLevelFromConfig(this._telemetryService, action.config); // Host customizations are self-managed by each agent's // PluginController via IAgentConfigurationService.onDidRootConfigChange. // Republish agent infos for non-customization schema changes diff --git a/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts b/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts index c29d1b541efe0..c7adda57eb78c 100644 --- a/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts +++ b/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts @@ -114,13 +114,18 @@ export class NodeAgentHostStarter extends Disposable implements IAgentHostStarte } } + const args = [ + '--type=agentHost', + '--logsPath', this._environmentService.logsHome.with({ scheme: Schemas.file }).fsPath, + '--user-data-dir', this._environmentService.userDataPath, + ]; + if (this._environmentService.disableTelemetry) { + args.push('--disable-telemetry'); + } + const opts: IIPCOptions = { serverName: 'Agent Host', - args: [ - '--type=agentHost', - '--logsPath', this._environmentService.logsHome.with({ scheme: Schemas.file }).fsPath, - '--user-data-dir', this._environmentService.userDataPath, - ], + args, env, }; diff --git a/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts b/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts index 8c62aa92a7ed9..2214e5df51a33 100644 --- a/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts +++ b/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts @@ -25,6 +25,9 @@ import { hasKey } from '../../../../base/common/types.js'; import { mainWindow } from '../../../../base/browser/window.js'; import { ROOT_STATE_URI, StateComponents } from '../../common/state/sessionState.js'; import type { IClientTransport, IProtocolTransport } from '../../common/state/sessionTransport.js'; +import { TestConfigurationService } from '../../../configuration/test/common/testConfigurationService.js'; +import { TelemetryLevel } from '../../../telemetry/common/telemetry.js'; +import { AgentHostTelemetryLevelConfigKey, telemetryLevelToAgentHostConfigValue } from '../../common/agentHostSchema.js'; type ProtocolTransportMessage = ProtocolMessage | AhpServerNotification | JsonRpcNotification | JsonRpcResponse | JsonRpcRequest; @@ -88,7 +91,7 @@ suite('RemoteAgentHostProtocolClient', () => { function createClient(transport = disposables.add(new TestProtocolTransport()), permissionService = createPermissionService(), loadEstimator?: { hasHighLoad(): boolean }): { client: RemoteAgentHostProtocolClient; transport: TestProtocolTransport } { const fileService = disposables.add(new FileService(new NullLogService())); - const client = disposables.add(new RemoteAgentHostProtocolClient('test.example:1234', transport, loadEstimator, new NullLogService(), fileService, permissionService)); + const client = disposables.add(new RemoteAgentHostProtocolClient('test.example:1234', transport, loadEstimator, new NullLogService(), fileService, permissionService, new TestConfigurationService())); return { client, transport }; } @@ -381,6 +384,18 @@ suite('RemoteAgentHostProtocolClient', () => { result: { protocolVersion: PROTOCOL_VERSION, serverSeq: 0, snapshots: [] }, }); await connectPromise; + const telemetryLevel = transport.sentMessages[1] as JsonRpcNotification; + assert.deepStrictEqual(telemetryLevel, { + jsonrpc: '2.0', + method: 'dispatchAction', + params: { + clientSeq: 0, + action: { + type: ActionType.RootConfigChanged, + config: { [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(TelemetryLevel.USAGE) }, + }, + }, + }); }); test('rejects connect when host returns UnsupportedProtocolVersion (-32005)', async () => { @@ -719,6 +734,15 @@ suite('RemoteAgentHostProtocolClient', () => { ); } + function findDispatchAction(transport: TestProtocolTransport, actionType: ActionType): JsonRpcNotification | undefined { + return transport.sentMessages.find( + (m): m is JsonRpcNotification => 'method' in m + && (m as JsonRpcNotification).method === 'dispatchAction' + && !('id' in m) + && ((m as JsonRpcNotification).params as { action?: { type?: unknown } } | undefined)?.action?.type === actionType, + ); + } + async function flushMicrotasks(): Promise { // `await Promise.resolve()` only advances one microtask; loop a few times to // drain chained .then handlers without resorting to fake timers. @@ -769,7 +793,7 @@ suite('RemoteAgentHostProtocolClient', () => { }; const fileService = disposables.add(new FileService(new NullLogService())); const client = disposables.add(new RemoteAgentHostProtocolClient( - 'test.example:1234', factory, undefined, new NullLogService(), fileService, createPermissionService(), + 'test.example:1234', factory, undefined, new NullLogService(), fileService, createPermissionService(), new TestConfigurationService(), )); return { client, transports }; } @@ -841,7 +865,7 @@ suite('RemoteAgentHostProtocolClient', () => { title: 'Renamed by user', }; client.dispatch(action); - const initialDispatch = findNotification(transports[0], 'dispatchAction'); + const initialDispatch = findDispatchAction(transports[0], ActionType.SessionTitleChanged); assert.ok(initialDispatch, 'optimistic dispatch should reach the original transport'); const initialSeq = (initialDispatch.params as { clientSeq: number }).clientSeq; @@ -858,7 +882,7 @@ suite('RemoteAgentHostProtocolClient', () => { }); await flushMicrotasks(); - const replayed = findNotification(reconnectTransport, 'dispatchAction'); + const replayed = findDispatchAction(reconnectTransport, ActionType.SessionTitleChanged); assert.ok(replayed, 'pending optimistic action should be re-sent after reconnect'); assert.strictEqual((replayed.params as { clientSeq: number }).clientSeq, initialSeq, 'replayed dispatch must reuse the original clientSeq'); @@ -889,7 +913,7 @@ suite('RemoteAgentHostProtocolClient', () => { title: 'Echoed back', }; client.dispatch(action); - const initialDispatch = findNotification(transports[0], 'dispatchAction')!; + const initialDispatch = findDispatchAction(transports[0], ActionType.SessionTitleChanged)!; const initialSeq = (initialDispatch.params as { clientSeq: number }).clientSeq; transports[0].fireClose(); @@ -914,7 +938,7 @@ suite('RemoteAgentHostProtocolClient', () => { }); await flushMicrotasks(); - assert.strictEqual(findNotification(reconnectTransport, 'dispatchAction'), undefined, + assert.strictEqual(findDispatchAction(reconnectTransport, ActionType.SessionTitleChanged), undefined, 'action echoed back via replay buffer must not be re-sent'); subRef.dispose(); @@ -979,7 +1003,7 @@ suite('RemoteAgentHostProtocolClient', () => { title: 'Rejected change', }; client.dispatch(action); - const initialDispatch = findNotification(transports[0], 'dispatchAction')!; + const initialDispatch = findDispatchAction(transports[0], ActionType.SessionTitleChanged)!; const initialSeq = (initialDispatch.params as { clientSeq: number }).clientSeq; transports[0].fireClose(); @@ -1008,7 +1032,7 @@ suite('RemoteAgentHostProtocolClient', () => { assert.ok(sessionState, 'session state should be hydrated'); assert.strictEqual(sessionState.summary.title, 'Original', 'rejected action must not have been applied to confirmed state'); - assert.strictEqual(findNotification(reconnectTransport, 'dispatchAction'), undefined, + assert.strictEqual(findDispatchAction(reconnectTransport, ActionType.SessionTitleChanged), undefined, 'rejected action must not be re-dispatched'); subRef.dispose(); diff --git a/src/vs/platform/agentHost/test/node/agentHostTelemetryService.test.ts b/src/vs/platform/agentHost/test/node/agentHostTelemetryService.test.ts new file mode 100644 index 0000000000000..35050bc512621 --- /dev/null +++ b/src/vs/platform/agentHost/test/node/agentHostTelemetryService.test.ts @@ -0,0 +1,91 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; +import { ITelemetryData, ITelemetryService, TelemetryLevel } from '../../../telemetry/common/telemetry.js'; +import { AgentHostTelemetryLevelConfigKey, telemetryLevelToAgentHostConfigValue } from '../../common/agentHostSchema.js'; +import { AgentHostTelemetryService, updateAgentHostTelemetryLevelFromConfig } from '../../node/agentHostTelemetryService.js'; + +class TestTelemetryService implements ITelemetryService { + declare readonly _serviceBrand: undefined; + + telemetryLevel = TelemetryLevel.USAGE; + sendErrorTelemetry = true; + sessionId = 'sessionId'; + machineId = 'machineId'; + sqmId = 'sqmId'; + devDeviceId = 'devDeviceId'; + firstSessionDate = 'firstSessionDate'; + readonly events: { eventName: string; data: ITelemetryData | undefined }[] = []; + readonly errorEvents: { eventName: string; data: ITelemetryData | undefined }[] = []; + + publicLog(eventName: string, data?: ITelemetryData): void { + this.events.push({ eventName, data }); + } + + publicLogError(eventName: string, data?: ITelemetryData): void { + this.errorEvents.push({ eventName, data }); + } + + publicLog2(eventName: string, data?: ITelemetryData): void { + this.events.push({ eventName, data }); + } + + publicLogError2(eventName: string, data?: ITelemetryData): void { + this.errorEvents.push({ eventName, data }); + } + + setExperimentProperty(): void { } + setCommonProperty(): void { } +} + +suite('AgentHostTelemetryService', () => { + const disposables = ensureNoDisposablesAreLeakedInTestSuite(); + + test('permanently disables usage and error telemetry after TelemetryLevel.NONE', async () => { + const delegate = new TestTelemetryService(); + const service = disposables.add(new AgentHostTelemetryService(delegate)); + + service.publicLog('beforeDisable', { count: 1 }); + service.updateTelemetryLevel(TelemetryLevel.NONE); + service.updateTelemetryLevel(TelemetryLevel.USAGE); + service.publicLog2('afterDisable'); + service.publicLogError2('afterDisableError'); + service.publicLog('afterDisableAsync', { count: 4 }); + service.publicLogError('afterDisableErrorAsync', { count: 5 }); + + assert.deepStrictEqual({ + telemetryLevel: service.telemetryLevel, + sendErrorTelemetry: service.sendErrorTelemetry, + events: delegate.events, + errorEvents: delegate.errorEvents, + }, { + telemetryLevel: TelemetryLevel.NONE, + sendErrorTelemetry: false, + events: [{ eventName: 'beforeDisable', data: { count: 1 } }], + errorEvents: [], + }); + }); + + test('uses most restrictive client telemetry level', () => { + const service = disposables.add(new AgentHostTelemetryService(new TestTelemetryService())); + + service.updateTelemetryLevel(TelemetryLevel.ERROR); + service.updateTelemetryLevel(TelemetryLevel.USAGE); + + assert.strictEqual(service.telemetryLevel, TelemetryLevel.ERROR); + }); + + test('updates telemetry level from root config string enum', () => { + const service = disposables.add(new AgentHostTelemetryService(new TestTelemetryService())); + + updateAgentHostTelemetryLevelFromConfig(service, { + [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(TelemetryLevel.ERROR), + }); + + assert.strictEqual(service.telemetryLevel, TelemetryLevel.ERROR); + }); +}); diff --git a/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts b/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts index b1213710ed2d9..b72e23efa5133 100644 --- a/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts +++ b/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts @@ -26,7 +26,9 @@ import { buildSubagentSessionUri, MessageAttachmentKind, PendingMessageKind, Res import { IProductService } from '../../../product/common/productService.js'; import { ITelemetryService, TelemetryLevel } from '../../../telemetry/common/telemetry.js'; import { NullTelemetryService } from '../../../telemetry/common/telemetryUtils.js'; +import { AgentHostTelemetryLevelConfigKey, telemetryLevelToAgentHostConfigValue } from '../../common/agentHostSchema.js'; import { AgentConfigurationService, IAgentConfigurationService } from '../../node/agentConfigurationService.js'; +import { AgentHostTelemetryService } from '../../node/agentHostTelemetryService.js'; import { IAgentHostGitService } from '../../node/agentHostGitService.js'; import { AgentService } from '../../node/agentService.js'; import { AgentSideEffects, IAgentSideEffectsOptions } from '../../node/agentSideEffects.js'; @@ -128,7 +130,7 @@ suite('AgentSideEffects', () => { agents: agentList, sessionDataService: createNullSessionDataService(), onTurnComplete: () => { }, - }, undefined, telemetryService); + }, undefined, disposables.add(new AgentHostTelemetryService(telemetryService))); }); teardown(() => { @@ -953,6 +955,24 @@ suite('AgentSideEffects', () => { status: CustomizationStatus.Loaded, }]); }); + + test('updates telemetry level from root config', () => { + setupSession(); + const action: RootConfigChangedAction = { + type: ActionType.RootConfigChanged, + config: { [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(TelemetryLevel.NONE) }, + }; + + sideEffects.handleAction(action); + sideEffects.handleAction({ + type: ActionType.SessionTurnStarted, + session: sessionUri.toString(), + turnId: 'turn-1', + userMessage: { text: 'hello world' }, + }); + + assert.deepStrictEqual(telemetryService.events, []); + }); }); // ---- onDidCustomizationsChange integration -------------------------- From 16f728c2e7f30b1a9b121c041f5c2633cbe834ca Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Sat, 16 May 2026 16:22:50 -0700 Subject: [PATCH 3/6] Extract agent host telemetry reporter Move the agentHostUserMessageSent event payload, classification, and publicLog2 call out of AgentSideEffects into a focused helper class. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../node/agentHostTelemetryReporter.ts | 40 +++++++++++++++++++ .../agentHost/node/agentSideEffects.ts | 37 +++-------------- 2 files changed, 46 insertions(+), 31 deletions(-) create mode 100644 src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts new file mode 100644 index 0000000000000..e4237860373ad --- /dev/null +++ b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts @@ -0,0 +1,40 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { ITelemetryService } from '../../telemetry/common/telemetry.js'; +import type { MessageAttachment } from '../common/state/protocol/state.js'; + +export type AgentHostUserMessageSentSource = 'direct' | 'queued'; + +export interface IAgentHostUserMessageSentEvent { + provider: string; + source: AgentHostUserMessageSentSource; + hasAttachments: boolean; + attachmentCount: number; +} + +export type IAgentHostUserMessageSentClassification = { + provider: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The provider handling the agent host session.' }; + source: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user message was sent directly or from the queued-message flow.' }; + hasAttachments: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the user message included attachments.' }; + attachmentCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of attachments included with the user message.' }; + owner: 'roblourens'; + comment: 'Tracks user messages sent from the agent host process to an agent provider.'; +}; + +export class AgentHostTelemetryReporter { + + constructor(private readonly _telemetryService: ITelemetryService) { } + + userMessageSent(provider: string, source: AgentHostUserMessageSentSource, attachments: readonly MessageAttachment[] | undefined): void { + const attachmentCount = attachments?.length ?? 0; + this._telemetryService.publicLog2('agentHostUserMessageSent', { + provider, + source, + hasAttachments: attachmentCount > 0, + attachmentCount, + }); + } +} diff --git a/src/vs/platform/agentHost/node/agentSideEffects.ts b/src/vs/platform/agentHost/node/agentSideEffects.ts index ced6f54862207..0d54ef0e20eaa 100644 --- a/src/vs/platform/agentHost/node/agentSideEffects.ts +++ b/src/vs/platform/agentHost/node/agentSideEffects.ts @@ -15,7 +15,7 @@ import { ILogService } from '../../log/common/log.js'; import { AgentSignal, IAgent, IAgentToolPendingConfirmationSignal } from '../common/agentService.js'; import { IDiffComputeService } from '../common/diffComputeService.js'; import { ISessionDatabase, ISessionDataService } from '../common/sessionDataService.js'; -import type { AgentInfo, MessageAttachment } from '../common/state/protocol/state.js'; +import type { AgentInfo } from '../common/state/protocol/state.js'; import { ActionType, isSessionAction, StateAction, type SessionToolCallCompleteAction } from '../common/state/sessionActions.js'; import { buildSubagentSessionUri, @@ -37,24 +37,7 @@ import { computeSessionDiffs, type IIncrementalDiffOptions } from './sessionDiff import { SessionPermissionManager } from './sessionPermissions.js'; import { ITelemetryService } from '../../telemetry/common/telemetry.js'; import { updateAgentHostTelemetryLevelFromConfig } from './agentHostTelemetryService.js'; - -type AgentHostUserMessageSentSource = 'direct' | 'queued'; - -type AgentHostUserMessageSentEvent = { - provider: string; - source: AgentHostUserMessageSentSource; - hasAttachments: boolean; - attachmentCount: number; -}; - -type AgentHostUserMessageSentClassification = { - provider: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The provider handling the agent host session.' }; - source: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user message was sent directly or from the queued-message flow.' }; - hasAttachments: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the user message included attachments.' }; - attachmentCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of attachments included with the user message.' }; - owner: 'roblourens'; - comment: 'Tracks user messages sent from the agent host process to an agent provider.'; -}; +import { AgentHostTelemetryReporter } from './agentHostTelemetryReporter.js'; /** * Options for constructing an {@link AgentSideEffects} instance. @@ -123,6 +106,7 @@ export class AgentSideEffects extends Disposable { * Key: `${parentSession}:${parentToolCallId}`. */ private readonly _pendingSubagentSignals = new Map(); + private readonly _telemetryReporter: AgentHostTelemetryReporter; constructor( private readonly _stateManager: AgentHostStateManager, @@ -133,6 +117,7 @@ export class AgentSideEffects extends Disposable { @ITelemetryService private readonly _telemetryService: ITelemetryService, ) { super(); + this._telemetryReporter = new AgentHostTelemetryReporter(this._telemetryService); this._diffComputeService = this._register(new NodeWorkerDiffComputeService(this._logService)); this._permissionManager = this._register(instantiationService.createInstance(SessionPermissionManager, this._stateManager)); @@ -733,7 +718,7 @@ export class AgentSideEffects extends Disposable { return; } const attachments = action.userMessage.attachments; - this._logUserMessageSent(agent.id, 'direct', attachments); + this._telemetryReporter.userMessageSent(agent.id, 'direct', attachments); agent.sendMessage(URI.parse(action.session), action.userMessage.text, attachments, action.turnId).catch(err => { const errCode = (err as { code?: number })?.code; this._logService.error(`[AgentSideEffects] sendMessage failed for session=${action.session}: code=${errCode}, message=${err instanceof Error ? err.message : String(err)}, type=${err?.constructor?.name}`, err); @@ -966,7 +951,7 @@ export class AgentSideEffects extends Disposable { return; } const attachments = msg.userMessage.attachments; - this._logUserMessageSent(agent.id, 'queued', attachments); + this._telemetryReporter.userMessageSent(agent.id, 'queued', attachments); agent.sendMessage(URI.parse(session), msg.userMessage.text, attachments, turnId).catch(err => { this._logService.error('[AgentSideEffects] sendMessage failed (queued)', err); this._stateManager.dispatchServerAction({ @@ -978,16 +963,6 @@ export class AgentSideEffects extends Disposable { }); } - private _logUserMessageSent(provider: string, source: AgentHostUserMessageSentSource, attachments: readonly MessageAttachment[] | undefined): void { - const attachmentCount = attachments?.length ?? 0; - this._telemetryService.publicLog2('agentHostUserMessageSent', { - provider, - source, - hasAttachments: attachmentCount > 0, - attachmentCount, - }); - } - // ---- Session diff computation ---------------------------------------------- /** From 00abecf36e439e9bbc0af1e27a07d4f36f560f08 Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Sat, 16 May 2026 16:29:48 -0700 Subject: [PATCH 4/6] Add agent host user message telemetry metadata Rename the user-message event to agentHost.userMessageSent and include safe session and active-client metadata that is available at send time. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../node/agentHostTelemetryReporter.ts | 33 +++++++++++++++++-- .../agentHost/node/agentSideEffects.ts | 4 +-- .../test/node/agentSideEffects.test.ts | 28 ++++++++++++++-- 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts index e4237860373ad..913a82a453280 100644 --- a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts +++ b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts @@ -4,20 +4,38 @@ *--------------------------------------------------------------------------------------------*/ import type { ITelemetryService } from '../../telemetry/common/telemetry.js'; +import { AgentSession } from '../common/agentService.js'; import type { MessageAttachment } from '../common/state/protocol/state.js'; +import { isSubagentSession, type SessionState } from '../common/state/sessionState.js'; export type AgentHostUserMessageSentSource = 'direct' | 'queued'; export interface IAgentHostUserMessageSentEvent { provider: string; + agentSessionId: string; source: AgentHostUserMessageSentSource; + isSubagentSession: boolean; + isInitialTurn: boolean; + turnCount: number; + hasActiveClient: boolean; + activeClientId?: string; + activeClientToolCount?: number; + activeClientCustomizationCount?: number; hasAttachments: boolean; attachmentCount: number; } export type IAgentHostUserMessageSentClassification = { provider: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The provider handling the agent host session.' }; + agentSessionId: { classification: 'EndUserPseudonymizedInformation'; purpose: 'FeatureInsight'; comment: 'The agent host session identifier.' }; source: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user message was sent directly or from the queued-message flow.' }; + isSubagentSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the message was sent to a subagent session.' }; + isInitialTurn: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether this is the first user turn in the session.' }; + turnCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of completed turns in the session when the message was sent.' }; + hasActiveClient: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the session had an active client when the message was sent.' }; + activeClientId?: { classification: 'EndUserPseudonymizedInformation'; purpose: 'FeatureInsight'; comment: 'The active client identifier for the session, if any.' }; + activeClientToolCount?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of tools provided by the active client, if any.' }; + activeClientCustomizationCount?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of customizations provided by the active client, if any.' }; hasAttachments: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the user message included attachments.' }; attachmentCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of attachments included with the user message.' }; owner: 'roblourens'; @@ -28,11 +46,22 @@ export class AgentHostTelemetryReporter { constructor(private readonly _telemetryService: ITelemetryService) { } - userMessageSent(provider: string, source: AgentHostUserMessageSentSource, attachments: readonly MessageAttachment[] | undefined): void { + userMessageSent(provider: string, session: string, sessionState: SessionState | undefined, source: AgentHostUserMessageSentSource, attachments: readonly MessageAttachment[] | undefined): void { const attachmentCount = attachments?.length ?? 0; - this._telemetryService.publicLog2('agentHostUserMessageSent', { + const activeClient = sessionState?.activeClient; + this._telemetryService.publicLog2('agentHost.userMessageSent', { provider, + agentSessionId: AgentSession.id(session), source, + isSubagentSession: isSubagentSession(session), + isInitialTurn: (sessionState?.turns.length ?? 0) === 0, + turnCount: sessionState?.turns.length ?? 0, + hasActiveClient: !!activeClient, + ...(activeClient ? { + activeClientId: activeClient.clientId, + activeClientToolCount: activeClient.tools.length, + activeClientCustomizationCount: activeClient.customizations?.length ?? 0, + } : {}), hasAttachments: attachmentCount > 0, attachmentCount, }); diff --git a/src/vs/platform/agentHost/node/agentSideEffects.ts b/src/vs/platform/agentHost/node/agentSideEffects.ts index 0d54ef0e20eaa..0f559098e57a8 100644 --- a/src/vs/platform/agentHost/node/agentSideEffects.ts +++ b/src/vs/platform/agentHost/node/agentSideEffects.ts @@ -718,7 +718,7 @@ export class AgentSideEffects extends Disposable { return; } const attachments = action.userMessage.attachments; - this._telemetryReporter.userMessageSent(agent.id, 'direct', attachments); + this._telemetryReporter.userMessageSent(agent.id, action.session, state, 'direct', attachments); agent.sendMessage(URI.parse(action.session), action.userMessage.text, attachments, action.turnId).catch(err => { const errCode = (err as { code?: number })?.code; this._logService.error(`[AgentSideEffects] sendMessage failed for session=${action.session}: code=${errCode}, message=${err instanceof Error ? err.message : String(err)}, type=${err?.constructor?.name}`, err); @@ -951,7 +951,7 @@ export class AgentSideEffects extends Disposable { return; } const attachments = msg.userMessage.attachments; - this._telemetryReporter.userMessageSent(agent.id, 'queued', attachments); + this._telemetryReporter.userMessageSent(agent.id, session, this._stateManager.getSessionState(session), 'queued', attachments); agent.sendMessage(URI.parse(session), msg.userMessage.text, attachments, turnId).catch(err => { this._logService.error('[AgentSideEffects] sendMessage failed (queued)', err); this._stateManager.dispatchServerAction({ diff --git a/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts b/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts index b72e23efa5133..0b2e869dace08 100644 --- a/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts +++ b/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts @@ -160,6 +160,17 @@ suite('AgentSideEffects', () => { test('logs telemetry when sending a direct user message', () => { setupSession(); + const activeClientAction: SessionAction = { + type: ActionType.SessionActiveClientChanged, + session: sessionUri.toString(), + activeClient: { + clientId: 'test-client', + tools: [{ name: 'testTool', inputSchema: { type: 'object' } }], + customizations: [{ uri: 'file:///customizations/SKILL.md', displayName: 'Test Skill' }], + }, + }; + stateManager.dispatchClientAction(activeClientAction, { clientId: 'test', clientSeq: 1 }); + sideEffects.handleAction(activeClientAction); const fileUri = URI.file('/workspace/direct.ts'); sideEffects.handleAction({ type: ActionType.SessionTurnStarted, @@ -169,10 +180,18 @@ suite('AgentSideEffects', () => { }); assert.deepStrictEqual(telemetryService.events, [{ - eventName: 'agentHostUserMessageSent', + eventName: 'agentHost.userMessageSent', data: { provider: 'mock', + agentSessionId: 'session-1', source: 'direct', + isSubagentSession: false, + isInitialTurn: true, + turnCount: 0, + hasActiveClient: true, + activeClientId: 'test-client', + activeClientToolCount: 1, + activeClientCustomizationCount: 1, hasAttachments: true, attachmentCount: 1, }, @@ -640,10 +659,15 @@ suite('AgentSideEffects', () => { sideEffects.handleAction(action); assert.deepStrictEqual(telemetryService.events, [{ - eventName: 'agentHostUserMessageSent', + eventName: 'agentHost.userMessageSent', data: { provider: 'mock', + agentSessionId: 'session-1', source: 'queued', + isSubagentSession: false, + isInitialTurn: true, + turnCount: 0, + hasActiveClient: false, hasAttachments: false, attachmentCount: 0, }, From b160fbdbde526a3167febf3b4c29f3060e9a9965 Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Sat, 16 May 2026 16:35:21 -0700 Subject: [PATCH 5/6] Trim agent host user message telemetry fields Remove boolean fields that can be derived from turnCount, activeClientId, and attachmentCount. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agentHost/node/agentHostTelemetryReporter.ts | 9 --------- .../agentHost/test/node/agentSideEffects.test.ts | 6 ------ 2 files changed, 15 deletions(-) diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts index 913a82a453280..db130d63762b5 100644 --- a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts +++ b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts @@ -15,13 +15,10 @@ export interface IAgentHostUserMessageSentEvent { agentSessionId: string; source: AgentHostUserMessageSentSource; isSubagentSession: boolean; - isInitialTurn: boolean; turnCount: number; - hasActiveClient: boolean; activeClientId?: string; activeClientToolCount?: number; activeClientCustomizationCount?: number; - hasAttachments: boolean; attachmentCount: number; } @@ -30,13 +27,10 @@ export type IAgentHostUserMessageSentClassification = { agentSessionId: { classification: 'EndUserPseudonymizedInformation'; purpose: 'FeatureInsight'; comment: 'The agent host session identifier.' }; source: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user message was sent directly or from the queued-message flow.' }; isSubagentSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the message was sent to a subagent session.' }; - isInitialTurn: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether this is the first user turn in the session.' }; turnCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of completed turns in the session when the message was sent.' }; - hasActiveClient: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the session had an active client when the message was sent.' }; activeClientId?: { classification: 'EndUserPseudonymizedInformation'; purpose: 'FeatureInsight'; comment: 'The active client identifier for the session, if any.' }; activeClientToolCount?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of tools provided by the active client, if any.' }; activeClientCustomizationCount?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of customizations provided by the active client, if any.' }; - hasAttachments: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the user message included attachments.' }; attachmentCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of attachments included with the user message.' }; owner: 'roblourens'; comment: 'Tracks user messages sent from the agent host process to an agent provider.'; @@ -54,15 +48,12 @@ export class AgentHostTelemetryReporter { agentSessionId: AgentSession.id(session), source, isSubagentSession: isSubagentSession(session), - isInitialTurn: (sessionState?.turns.length ?? 0) === 0, turnCount: sessionState?.turns.length ?? 0, - hasActiveClient: !!activeClient, ...(activeClient ? { activeClientId: activeClient.clientId, activeClientToolCount: activeClient.tools.length, activeClientCustomizationCount: activeClient.customizations?.length ?? 0, } : {}), - hasAttachments: attachmentCount > 0, attachmentCount, }); } diff --git a/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts b/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts index 0b2e869dace08..8979a527f59e2 100644 --- a/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts +++ b/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts @@ -186,13 +186,10 @@ suite('AgentSideEffects', () => { agentSessionId: 'session-1', source: 'direct', isSubagentSession: false, - isInitialTurn: true, turnCount: 0, - hasActiveClient: true, activeClientId: 'test-client', activeClientToolCount: 1, activeClientCustomizationCount: 1, - hasAttachments: true, attachmentCount: 1, }, }]); @@ -665,10 +662,7 @@ suite('AgentSideEffects', () => { agentSessionId: 'session-1', source: 'queued', isSubagentSession: false, - isInitialTurn: true, turnCount: 0, - hasActiveClient: false, - hasAttachments: false, attachmentCount: 0, }, }]); From 73f8f98fefbf56966eaddee2899630b1cc48326b Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Sat, 16 May 2026 16:55:37 -0700 Subject: [PATCH 6/6] Fix agent host telemetry ID classifications Classify Agent Host protocol IDs as system metadata instead of end-user pseudonymized information. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts index db130d63762b5..63b834877187c 100644 --- a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts +++ b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts @@ -24,11 +24,11 @@ export interface IAgentHostUserMessageSentEvent { export type IAgentHostUserMessageSentClassification = { provider: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The provider handling the agent host session.' }; - agentSessionId: { classification: 'EndUserPseudonymizedInformation'; purpose: 'FeatureInsight'; comment: 'The agent host session identifier.' }; + agentSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The agent host session identifier.' }; source: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user message was sent directly or from the queued-message flow.' }; isSubagentSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the message was sent to a subagent session.' }; turnCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of completed turns in the session when the message was sent.' }; - activeClientId?: { classification: 'EndUserPseudonymizedInformation'; purpose: 'FeatureInsight'; comment: 'The active client identifier for the session, if any.' }; + activeClientId?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The active client identifier for the session, if any.' }; activeClientToolCount?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of tools provided by the active client, if any.' }; activeClientCustomizationCount?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of customizations provided by the active client, if any.' }; attachmentCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of attachments included with the user message.' };