diff --git a/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts b/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts index 3266092aa1515..0913ecf054af1 100644 --- a/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts +++ b/src/vs/platform/agentHost/browser/remoteAgentHostProtocolClient.ts @@ -17,6 +17,7 @@ import { URI } from '../../../base/common/uri.js'; import { generateUuid } from '../../../base/common/uuid.js'; import { ILogService } from '../../log/common/log.js'; import { FileSystemProviderErrorCode, IFileService, toFileSystemProviderErrorCode } from '../../files/common/files.js'; +import { IConfigurationService } from '../../configuration/common/configuration.js'; import { AgentSession, IAgentConnection, IAgentCreateSessionConfig, IAgentResolveSessionConfigParams, IAgentSessionConfigCompletionsParams, IAgentSessionMetadata, AuthenticateParams, AuthenticateResult } from '../common/agentService.js'; import { AgentSubscriptionManager, type IAgentSubscription } from '../common/state/agentSubscription.js'; import { agentHostAuthority, fromAgentHostUri, toAgentHostUri } from '../common/agentHostUri.js'; @@ -32,6 +33,9 @@ import { AhpErrorCodes } from '../common/state/protocol/errors.js'; import { ContentEncoding, ResourceRequestParams, type CompletionsParams, type CompletionsResult, type CreateTerminalParams, type ResolveSessionConfigResult, type SessionConfigCompletionsResult } from '../common/state/protocol/commands.js'; import { decodeBase64, encodeBase64, VSBuffer } from '../../../base/common/buffer.js'; import { ILoadEstimator, LoadEstimator } from '../../../base/parts/ipc/common/ipc.net.js'; +import { TELEMETRY_CRASH_REPORTER_SETTING_ID, TELEMETRY_OLD_SETTING_ID, TELEMETRY_SETTING_ID } from '../../telemetry/common/telemetry.js'; +import { getTelemetryLevel } from '../../telemetry/common/telemetryUtils.js'; +import { AgentHostTelemetryLevelConfigKey, telemetryLevelToAgentHostConfigValue } from '../common/agentHostSchema.js'; const AHP_CLIENT_CONNECTION_CLOSED = -32000; @@ -248,6 +252,7 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC @ILogService private readonly _logService: ILogService, @IFileService private readonly _fileService: IFileService, @IAgentHostPermissionService private readonly _permissionService: IAgentHostPermissionService, + @IConfigurationService private readonly _configurationService: IConfigurationService, ) { super(); this._address = address; @@ -275,6 +280,15 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC this._subscriptionManager.receiveEnvelope(envelope); })); + this._register(this._configurationService.onDidChangeConfiguration(e => { + if (e.affectsConfiguration(TELEMETRY_SETTING_ID) || e.affectsConfiguration(TELEMETRY_OLD_SETTING_ID) || e.affectsConfiguration(TELEMETRY_CRASH_REPORTER_SETTING_ID)) { + if (this._state.kind !== AgentHostClientState.Connected) { + return; + } + this._updateTelemetryLevel(); + } + })); + // Detect silently-dead transports — see {@link _resetLivenessTimers}. this._resetLivenessTimers(); } @@ -357,6 +371,7 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC } this._completionTriggerCharacters = result.completionTriggerCharacters ?? []; + this._updateTelemetryLevel(); this._transitionTo({ kind: AgentHostClientState.Connected }); } @@ -1119,6 +1134,13 @@ export class RemoteAgentHostProtocolClient extends Disposable implements IAgentC return this._dispatchRequest(method, params); } + private _updateTelemetryLevel(): void { + this.dispatchAction({ + type: ActionType.RootConfigChanged, + config: { [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(getTelemetryLevel(this._configurationService)) }, + }, this._clientId, 0); + } + /** * Common path for outgoing JSON-RPC requests: gate on any in-flight * reconnect (unless explicitly bypassed for the `reconnect` RPC itself), diff --git a/src/vs/platform/agentHost/common/agentHostSchema.ts b/src/vs/platform/agentHost/common/agentHostSchema.ts index 1136c5fd147c6..9b36c902f4eb0 100644 --- a/src/vs/platform/agentHost/common/agentHostSchema.ts +++ b/src/vs/platform/agentHost/common/agentHostSchema.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { localize } from '../../../nls.js'; +import { TelemetryConfiguration, TelemetryLevel } from '../../telemetry/common/telemetry.js'; import { SessionConfigKey } from './sessionConfigKeys.js'; import type { SessionConfigPropertySchema, SessionConfigSchema } from './state/protocol/commands.js'; import { JsonRpcErrorCodes, ProtocolError } from './state/sessionProtocol.js'; @@ -349,6 +350,43 @@ export const platformSessionSchema = createSchema({ * auto-approval. See `SessionPermissionManager` for the evaluation * rules. */ +export const AgentHostTelemetryLevelConfigKey = 'telemetryLevel'; + +export function telemetryLevelToAgentHostConfigValue(telemetryLevel: TelemetryLevel): TelemetryConfiguration { + switch (telemetryLevel) { + case TelemetryLevel.NONE: + return TelemetryConfiguration.OFF; + case TelemetryLevel.CRASH: + return TelemetryConfiguration.CRASH; + case TelemetryLevel.ERROR: + return TelemetryConfiguration.ERROR; + case TelemetryLevel.USAGE: + return TelemetryConfiguration.ON; + } +} + +export function agentHostConfigValueToTelemetryLevel(value: unknown): TelemetryLevel | undefined { + switch (value) { + case TelemetryConfiguration.OFF: + return TelemetryLevel.NONE; + case TelemetryConfiguration.CRASH: + return TelemetryLevel.CRASH; + case TelemetryConfiguration.ERROR: + return TelemetryLevel.ERROR; + case TelemetryConfiguration.ON: + return TelemetryLevel.USAGE; + default: + return undefined; + } +} + export const platformRootSchema = createSchema({ [SessionConfigKey.Permissions]: permissionsProperty, + [AgentHostTelemetryLevelConfigKey]: schemaProperty({ + type: 'string', + title: localize('agentHost.config.telemetryLevel.title', "Telemetry Level"), + description: localize('agentHost.config.telemetryLevel.description', "Most restrictive telemetry level requested by connected clients."), + enum: [TelemetryConfiguration.ON, TelemetryConfiguration.ERROR, TelemetryConfiguration.CRASH, TelemetryConfiguration.OFF], + default: TelemetryConfiguration.ON, + }), }); diff --git a/src/vs/platform/agentHost/electron-browser/localAgentHostService.ts b/src/vs/platform/agentHost/electron-browser/localAgentHostService.ts index 9f86d14de1508..347625c486262 100644 --- a/src/vs/platform/agentHost/electron-browser/localAgentHostService.ts +++ b/src/vs/platform/agentHost/electron-browser/localAgentHostService.ts @@ -20,13 +20,16 @@ import { AhpJsonlLogger, getAhpLogByteLength } from '../common/ahpJsonlLogger.js import { wrapAgentServiceWithAhpLogging } from './localAhpJsonlLogging.js'; import { AgentSubscriptionManager, type IAgentSubscription } from '../common/state/agentSubscription.js'; import type { CompletionsParams, CompletionsResult, CreateTerminalParams, ResolveSessionConfigResult, SessionConfigCompletionsResult } from '../common/state/protocol/commands.js'; -import type { ActionEnvelope, INotification, IRootConfigChangedAction, SessionAction, TerminalAction } from '../common/state/sessionActions.js'; +import { ActionType, type ActionEnvelope, type INotification, type IRootConfigChangedAction, type SessionAction, type TerminalAction } from '../common/state/sessionActions.js'; import type { ResourceCopyParams, ResourceCopyResult, ResourceDeleteParams, ResourceDeleteResult, ResourceListResult, ResourceMoveParams, ResourceMoveResult, ResourceReadResult, ResourceWriteParams, ResourceWriteResult, IStateSnapshot } from '../common/state/sessionProtocol.js'; import { StateComponents, ROOT_STATE_URI, type RootState } from '../common/state/sessionState.js'; import { revive } from '../../../base/common/marshalling.js'; import { URI } from '../../../base/common/uri.js'; import { IFileService } from '../../files/common/files.js'; import { AGENT_HOST_CLIENT_RESOURCE_CHANNEL, AgentHostClientResourceChannel } from '../common/agentHostClientResourceChannel.js'; +import { TELEMETRY_CRASH_REPORTER_SETTING_ID, TELEMETRY_OLD_SETTING_ID, TELEMETRY_SETTING_ID } from '../../telemetry/common/telemetry.js'; +import { getTelemetryLevel } from '../../telemetry/common/telemetryUtils.js'; +import { AgentHostTelemetryLevelConfigKey, telemetryLevelToAgentHostConfigValue } from '../common/agentHostSchema.js'; /** * Renderer-side implementation of {@link IAgentHostService} that connects @@ -76,7 +79,7 @@ export class LocalAgentHostServiceClient extends Disposable implements IAgentHos constructor( @ILogService private readonly _logService: ILogService, - @IConfigurationService configurationService: IConfigurationService, + @IConfigurationService private readonly _configurationService: IConfigurationService, @IFileService private readonly _fileService: IFileService, @IEnvironmentService environmentService: IEnvironmentService, @IInstantiationService instantiationService: IInstantiationService, @@ -92,7 +95,7 @@ export class LocalAgentHostServiceClient extends Disposable implements IAgentHos // Optionally wrap the proxy with a logging layer that synthesizes JSON-RPC // frames for every request/response/notification on the in-process MessagePort // channel, mirroring the AHP transport JSONL logs produced by remote agent hosts. - this._ahpLogger = configurationService.getValue(AgentHostAhpJsonlLoggingSettingId) + this._ahpLogger = this._configurationService.getValue(AgentHostAhpJsonlLoggingSettingId) ? this._register(instantiationService.createInstance(AhpJsonlLogger, { logsHome: environmentService.logsHome, connectionId: this.clientId, @@ -113,7 +116,13 @@ export class LocalAgentHostServiceClient extends Disposable implements IAgentHos resource => this.unsubscribe(resource), )); - if (configurationService.getValue(AgentHostEnabledSettingId)) { + this._register(this._configurationService.onDidChangeConfiguration(e => { + if (e.affectsConfiguration(TELEMETRY_SETTING_ID) || e.affectsConfiguration(TELEMETRY_OLD_SETTING_ID) || e.affectsConfiguration(TELEMETRY_CRASH_REPORTER_SETTING_ID)) { + this._updateTelemetryLevel(); + } + })); + + if (this._configurationService.getValue(AgentHostEnabledSettingId)) { this._connect(); } } @@ -133,6 +142,7 @@ export class LocalAgentHostServiceClient extends Disposable implements IAgentHos // AgentHostClientFileSystemProvider that calls back through this channel. client.registerChannel(AGENT_HOST_CLIENT_RESOURCE_CHANNEL, new AgentHostClientResourceChannel(this._fileService, this._ahpLogger)); this._clientEventually.complete(client); + this._updateTelemetryLevel(); store.add(this._proxy.onDidAction(e => { const revived = revive(e) as ActionEnvelope; @@ -161,6 +171,13 @@ export class LocalAgentHostServiceClient extends Disposable implements IAgentHos }); } + private _updateTelemetryLevel(): void { + this.dispatchAction({ + type: ActionType.RootConfigChanged, + config: { [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(getTelemetryLevel(this._configurationService)) }, + }, this.clientId, 0); + } + // ---- IAgentService forwarding (no await needed, delayed channel handles queuing) ---- authenticate(params: AuthenticateParams): Promise { diff --git a/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts b/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts index de12b6329ca37..db790842723e6 100644 --- a/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts +++ b/src/vs/platform/agentHost/electron-main/electronAgentHostStarter.ts @@ -85,15 +85,20 @@ export class ElectronAgentHostStarter extends Disposable implements IAgentHostSt dbSpanExporterEnabled: this._configurationService.getValue(AgentHostOTelDbSpanExporterEnabledSettingId), }, process.env); + const args = [ + '--logsPath', this._environmentMainService.logsHome.with({ scheme: Schemas.file }).fsPath, + '--user-data-dir', this._environmentMainService.userDataPath, + ]; + if (this._environmentMainService.disableTelemetry) { + args.push('--disable-telemetry'); + } + this.utilityProcess.start({ type: 'agentHost', name: 'agent-host', entryPoint: 'vs/platform/agentHost/node/agentHostMain', execArgv, - args: [ - '--logsPath', this._environmentMainService.logsHome.with({ scheme: Schemas.file }).fsPath, - '--user-data-dir', this._environmentMainService.userDataPath, - ], + args, env: { ...deepClone(process.env), ...shellEnv, diff --git a/src/vs/platform/agentHost/node/agentHostMain.ts b/src/vs/platform/agentHost/node/agentHostMain.ts index 8bcfefc7b4629..2dc1125a8d858 100644 --- a/src/vs/platform/agentHost/node/agentHostMain.ts +++ b/src/vs/platform/agentHost/node/agentHostMain.ts @@ -59,15 +59,20 @@ import { AgentPluginManager } from './agentPluginManager.js'; import { AgentHostGitService, IAgentHostGitService } from './agentHostGitService.js'; import { registerPendingEditContentProvider } from './copilot/pendingEditContentStore.js'; import { join } from '../../../base/common/path.js'; +import { createAgentHostTelemetryService } from './agentHostTelemetryService.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; // Entry point for the agent host utility process. // Sets up IPC, logging, and registers agent providers (Copilot). // When VSCODE_AGENT_HOST_PORT or VSCODE_AGENT_HOST_SOCKET_PATH env vars // are set, also starts a WebSocket server for external clients. -startAgentHost(); +void startAgentHost().catch(err => { + console.error(err); + process.exit(1); +}); -function startAgentHost(): void { +async function startAgentHost(): Promise { // Setup RPC - supports both Electron utility process and Node child process let server: ChildProcessServer | UtilityProcessServer; if (isUtilityProcess(process)) { @@ -100,6 +105,7 @@ function startAgentHost(): void { // Session data service const sessionDataService = new SessionDataService(URI.file(environmentService.userDataPath), fileService, logService); const rootConfigResource = joinPath(environmentService.appSettingsHome, 'globalStorage', 'agent-host-config.json'); + const telemetryService = await createAgentHostTelemetryService({ environmentService, productService, fileService, loggerService, logService, disposables }); // Create the real service implementation that lives in this process let agentService: AgentService; @@ -113,6 +119,7 @@ function startAgentHost(): void { diServices.set(IFileService, fileService); diServices.set(ISessionDataService, sessionDataService); diServices.set(IProductService, productService); + diServices.set(ITelemetryService, telemetryService); instantiationService = new InstantiationService(diServices); const gitService = instantiationService.createInstance(AgentHostGitService); diServices.set(IAgentHostGitService, gitService); @@ -124,7 +131,7 @@ function startAgentHost(): void { diServices.set(IClaudeAgentSdkService, claudeAgentSdkService); const agentHostOTelService = disposables.add(instantiationService.createInstance(AgentHostOTelService)); diServices.set(IAgentHostOTelService, agentHostOTelService); - agentService = new AgentService(logService, fileService, sessionDataService, productService, gitService, rootConfigResource); + agentService = new AgentService(logService, fileService, sessionDataService, productService, gitService, rootConfigResource, telemetryService); const pluginManager = new AgentPluginManager(URI.file(environmentService.userDataPath), fileService, logService); diServices.set(IAgentPluginManager, pluginManager); const diffComputeService = disposables.add(new NodeWorkerDiffComputeService(logService)); diff --git a/src/vs/platform/agentHost/node/agentHostServerMain.ts b/src/vs/platform/agentHost/node/agentHostServerMain.ts index aa3ceba14d706..5f8f2c8814fe6 100644 --- a/src/vs/platform/agentHost/node/agentHostServerMain.ts +++ b/src/vs/platform/agentHost/node/agentHostServerMain.ts @@ -60,6 +60,8 @@ import { AgentPluginManager } from './agentPluginManager.js'; import { IAgentPluginManager } from '../common/agentPluginManager.js'; import { registerPendingEditContentProvider } from './copilot/pendingEditContentStore.js'; import { AgentHostGitService, IAgentHostGitService } from './agentHostGitService.js'; +import { createAgentHostTelemetryService } from './agentHostTelemetryService.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; /** Log to stderr so messages appear in the terminal alongside the process. */ function log(msg: string): void { @@ -186,17 +188,19 @@ async function main(): Promise { // `createInstance` (it needs IFileService + INativeEnvironmentService). // The git service is shared by AgentService (for diff computation + // showBlob) and the production agent registration path. + const telemetryService = await createAgentHostTelemetryService({ environmentService, productService, fileService, loggerService, logService, disposables, disableTelemetry: options.quiet }); const diServices = new ServiceCollection(); diServices.set(IProductService, productService); diServices.set(INativeEnvironmentService, environmentService); diServices.set(ILogService, logService); diServices.set(IFileService, fileService); diServices.set(ISessionDataService, sessionDataService); + diServices.set(ITelemetryService, telemetryService); const instantiationService = new InstantiationService(diServices); const gitService = instantiationService.createInstance(AgentHostGitService); // Create the agent service (owns AgentHostStateManager + AgentSideEffects internally) - const agentService = new AgentService(logService, fileService, sessionDataService, productService, gitService, rootConfigResource); + const agentService = new AgentService(logService, fileService, sessionDataService, productService, gitService, rootConfigResource, telemetryService); disposables.add(agentService); // Register agents diff --git a/src/vs/platform/agentHost/node/agentHostStateManager.ts b/src/vs/platform/agentHost/node/agentHostStateManager.ts index a6705f0f56959..3dc7e1f58bdbe 100644 --- a/src/vs/platform/agentHost/node/agentHostStateManager.ts +++ b/src/vs/platform/agentHost/node/agentHostStateManager.ts @@ -8,11 +8,12 @@ import { Emitter, Event } from '../../../base/common/event.js'; import { Disposable } from '../../../base/common/lifecycle.js'; import { equals } from '../../../base/common/objects.js'; import { ILogService } from '../../log/common/log.js'; +import { TelemetryLevel } from '../../telemetry/common/telemetry.js'; import { ActionType, NotificationType, ActionEnvelope, ActionOrigin, INotification, IRootConfigChangedAction, SessionAction, RootAction, StateAction, TerminalAction, isRootAction, isSessionAction } from '../common/state/sessionActions.js'; import type { IStateSnapshot } from '../common/state/sessionProtocol.js'; import { rootReducer, sessionReducer } from '../common/state/sessionReducers.js'; import { createRootState, createSessionState, SessionLifecycle, type RootState, type SessionMeta, type SessionState, type SessionSummary, type Turn, type URI, ROOT_STATE_URI } from '../common/state/sessionState.js'; -import { IPermissionsValue, platformRootSchema } from '../common/agentHostSchema.js'; +import { AgentHostTelemetryLevelConfigKey, IPermissionsValue, platformRootSchema, telemetryLevelToAgentHostConfigValue } from '../common/agentHostSchema.js'; import { SessionConfigKey } from '../common/sessionConfigKeys.js'; /** @@ -67,6 +68,7 @@ export class AgentHostStateManager extends Disposable { schema: platformRootSchema.toProtocol(), values: platformRootSchema.validateOrDefault({}, { [SessionConfigKey.Permissions]: { allow: [], deny: [] } satisfies IPermissionsValue, + [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(TelemetryLevel.USAGE), }), }, }; diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts new file mode 100644 index 0000000000000..63b834877187c --- /dev/null +++ b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts @@ -0,0 +1,60 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { ITelemetryService } from '../../telemetry/common/telemetry.js'; +import { AgentSession } from '../common/agentService.js'; +import type { MessageAttachment } from '../common/state/protocol/state.js'; +import { isSubagentSession, type SessionState } from '../common/state/sessionState.js'; + +export type AgentHostUserMessageSentSource = 'direct' | 'queued'; + +export interface IAgentHostUserMessageSentEvent { + provider: string; + agentSessionId: string; + source: AgentHostUserMessageSentSource; + isSubagentSession: boolean; + turnCount: number; + activeClientId?: string; + activeClientToolCount?: number; + activeClientCustomizationCount?: number; + attachmentCount: number; +} + +export type IAgentHostUserMessageSentClassification = { + provider: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The provider handling the agent host session.' }; + agentSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The agent host session identifier.' }; + source: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user message was sent directly or from the queued-message flow.' }; + isSubagentSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the message was sent to a subagent session.' }; + turnCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of completed turns in the session when the message was sent.' }; + activeClientId?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The active client identifier for the session, if any.' }; + activeClientToolCount?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of tools provided by the active client, if any.' }; + activeClientCustomizationCount?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of customizations provided by the active client, if any.' }; + attachmentCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of attachments included with the user message.' }; + owner: 'roblourens'; + comment: 'Tracks user messages sent from the agent host process to an agent provider.'; +}; + +export class AgentHostTelemetryReporter { + + constructor(private readonly _telemetryService: ITelemetryService) { } + + userMessageSent(provider: string, session: string, sessionState: SessionState | undefined, source: AgentHostUserMessageSentSource, attachments: readonly MessageAttachment[] | undefined): void { + const attachmentCount = attachments?.length ?? 0; + const activeClient = sessionState?.activeClient; + this._telemetryService.publicLog2('agentHost.userMessageSent', { + provider, + agentSessionId: AgentSession.id(session), + source, + isSubagentSession: isSubagentSession(session), + turnCount: sessionState?.turns.length ?? 0, + ...(activeClient ? { + activeClientId: activeClient.clientId, + activeClientToolCount: activeClient.tools.length, + activeClientCustomizationCount: activeClient.customizations?.length ?? 0, + } : {}), + attachmentCount, + }); + } +} diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryService.ts b/src/vs/platform/agentHost/node/agentHostTelemetryService.ts new file mode 100644 index 0000000000000..295d0d14f9648 --- /dev/null +++ b/src/vs/platform/agentHost/node/agentHostTelemetryService.ts @@ -0,0 +1,170 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { hostname, release } from 'os'; +import { Disposable, isDisposable, toDisposable, type DisposableStore } from '../../../base/common/lifecycle.js'; +import { joinPath } from '../../../base/common/resources.js'; +import { getDevDeviceId, getMachineId, getSqmMachineId } from '../../../base/node/id.js'; +import { ConfigurationService } from '../../configuration/common/configurationService.js'; +import { INativeEnvironmentService } from '../../environment/common/environment.js'; +import { IFileService } from '../../files/common/files.js'; +import { ILogService, ILoggerService } from '../../log/common/log.js'; +import { NullPolicyService } from '../../policy/common/policy.js'; +import { IProductService } from '../../product/common/productService.js'; +import { OneDataSystemAppender } from '../../telemetry/node/1dsAppender.js'; +import { resolveCommonProperties } from '../../telemetry/common/commonProperties.js'; +import { ClassifiedEvent, IGDPRProperty, OmitMetadata, StrictPropertyCheck } from '../../telemetry/common/gdprTypings.js'; +import { ITelemetryData, ITelemetryService, TelemetryLevel } from '../../telemetry/common/telemetry.js'; +import { TelemetryLogAppender } from '../../telemetry/common/telemetryLogAppender.js'; +import { TelemetryService } from '../../telemetry/common/telemetryService.js'; +import { getPiiPathsFromEnvironment, isInternalTelemetry, isLoggingOnly, NullTelemetryService, supportsTelemetry, type ITelemetryAppender } from '../../telemetry/common/telemetryUtils.js'; +import { AgentHostTelemetryLevelConfigKey, agentHostConfigValueToTelemetryLevel } from '../common/agentHostSchema.js'; + +export interface IAgentHostTelemetryServiceOptions { + readonly environmentService: INativeEnvironmentService; + readonly productService: IProductService; + readonly fileService: IFileService; + readonly loggerService: ILoggerService | undefined; + readonly logService: ILogService; + readonly disposables: DisposableStore; + readonly disableTelemetry?: boolean; +} + +export interface IAgentHostTelemetryService extends ITelemetryService { + updateTelemetryLevel(telemetryLevel: TelemetryLevel): void; +} + +export class AgentHostTelemetryService extends Disposable implements IAgentHostTelemetryService { + declare readonly _serviceBrand: undefined; + + private _telemetryLevel = TelemetryLevel.USAGE; + + constructor(private readonly _delegate: ITelemetryService) { + super(); + if (isDisposable(_delegate)) { + this._register(_delegate); + } + } + + get telemetryLevel(): TelemetryLevel { + return Math.min(this._delegate.telemetryLevel, this._telemetryLevel); + } + + get sendErrorTelemetry(): boolean { + return this.telemetryLevel >= TelemetryLevel.ERROR && this._delegate.sendErrorTelemetry; + } + + get sessionId(): string { + return this._delegate.sessionId; + } + + get machineId(): string { + return this._delegate.machineId; + } + + get sqmId(): string { + return this._delegate.sqmId; + } + + get devDeviceId(): string { + return this._delegate.devDeviceId; + } + + get firstSessionDate(): string { + return this._delegate.firstSessionDate; + } + + get msftInternal(): boolean | undefined { + return this._delegate.msftInternal; + } + + publicLog(eventName: string, data?: ITelemetryData): void { + if (this.telemetryLevel < TelemetryLevel.USAGE) { + return; + } + this._delegate.publicLog(eventName, data); + } + + publicLogError(eventName: string, data?: ITelemetryData): void { + if (this.telemetryLevel < TelemetryLevel.ERROR) { + return; + } + this._delegate.publicLogError(eventName, data); + } + + publicLog2> = never, T extends IGDPRProperty = never>(eventName: string, data?: StrictPropertyCheck): void { + if (this.telemetryLevel < TelemetryLevel.USAGE) { + return; + } + this._delegate.publicLog2(eventName, data); + } + + publicLogError2> = never, T extends IGDPRProperty = never>(eventName: string, data?: StrictPropertyCheck): void { + if (this.telemetryLevel < TelemetryLevel.ERROR) { + return; + } + this._delegate.publicLogError2(eventName, data); + } + + setExperimentProperty(name: string, value: string): void { + this._delegate.setExperimentProperty(name, value); + } + + setCommonProperty(name: string, value: string): void { + this._delegate.setCommonProperty(name, value); + } + + updateTelemetryLevel(telemetryLevel: TelemetryLevel): void { + this._telemetryLevel = Math.min(this._telemetryLevel, telemetryLevel); + } +} + +export function updateAgentHostTelemetryLevelFromConfig(telemetryService: ITelemetryService, config: Record | undefined): void { + const telemetryLevel = config?.[AgentHostTelemetryLevelConfigKey]; + const telemetryLevelValue = agentHostConfigValueToTelemetryLevel(telemetryLevel); + if (!isAgentHostTelemetryService(telemetryService) || telemetryLevelValue === undefined) { + return; + } + telemetryService.updateTelemetryLevel(telemetryLevelValue); +} + +function isAgentHostTelemetryService(telemetryService: ITelemetryService): telemetryService is IAgentHostTelemetryService { + return typeof (telemetryService as IAgentHostTelemetryService).updateTelemetryLevel === 'function'; +} + +export async function createAgentHostTelemetryService(options: IAgentHostTelemetryServiceOptions): Promise { + const { environmentService, productService, fileService, loggerService, logService, disposables } = options; + if (options.disableTelemetry || !loggerService || !supportsTelemetry(productService, environmentService)) { + return disposables.add(new AgentHostTelemetryService(NullTelemetryService)); + } + + const configurationService = disposables.add(new ConfigurationService(joinPath(environmentService.appSettingsHome, 'settings.json'), fileService, new NullPolicyService(), logService)); + await configurationService.initialize(); + + const appenders: ITelemetryAppender[] = [ + disposables.add(new TelemetryLogAppender('', false, loggerService, environmentService, productService)), + ]; + const internalTelemetry = isInternalTelemetry(productService, configurationService); + if (!isLoggingOnly(productService, environmentService) && productService.aiConfig?.ariaKey) { + const collectorAppender = new OneDataSystemAppender(undefined, internalTelemetry, 'monacoworkbench', null, productService.aiConfig.ariaKey); + disposables.add(toDisposable(() => { void collectorAppender.flush(); })); + appenders.push(collectorAppender); + } + + const [machineId, sqmId, devDeviceId] = await Promise.all([ + getMachineId(error => logService.error(error)), + getSqmMachineId(error => logService.error(error)), + getDevDeviceId(error => logService.error(error)), + ]); + + const telemetryService = new TelemetryService({ + appenders, + sendErrorTelemetry: true, + commonProperties: resolveCommonProperties(release(), hostname(), process.arch, productService.commit, productService.version, machineId, sqmId, devDeviceId, internalTelemetry, productService.date), + piiPaths: getPiiPathsFromEnvironment(environmentService), + }, configurationService, productService); + + return disposables.add(new AgentHostTelemetryService(telemetryService)); +} diff --git a/src/vs/platform/agentHost/node/agentService.ts b/src/vs/platform/agentHost/node/agentService.ts index b7dc4d4abd02f..53c463e653191 100644 --- a/src/vs/platform/agentHost/node/agentService.ts +++ b/src/vs/platform/agentHost/node/agentService.ts @@ -40,6 +40,9 @@ import { AgentHostFileCompletionProvider } from './agentHostFileCompletionProvid import { AgentHostSkillCompletionProvider } from './agentHostSkillCompletionProvider.js'; import { AgentHostWorkspaceFiles } from './agentHostWorkspaceFiles.js'; import { toAgentClientUri } from '../common/agentClientUri.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; +import { NullTelemetryService } from '../../telemetry/common/telemetryUtils.js'; +import { updateAgentHostTelemetryLevelFromConfig } from './agentHostTelemetryService.js'; /** * Grace period before an empty, unsubscribed session is garbage-collected @@ -131,6 +134,7 @@ export class AgentService extends Disposable implements IAgentService { private readonly _productService: IProductService, private readonly _gitService: IAgentHostGitService, private readonly _rootConfigResource?: URI, + private readonly _telemetryService: ITelemetryService = NullTelemetryService, ) { super(); this._logService.info('AgentService initialized'); @@ -143,11 +147,13 @@ export class AgentService extends Disposable implements IAgentService { // via DI rather than being plumbed plain-class references. const configurationService: IAgentConfigurationService = this._register(new AgentConfigurationService(this._stateManager, this._logService, this._rootConfigResource)); this._configurationService = configurationService; + updateAgentHostTelemetryLevelFromConfig(this._telemetryService, this._stateManager.rootState.config?.values); const services = new ServiceCollection( [ILogService, this._logService], [IProductService, this._productService], [IAgentConfigurationService, configurationService], [IAgentHostGitService, this._gitService], + [ITelemetryService, this._telemetryService], ); const instantiationService = this._register(new InstantiationService(services, /*strict*/ true)); diff --git a/src/vs/platform/agentHost/node/agentSideEffects.ts b/src/vs/platform/agentHost/node/agentSideEffects.ts index e5b35815c8677..0f559098e57a8 100644 --- a/src/vs/platform/agentHost/node/agentSideEffects.ts +++ b/src/vs/platform/agentHost/node/agentSideEffects.ts @@ -35,6 +35,9 @@ import { AgentHostStateManager } from './agentHostStateManager.js'; import { NodeWorkerDiffComputeService } from './diffComputeService.js'; import { computeSessionDiffs, type IIncrementalDiffOptions } from './sessionDiffAggregator.js'; import { SessionPermissionManager } from './sessionPermissions.js'; +import { ITelemetryService } from '../../telemetry/common/telemetry.js'; +import { updateAgentHostTelemetryLevelFromConfig } from './agentHostTelemetryService.js'; +import { AgentHostTelemetryReporter } from './agentHostTelemetryReporter.js'; /** * Options for constructing an {@link AgentSideEffects} instance. @@ -103,6 +106,7 @@ export class AgentSideEffects extends Disposable { * Key: `${parentSession}:${parentToolCallId}`. */ private readonly _pendingSubagentSignals = new Map(); + private readonly _telemetryReporter: AgentHostTelemetryReporter; constructor( private readonly _stateManager: AgentHostStateManager, @@ -110,8 +114,10 @@ export class AgentSideEffects extends Disposable { @IInstantiationService instantiationService: IInstantiationService, @ILogService private readonly _logService: ILogService, @IAgentHostGitService private readonly _gitService: IAgentHostGitService, + @ITelemetryService private readonly _telemetryService: ITelemetryService, ) { super(); + this._telemetryReporter = new AgentHostTelemetryReporter(this._telemetryService); this._diffComputeService = this._register(new NodeWorkerDiffComputeService(this._logService)); this._permissionManager = this._register(instantiationService.createInstance(SessionPermissionManager, this._stateManager)); @@ -712,6 +718,7 @@ export class AgentSideEffects extends Disposable { return; } const attachments = action.userMessage.attachments; + this._telemetryReporter.userMessageSent(agent.id, action.session, state, 'direct', attachments); agent.sendMessage(URI.parse(action.session), action.userMessage.text, attachments, action.turnId).catch(err => { const errCode = (err as { code?: number })?.code; this._logService.error(`[AgentSideEffects] sendMessage failed for session=${action.session}: code=${errCode}, message=${err instanceof Error ? err.message : String(err)}, type=${err?.constructor?.name}`, err); @@ -806,6 +813,7 @@ export class AgentSideEffects extends Disposable { break; } case ActionType.RootConfigChanged: { + updateAgentHostTelemetryLevelFromConfig(this._telemetryService, action.config); // Host customizations are self-managed by each agent's // PluginController via IAgentConfigurationService.onDidRootConfigChange. // Republish agent infos for non-customization schema changes @@ -943,6 +951,7 @@ export class AgentSideEffects extends Disposable { return; } const attachments = msg.userMessage.attachments; + this._telemetryReporter.userMessageSent(agent.id, session, this._stateManager.getSessionState(session), 'queued', attachments); agent.sendMessage(URI.parse(session), msg.userMessage.text, attachments, turnId).catch(err => { this._logService.error('[AgentSideEffects] sendMessage failed (queued)', err); this._stateManager.dispatchServerAction({ diff --git a/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts b/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts index c29d1b541efe0..c7adda57eb78c 100644 --- a/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts +++ b/src/vs/platform/agentHost/node/nodeAgentHostStarter.ts @@ -114,13 +114,18 @@ export class NodeAgentHostStarter extends Disposable implements IAgentHostStarte } } + const args = [ + '--type=agentHost', + '--logsPath', this._environmentService.logsHome.with({ scheme: Schemas.file }).fsPath, + '--user-data-dir', this._environmentService.userDataPath, + ]; + if (this._environmentService.disableTelemetry) { + args.push('--disable-telemetry'); + } + const opts: IIPCOptions = { serverName: 'Agent Host', - args: [ - '--type=agentHost', - '--logsPath', this._environmentService.logsHome.with({ scheme: Schemas.file }).fsPath, - '--user-data-dir', this._environmentService.userDataPath, - ], + args, env, }; diff --git a/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts b/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts index 8c62aa92a7ed9..2214e5df51a33 100644 --- a/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts +++ b/src/vs/platform/agentHost/test/electron-browser/remoteAgentHostProtocolClient.test.ts @@ -25,6 +25,9 @@ import { hasKey } from '../../../../base/common/types.js'; import { mainWindow } from '../../../../base/browser/window.js'; import { ROOT_STATE_URI, StateComponents } from '../../common/state/sessionState.js'; import type { IClientTransport, IProtocolTransport } from '../../common/state/sessionTransport.js'; +import { TestConfigurationService } from '../../../configuration/test/common/testConfigurationService.js'; +import { TelemetryLevel } from '../../../telemetry/common/telemetry.js'; +import { AgentHostTelemetryLevelConfigKey, telemetryLevelToAgentHostConfigValue } from '../../common/agentHostSchema.js'; type ProtocolTransportMessage = ProtocolMessage | AhpServerNotification | JsonRpcNotification | JsonRpcResponse | JsonRpcRequest; @@ -88,7 +91,7 @@ suite('RemoteAgentHostProtocolClient', () => { function createClient(transport = disposables.add(new TestProtocolTransport()), permissionService = createPermissionService(), loadEstimator?: { hasHighLoad(): boolean }): { client: RemoteAgentHostProtocolClient; transport: TestProtocolTransport } { const fileService = disposables.add(new FileService(new NullLogService())); - const client = disposables.add(new RemoteAgentHostProtocolClient('test.example:1234', transport, loadEstimator, new NullLogService(), fileService, permissionService)); + const client = disposables.add(new RemoteAgentHostProtocolClient('test.example:1234', transport, loadEstimator, new NullLogService(), fileService, permissionService, new TestConfigurationService())); return { client, transport }; } @@ -381,6 +384,18 @@ suite('RemoteAgentHostProtocolClient', () => { result: { protocolVersion: PROTOCOL_VERSION, serverSeq: 0, snapshots: [] }, }); await connectPromise; + const telemetryLevel = transport.sentMessages[1] as JsonRpcNotification; + assert.deepStrictEqual(telemetryLevel, { + jsonrpc: '2.0', + method: 'dispatchAction', + params: { + clientSeq: 0, + action: { + type: ActionType.RootConfigChanged, + config: { [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(TelemetryLevel.USAGE) }, + }, + }, + }); }); test('rejects connect when host returns UnsupportedProtocolVersion (-32005)', async () => { @@ -719,6 +734,15 @@ suite('RemoteAgentHostProtocolClient', () => { ); } + function findDispatchAction(transport: TestProtocolTransport, actionType: ActionType): JsonRpcNotification | undefined { + return transport.sentMessages.find( + (m): m is JsonRpcNotification => 'method' in m + && (m as JsonRpcNotification).method === 'dispatchAction' + && !('id' in m) + && ((m as JsonRpcNotification).params as { action?: { type?: unknown } } | undefined)?.action?.type === actionType, + ); + } + async function flushMicrotasks(): Promise { // `await Promise.resolve()` only advances one microtask; loop a few times to // drain chained .then handlers without resorting to fake timers. @@ -769,7 +793,7 @@ suite('RemoteAgentHostProtocolClient', () => { }; const fileService = disposables.add(new FileService(new NullLogService())); const client = disposables.add(new RemoteAgentHostProtocolClient( - 'test.example:1234', factory, undefined, new NullLogService(), fileService, createPermissionService(), + 'test.example:1234', factory, undefined, new NullLogService(), fileService, createPermissionService(), new TestConfigurationService(), )); return { client, transports }; } @@ -841,7 +865,7 @@ suite('RemoteAgentHostProtocolClient', () => { title: 'Renamed by user', }; client.dispatch(action); - const initialDispatch = findNotification(transports[0], 'dispatchAction'); + const initialDispatch = findDispatchAction(transports[0], ActionType.SessionTitleChanged); assert.ok(initialDispatch, 'optimistic dispatch should reach the original transport'); const initialSeq = (initialDispatch.params as { clientSeq: number }).clientSeq; @@ -858,7 +882,7 @@ suite('RemoteAgentHostProtocolClient', () => { }); await flushMicrotasks(); - const replayed = findNotification(reconnectTransport, 'dispatchAction'); + const replayed = findDispatchAction(reconnectTransport, ActionType.SessionTitleChanged); assert.ok(replayed, 'pending optimistic action should be re-sent after reconnect'); assert.strictEqual((replayed.params as { clientSeq: number }).clientSeq, initialSeq, 'replayed dispatch must reuse the original clientSeq'); @@ -889,7 +913,7 @@ suite('RemoteAgentHostProtocolClient', () => { title: 'Echoed back', }; client.dispatch(action); - const initialDispatch = findNotification(transports[0], 'dispatchAction')!; + const initialDispatch = findDispatchAction(transports[0], ActionType.SessionTitleChanged)!; const initialSeq = (initialDispatch.params as { clientSeq: number }).clientSeq; transports[0].fireClose(); @@ -914,7 +938,7 @@ suite('RemoteAgentHostProtocolClient', () => { }); await flushMicrotasks(); - assert.strictEqual(findNotification(reconnectTransport, 'dispatchAction'), undefined, + assert.strictEqual(findDispatchAction(reconnectTransport, ActionType.SessionTitleChanged), undefined, 'action echoed back via replay buffer must not be re-sent'); subRef.dispose(); @@ -979,7 +1003,7 @@ suite('RemoteAgentHostProtocolClient', () => { title: 'Rejected change', }; client.dispatch(action); - const initialDispatch = findNotification(transports[0], 'dispatchAction')!; + const initialDispatch = findDispatchAction(transports[0], ActionType.SessionTitleChanged)!; const initialSeq = (initialDispatch.params as { clientSeq: number }).clientSeq; transports[0].fireClose(); @@ -1008,7 +1032,7 @@ suite('RemoteAgentHostProtocolClient', () => { assert.ok(sessionState, 'session state should be hydrated'); assert.strictEqual(sessionState.summary.title, 'Original', 'rejected action must not have been applied to confirmed state'); - assert.strictEqual(findNotification(reconnectTransport, 'dispatchAction'), undefined, + assert.strictEqual(findDispatchAction(reconnectTransport, ActionType.SessionTitleChanged), undefined, 'rejected action must not be re-dispatched'); subRef.dispose(); diff --git a/src/vs/platform/agentHost/test/node/agentHostTelemetryService.test.ts b/src/vs/platform/agentHost/test/node/agentHostTelemetryService.test.ts new file mode 100644 index 0000000000000..35050bc512621 --- /dev/null +++ b/src/vs/platform/agentHost/test/node/agentHostTelemetryService.test.ts @@ -0,0 +1,91 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; +import { ITelemetryData, ITelemetryService, TelemetryLevel } from '../../../telemetry/common/telemetry.js'; +import { AgentHostTelemetryLevelConfigKey, telemetryLevelToAgentHostConfigValue } from '../../common/agentHostSchema.js'; +import { AgentHostTelemetryService, updateAgentHostTelemetryLevelFromConfig } from '../../node/agentHostTelemetryService.js'; + +class TestTelemetryService implements ITelemetryService { + declare readonly _serviceBrand: undefined; + + telemetryLevel = TelemetryLevel.USAGE; + sendErrorTelemetry = true; + sessionId = 'sessionId'; + machineId = 'machineId'; + sqmId = 'sqmId'; + devDeviceId = 'devDeviceId'; + firstSessionDate = 'firstSessionDate'; + readonly events: { eventName: string; data: ITelemetryData | undefined }[] = []; + readonly errorEvents: { eventName: string; data: ITelemetryData | undefined }[] = []; + + publicLog(eventName: string, data?: ITelemetryData): void { + this.events.push({ eventName, data }); + } + + publicLogError(eventName: string, data?: ITelemetryData): void { + this.errorEvents.push({ eventName, data }); + } + + publicLog2(eventName: string, data?: ITelemetryData): void { + this.events.push({ eventName, data }); + } + + publicLogError2(eventName: string, data?: ITelemetryData): void { + this.errorEvents.push({ eventName, data }); + } + + setExperimentProperty(): void { } + setCommonProperty(): void { } +} + +suite('AgentHostTelemetryService', () => { + const disposables = ensureNoDisposablesAreLeakedInTestSuite(); + + test('permanently disables usage and error telemetry after TelemetryLevel.NONE', async () => { + const delegate = new TestTelemetryService(); + const service = disposables.add(new AgentHostTelemetryService(delegate)); + + service.publicLog('beforeDisable', { count: 1 }); + service.updateTelemetryLevel(TelemetryLevel.NONE); + service.updateTelemetryLevel(TelemetryLevel.USAGE); + service.publicLog2('afterDisable'); + service.publicLogError2('afterDisableError'); + service.publicLog('afterDisableAsync', { count: 4 }); + service.publicLogError('afterDisableErrorAsync', { count: 5 }); + + assert.deepStrictEqual({ + telemetryLevel: service.telemetryLevel, + sendErrorTelemetry: service.sendErrorTelemetry, + events: delegate.events, + errorEvents: delegate.errorEvents, + }, { + telemetryLevel: TelemetryLevel.NONE, + sendErrorTelemetry: false, + events: [{ eventName: 'beforeDisable', data: { count: 1 } }], + errorEvents: [], + }); + }); + + test('uses most restrictive client telemetry level', () => { + const service = disposables.add(new AgentHostTelemetryService(new TestTelemetryService())); + + service.updateTelemetryLevel(TelemetryLevel.ERROR); + service.updateTelemetryLevel(TelemetryLevel.USAGE); + + assert.strictEqual(service.telemetryLevel, TelemetryLevel.ERROR); + }); + + test('updates telemetry level from root config string enum', () => { + const service = disposables.add(new AgentHostTelemetryService(new TestTelemetryService())); + + updateAgentHostTelemetryLevelFromConfig(service, { + [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(TelemetryLevel.ERROR), + }); + + assert.strictEqual(service.telemetryLevel, TelemetryLevel.ERROR); + }); +}); diff --git a/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts b/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts index 11423965c64a0..8979a527f59e2 100644 --- a/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts +++ b/src/vs/platform/agentHost/test/node/agentSideEffects.test.ts @@ -24,7 +24,11 @@ import { CustomizationStatus } from '../../common/state/protocol/state.js'; import { ActionType, ActionEnvelope, SessionAction } from '../../common/state/sessionActions.js'; import { buildSubagentSessionUri, MessageAttachmentKind, PendingMessageKind, ResponsePartKind, SessionStatus, ToolCallConfirmationReason, ToolCallStatus, ToolResultContentType } from '../../common/state/sessionState.js'; import { IProductService } from '../../../product/common/productService.js'; +import { ITelemetryService, TelemetryLevel } from '../../../telemetry/common/telemetry.js'; +import { NullTelemetryService } from '../../../telemetry/common/telemetryUtils.js'; +import { AgentHostTelemetryLevelConfigKey, telemetryLevelToAgentHostConfigValue } from '../../common/agentHostSchema.js'; import { AgentConfigurationService, IAgentConfigurationService } from '../../node/agentConfigurationService.js'; +import { AgentHostTelemetryService } from '../../node/agentHostTelemetryService.js'; import { IAgentHostGitService } from '../../node/agentHostGitService.js'; import { AgentService } from '../../node/agentService.js'; import { AgentSideEffects, IAgentSideEffectsOptions } from '../../node/agentSideEffects.js'; @@ -40,17 +44,39 @@ import { MockAgent } from './mockAgent.js'; * scope that satisfies its {@link IAgentConfigurationService} / * {@link ILogService} / {@link IAgentHostGitService} dependencies. */ -function createTestSideEffects(disposables: DisposableStore, stateManager: AgentHostStateManager, options: IAgentSideEffectsOptions, gitService?: IAgentHostGitService): AgentSideEffects { +function createTestSideEffects(disposables: DisposableStore, stateManager: AgentHostStateManager, options: IAgentSideEffectsOptions, gitService?: IAgentHostGitService, telemetryService: ITelemetryService = NullTelemetryService): AgentSideEffects { const logService = new NullLogService(); const configService = disposables.add(new AgentConfigurationService(stateManager, logService)); const instantiationService = disposables.add(new InstantiationService(new ServiceCollection( [ILogService, logService], [IAgentConfigurationService, configService], [IAgentHostGitService, gitService ?? createNoopGitService()], + [ITelemetryService, telemetryService], ), /*strict*/ true)); return disposables.add(instantiationService.createInstance(AgentSideEffects, stateManager, options)); } +class TestTelemetryService implements ITelemetryService { + declare readonly _serviceBrand: undefined; + readonly telemetryLevel = TelemetryLevel.USAGE; + readonly sessionId = 'test-session'; + readonly machineId = 'test-machine'; + readonly sqmId = 'test-sqm'; + readonly devDeviceId = 'test-dev-device'; + readonly firstSessionDate = 'test-first-session-date'; + readonly sendErrorTelemetry = false; + readonly events: { eventName: string; data: unknown }[] = []; + + publicLog(): void { } + publicLog2(eventName: string, data?: unknown): void { + this.events.push({ eventName, data }); + } + publicLogError(): void { } + publicLogError2(): void { } + setExperimentProperty(): void { } + setCommonProperty(): void { } +} + suite('AgentSideEffects', () => { const disposables = new DisposableStore(); @@ -59,6 +85,7 @@ suite('AgentSideEffects', () => { let agent: MockAgent; let sideEffects: AgentSideEffects; let agentList: ReturnType>; + let telemetryService: TestTelemetryService; const sessionUri = AgentSession.uri('mock', 'session-1'); @@ -97,12 +124,13 @@ suite('AgentSideEffects', () => { disposables.add(toDisposable(() => agent.dispose())); stateManager = disposables.add(new AgentHostStateManager(new NullLogService())); agentList = observableValue('agents', [agent]); + telemetryService = new TestTelemetryService(); sideEffects = createTestSideEffects(disposables, stateManager, { getAgent: () => agent, agents: agentList, sessionDataService: createNullSessionDataService(), onTurnComplete: () => { }, - }); + }, undefined, disposables.add(new AgentHostTelemetryService(telemetryService))); }); teardown(() => { @@ -130,6 +158,43 @@ suite('AgentSideEffects', () => { assert.deepStrictEqual(agent.sendMessageCalls, [{ session: URI.parse(sessionUri.toString()), prompt: 'hello world', attachments: undefined }]); }); + test('logs telemetry when sending a direct user message', () => { + setupSession(); + const activeClientAction: SessionAction = { + type: ActionType.SessionActiveClientChanged, + session: sessionUri.toString(), + activeClient: { + clientId: 'test-client', + tools: [{ name: 'testTool', inputSchema: { type: 'object' } }], + customizations: [{ uri: 'file:///customizations/SKILL.md', displayName: 'Test Skill' }], + }, + }; + stateManager.dispatchClientAction(activeClientAction, { clientId: 'test', clientSeq: 1 }); + sideEffects.handleAction(activeClientAction); + const fileUri = URI.file('/workspace/direct.ts'); + sideEffects.handleAction({ + type: ActionType.SessionTurnStarted, + session: sessionUri.toString(), + turnId: 'turn-1', + userMessage: { text: 'hello world', attachments: [{ type: MessageAttachmentKind.Resource, uri: fileUri.toString(), label: 'direct.ts', displayKind: 'document' }] }, + }); + + assert.deepStrictEqual(telemetryService.events, [{ + eventName: 'agentHost.userMessageSent', + data: { + provider: 'mock', + agentSessionId: 'session-1', + source: 'direct', + isSubagentSession: false, + turnCount: 0, + activeClientId: 'test-client', + activeClientToolCount: 1, + activeClientCustomizationCount: 1, + attachmentCount: 1, + }, + }]); + }); + test('parses protocol attachment URI strings before passing them to the agent', () => { setupSession(); const fileUri = URI.file('/workspace/test.ts'); @@ -577,6 +642,32 @@ suite('AgentSideEffects', () => { }]); }); + test('logs telemetry when sending a queued user message', () => { + setupSession(); + + const action = { + type: ActionType.SessionPendingMessageSet as const, + session: sessionUri.toString(), + kind: PendingMessageKind.Queued, + id: 'q-telemetry', + userMessage: { text: 'queued message' }, + }; + stateManager.dispatchClientAction(action, { clientId: 'test', clientSeq: 1 }); + sideEffects.handleAction(action); + + assert.deepStrictEqual(telemetryService.events, [{ + eventName: 'agentHost.userMessageSent', + data: { + provider: 'mock', + agentSessionId: 'session-1', + source: 'queued', + isSubagentSession: false, + turnCount: 0, + attachmentCount: 0, + }, + }]); + }); + test('syncs on SessionPendingMessageRemoved', () => { setupSession(); @@ -882,6 +973,24 @@ suite('AgentSideEffects', () => { status: CustomizationStatus.Loaded, }]); }); + + test('updates telemetry level from root config', () => { + setupSession(); + const action: RootConfigChangedAction = { + type: ActionType.RootConfigChanged, + config: { [AgentHostTelemetryLevelConfigKey]: telemetryLevelToAgentHostConfigValue(TelemetryLevel.NONE) }, + }; + + sideEffects.handleAction(action); + sideEffects.handleAction({ + type: ActionType.SessionTurnStarted, + session: sessionUri.toString(), + turnId: 'turn-1', + userMessage: { text: 'hello world' }, + }); + + assert.deepStrictEqual(telemetryService.events, []); + }); }); // ---- onDidCustomizationsChange integration --------------------------