diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 25ad70c2815c..01e78f5f35d8 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -813,6 +813,14 @@ components: schema: example: "my-job" type: string + LLMObsAccountIDPathParameter: + description: The ID of the integration account. + example: "account-abc123" + in: path + name: account_id + required: true + schema: + type: string LLMObsAnnotationQueueIDPathParameter: description: The ID of the LLM Observability annotation queue. example: "00000000-0000-0000-0000-000000000001" @@ -845,6 +853,14 @@ components: required: true schema: type: string + LLMObsIntegrationPathParameter: + description: The name of the LLM integration. + example: openai + in: path + name: integration + required: true + schema: + $ref: "#/components/schemas/LLMObsIntegrationName" LLMObsProjectIDPathParameter: description: The ID of the LLM Observability project. example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" @@ -41879,6 +41895,80 @@ components: required: - label_schemas type: object + LLMObsAnthropicEffort: + description: The effort level for Anthropic inference. + enum: + - low + - medium + - high + - max + example: medium + nullable: true + type: string + x-enum-varnames: + - LOW + - MEDIUM + - HIGH + - MAX + LLMObsAnthropicMetadata: + description: Anthropic-specific metadata for an inference request. + properties: + effort: + $ref: "#/components/schemas/LLMObsAnthropicEffort" + thinking: + $ref: "#/components/schemas/LLMObsAnthropicThinkingConfig" + nullable: true + type: object + LLMObsAnthropicThinkingConfig: + description: Configuration for Anthropic extended thinking feature. + properties: + budget_tokens: + description: Maximum token budget for extended thinking. Required when type is `enabled`. + example: 1024 + format: int64 + nullable: true + type: integer + type: + $ref: "#/components/schemas/LLMObsAnthropicThinkingType" + required: + - type + type: object + LLMObsAnthropicThinkingType: + description: The thinking mode for Anthropic extended thinking. + enum: + - enabled + - disabled + - adaptive + example: enabled + type: string + x-enum-varnames: + - ENABLED + - DISABLED + - ADAPTIVE + LLMObsAzureOpenAIMetadata: + description: Azure OpenAI-specific metadata for an integration account or inference request. + properties: + deployment_id: + description: The Azure OpenAI deployment ID. + example: "my-gpt4-deployment" + type: string + model_version: + description: The model version deployed in Azure. + example: "0613" + type: string + resource_name: + description: The Azure OpenAI resource name. + example: "my-azure-resource" + type: string + type: object + LLMObsBedrockMetadata: + description: Amazon Bedrock-specific metadata for an inference request. + properties: + region: + description: The AWS region for the Bedrock request. + example: "us-east-1" + type: string + type: object LLMObsContentBlock: description: |- A single content block rendered inside a `display_block` interaction. @@ -44204,6 +44294,536 @@ components: required: - data type: object + LLMObsInferenceCode: + description: A generated code snippet for running an inference request programmatically. + properties: + code: + description: The generated code content. + example: "import openai\nclient = openai.OpenAI()\n..." + type: string + id: + description: Unique identifier for the code snippet. + example: "code-python-001" + type: string + type: + description: The programming language or SDK type of the code snippet. + example: "python" + type: string + required: + - id + - type + - code + type: object + LLMObsInferenceContent: + description: A structured content block within a message. + properties: + type: + description: The content block type. + example: "text" + type: string + value: + $ref: "#/components/schemas/LLMObsInferenceContentValue" + required: + - type + - value + type: object + LLMObsInferenceContentList: + description: List of structured content blocks in a message. + items: + $ref: "#/components/schemas/LLMObsInferenceContent" + type: array + LLMObsInferenceContentValue: + description: The typed value of a message content block. + properties: + text: + description: Plain text content. + example: "Hello, how can I help you?" + type: string + tool_call: + $ref: "#/components/schemas/LLMObsInferenceToolCall" + tool_call_result: + $ref: "#/components/schemas/LLMObsInferenceToolResult" + type: object + LLMObsInferenceErrorResponse: + description: Error details returned when an inference provider returns an error. + properties: + message: + description: A human-readable description of the error. + example: "The model does not exist." + type: string + type: + description: The provider-specific error type. + example: "invalid_request_error" + type: string + required: + - type + - message + type: object + LLMObsInferenceFunction: + description: A function definition for a tool available to the model. + properties: + description: + description: A description of what the function does. + example: "Get the current weather for a location." + type: string + name: + description: The name of the function. + example: "get_weather" + type: string + parameters: + additionalProperties: {} + description: JSON schema describing the function parameters. + example: + properties: + location: + type: string + type: object + type: object + required: + - name + - parameters + type: object + LLMObsInferenceMessage: + description: A single message in an LLM inference conversation. + properties: + content: + description: Plain text content of the message. + example: "What is the capital of France?" + type: string + contents: + $ref: "#/components/schemas/LLMObsInferenceContentList" + id: + description: Unique identifier for the message. + example: "msg_001" + type: string + role: + description: The role of the message author. + example: "user" + type: string + tool_calls: + $ref: "#/components/schemas/LLMObsInferenceToolCallsList" + tool_results: + $ref: "#/components/schemas/LLMObsInferenceToolResultsList" + type: object + LLMObsInferenceMessagesList: + description: List of messages in an inference conversation. + items: + $ref: "#/components/schemas/LLMObsInferenceMessage" + type: array + LLMObsInferenceRunResult: + description: The output of a completed LLM inference call. + properties: + assessment: + description: An optional assessment of the inference output quality. + example: "pass" + nullable: true + type: string + content: + description: The text content of the model response. + example: "The capital of France is Paris." + type: string + finish_reason: + description: The reason the model stopped generating tokens. + example: "stop" + type: string + inference_codes: + $ref: "#/components/schemas/LLMObsIntegrationInferenceCodesResponse" + input_tokens: + description: Number of input tokens consumed. + example: 15 + format: int64 + type: integer + internal_reasoning: + $ref: "#/components/schemas/LLMObsInternalReasoning" + nullable: true + latency: + description: Request latency in milliseconds. + example: 843 + format: int64 + type: integer + output_tokens: + description: Number of output tokens generated. + example: 10 + format: int64 + type: integer + tools: + $ref: "#/components/schemas/LLMObsInferenceToolsList" + total_tokens: + description: Total tokens used (input plus output). + example: 25 + format: int64 + type: integer + required: + - content + - input_tokens + - output_tokens + - total_tokens + - latency + - finish_reason + - inference_codes + - tools + - assessment + type: object + LLMObsInferenceTool: + description: A tool definition available to the model during inference. + properties: + function: + $ref: "#/components/schemas/LLMObsInferenceFunction" + type: + description: The type of tool. + example: "function" + type: string + required: + - type + - function + type: object + LLMObsInferenceToolCall: + description: A tool call made during LLM inference. + properties: + arguments: + additionalProperties: {} + description: The arguments passed to the tool. + example: + location: "San Francisco" + type: object + name: + description: The name of the tool being called. + example: "get_weather" + type: string + tool_id: + description: Unique identifier for the tool call. + example: "call_abc123" + type: string + type: + description: The type of tool call. + example: "function" + type: string + type: object + LLMObsInferenceToolCallsList: + description: List of tool calls in a message. + items: + $ref: "#/components/schemas/LLMObsInferenceToolCall" + type: array + LLMObsInferenceToolResult: + description: The result returned by a tool call during LLM inference. + properties: + name: + description: The name of the tool that produced this result. + example: "get_weather" + type: string + result: + description: The result content returned by the tool. + example: "The weather in San Francisco is 68°F and sunny." + type: string + tool_id: + description: Identifier matching the corresponding tool call. + example: "call_abc123" + type: string + type: + description: The type of tool result. + example: "function" + type: string + type: object + LLMObsInferenceToolResultsList: + description: List of tool results in a message. + items: + $ref: "#/components/schemas/LLMObsInferenceToolResult" + type: array + LLMObsInferenceToolsList: + description: List of tools available to the model. + items: + $ref: "#/components/schemas/LLMObsInferenceTool" + type: array + LLMObsIntegrationAccount: + description: A configured account for an LLM provider integration. + properties: + account_id: + description: Provider-specific account identifier. + example: "org-XYZ123" + type: string + account_name: + description: Human-readable name for the integration account. + example: "Production OpenAI" + type: string + account_region: + description: Provider region associated with the account, if applicable. + example: "us-east-1" + type: string + azure_openai_metadata: + $ref: "#/components/schemas/LLMObsAzureOpenAIMetadata" + id: + description: Unique identifier for the integration account. + example: "account-abc123" + type: string + integration: + description: The name of the LLM provider integration. + example: openai + type: string + vertex_ai_metadata: + $ref: "#/components/schemas/LLMObsVertexAIMetadata" + required: + - id + - account_id + - account_name + - integration + type: object + LLMObsIntegrationInferenceCodesResponse: + description: List of generated code snippets for the inference configuration. + items: + $ref: "#/components/schemas/LLMObsInferenceCode" + type: array + LLMObsIntegrationInferenceRequest: + description: Parameters for an LLM inference request. + properties: + anthropic_metadata: + $ref: "#/components/schemas/LLMObsAnthropicMetadata" + nullable: true + azure_openai_metadata: + $ref: "#/components/schemas/LLMObsAzureOpenAIMetadata" + nullable: true + bedrock_metadata: + $ref: "#/components/schemas/LLMObsBedrockMetadata" + nullable: true + frequency_penalty: + description: Penalty for token frequency to reduce repetition. + example: 0.0 + format: double + nullable: true + type: number + json_schema: + description: JSON schema for structured output, if supported by the model. + example: '{"type":"object","properties":{"answer":{"type":"string"}}}' + nullable: true + type: string + max_completion_tokens: + description: Maximum number of completion tokens to generate (alternative to max_tokens for some providers). + example: 1024 + format: int64 + nullable: true + type: integer + max_tokens: + description: Maximum number of tokens to generate. + example: 1024 + format: int64 + nullable: true + type: integer + messages: + $ref: "#/components/schemas/LLMObsInferenceMessagesList" + model_id: + description: The model identifier to use for inference. + example: "gpt-4o" + type: string + openai_metadata: + $ref: "#/components/schemas/LLMObsOpenAIMetadata" + nullable: true + presence_penalty: + description: Penalty for token presence to encourage topic diversity. + example: 0.0 + format: double + nullable: true + type: number + temperature: + description: Sampling temperature between 0 and 2. Higher values produce more random output. + example: 0.7 + format: double + nullable: true + type: number + tools: + $ref: "#/components/schemas/LLMObsInferenceToolsList" + top_k: + description: Top-K sampling parameter. + example: 50 + format: int64 + nullable: true + type: integer + top_p: + description: Nucleus sampling probability mass. + example: 1.0 + format: double + nullable: true + type: number + vertex_ai_metadata: + $ref: "#/components/schemas/LLMObsVertexAIMetadata" + nullable: true + required: + - model_id + - messages + type: object + LLMObsIntegrationInferenceResponse: + description: The result of an LLM inference request, including input parameters and the model response. + properties: + anthropic_metadata: + $ref: "#/components/schemas/LLMObsAnthropicMetadata" + nullable: true + azure_openai_metadata: + $ref: "#/components/schemas/LLMObsAzureOpenAIMetadata" + nullable: true + bedrock_metadata: + $ref: "#/components/schemas/LLMObsBedrockMetadata" + nullable: true + error_response: + $ref: "#/components/schemas/LLMObsInferenceErrorResponse" + frequency_penalty: + description: Frequency penalty that was applied. + example: 0.0 + format: double + nullable: true + type: number + json_schema: + description: JSON schema that was applied for structured output. + example: '{"type":"object","properties":{"answer":{"type":"string"}}}' + nullable: true + type: string + max_completion_tokens: + description: Maximum number of completion tokens that were configured. + example: 1024 + format: int64 + nullable: true + type: integer + max_tokens: + description: Maximum number of tokens that were configured. + example: 1024 + format: int64 + nullable: true + type: integer + messages: + $ref: "#/components/schemas/LLMObsInferenceMessagesList" + model_id: + description: The model identifier used for inference. + example: "gpt-4o" + type: string + openai_metadata: + $ref: "#/components/schemas/LLMObsOpenAIMetadata" + nullable: true + presence_penalty: + description: Presence penalty that was applied. + example: 0.0 + format: double + nullable: true + type: number + response: + $ref: "#/components/schemas/LLMObsInferenceRunResult" + temperature: + description: Sampling temperature that was used. + example: 0.7 + format: double + nullable: true + type: number + tools: + $ref: "#/components/schemas/LLMObsInferenceToolsList" + top_k: + description: Top-K sampling parameter that was used. + example: 50 + format: int64 + nullable: true + type: integer + top_p: + description: Nucleus sampling parameter that was used. + example: 1.0 + format: double + nullable: true + type: number + vertex_ai_metadata: + $ref: "#/components/schemas/LLMObsVertexAIMetadata" + nullable: true + required: + - model_id + - messages + - response + type: object + LLMObsIntegrationModel: + description: A model available for a given LLM provider integration and account. + properties: + has_access: + description: Whether the account has access to this model. + example: true + type: boolean + id: + description: Unique identifier for the model entry. + example: "gpt-4o" + type: string + integration: + description: The name of the LLM provider integration. + example: "openai" + type: string + integration_display_name: + description: Human-readable name of the LLM provider integration. + example: "OpenAI" + type: string + json_schema: + description: Whether the model supports structured output via JSON schema. + example: true + type: boolean + model_display_name: + description: Human-readable model name. + example: "GPT-4o" + type: string + model_id: + description: Provider-specific model identifier used in inference calls. + example: "gpt-4o" + type: string + provider: + description: The underlying model provider. + example: "openai" + type: string + provider_display_name: + description: Human-readable name of the underlying model provider. + example: "OpenAI" + type: string + region_prefix_overrides: + $ref: "#/components/schemas/LLMObsIntegrationModelRegionPrefixOverrides" + required: + - id + - model_id + - model_display_name + - integration + - integration_display_name + - provider + - provider_display_name + - json_schema + - has_access + type: object + LLMObsIntegrationModelRegionPrefixOverrides: + additionalProperties: + type: string + description: Map of region-specific model ID prefix overrides. + example: + us-east-1: "us." + type: object + LLMObsIntegrationName: + description: The name of a supported LLM provider integration. + enum: + - openai + - amazon_bedrock + - anthropic + - azure_openai + - vertex_ai + - llmproxy + example: openai + type: string + x-enum-varnames: + - OPENAI + - AMAZON_BEDROCK + - ANTHROPIC + - AZURE_OPENAI + - VERTEX_AI + - LLMPROXY + LLMObsInternalReasoning: + description: The model's internal reasoning or thinking output, if available. + properties: + reasoning_tokens: + description: Number of tokens used for internal reasoning. + example: 256 + format: int64 + nullable: true + type: integer + text: + description: The reasoning text produced by the model. + example: "Let me think about this step by step..." + type: string + required: + - text + type: object LLMObsLabelSchema: description: Schema definition for a single label in an annotation queue. properties: @@ -44303,6 +44923,44 @@ components: - CATEGORICAL - BOOLEAN - JSON + LLMObsOpenAIMetadata: + description: OpenAI-specific metadata for an inference request. + properties: + reasoning_effort: + $ref: "#/components/schemas/LLMObsOpenAIReasoningEffort" + reasoning_summary: + $ref: "#/components/schemas/LLMObsOpenAIReasoningSummary" + type: object + LLMObsOpenAIReasoningEffort: + description: The reasoning effort level for OpenAI models that support it. + enum: + - none + - low + - medium + - high + - xhigh + example: medium + nullable: true + type: string + x-enum-varnames: + - NONE + - LOW + - MEDIUM + - HIGH + - XHIGH + LLMObsOpenAIReasoningSummary: + description: The verbosity of the reasoning summary. + enum: + - auto + - concise + - detailed + example: auto + nullable: true + type: string + x-enum-varnames: + - AUTO + - CONCISE + - DETAILED LLMObsProjectDataAttributesRequest: description: Attributes for creating an LLM Observability project. properties: @@ -44930,6 +45588,25 @@ components: - TRACE - EXPERIMENT_TRACE - SESSION + LLMObsVertexAIMetadata: + description: Vertex AI-specific metadata for an integration account or inference request. + properties: + location: + description: The Vertex AI region. + example: "us-central1" + type: string + project: + description: The Google Cloud project ID. + example: "my-gcp-project" + type: string + project_ids: + description: List of Google Cloud project IDs available to the service account. + example: + - "my-gcp-project" + items: + type: string + type: array + type: object Language: description: Programming language enum: @@ -121444,6 +122121,199 @@ paths: x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/integrations/{integration}/accounts: + get: + description: Retrieve the list of configured accounts for the specified LLM provider integration. + operationId: ListLLMObsIntegrationAccounts + parameters: + - $ref: "#/components/parameters/LLMObsIntegrationPathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + - account_id: "org-XYZ123" + account_name: "Production OpenAI" + account_region: "" + id: "account-abc123" + integration: "openai" + schema: + items: + $ref: "#/components/schemas/LLMObsIntegrationAccount" + type: array + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List LLM integration accounts + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in Preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/integrations/{integration}/{account_id}/inference: + post: + description: Run an LLM inference request through the specified integration and account, returning the model response and token usage. + operationId: CreateLLMObsIntegrationInference + parameters: + - $ref: "#/components/parameters/LLMObsIntegrationPathParameter" + - $ref: "#/components/parameters/LLMObsAccountIDPathParameter" + requestBody: + content: + application/json: + examples: + default: + value: + max_tokens: 256 + messages: + - content: "What is the capital of France?" + role: "user" + model_id: "gpt-4o" + temperature: 0.7 + schema: + $ref: "#/components/schemas/LLMObsIntegrationInferenceRequest" + description: Inference request parameters. + required: true + responses: + "200": + content: + application/json: + examples: + default: + value: + max_tokens: 256 + messages: + - content: "What is the capital of France?" + role: "user" + model_id: "gpt-4o" + response: + assessment: "pass" + content: "The capital of France is Paris." + finish_reason: "stop" + inference_codes: [] + input_tokens: 15 + latency: 843 + output_tokens: 9 + tools: [] + total_tokens: 24 + temperature: 0.7 + schema: + $ref: "#/components/schemas/LLMObsIntegrationInferenceResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Run an LLM inference + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in Preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/integrations/{integration}/{account_id}/models: + get: + description: Retrieve the list of models available for the specified LLM provider integration and account. + operationId: ListLLMObsIntegrationModels + parameters: + - $ref: "#/components/parameters/LLMObsIntegrationPathParameter" + - $ref: "#/components/parameters/LLMObsAccountIDPathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + - has_access: true + id: "gpt-4o" + integration: "openai" + integration_display_name: "OpenAI" + json_schema: true + model_display_name: "GPT-4o" + model_id: "gpt-4o" + provider: "openai" + provider_display_name: "OpenAI" + schema: + items: + $ref: "#/components/schemas/LLMObsIntegrationModel" + type: array + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List LLM integration models + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in Preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/llm-obs/v1/projects: get: description: List all LLM Observability projects sorted by creation date, newest first. diff --git a/features/v2/llm_observability.feature b/features/v2/llm_observability.feature index ec671390145d..0e1a7e984fa0 100644 --- a/features/v2/llm_observability.feature +++ b/features/v2/llm_observability.feature @@ -588,6 +588,40 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM integration accounts returns "Bad Request" response + Given operation "ListLLMObsIntegrationAccounts" enabled + And new "ListLLMObsIntegrationAccounts" request + And request contains "integration" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM integration accounts returns "OK" response + Given operation "ListLLMObsIntegrationAccounts" enabled + And new "ListLLMObsIntegrationAccounts" request + And request contains "integration" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM integration models returns "Bad Request" response + Given operation "ListLLMObsIntegrationModels" enabled + And new "ListLLMObsIntegrationModels" request + And request contains "integration" parameter from "REPLACE.ME" + And request contains "account_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM integration models returns "OK" response + Given operation "ListLLMObsIntegrationModels" enabled + And new "ListLLMObsIntegrationModels" request + And request contains "integration" parameter from "REPLACE.ME" + And request contains "account_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: List events for an LLM Observability experiment returns "Bad Request" response Given operation "ListLLMObsExperimentEvents" enabled @@ -639,6 +673,26 @@ Feature: LLM Observability When the request is sent Then the response status is 404 Not Found + @generated @skip @team:DataDog/ml-observability + Scenario: Run an LLM inference returns "Bad Request" response + Given operation "CreateLLMObsIntegrationInference" enabled + And new "CreateLLMObsIntegrationInference" request + And request contains "integration" parameter from "REPLACE.ME" + And request contains "account_id" parameter from "REPLACE.ME" + And body with value {"anthropic_metadata": {"effort": "medium", "thinking": {"budget_tokens": 1024, "type": "enabled"}}, "azure_openai_metadata": {"deployment_id": "my-gpt4-deployment", "model_version": "0613", "resource_name": "my-azure-resource"}, "bedrock_metadata": {"region": "us-east-1"}, "frequency_penalty": 0.0, "json_schema": "{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}}}", "max_completion_tokens": 1024, "max_tokens": 1024, "messages": [{"content": "What is the capital of France?", "contents": [{"type": "text", "value": {"text": "Hello, how can I help you?", "tool_call": {"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}}}], "id": "msg_001", "role": "user", "tool_calls": [{"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}], "tool_results": [{"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}]}], "model_id": "gpt-4o", "openai_metadata": {"reasoning_effort": "medium", "reasoning_summary": "auto"}, "presence_penalty": 0.0, "temperature": 0.7, "tools": [{"function": {"description": "Get the current weather for a location.", "name": "get_weather", "parameters": {"properties": {"location": {"type": "string"}}, "type": "object"}}, "type": "function"}], "top_k": 50, "top_p": 1.0, "vertex_ai_metadata": {"location": "us-central1", "project": "my-gcp-project", "project_ids": ["my-gcp-project"]}} + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Run an LLM inference returns "OK" response + Given operation "CreateLLMObsIntegrationInference" enabled + And new "CreateLLMObsIntegrationInference" request + And request contains "integration" parameter from "REPLACE.ME" + And request contains "account_id" parameter from "REPLACE.ME" + And body with value {"anthropic_metadata": {"effort": "medium", "thinking": {"budget_tokens": 1024, "type": "enabled"}}, "azure_openai_metadata": {"deployment_id": "my-gpt4-deployment", "model_version": "0613", "resource_name": "my-azure-resource"}, "bedrock_metadata": {"region": "us-east-1"}, "frequency_penalty": 0.0, "json_schema": "{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}}}", "max_completion_tokens": 1024, "max_tokens": 1024, "messages": [{"content": "What is the capital of France?", "contents": [{"type": "text", "value": {"text": "Hello, how can I help you?", "tool_call": {"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}}}], "id": "msg_001", "role": "user", "tool_calls": [{"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}], "tool_results": [{"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}]}], "model_id": "gpt-4o", "openai_metadata": {"reasoning_effort": "medium", "reasoning_summary": "auto"}, "presence_penalty": 0.0, "temperature": 0.7, "tools": [{"function": {"description": "Get the current weather for a location.", "name": "get_weather", "parameters": {"properties": {"location": {"type": "string"}}, "type": "object"}}, "type": "function"}], "top_k": 50, "top_p": 1.0, "vertex_ai_metadata": {"location": "us-central1", "project": "my-gcp-project", "project_ids": ["my-gcp-project"]}} + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Search LLM Observability experimentation entities returns "Bad Request" response Given operation "SearchLLMObsExperimentation" enabled diff --git a/features/v2/undo.json b/features/v2/undo.json index 02753f1a80a7..9c1afc7d96bb 100644 --- a/features/v2/undo.json +++ b/features/v2/undo.json @@ -3797,6 +3797,24 @@ "type": "unsafe" } }, + "ListLLMObsIntegrationAccounts": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, + "CreateLLMObsIntegrationInference": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, + "ListLLMObsIntegrationModels": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, "ListLLMObsProjects": { "tag": "LLM Observability", "undo": { diff --git a/private/bdd_runner/src/support/scenarios_model_mapping.ts b/private/bdd_runner/src/support/scenarios_model_mapping.ts index e5be093fdc5a..0495153fd3ec 100644 --- a/private/bdd_runner/src/support/scenarios_model_mapping.ts +++ b/private/bdd_runner/src/support/scenarios_model_mapping.ts @@ -2755,6 +2755,39 @@ export const ScenariosModelMappings: { [key: string]: OperationMapping } = { }, operationResponseType: "{}", }, + "LLMObservabilityApi.V2.ListLLMObsIntegrationAccounts": { + integration: { + type: "LLMObsIntegrationName", + format: "", + }, + operationResponseType: "Array", + }, + "LLMObservabilityApi.V2.CreateLLMObsIntegrationInference": { + integration: { + type: "LLMObsIntegrationName", + format: "", + }, + accountId: { + type: "string", + format: "", + }, + body: { + type: "LLMObsIntegrationInferenceRequest", + format: "", + }, + operationResponseType: "LLMObsIntegrationInferenceResponse", + }, + "LLMObservabilityApi.V2.ListLLMObsIntegrationModels": { + integration: { + type: "LLMObsIntegrationName", + format: "", + }, + accountId: { + type: "string", + format: "", + }, + operationResponseType: "Array", + }, "LLMObservabilityApi.V2.ListLLMObsProjects": { filterId: { type: "string", diff --git a/services/llm_observability/src/v2/LLMObservabilityApi.ts b/services/llm_observability/src/v2/LLMObservabilityApi.ts index 3a972f7ef96e..012d5f6e46df 100644 --- a/services/llm_observability/src/v2/LLMObservabilityApi.ts +++ b/services/llm_observability/src/v2/LLMObservabilityApi.ts @@ -62,6 +62,11 @@ import { LLMObsExperimentRequest } from "./models/LLMObsExperimentRequest"; import { LLMObsExperimentResponse } from "./models/LLMObsExperimentResponse"; import { LLMObsExperimentsResponse } from "./models/LLMObsExperimentsResponse"; import { LLMObsExperimentUpdateRequest } from "./models/LLMObsExperimentUpdateRequest"; +import { LLMObsIntegrationAccount } from "./models/LLMObsIntegrationAccount"; +import { LLMObsIntegrationInferenceRequest } from "./models/LLMObsIntegrationInferenceRequest"; +import { LLMObsIntegrationInferenceResponse } from "./models/LLMObsIntegrationInferenceResponse"; +import { LLMObsIntegrationModel } from "./models/LLMObsIntegrationModel"; +import { LLMObsIntegrationName } from "./models/LLMObsIntegrationName"; import { LLMObsProjectRequest } from "./models/LLMObsProjectRequest"; import { LLMObsProjectResponse } from "./models/LLMObsProjectResponse"; import { LLMObsProjectsResponse } from "./models/LLMObsProjectsResponse"; @@ -552,6 +557,84 @@ export class LLMObservabilityApiRequestFactory extends BaseAPIRequestFactory { return requestContext; } + public async createLLMObsIntegrationInference( + integration: LLMObsIntegrationName, + accountId: string, + body: LLMObsIntegrationInferenceRequest, + _options?: Configuration, + ): Promise { + const _config = _options || this.configuration; + + if ( + !_config.unstableOperations[ + "LLMObservabilityApi.v2.createLLMObsIntegrationInference" + ] + ) { + throw new Error( + "Unstable operation 'createLLMObsIntegrationInference' is disabled. Enable it by setting `configuration.unstableOperations['LLMObservabilityApi.v2.createLLMObsIntegrationInference'] = true`", + ); + } + + // verify required parameter 'integration' is not null or undefined + if (integration === null || integration === undefined) { + throw new RequiredError( + "integration", + "createLLMObsIntegrationInference", + ); + } + + // verify required parameter 'accountId' is not null or undefined + if (accountId === null || accountId === undefined) { + throw new RequiredError("accountId", "createLLMObsIntegrationInference"); + } + + // verify required parameter 'body' is not null or undefined + if (body === null || body === undefined) { + throw new RequiredError("body", "createLLMObsIntegrationInference"); + } + + // Path Params + const localVarPath = + "/api/v2/llm-obs/v1/integrations/{integration}/{account_id}/inference" + .replace("{integration}", encodeURIComponent(String(integration))) + .replace("{account_id}", encodeURIComponent(String(accountId))); + + // Make Request Context + const { server, overrides } = _config.getServerAndOverrides( + "LLMObservabilityApi.v2.createLLMObsIntegrationInference", + LLMObservabilityApi.operationServers, + ); + const requestContext = server.makeRequestContext( + localVarPath, + HttpMethod.POST, + overrides, + ); + requestContext.setHeaderParam("Accept", "application/json"); + requestContext.setHttpConfig(_config.httpConfig); + + // Set User-Agent + if (this.userAgent) { + requestContext.setHeaderParam("User-Agent", this.userAgent); + } + + // Body Params + const contentType = getPreferredMediaType(["application/json"]); + requestContext.setHeaderParam("Content-Type", contentType); + const serializedBody = stringify( + serialize(body, TypingInfo, "LLMObsIntegrationInferenceRequest", ""), + contentType, + ); + requestContext.setBody(serializedBody); + + // Apply auth methods + applySecurityAuthentication(_config, requestContext, [ + "apiKeyAuth", + "appKeyAuth", + ]); + + return requestContext; + } + public async createLLMObsProject( body: LLMObsProjectRequest, _options?: Configuration, @@ -1680,6 +1763,121 @@ export class LLMObservabilityApiRequestFactory extends BaseAPIRequestFactory { return requestContext; } + public async listLLMObsIntegrationAccounts( + integration: LLMObsIntegrationName, + _options?: Configuration, + ): Promise { + const _config = _options || this.configuration; + + if ( + !_config.unstableOperations[ + "LLMObservabilityApi.v2.listLLMObsIntegrationAccounts" + ] + ) { + throw new Error( + "Unstable operation 'listLLMObsIntegrationAccounts' is disabled. Enable it by setting `configuration.unstableOperations['LLMObservabilityApi.v2.listLLMObsIntegrationAccounts'] = true`", + ); + } + + // verify required parameter 'integration' is not null or undefined + if (integration === null || integration === undefined) { + throw new RequiredError("integration", "listLLMObsIntegrationAccounts"); + } + + // Path Params + const localVarPath = + "/api/v2/llm-obs/v1/integrations/{integration}/accounts".replace( + "{integration}", + encodeURIComponent(String(integration)), + ); + + // Make Request Context + const { server, overrides } = _config.getServerAndOverrides( + "LLMObservabilityApi.v2.listLLMObsIntegrationAccounts", + LLMObservabilityApi.operationServers, + ); + const requestContext = server.makeRequestContext( + localVarPath, + HttpMethod.GET, + overrides, + ); + requestContext.setHeaderParam("Accept", "application/json"); + requestContext.setHttpConfig(_config.httpConfig); + + // Set User-Agent + if (this.userAgent) { + requestContext.setHeaderParam("User-Agent", this.userAgent); + } + + // Apply auth methods + applySecurityAuthentication(_config, requestContext, [ + "apiKeyAuth", + "appKeyAuth", + ]); + + return requestContext; + } + + public async listLLMObsIntegrationModels( + integration: LLMObsIntegrationName, + accountId: string, + _options?: Configuration, + ): Promise { + const _config = _options || this.configuration; + + if ( + !_config.unstableOperations[ + "LLMObservabilityApi.v2.listLLMObsIntegrationModels" + ] + ) { + throw new Error( + "Unstable operation 'listLLMObsIntegrationModels' is disabled. Enable it by setting `configuration.unstableOperations['LLMObservabilityApi.v2.listLLMObsIntegrationModels'] = true`", + ); + } + + // verify required parameter 'integration' is not null or undefined + if (integration === null || integration === undefined) { + throw new RequiredError("integration", "listLLMObsIntegrationModels"); + } + + // verify required parameter 'accountId' is not null or undefined + if (accountId === null || accountId === undefined) { + throw new RequiredError("accountId", "listLLMObsIntegrationModels"); + } + + // Path Params + const localVarPath = + "/api/v2/llm-obs/v1/integrations/{integration}/{account_id}/models" + .replace("{integration}", encodeURIComponent(String(integration))) + .replace("{account_id}", encodeURIComponent(String(accountId))); + + // Make Request Context + const { server, overrides } = _config.getServerAndOverrides( + "LLMObservabilityApi.v2.listLLMObsIntegrationModels", + LLMObservabilityApi.operationServers, + ); + const requestContext = server.makeRequestContext( + localVarPath, + HttpMethod.GET, + overrides, + ); + requestContext.setHeaderParam("Accept", "application/json"); + requestContext.setHttpConfig(_config.httpConfig); + + // Set User-Agent + if (this.userAgent) { + requestContext.setHeaderParam("User-Agent", this.userAgent); + } + + // Apply auth methods + applySecurityAuthentication(_config, requestContext, [ + "apiKeyAuth", + "appKeyAuth", + ]); + + return requestContext; + } + public async listLLMObsProjects( filterId?: string, filterName?: string, @@ -3144,6 +3342,88 @@ export class LLMObservabilityApiResponseProcessor { ); } + /** + * Unwraps the actual response sent by the server from the response context and deserializes the response content + * to the expected objects + * + * @params response Response returned by the server for a request to createLLMObsIntegrationInference + * @throws ApiException if the response code was not in [200, 299] + */ + public async createLLMObsIntegrationInference( + response: ResponseContext, + ): Promise { + const contentType = normalizeMediaType(response.headers["content-type"]); + if (response.httpStatusCode === 200) { + const body: LLMObsIntegrationInferenceResponse = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "LLMObsIntegrationInferenceResponse", + ) as LLMObsIntegrationInferenceResponse; + return body; + } + if ( + response.httpStatusCode === 400 || + response.httpStatusCode === 401 || + response.httpStatusCode === 403 || + response.httpStatusCode === 500 + ) { + const bodyText = parse(await response.body.text(), contentType); + let body: JSONAPIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "JSONAPIErrorResponse", + ) as JSONAPIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException( + response.httpStatusCode, + body, + ); + } + if (response.httpStatusCode === 429) { + const bodyText = parse(await response.body.text(), contentType); + let body: APIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "APIErrorResponse", + ) as APIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException(response.httpStatusCode, body); + } + + // Work around for missing responses in specification, e.g. for petstore.yaml + if (response.httpStatusCode >= 200 && response.httpStatusCode <= 299) { + const body: LLMObsIntegrationInferenceResponse = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "LLMObsIntegrationInferenceResponse", + "", + ) as LLMObsIntegrationInferenceResponse; + return body; + } + + const body = (await response.body.text()) || ""; + throw new ApiException( + response.httpStatusCode, + 'Unknown API Status Code!\nBody: "' + body + '"', + ); + } + /** * Unwraps the actual response sent by the server from the response context and deserializes the response content * to the expected objects @@ -4451,6 +4731,168 @@ export class LLMObservabilityApiResponseProcessor { ); } + /** + * Unwraps the actual response sent by the server from the response context and deserializes the response content + * to the expected objects + * + * @params response Response returned by the server for a request to listLLMObsIntegrationAccounts + * @throws ApiException if the response code was not in [200, 299] + */ + public async listLLMObsIntegrationAccounts( + response: ResponseContext, + ): Promise> { + const contentType = normalizeMediaType(response.headers["content-type"]); + if (response.httpStatusCode === 200) { + const body: Array = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "Array", + ) as Array; + return body; + } + if ( + response.httpStatusCode === 400 || + response.httpStatusCode === 401 || + response.httpStatusCode === 403 + ) { + const bodyText = parse(await response.body.text(), contentType); + let body: JSONAPIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "JSONAPIErrorResponse", + ) as JSONAPIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException( + response.httpStatusCode, + body, + ); + } + if (response.httpStatusCode === 429) { + const bodyText = parse(await response.body.text(), contentType); + let body: APIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "APIErrorResponse", + ) as APIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException(response.httpStatusCode, body); + } + + // Work around for missing responses in specification, e.g. for petstore.yaml + if (response.httpStatusCode >= 200 && response.httpStatusCode <= 299) { + const body: Array = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "Array", + "", + ) as Array; + return body; + } + + const body = (await response.body.text()) || ""; + throw new ApiException( + response.httpStatusCode, + 'Unknown API Status Code!\nBody: "' + body + '"', + ); + } + + /** + * Unwraps the actual response sent by the server from the response context and deserializes the response content + * to the expected objects + * + * @params response Response returned by the server for a request to listLLMObsIntegrationModels + * @throws ApiException if the response code was not in [200, 299] + */ + public async listLLMObsIntegrationModels( + response: ResponseContext, + ): Promise> { + const contentType = normalizeMediaType(response.headers["content-type"]); + if (response.httpStatusCode === 200) { + const body: Array = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "Array", + ) as Array; + return body; + } + if ( + response.httpStatusCode === 400 || + response.httpStatusCode === 401 || + response.httpStatusCode === 403 + ) { + const bodyText = parse(await response.body.text(), contentType); + let body: JSONAPIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "JSONAPIErrorResponse", + ) as JSONAPIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException( + response.httpStatusCode, + body, + ); + } + if (response.httpStatusCode === 429) { + const bodyText = parse(await response.body.text(), contentType); + let body: APIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "APIErrorResponse", + ) as APIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException(response.httpStatusCode, body); + } + + // Work around for missing responses in specification, e.g. for petstore.yaml + if (response.httpStatusCode >= 200 && response.httpStatusCode <= 299) { + const body: Array = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "Array", + "", + ) as Array; + return body; + } + + const body = (await response.body.text()) || ""; + throw new ApiException( + response.httpStatusCode, + 'Unknown API Status Code!\nBody: "' + body + '"', + ); + } + /** * Unwraps the actual response sent by the server from the response context and deserializes the response content * to the expected objects @@ -5505,6 +5947,24 @@ export interface LLMObservabilityApiCreateLLMObsExperimentEventsRequest { body: LLMObsExperimentEventsRequest; } +export interface LLMObservabilityApiCreateLLMObsIntegrationInferenceRequest { + /** + * The name of the LLM integration. + * @type LLMObsIntegrationName + */ + integration: LLMObsIntegrationName; + /** + * The ID of the integration account. + * @type string + */ + accountId: string; + /** + * Inference request parameters. + * @type LLMObsIntegrationInferenceRequest + */ + body: LLMObsIntegrationInferenceRequest; +} + export interface LLMObservabilityApiCreateLLMObsProjectRequest { /** * Create project payload. @@ -5736,6 +6196,27 @@ export interface LLMObservabilityApiListLLMObsExperimentsRequest { pageLimit?: number; } +export interface LLMObservabilityApiListLLMObsIntegrationAccountsRequest { + /** + * The name of the LLM integration. + * @type LLMObsIntegrationName + */ + integration: LLMObsIntegrationName; +} + +export interface LLMObservabilityApiListLLMObsIntegrationModelsRequest { + /** + * The name of the LLM integration. + * @type LLMObsIntegrationName + */ + integration: LLMObsIntegrationName; + /** + * The ID of the integration account. + * @type string + */ + accountId: string; +} + export interface LLMObservabilityApiListLLMObsProjectsRequest { /** * Filter projects by project ID. @@ -6143,6 +6624,32 @@ export class LLMObservabilityApi { }); } + /** + * Run an LLM inference request through the specified integration and account, returning the model response and token usage. + * @param param The request object + */ + public createLLMObsIntegrationInference( + param: LLMObservabilityApiCreateLLMObsIntegrationInferenceRequest, + options?: Configuration, + ): Promise { + const requestContextPromise = + this.requestFactory.createLLMObsIntegrationInference( + param.integration, + param.accountId, + param.body, + options, + ); + return requestContextPromise.then((requestContext) => { + return this.configuration.httpApi + .send(requestContext) + .then((responseContext) => { + return this.responseProcessor.createLLMObsIntegrationInference( + responseContext, + ); + }); + }); + } + /** * Create a new LLM Observability project. Returns the existing project if a name conflict occurs. * @param param The request object @@ -6544,6 +7051,55 @@ export class LLMObservabilityApi { }); } + /** + * Retrieve the list of configured accounts for the specified LLM provider integration. + * @param param The request object + */ + public listLLMObsIntegrationAccounts( + param: LLMObservabilityApiListLLMObsIntegrationAccountsRequest, + options?: Configuration, + ): Promise> { + const requestContextPromise = + this.requestFactory.listLLMObsIntegrationAccounts( + param.integration, + options, + ); + return requestContextPromise.then((requestContext) => { + return this.configuration.httpApi + .send(requestContext) + .then((responseContext) => { + return this.responseProcessor.listLLMObsIntegrationAccounts( + responseContext, + ); + }); + }); + } + + /** + * Retrieve the list of models available for the specified LLM provider integration and account. + * @param param The request object + */ + public listLLMObsIntegrationModels( + param: LLMObservabilityApiListLLMObsIntegrationModelsRequest, + options?: Configuration, + ): Promise> { + const requestContextPromise = + this.requestFactory.listLLMObsIntegrationModels( + param.integration, + param.accountId, + options, + ); + return requestContextPromise.then((requestContext) => { + return this.configuration.httpApi + .send(requestContext) + .then((responseContext) => { + return this.responseProcessor.listLLMObsIntegrationModels( + responseContext, + ); + }); + }); + } + /** * List all LLM Observability projects sorted by creation date, newest first. * @param param The request object diff --git a/services/llm_observability/src/v2/index.ts b/services/llm_observability/src/v2/index.ts index 7c44ed647d16..c55bf17f09f9 100644 --- a/services/llm_observability/src/v2/index.ts +++ b/services/llm_observability/src/v2/index.ts @@ -6,6 +6,7 @@ export { LLMObservabilityApiCreateLLMObsDatasetRecordsRequest, LLMObservabilityApiCreateLLMObsExperimentRequest, LLMObservabilityApiCreateLLMObsExperimentEventsRequest, + LLMObservabilityApiCreateLLMObsIntegrationInferenceRequest, LLMObservabilityApiCreateLLMObsProjectRequest, LLMObservabilityApiDeleteLLMObsAnnotationQueueRequest, LLMObservabilityApiDeleteLLMObsAnnotationQueueInteractionsRequest, @@ -23,6 +24,8 @@ export { LLMObservabilityApiListLLMObsDatasetsRequest, LLMObservabilityApiListLLMObsExperimentEventsRequest, LLMObservabilityApiListLLMObsExperimentsRequest, + LLMObservabilityApiListLLMObsIntegrationAccountsRequest, + LLMObservabilityApiListLLMObsIntegrationModelsRequest, LLMObservabilityApiListLLMObsProjectsRequest, LLMObservabilityApiListLLMObsSpansRequest, LLMObservabilityApiSearchLLMObsExperimentationRequest, @@ -77,6 +80,12 @@ export { LLMObsAnnotationQueueUpdateDataAttributesRequest } from "./models/LLMOb export { LLMObsAnnotationQueueUpdateDataRequest } from "./models/LLMObsAnnotationQueueUpdateDataRequest"; export { LLMObsAnnotationQueueUpdateRequest } from "./models/LLMObsAnnotationQueueUpdateRequest"; export { LLMObsAnnotationSchema } from "./models/LLMObsAnnotationSchema"; +export { LLMObsAnthropicEffort } from "./models/LLMObsAnthropicEffort"; +export { LLMObsAnthropicMetadata } from "./models/LLMObsAnthropicMetadata"; +export { LLMObsAnthropicThinkingConfig } from "./models/LLMObsAnthropicThinkingConfig"; +export { LLMObsAnthropicThinkingType } from "./models/LLMObsAnthropicThinkingType"; +export { LLMObsAzureOpenAIMetadata } from "./models/LLMObsAzureOpenAIMetadata"; +export { LLMObsBedrockMetadata } from "./models/LLMObsBedrockMetadata"; export { LLMObsContentBlock } from "./models/LLMObsContentBlock"; export { LLMObsContentBlockHeaderLevel } from "./models/LLMObsContentBlockHeaderLevel"; export { LLMObsContentBlockLLMObsTraceInteractionType } from "./models/LLMObsContentBlockLLMObsTraceInteractionType"; @@ -218,10 +227,29 @@ export { LLMObsExperimentType } from "./models/LLMObsExperimentType"; export { LLMObsExperimentUpdateDataAttributesRequest } from "./models/LLMObsExperimentUpdateDataAttributesRequest"; export { LLMObsExperimentUpdateDataRequest } from "./models/LLMObsExperimentUpdateDataRequest"; export { LLMObsExperimentUpdateRequest } from "./models/LLMObsExperimentUpdateRequest"; +export { LLMObsInferenceCode } from "./models/LLMObsInferenceCode"; +export { LLMObsInferenceContent } from "./models/LLMObsInferenceContent"; +export { LLMObsInferenceContentValue } from "./models/LLMObsInferenceContentValue"; +export { LLMObsInferenceErrorResponse } from "./models/LLMObsInferenceErrorResponse"; +export { LLMObsInferenceFunction } from "./models/LLMObsInferenceFunction"; +export { LLMObsInferenceMessage } from "./models/LLMObsInferenceMessage"; +export { LLMObsInferenceRunResult } from "./models/LLMObsInferenceRunResult"; +export { LLMObsInferenceTool } from "./models/LLMObsInferenceTool"; +export { LLMObsInferenceToolCall } from "./models/LLMObsInferenceToolCall"; +export { LLMObsInferenceToolResult } from "./models/LLMObsInferenceToolResult"; +export { LLMObsIntegrationAccount } from "./models/LLMObsIntegrationAccount"; +export { LLMObsIntegrationInferenceRequest } from "./models/LLMObsIntegrationInferenceRequest"; +export { LLMObsIntegrationInferenceResponse } from "./models/LLMObsIntegrationInferenceResponse"; +export { LLMObsIntegrationModel } from "./models/LLMObsIntegrationModel"; +export { LLMObsIntegrationName } from "./models/LLMObsIntegrationName"; +export { LLMObsInternalReasoning } from "./models/LLMObsInternalReasoning"; export { LLMObsLabelSchema } from "./models/LLMObsLabelSchema"; export { LLMObsLabelSchemaType } from "./models/LLMObsLabelSchemaType"; export { LLMObsMetricAssessment } from "./models/LLMObsMetricAssessment"; export { LLMObsMetricScoreType } from "./models/LLMObsMetricScoreType"; +export { LLMObsOpenAIMetadata } from "./models/LLMObsOpenAIMetadata"; +export { LLMObsOpenAIReasoningEffort } from "./models/LLMObsOpenAIReasoningEffort"; +export { LLMObsOpenAIReasoningSummary } from "./models/LLMObsOpenAIReasoningSummary"; export { LLMObsProjectDataAttributesRequest } from "./models/LLMObsProjectDataAttributesRequest"; export { LLMObsProjectDataAttributesResponse } from "./models/LLMObsProjectDataAttributesResponse"; export { LLMObsProjectDataRequest } from "./models/LLMObsProjectDataRequest"; @@ -258,3 +286,4 @@ export { LLMObsTraceAnnotatedInteractionItem } from "./models/LLMObsTraceAnnotat export { LLMObsTraceInteractionItem } from "./models/LLMObsTraceInteractionItem"; export { LLMObsTraceInteractionResponseItem } from "./models/LLMObsTraceInteractionResponseItem"; export { LLMObsTraceInteractionType } from "./models/LLMObsTraceInteractionType"; +export { LLMObsVertexAIMetadata } from "./models/LLMObsVertexAIMetadata"; diff --git a/services/llm_observability/src/v2/models/LLMObsAnthropicEffort.ts b/services/llm_observability/src/v2/models/LLMObsAnthropicEffort.ts new file mode 100644 index 000000000000..f51482d75c76 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsAnthropicEffort.ts @@ -0,0 +1,15 @@ +import { UnparsedObject } from "@datadog/datadog-api-client"; + +/** + * The effort level for Anthropic inference. + */ +export type LLMObsAnthropicEffort = + | typeof LOW + | typeof MEDIUM + | typeof HIGH + | typeof MAX + | UnparsedObject; +export const LOW = "low"; +export const MEDIUM = "medium"; +export const HIGH = "high"; +export const MAX = "max"; diff --git a/services/llm_observability/src/v2/models/LLMObsAnthropicMetadata.ts b/services/llm_observability/src/v2/models/LLMObsAnthropicMetadata.ts new file mode 100644 index 000000000000..6d76eefb144c --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsAnthropicMetadata.ts @@ -0,0 +1,55 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsAnthropicEffort } from "./LLMObsAnthropicEffort"; +import { LLMObsAnthropicThinkingConfig } from "./LLMObsAnthropicThinkingConfig"; + +/** + * Anthropic-specific metadata for an inference request. + */ +export class LLMObsAnthropicMetadata { + /** + * The effort level for Anthropic inference. + */ + "effort"?: LLMObsAnthropicEffort; + /** + * Configuration for Anthropic extended thinking feature. + */ + "thinking"?: LLMObsAnthropicThinkingConfig; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + effort: { + baseName: "effort", + type: "LLMObsAnthropicEffort", + }, + thinking: { + baseName: "thinking", + type: "LLMObsAnthropicThinkingConfig", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsAnthropicMetadata.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsAnthropicThinkingConfig.ts b/services/llm_observability/src/v2/models/LLMObsAnthropicThinkingConfig.ts new file mode 100644 index 000000000000..2258d1894dd2 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsAnthropicThinkingConfig.ts @@ -0,0 +1,56 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsAnthropicThinkingType } from "./LLMObsAnthropicThinkingType"; + +/** + * Configuration for Anthropic extended thinking feature. + */ +export class LLMObsAnthropicThinkingConfig { + /** + * Maximum token budget for extended thinking. Required when type is `enabled`. + */ + "budgetTokens"?: number; + /** + * The thinking mode for Anthropic extended thinking. + */ + "type": LLMObsAnthropicThinkingType; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + budgetTokens: { + baseName: "budget_tokens", + type: "number", + format: "int64", + }, + type: { + baseName: "type", + type: "LLMObsAnthropicThinkingType", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsAnthropicThinkingConfig.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsAnthropicThinkingType.ts b/services/llm_observability/src/v2/models/LLMObsAnthropicThinkingType.ts new file mode 100644 index 000000000000..2a530529064e --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsAnthropicThinkingType.ts @@ -0,0 +1,13 @@ +import { UnparsedObject } from "@datadog/datadog-api-client"; + +/** + * The thinking mode for Anthropic extended thinking. + */ +export type LLMObsAnthropicThinkingType = + | typeof ENABLED + | typeof DISABLED + | typeof ADAPTIVE + | UnparsedObject; +export const ENABLED = "enabled"; +export const DISABLED = "disabled"; +export const ADAPTIVE = "adaptive"; diff --git a/services/llm_observability/src/v2/models/LLMObsAzureOpenAIMetadata.ts b/services/llm_observability/src/v2/models/LLMObsAzureOpenAIMetadata.ts new file mode 100644 index 000000000000..c3a338d4da8f --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsAzureOpenAIMetadata.ts @@ -0,0 +1,60 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * Azure OpenAI-specific metadata for an integration account or inference request. + */ +export class LLMObsAzureOpenAIMetadata { + /** + * The Azure OpenAI deployment ID. + */ + "deploymentId"?: string; + /** + * The model version deployed in Azure. + */ + "modelVersion"?: string; + /** + * The Azure OpenAI resource name. + */ + "resourceName"?: string; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + deploymentId: { + baseName: "deployment_id", + type: "string", + }, + modelVersion: { + baseName: "model_version", + type: "string", + }, + resourceName: { + baseName: "resource_name", + type: "string", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsAzureOpenAIMetadata.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsBedrockMetadata.ts b/services/llm_observability/src/v2/models/LLMObsBedrockMetadata.ts new file mode 100644 index 000000000000..569ac63b71de --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsBedrockMetadata.ts @@ -0,0 +1,44 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * Amazon Bedrock-specific metadata for an inference request. + */ +export class LLMObsBedrockMetadata { + /** + * The AWS region for the Bedrock request. + */ + "region"?: string; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + region: { + baseName: "region", + type: "string", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsBedrockMetadata.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsInferenceCode.ts b/services/llm_observability/src/v2/models/LLMObsInferenceCode.ts new file mode 100644 index 000000000000..251c2c9a7db5 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsInferenceCode.ts @@ -0,0 +1,63 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * A generated code snippet for running an inference request programmatically. + */ +export class LLMObsInferenceCode { + /** + * The generated code content. + */ + "code": string; + /** + * Unique identifier for the code snippet. + */ + "id": string; + /** + * The programming language or SDK type of the code snippet. + */ + "type": string; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + code: { + baseName: "code", + type: "string", + required: true, + }, + id: { + baseName: "id", + type: "string", + required: true, + }, + type: { + baseName: "type", + type: "string", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsInferenceCode.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsInferenceContent.ts b/services/llm_observability/src/v2/models/LLMObsInferenceContent.ts new file mode 100644 index 000000000000..bab998db7bf6 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsInferenceContent.ts @@ -0,0 +1,56 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsInferenceContentValue } from "./LLMObsInferenceContentValue"; + +/** + * A structured content block within a message. + */ +export class LLMObsInferenceContent { + /** + * The content block type. + */ + "type": string; + /** + * The typed value of a message content block. + */ + "value": LLMObsInferenceContentValue; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + type: { + baseName: "type", + type: "string", + required: true, + }, + value: { + baseName: "value", + type: "LLMObsInferenceContentValue", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsInferenceContent.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsInferenceContentValue.ts b/services/llm_observability/src/v2/models/LLMObsInferenceContentValue.ts new file mode 100644 index 000000000000..b184d4f83ba7 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsInferenceContentValue.ts @@ -0,0 +1,63 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsInferenceToolCall } from "./LLMObsInferenceToolCall"; +import { LLMObsInferenceToolResult } from "./LLMObsInferenceToolResult"; + +/** + * The typed value of a message content block. + */ +export class LLMObsInferenceContentValue { + /** + * Plain text content. + */ + "text"?: string; + /** + * A tool call made during LLM inference. + */ + "toolCall"?: LLMObsInferenceToolCall; + /** + * The result returned by a tool call during LLM inference. + */ + "toolCallResult"?: LLMObsInferenceToolResult; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + text: { + baseName: "text", + type: "string", + }, + toolCall: { + baseName: "tool_call", + type: "LLMObsInferenceToolCall", + }, + toolCallResult: { + baseName: "tool_call_result", + type: "LLMObsInferenceToolResult", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsInferenceContentValue.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsInferenceErrorResponse.ts b/services/llm_observability/src/v2/models/LLMObsInferenceErrorResponse.ts new file mode 100644 index 000000000000..bcac7aced97f --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsInferenceErrorResponse.ts @@ -0,0 +1,54 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * Error details returned when an inference provider returns an error. + */ +export class LLMObsInferenceErrorResponse { + /** + * A human-readable description of the error. + */ + "message": string; + /** + * The provider-specific error type. + */ + "type": string; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + message: { + baseName: "message", + type: "string", + required: true, + }, + type: { + baseName: "type", + type: "string", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsInferenceErrorResponse.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsInferenceFunction.ts b/services/llm_observability/src/v2/models/LLMObsInferenceFunction.ts new file mode 100644 index 000000000000..645100d58f0d --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsInferenceFunction.ts @@ -0,0 +1,62 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * A function definition for a tool available to the model. + */ +export class LLMObsInferenceFunction { + /** + * A description of what the function does. + */ + "description"?: string; + /** + * The name of the function. + */ + "name": string; + /** + * JSON schema describing the function parameters. + */ + "parameters": { [key: string]: any }; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + description: { + baseName: "description", + type: "string", + }, + name: { + baseName: "name", + type: "string", + required: true, + }, + parameters: { + baseName: "parameters", + type: "{ [key: string]: any; }", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsInferenceFunction.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsInferenceMessage.ts b/services/llm_observability/src/v2/models/LLMObsInferenceMessage.ts new file mode 100644 index 000000000000..bb711e2a27ca --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsInferenceMessage.ts @@ -0,0 +1,88 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsInferenceContent } from "./LLMObsInferenceContent"; +import { LLMObsInferenceToolCall } from "./LLMObsInferenceToolCall"; +import { LLMObsInferenceToolResult } from "./LLMObsInferenceToolResult"; + +/** + * A single message in an LLM inference conversation. + */ +export class LLMObsInferenceMessage { + /** + * Plain text content of the message. + */ + "content"?: string; + /** + * List of structured content blocks in a message. + */ + "contents"?: Array; + /** + * Unique identifier for the message. + */ + "id"?: string; + /** + * The role of the message author. + */ + "role"?: string; + /** + * List of tool calls in a message. + */ + "toolCalls"?: Array; + /** + * List of tool results in a message. + */ + "toolResults"?: Array; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + content: { + baseName: "content", + type: "string", + }, + contents: { + baseName: "contents", + type: "Array", + }, + id: { + baseName: "id", + type: "string", + }, + role: { + baseName: "role", + type: "string", + }, + toolCalls: { + baseName: "tool_calls", + type: "Array", + }, + toolResults: { + baseName: "tool_results", + type: "Array", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsInferenceMessage.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsInferenceRunResult.ts b/services/llm_observability/src/v2/models/LLMObsInferenceRunResult.ts new file mode 100644 index 000000000000..388240534b93 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsInferenceRunResult.ts @@ -0,0 +1,133 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsInferenceCode } from "./LLMObsInferenceCode"; +import { LLMObsInferenceTool } from "./LLMObsInferenceTool"; +import { LLMObsInternalReasoning } from "./LLMObsInternalReasoning"; + +/** + * The output of a completed LLM inference call. + */ +export class LLMObsInferenceRunResult { + /** + * An optional assessment of the inference output quality. + */ + "assessment": string | null; + /** + * The text content of the model response. + */ + "content": string; + /** + * The reason the model stopped generating tokens. + */ + "finishReason": string; + /** + * List of generated code snippets for the inference configuration. + */ + "inferenceCodes": Array; + /** + * Number of input tokens consumed. + */ + "inputTokens": number; + /** + * The model's internal reasoning or thinking output, if available. + */ + "internalReasoning"?: LLMObsInternalReasoning; + /** + * Request latency in milliseconds. + */ + "latency": number; + /** + * Number of output tokens generated. + */ + "outputTokens": number; + /** + * List of tools available to the model. + */ + "tools": Array; + /** + * Total tokens used (input plus output). + */ + "totalTokens": number; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + assessment: { + baseName: "assessment", + type: "string", + required: true, + }, + content: { + baseName: "content", + type: "string", + required: true, + }, + finishReason: { + baseName: "finish_reason", + type: "string", + required: true, + }, + inferenceCodes: { + baseName: "inference_codes", + type: "Array", + required: true, + }, + inputTokens: { + baseName: "input_tokens", + type: "number", + required: true, + format: "int64", + }, + internalReasoning: { + baseName: "internal_reasoning", + type: "LLMObsInternalReasoning", + }, + latency: { + baseName: "latency", + type: "number", + required: true, + format: "int64", + }, + outputTokens: { + baseName: "output_tokens", + type: "number", + required: true, + format: "int64", + }, + tools: { + baseName: "tools", + type: "Array", + required: true, + }, + totalTokens: { + baseName: "total_tokens", + type: "number", + required: true, + format: "int64", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsInferenceRunResult.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsInferenceTool.ts b/services/llm_observability/src/v2/models/LLMObsInferenceTool.ts new file mode 100644 index 000000000000..2dd9cf6c5ebe --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsInferenceTool.ts @@ -0,0 +1,56 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsInferenceFunction } from "./LLMObsInferenceFunction"; + +/** + * A tool definition available to the model during inference. + */ +export class LLMObsInferenceTool { + /** + * A function definition for a tool available to the model. + */ + "_function": LLMObsInferenceFunction; + /** + * The type of tool. + */ + "type": string; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + _function: { + baseName: "function", + type: "LLMObsInferenceFunction", + required: true, + }, + type: { + baseName: "type", + type: "string", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsInferenceTool.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsInferenceToolCall.ts b/services/llm_observability/src/v2/models/LLMObsInferenceToolCall.ts new file mode 100644 index 000000000000..3a8628414b7a --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsInferenceToolCall.ts @@ -0,0 +1,68 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * A tool call made during LLM inference. + */ +export class LLMObsInferenceToolCall { + /** + * The arguments passed to the tool. + */ + "arguments"?: { [key: string]: any }; + /** + * The name of the tool being called. + */ + "name"?: string; + /** + * Unique identifier for the tool call. + */ + "toolId"?: string; + /** + * The type of tool call. + */ + "type"?: string; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + arguments: { + baseName: "arguments", + type: "{ [key: string]: any; }", + }, + name: { + baseName: "name", + type: "string", + }, + toolId: { + baseName: "tool_id", + type: "string", + }, + type: { + baseName: "type", + type: "string", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsInferenceToolCall.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsInferenceToolResult.ts b/services/llm_observability/src/v2/models/LLMObsInferenceToolResult.ts new file mode 100644 index 000000000000..8eb1617150bf --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsInferenceToolResult.ts @@ -0,0 +1,68 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * The result returned by a tool call during LLM inference. + */ +export class LLMObsInferenceToolResult { + /** + * The name of the tool that produced this result. + */ + "name"?: string; + /** + * The result content returned by the tool. + */ + "result"?: string; + /** + * Identifier matching the corresponding tool call. + */ + "toolId"?: string; + /** + * The type of tool result. + */ + "type"?: string; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + name: { + baseName: "name", + type: "string", + }, + result: { + baseName: "result", + type: "string", + }, + toolId: { + baseName: "tool_id", + type: "string", + }, + type: { + baseName: "type", + type: "string", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsInferenceToolResult.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsIntegrationAccount.ts b/services/llm_observability/src/v2/models/LLMObsIntegrationAccount.ts new file mode 100644 index 000000000000..d1cbf4d8c53f --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsIntegrationAccount.ts @@ -0,0 +1,99 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsAzureOpenAIMetadata } from "./LLMObsAzureOpenAIMetadata"; +import { LLMObsVertexAIMetadata } from "./LLMObsVertexAIMetadata"; + +/** + * A configured account for an LLM provider integration. + */ +export class LLMObsIntegrationAccount { + /** + * Provider-specific account identifier. + */ + "accountId": string; + /** + * Human-readable name for the integration account. + */ + "accountName": string; + /** + * Provider region associated with the account, if applicable. + */ + "accountRegion"?: string; + /** + * Azure OpenAI-specific metadata for an integration account or inference request. + */ + "azureOpenaiMetadata"?: LLMObsAzureOpenAIMetadata; + /** + * Unique identifier for the integration account. + */ + "id": string; + /** + * The name of the LLM provider integration. + */ + "integration": string; + /** + * Vertex AI-specific metadata for an integration account or inference request. + */ + "vertexAiMetadata"?: LLMObsVertexAIMetadata; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + accountId: { + baseName: "account_id", + type: "string", + required: true, + }, + accountName: { + baseName: "account_name", + type: "string", + required: true, + }, + accountRegion: { + baseName: "account_region", + type: "string", + }, + azureOpenaiMetadata: { + baseName: "azure_openai_metadata", + type: "LLMObsAzureOpenAIMetadata", + }, + id: { + baseName: "id", + type: "string", + required: true, + }, + integration: { + baseName: "integration", + type: "string", + required: true, + }, + vertexAiMetadata: { + baseName: "vertex_ai_metadata", + type: "LLMObsVertexAIMetadata", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsIntegrationAccount.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsIntegrationInferenceRequest.ts b/services/llm_observability/src/v2/models/LLMObsIntegrationInferenceRequest.ts new file mode 100644 index 000000000000..b41828751246 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsIntegrationInferenceRequest.ts @@ -0,0 +1,181 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsAnthropicMetadata } from "./LLMObsAnthropicMetadata"; +import { LLMObsAzureOpenAIMetadata } from "./LLMObsAzureOpenAIMetadata"; +import { LLMObsBedrockMetadata } from "./LLMObsBedrockMetadata"; +import { LLMObsInferenceMessage } from "./LLMObsInferenceMessage"; +import { LLMObsInferenceTool } from "./LLMObsInferenceTool"; +import { LLMObsOpenAIMetadata } from "./LLMObsOpenAIMetadata"; +import { LLMObsVertexAIMetadata } from "./LLMObsVertexAIMetadata"; + +/** + * Parameters for an LLM inference request. + */ +export class LLMObsIntegrationInferenceRequest { + /** + * Anthropic-specific metadata for an inference request. + */ + "anthropicMetadata"?: LLMObsAnthropicMetadata; + /** + * Azure OpenAI-specific metadata for an integration account or inference request. + */ + "azureOpenaiMetadata"?: LLMObsAzureOpenAIMetadata; + /** + * Amazon Bedrock-specific metadata for an inference request. + */ + "bedrockMetadata"?: LLMObsBedrockMetadata; + /** + * Penalty for token frequency to reduce repetition. + */ + "frequencyPenalty"?: number; + /** + * JSON schema for structured output, if supported by the model. + */ + "jsonSchema"?: string; + /** + * Maximum number of completion tokens to generate (alternative to max_tokens for some providers). + */ + "maxCompletionTokens"?: number; + /** + * Maximum number of tokens to generate. + */ + "maxTokens"?: number; + /** + * List of messages in an inference conversation. + */ + "messages": Array; + /** + * The model identifier to use for inference. + */ + "modelId": string; + /** + * OpenAI-specific metadata for an inference request. + */ + "openaiMetadata"?: LLMObsOpenAIMetadata; + /** + * Penalty for token presence to encourage topic diversity. + */ + "presencePenalty"?: number; + /** + * Sampling temperature between 0 and 2. Higher values produce more random output. + */ + "temperature"?: number; + /** + * List of tools available to the model. + */ + "tools"?: Array; + /** + * Top-K sampling parameter. + */ + "topK"?: number; + /** + * Nucleus sampling probability mass. + */ + "topP"?: number; + /** + * Vertex AI-specific metadata for an integration account or inference request. + */ + "vertexAiMetadata"?: LLMObsVertexAIMetadata; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + anthropicMetadata: { + baseName: "anthropic_metadata", + type: "LLMObsAnthropicMetadata", + }, + azureOpenaiMetadata: { + baseName: "azure_openai_metadata", + type: "LLMObsAzureOpenAIMetadata", + }, + bedrockMetadata: { + baseName: "bedrock_metadata", + type: "LLMObsBedrockMetadata", + }, + frequencyPenalty: { + baseName: "frequency_penalty", + type: "number", + format: "double", + }, + jsonSchema: { + baseName: "json_schema", + type: "string", + }, + maxCompletionTokens: { + baseName: "max_completion_tokens", + type: "number", + format: "int64", + }, + maxTokens: { + baseName: "max_tokens", + type: "number", + format: "int64", + }, + messages: { + baseName: "messages", + type: "Array", + required: true, + }, + modelId: { + baseName: "model_id", + type: "string", + required: true, + }, + openaiMetadata: { + baseName: "openai_metadata", + type: "LLMObsOpenAIMetadata", + }, + presencePenalty: { + baseName: "presence_penalty", + type: "number", + format: "double", + }, + temperature: { + baseName: "temperature", + type: "number", + format: "double", + }, + tools: { + baseName: "tools", + type: "Array", + }, + topK: { + baseName: "top_k", + type: "number", + format: "int64", + }, + topP: { + baseName: "top_p", + type: "number", + format: "double", + }, + vertexAiMetadata: { + baseName: "vertex_ai_metadata", + type: "LLMObsVertexAIMetadata", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsIntegrationInferenceRequest.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsIntegrationInferenceResponse.ts b/services/llm_observability/src/v2/models/LLMObsIntegrationInferenceResponse.ts new file mode 100644 index 000000000000..e118214dea05 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsIntegrationInferenceResponse.ts @@ -0,0 +1,200 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsAnthropicMetadata } from "./LLMObsAnthropicMetadata"; +import { LLMObsAzureOpenAIMetadata } from "./LLMObsAzureOpenAIMetadata"; +import { LLMObsBedrockMetadata } from "./LLMObsBedrockMetadata"; +import { LLMObsInferenceErrorResponse } from "./LLMObsInferenceErrorResponse"; +import { LLMObsInferenceMessage } from "./LLMObsInferenceMessage"; +import { LLMObsInferenceRunResult } from "./LLMObsInferenceRunResult"; +import { LLMObsInferenceTool } from "./LLMObsInferenceTool"; +import { LLMObsOpenAIMetadata } from "./LLMObsOpenAIMetadata"; +import { LLMObsVertexAIMetadata } from "./LLMObsVertexAIMetadata"; + +/** + * The result of an LLM inference request, including input parameters and the model response. + */ +export class LLMObsIntegrationInferenceResponse { + /** + * Anthropic-specific metadata for an inference request. + */ + "anthropicMetadata"?: LLMObsAnthropicMetadata; + /** + * Azure OpenAI-specific metadata for an integration account or inference request. + */ + "azureOpenaiMetadata"?: LLMObsAzureOpenAIMetadata; + /** + * Amazon Bedrock-specific metadata for an inference request. + */ + "bedrockMetadata"?: LLMObsBedrockMetadata; + /** + * Error details returned when an inference provider returns an error. + */ + "errorResponse"?: LLMObsInferenceErrorResponse; + /** + * Frequency penalty that was applied. + */ + "frequencyPenalty"?: number; + /** + * JSON schema that was applied for structured output. + */ + "jsonSchema"?: string; + /** + * Maximum number of completion tokens that were configured. + */ + "maxCompletionTokens"?: number; + /** + * Maximum number of tokens that were configured. + */ + "maxTokens"?: number; + /** + * List of messages in an inference conversation. + */ + "messages": Array; + /** + * The model identifier used for inference. + */ + "modelId": string; + /** + * OpenAI-specific metadata for an inference request. + */ + "openaiMetadata"?: LLMObsOpenAIMetadata; + /** + * Presence penalty that was applied. + */ + "presencePenalty"?: number; + /** + * The output of a completed LLM inference call. + */ + "response": LLMObsInferenceRunResult; + /** + * Sampling temperature that was used. + */ + "temperature"?: number; + /** + * List of tools available to the model. + */ + "tools"?: Array; + /** + * Top-K sampling parameter that was used. + */ + "topK"?: number; + /** + * Nucleus sampling parameter that was used. + */ + "topP"?: number; + /** + * Vertex AI-specific metadata for an integration account or inference request. + */ + "vertexAiMetadata"?: LLMObsVertexAIMetadata; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + anthropicMetadata: { + baseName: "anthropic_metadata", + type: "LLMObsAnthropicMetadata", + }, + azureOpenaiMetadata: { + baseName: "azure_openai_metadata", + type: "LLMObsAzureOpenAIMetadata", + }, + bedrockMetadata: { + baseName: "bedrock_metadata", + type: "LLMObsBedrockMetadata", + }, + errorResponse: { + baseName: "error_response", + type: "LLMObsInferenceErrorResponse", + }, + frequencyPenalty: { + baseName: "frequency_penalty", + type: "number", + format: "double", + }, + jsonSchema: { + baseName: "json_schema", + type: "string", + }, + maxCompletionTokens: { + baseName: "max_completion_tokens", + type: "number", + format: "int64", + }, + maxTokens: { + baseName: "max_tokens", + type: "number", + format: "int64", + }, + messages: { + baseName: "messages", + type: "Array", + required: true, + }, + modelId: { + baseName: "model_id", + type: "string", + required: true, + }, + openaiMetadata: { + baseName: "openai_metadata", + type: "LLMObsOpenAIMetadata", + }, + presencePenalty: { + baseName: "presence_penalty", + type: "number", + format: "double", + }, + response: { + baseName: "response", + type: "LLMObsInferenceRunResult", + required: true, + }, + temperature: { + baseName: "temperature", + type: "number", + format: "double", + }, + tools: { + baseName: "tools", + type: "Array", + }, + topK: { + baseName: "top_k", + type: "number", + format: "int64", + }, + topP: { + baseName: "top_p", + type: "number", + format: "double", + }, + vertexAiMetadata: { + baseName: "vertex_ai_metadata", + type: "LLMObsVertexAIMetadata", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsIntegrationInferenceResponse.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsIntegrationModel.ts b/services/llm_observability/src/v2/models/LLMObsIntegrationModel.ts new file mode 100644 index 000000000000..4f58e32dc038 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsIntegrationModel.ts @@ -0,0 +1,125 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * A model available for a given LLM provider integration and account. + */ +export class LLMObsIntegrationModel { + /** + * Whether the account has access to this model. + */ + "hasAccess": boolean; + /** + * Unique identifier for the model entry. + */ + "id": string; + /** + * The name of the LLM provider integration. + */ + "integration": string; + /** + * Human-readable name of the LLM provider integration. + */ + "integrationDisplayName": string; + /** + * Whether the model supports structured output via JSON schema. + */ + "jsonSchema": boolean; + /** + * Human-readable model name. + */ + "modelDisplayName": string; + /** + * Provider-specific model identifier used in inference calls. + */ + "modelId": string; + /** + * The underlying model provider. + */ + "provider": string; + /** + * Human-readable name of the underlying model provider. + */ + "providerDisplayName": string; + /** + * Map of region-specific model ID prefix overrides. + */ + "regionPrefixOverrides"?: { [key: string]: string }; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + hasAccess: { + baseName: "has_access", + type: "boolean", + required: true, + }, + id: { + baseName: "id", + type: "string", + required: true, + }, + integration: { + baseName: "integration", + type: "string", + required: true, + }, + integrationDisplayName: { + baseName: "integration_display_name", + type: "string", + required: true, + }, + jsonSchema: { + baseName: "json_schema", + type: "boolean", + required: true, + }, + modelDisplayName: { + baseName: "model_display_name", + type: "string", + required: true, + }, + modelId: { + baseName: "model_id", + type: "string", + required: true, + }, + provider: { + baseName: "provider", + type: "string", + required: true, + }, + providerDisplayName: { + baseName: "provider_display_name", + type: "string", + required: true, + }, + regionPrefixOverrides: { + baseName: "region_prefix_overrides", + type: "{ [key: string]: string; }", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsIntegrationModel.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsIntegrationName.ts b/services/llm_observability/src/v2/models/LLMObsIntegrationName.ts new file mode 100644 index 000000000000..12d47589a7e7 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsIntegrationName.ts @@ -0,0 +1,19 @@ +import { UnparsedObject } from "@datadog/datadog-api-client"; + +/** + * The name of a supported LLM provider integration. + */ +export type LLMObsIntegrationName = + | typeof OPENAI + | typeof AMAZON_BEDROCK + | typeof ANTHROPIC + | typeof AZURE_OPENAI + | typeof VERTEX_AI + | typeof LLMPROXY + | UnparsedObject; +export const OPENAI = "openai"; +export const AMAZON_BEDROCK = "amazon_bedrock"; +export const ANTHROPIC = "anthropic"; +export const AZURE_OPENAI = "azure_openai"; +export const VERTEX_AI = "vertex_ai"; +export const LLMPROXY = "llmproxy"; diff --git a/services/llm_observability/src/v2/models/LLMObsInternalReasoning.ts b/services/llm_observability/src/v2/models/LLMObsInternalReasoning.ts new file mode 100644 index 000000000000..071c2ca2b0d0 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsInternalReasoning.ts @@ -0,0 +1,54 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * The model's internal reasoning or thinking output, if available. + */ +export class LLMObsInternalReasoning { + /** + * Number of tokens used for internal reasoning. + */ + "reasoningTokens"?: number; + /** + * The reasoning text produced by the model. + */ + "text": string; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + reasoningTokens: { + baseName: "reasoning_tokens", + type: "number", + format: "int64", + }, + text: { + baseName: "text", + type: "string", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsInternalReasoning.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsOpenAIMetadata.ts b/services/llm_observability/src/v2/models/LLMObsOpenAIMetadata.ts new file mode 100644 index 000000000000..e53a7a579eae --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsOpenAIMetadata.ts @@ -0,0 +1,55 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsOpenAIReasoningEffort } from "./LLMObsOpenAIReasoningEffort"; +import { LLMObsOpenAIReasoningSummary } from "./LLMObsOpenAIReasoningSummary"; + +/** + * OpenAI-specific metadata for an inference request. + */ +export class LLMObsOpenAIMetadata { + /** + * The reasoning effort level for OpenAI models that support it. + */ + "reasoningEffort"?: LLMObsOpenAIReasoningEffort; + /** + * The verbosity of the reasoning summary. + */ + "reasoningSummary"?: LLMObsOpenAIReasoningSummary; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + reasoningEffort: { + baseName: "reasoning_effort", + type: "LLMObsOpenAIReasoningEffort", + }, + reasoningSummary: { + baseName: "reasoning_summary", + type: "LLMObsOpenAIReasoningSummary", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsOpenAIMetadata.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsOpenAIReasoningEffort.ts b/services/llm_observability/src/v2/models/LLMObsOpenAIReasoningEffort.ts new file mode 100644 index 000000000000..0f033933ceb2 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsOpenAIReasoningEffort.ts @@ -0,0 +1,17 @@ +import { UnparsedObject } from "@datadog/datadog-api-client"; + +/** + * The reasoning effort level for OpenAI models that support it. + */ +export type LLMObsOpenAIReasoningEffort = + | typeof NONE + | typeof LOW + | typeof MEDIUM + | typeof HIGH + | typeof XHIGH + | UnparsedObject; +export const NONE = "none"; +export const LOW = "low"; +export const MEDIUM = "medium"; +export const HIGH = "high"; +export const XHIGH = "xhigh"; diff --git a/services/llm_observability/src/v2/models/LLMObsOpenAIReasoningSummary.ts b/services/llm_observability/src/v2/models/LLMObsOpenAIReasoningSummary.ts new file mode 100644 index 000000000000..e6b9fe54d899 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsOpenAIReasoningSummary.ts @@ -0,0 +1,13 @@ +import { UnparsedObject } from "@datadog/datadog-api-client"; + +/** + * The verbosity of the reasoning summary. + */ +export type LLMObsOpenAIReasoningSummary = + | typeof AUTO + | typeof CONCISE + | typeof DETAILED + | UnparsedObject; +export const AUTO = "auto"; +export const CONCISE = "concise"; +export const DETAILED = "detailed"; diff --git a/services/llm_observability/src/v2/models/LLMObsVertexAIMetadata.ts b/services/llm_observability/src/v2/models/LLMObsVertexAIMetadata.ts new file mode 100644 index 000000000000..adb5167b4c69 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsVertexAIMetadata.ts @@ -0,0 +1,60 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * Vertex AI-specific metadata for an integration account or inference request. + */ +export class LLMObsVertexAIMetadata { + /** + * The Vertex AI region. + */ + "location"?: string; + /** + * The Google Cloud project ID. + */ + "project"?: string; + /** + * List of Google Cloud project IDs available to the service account. + */ + "projectIds"?: Array; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + location: { + baseName: "location", + type: "string", + }, + project: { + baseName: "project", + type: "string", + }, + projectIds: { + baseName: "project_ids", + type: "Array", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsVertexAIMetadata.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/TypingInfo.ts b/services/llm_observability/src/v2/models/TypingInfo.ts index 27f7d6e0051c..3ad79719235f 100644 --- a/services/llm_observability/src/v2/models/TypingInfo.ts +++ b/services/llm_observability/src/v2/models/TypingInfo.ts @@ -31,6 +31,10 @@ import { LLMObsAnnotationQueueUpdateDataRequest } from "./LLMObsAnnotationQueueU import { LLMObsAnnotationQueueUpdateRequest } from "./LLMObsAnnotationQueueUpdateRequest"; import { LLMObsAnnotationQueuesResponse } from "./LLMObsAnnotationQueuesResponse"; import { LLMObsAnnotationSchema } from "./LLMObsAnnotationSchema"; +import { LLMObsAnthropicMetadata } from "./LLMObsAnthropicMetadata"; +import { LLMObsAnthropicThinkingConfig } from "./LLMObsAnthropicThinkingConfig"; +import { LLMObsAzureOpenAIMetadata } from "./LLMObsAzureOpenAIMetadata"; +import { LLMObsBedrockMetadata } from "./LLMObsBedrockMetadata"; import { LLMObsContentBlock } from "./LLMObsContentBlock"; import { LLMObsContentBlockTimeFrame } from "./LLMObsContentBlockTimeFrame"; import { LLMObsCursorMeta } from "./LLMObsCursorMeta"; @@ -155,7 +159,23 @@ import { LLMObsExperimentationSimpleSearchRequest } from "./LLMObsExperimentatio import { LLMObsExperimentationSimpleSearchResponse } from "./LLMObsExperimentationSimpleSearchResponse"; import { LLMObsExperimentationSortField } from "./LLMObsExperimentationSortField"; import { LLMObsExperimentsResponse } from "./LLMObsExperimentsResponse"; +import { LLMObsInferenceCode } from "./LLMObsInferenceCode"; +import { LLMObsInferenceContent } from "./LLMObsInferenceContent"; +import { LLMObsInferenceContentValue } from "./LLMObsInferenceContentValue"; +import { LLMObsInferenceErrorResponse } from "./LLMObsInferenceErrorResponse"; +import { LLMObsInferenceFunction } from "./LLMObsInferenceFunction"; +import { LLMObsInferenceMessage } from "./LLMObsInferenceMessage"; +import { LLMObsInferenceRunResult } from "./LLMObsInferenceRunResult"; +import { LLMObsInferenceTool } from "./LLMObsInferenceTool"; +import { LLMObsInferenceToolCall } from "./LLMObsInferenceToolCall"; +import { LLMObsInferenceToolResult } from "./LLMObsInferenceToolResult"; +import { LLMObsIntegrationAccount } from "./LLMObsIntegrationAccount"; +import { LLMObsIntegrationInferenceRequest } from "./LLMObsIntegrationInferenceRequest"; +import { LLMObsIntegrationInferenceResponse } from "./LLMObsIntegrationInferenceResponse"; +import { LLMObsIntegrationModel } from "./LLMObsIntegrationModel"; +import { LLMObsInternalReasoning } from "./LLMObsInternalReasoning"; import { LLMObsLabelSchema } from "./LLMObsLabelSchema"; +import { LLMObsOpenAIMetadata } from "./LLMObsOpenAIMetadata"; import { LLMObsProjectDataAttributesRequest } from "./LLMObsProjectDataAttributesRequest"; import { LLMObsProjectDataAttributesResponse } from "./LLMObsProjectDataAttributesResponse"; import { LLMObsProjectDataRequest } from "./LLMObsProjectDataRequest"; @@ -187,12 +207,15 @@ import { LLMObsSpansResponsePage } from "./LLMObsSpansResponsePage"; import { LLMObsTraceAnnotatedInteractionItem } from "./LLMObsTraceAnnotatedInteractionItem"; import { LLMObsTraceInteractionItem } from "./LLMObsTraceInteractionItem"; import { LLMObsTraceInteractionResponseItem } from "./LLMObsTraceInteractionResponseItem"; +import { LLMObsVertexAIMetadata } from "./LLMObsVertexAIMetadata"; export const TypingInfo: ModelTypingInfo = { enumsMap: { LLMObsAnnotatedInteractionsType: ["annotated_interactions"], LLMObsAnnotationQueueInteractionsType: ["interactions"], LLMObsAnnotationQueueType: ["queues"], + LLMObsAnthropicEffort: ["low", "medium", "high", "max"], + LLMObsAnthropicThinkingType: ["enabled", "disabled", "adaptive"], LLMObsContentBlockHeaderLevel: ["sm", "md", "lg", "xl"], LLMObsContentBlockLLMObsTraceInteractionType: ["trace", "experiment_trace"], LLMObsContentBlockType: [ @@ -225,9 +248,19 @@ export const TypingInfo: ModelTypingInfo = { LLMObsExperimentType: ["experiments"], LLMObsExperimentationSortFieldDirection: ["asc", "desc"], LLMObsExperimentationType: ["experimentation"], + LLMObsIntegrationName: [ + "openai", + "amazon_bedrock", + "anthropic", + "azure_openai", + "vertex_ai", + "llmproxy", + ], LLMObsLabelSchemaType: ["score", "categorical", "boolean", "text"], LLMObsMetricAssessment: ["pass", "fail"], LLMObsMetricScoreType: ["score", "categorical", "boolean", "json"], + LLMObsOpenAIReasoningEffort: ["none", "low", "medium", "high", "xhigh"], + LLMObsOpenAIReasoningSummary: ["auto", "concise", "detailed"], LLMObsProjectType: ["projects"], LLMObsRecordType: ["records"], LLMObsSearchSpansRequestType: ["spans"], @@ -305,6 +338,10 @@ export const TypingInfo: ModelTypingInfo = { LLMObsAnnotationQueueUpdateRequest: LLMObsAnnotationQueueUpdateRequest, LLMObsAnnotationQueuesResponse: LLMObsAnnotationQueuesResponse, LLMObsAnnotationSchema: LLMObsAnnotationSchema, + LLMObsAnthropicMetadata: LLMObsAnthropicMetadata, + LLMObsAnthropicThinkingConfig: LLMObsAnthropicThinkingConfig, + LLMObsAzureOpenAIMetadata: LLMObsAzureOpenAIMetadata, + LLMObsBedrockMetadata: LLMObsBedrockMetadata, LLMObsContentBlock: LLMObsContentBlock, LLMObsContentBlockTimeFrame: LLMObsContentBlockTimeFrame, LLMObsCursorMeta: LLMObsCursorMeta, @@ -474,7 +511,23 @@ export const TypingInfo: ModelTypingInfo = { LLMObsExperimentationSimpleSearchResponse, LLMObsExperimentationSortField: LLMObsExperimentationSortField, LLMObsExperimentsResponse: LLMObsExperimentsResponse, + LLMObsInferenceCode: LLMObsInferenceCode, + LLMObsInferenceContent: LLMObsInferenceContent, + LLMObsInferenceContentValue: LLMObsInferenceContentValue, + LLMObsInferenceErrorResponse: LLMObsInferenceErrorResponse, + LLMObsInferenceFunction: LLMObsInferenceFunction, + LLMObsInferenceMessage: LLMObsInferenceMessage, + LLMObsInferenceRunResult: LLMObsInferenceRunResult, + LLMObsInferenceTool: LLMObsInferenceTool, + LLMObsInferenceToolCall: LLMObsInferenceToolCall, + LLMObsInferenceToolResult: LLMObsInferenceToolResult, + LLMObsIntegrationAccount: LLMObsIntegrationAccount, + LLMObsIntegrationInferenceRequest: LLMObsIntegrationInferenceRequest, + LLMObsIntegrationInferenceResponse: LLMObsIntegrationInferenceResponse, + LLMObsIntegrationModel: LLMObsIntegrationModel, + LLMObsInternalReasoning: LLMObsInternalReasoning, LLMObsLabelSchema: LLMObsLabelSchema, + LLMObsOpenAIMetadata: LLMObsOpenAIMetadata, LLMObsProjectDataAttributesRequest: LLMObsProjectDataAttributesRequest, LLMObsProjectDataAttributesResponse: LLMObsProjectDataAttributesResponse, LLMObsProjectDataRequest: LLMObsProjectDataRequest, @@ -507,5 +560,6 @@ export const TypingInfo: ModelTypingInfo = { LLMObsTraceAnnotatedInteractionItem: LLMObsTraceAnnotatedInteractionItem, LLMObsTraceInteractionItem: LLMObsTraceInteractionItem, LLMObsTraceInteractionResponseItem: LLMObsTraceInteractionResponseItem, + LLMObsVertexAIMetadata: LLMObsVertexAIMetadata, }, };