diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 9041f8a003..5b3f25722b 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -38071,6 +38071,8 @@ components: LLMObsAnnotationQueueDataAttributesRequest: description: Attributes for creating an LLM Observability annotation queue. properties: + annotation_schema: + $ref: "#/components/schemas/LLMObsAnnotationSchema" description: description: Description of the annotation queue. example: "Queue for annotating customer support traces" @@ -38090,6 +38092,8 @@ components: LLMObsAnnotationQueueDataAttributesResponse: description: Attributes of an LLM Observability annotation queue. properties: + annotation_schema: + $ref: "#/components/schemas/LLMObsAnnotationSchema" created_at: description: Timestamp when the queue was created. example: "2024-01-15T10:30:00Z" @@ -38271,6 +38275,65 @@ components: type: string x-enum-varnames: - INTERACTIONS + LLMObsAnnotationQueueLabelSchemaAttributes: + description: Attributes of an annotation queue label schema. + properties: + annotation_schema: + $ref: "#/components/schemas/LLMObsAnnotationSchema" + required: + - annotation_schema + type: object + LLMObsAnnotationQueueLabelSchemaData: + description: Data object for an annotation queue label schema. + properties: + attributes: + $ref: "#/components/schemas/LLMObsAnnotationQueueLabelSchemaAttributes" + id: + description: Unique identifier of the annotation queue. + example: "b5e7f3a1-9c2d-4f8b-a1e6-3d7e9f0a2b4c" + type: string + type: + $ref: "#/components/schemas/LLMObsAnnotationQueueType" + required: + - id + - type + - attributes + type: object + LLMObsAnnotationQueueLabelSchemaResponse: + description: Response containing the label schema of an annotation queue. + properties: + data: + $ref: "#/components/schemas/LLMObsAnnotationQueueLabelSchemaData" + required: + - data + type: object + LLMObsAnnotationQueueLabelSchemaUpdateAttributes: + description: Attributes for updating an annotation queue label schema. + properties: + annotation_schema: + $ref: "#/components/schemas/LLMObsAnnotationSchema" + required: + - annotation_schema + type: object + LLMObsAnnotationQueueLabelSchemaUpdateData: + description: Data object for updating an annotation queue label schema. + properties: + attributes: + $ref: "#/components/schemas/LLMObsAnnotationQueueLabelSchemaUpdateAttributes" + type: + $ref: "#/components/schemas/LLMObsAnnotationQueueType" + required: + - type + - attributes + type: object + LLMObsAnnotationQueueLabelSchemaUpdateRequest: + description: Request to update the label schema of an annotation queue. + properties: + data: + $ref: "#/components/schemas/LLMObsAnnotationQueueLabelSchemaUpdateData" + required: + - data + type: object LLMObsAnnotationQueueRequest: description: Request to create an LLM Observability annotation queue. properties: @@ -38298,6 +38361,8 @@ components: LLMObsAnnotationQueueUpdateDataAttributesRequest: description: Attributes for updating an LLM Observability annotation queue. All fields are optional. properties: + annotation_schema: + $ref: "#/components/schemas/LLMObsAnnotationSchema" description: description: Updated description of the annotation queue. example: "Updated description" @@ -38337,6 +38402,17 @@ components: required: - data type: object + LLMObsAnnotationSchema: + description: Schema defining the labels for an annotation queue. + properties: + label_schemas: + description: List of label schema definitions. + items: + $ref: "#/components/schemas/LLMObsLabelSchema" + type: array + required: + - label_schemas + type: object LLMObsCursorMeta: description: Pagination cursor metadata. properties: @@ -39634,6 +39710,81 @@ components: x-enum-varnames: - TRACE - EXPERIMENT_TRACE + LLMObsLabelSchema: + description: Schema definition for a single label in an annotation queue. + properties: + description: + description: Description of the label. + example: "Rating of the response quality." + type: string + has_assessment: + description: Whether this label includes an assessment field. + example: false + type: boolean + has_reasoning: + description: Whether this label includes a reasoning field. + example: false + type: boolean + id: + description: Unique identifier of the label schema. Assigned by the server if not provided. + example: "ab12cd34" + type: string + is_assessment: + description: Whether the boolean label represents an assessment. Requires `has_assessment` to be true. + example: false + type: boolean + is_integer: + description: Whether score values must be integers. Applicable to score-type labels. + example: false + type: boolean + is_required: + description: Whether this label is required for an annotation. + example: true + type: boolean + max: + description: Maximum value for score-type labels. + example: 5.0 + format: double + type: number + min: + description: Minimum value for score-type labels. + example: 0.0 + format: double + type: number + name: + description: Name of the label. Must match the pattern `^[a-zA-Z0-9_-]+$` and be unique within the queue. + example: "quality" + type: string + type: + $ref: "#/components/schemas/LLMObsLabelSchemaType" + values: + description: Allowed values for categorical-type labels. Must contain at least one non-empty, unique value. + example: + - "good" + - "bad" + - "neutral" + items: + description: An allowed value for a categorical label. + type: string + type: array + required: + - name + - type + type: object + LLMObsLabelSchemaType: + description: Type of a label in an annotation queue label schema. + enum: + - score + - categorical + - boolean + - text + example: score + type: string + x-enum-varnames: + - SCORE + - CATEGORICAL + - BOOLEAN + - TEXT LLMObsMetricAssessment: description: Assessment result for an LLM Observability experiment metric. enum: @@ -103821,8 +103972,10 @@ paths: If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). post: description: |- - Create a new annotation queue. Only `name`, `project_id`, and `description` are accepted. - Fields such as `created_by`, `owned_by`, `created_at`, `modified_by`, and `modified_at` are inferred by the backend. + Create an annotation queue. The `name` and `project_id` fields are required. + An optional `annotation_schema` can be provided to define the labels for the queue. + Fields such as `created_by`, `owned_by`, `created_at`, `modified_by`, + and `modified_at` are inferred by the backend. operationId: CreateLLMObsAnnotationQueue requestBody: content: @@ -103836,6 +103989,28 @@ paths: name: My annotation queue project_id: a33671aa-24fd-4dcd-9b33-a8ec7dde7751 type: queues + with_schema: + summary: Create queue with annotation schema + value: + data: + attributes: + annotation_schema: + label_schemas: + - is_required: true + max: 5.0 + min: 0.0 + name: quality + type: score + - name: sentiment + type: categorical + values: + - positive + - negative + - neutral + description: Queue for annotating customer support traces + name: My annotation queue + project_id: a33671aa-24fd-4dcd-9b33-a8ec7dde7751 + type: queues schema: $ref: "#/components/schemas/LLMObsAnnotationQueueRequest" description: Create annotation queue payload. @@ -103916,7 +104091,7 @@ paths: If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). patch: description: |- - Partially update an annotation queue. Only `name` and `description` can be updated. + Partially update an annotation queue. The `name`, `description`, and `annotation_schema` fields can be updated. operationId: UpdateLLMObsAnnotationQueue parameters: - $ref: "#/components/parameters/LLMObsAnnotationQueueIDPathParameter" @@ -104151,6 +104326,206 @@ paths: x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/annotation-queues/{queue_id}/label-schema: + get: + description: Retrieve the label schema for a given annotation queue. + operationId: GetLLMObsAnnotationQueueLabelSchema + parameters: + - $ref: "#/components/parameters/LLMObsAnnotationQueueIDPathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + annotation_schema: + label_schemas: + - id: "ab12cd34" + is_required: true + max: 5.0 + min: 0.0 + name: quality + type: score + - id: "ef56gh78" + name: sentiment + type: categorical + values: + - positive + - negative + - neutral + id: "b5e7f3a1-9c2d-4f8b-a1e6-3d7e9f0a2b4c" + type: queues + schema: + $ref: "#/components/schemas/LLMObsAnnotationQueueLabelSchemaResponse" + description: OK + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Get annotation queue label schema + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + put: + description: |- + Create or replace the label schema for a given annotation queue. + The label schema defines the labels annotators can apply to interactions in the queue. + Label names must be unique within the queue and match the pattern `^[a-zA-Z0-9_-]+$`. + Each label must have a valid type: score, categorical, boolean, or text. + operationId: UpdateLLMObsAnnotationQueueLabelSchema + parameters: + - $ref: "#/components/parameters/LLMObsAnnotationQueueIDPathParameter" + requestBody: + content: + application/json: + examples: + boolean_label: + summary: Boolean label schema + value: + data: + attributes: + annotation_schema: + label_schemas: + - has_assessment: true + is_assessment: true + is_required: true + name: is_correct + type: boolean + type: queues + categorical_label: + summary: Categorical label schema + value: + data: + attributes: + annotation_schema: + label_schemas: + - is_required: true + name: sentiment + type: categorical + values: + - positive + - negative + - neutral + type: queues + default: + value: + data: + attributes: + annotation_schema: + label_schemas: + - is_required: true + max: 5.0 + min: 0.0 + name: quality + type: score + type: queues + score_label: + summary: Score label schema + value: + data: + attributes: + annotation_schema: + label_schemas: + - has_reasoning: true + is_required: true + max: 5.0 + min: 0.0 + name: quality + type: score + type: queues + text_label: + summary: Text label schema + value: + data: + attributes: + annotation_schema: + label_schemas: + - is_required: false + name: feedback + type: text + type: queues + schema: + $ref: "#/components/schemas/LLMObsAnnotationQueueLabelSchemaUpdateRequest" + description: Update label schema payload. + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/LLMObsAnnotationQueueLabelSchemaResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Update annotation queue label schema + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/llm-obs/v1/experiments: get: description: List all LLM Observability experiments sorted by creation date, newest first. diff --git a/docs/datadog_api_client.v2.model.rst b/docs/datadog_api_client.v2.model.rst index 32967b25dd..20a44c2acf 100644 --- a/docs/datadog_api_client.v2.model.rst +++ b/docs/datadog_api_client.v2.model.rst @@ -16916,6 +16916,48 @@ datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_interactions\_type mo :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_label\_schema\_attributes module +------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_label\_schema\_data module +------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_label\_schema\_response module +----------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_label\_schema\_update\_attributes module +--------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_label\_schema\_update\_data module +--------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_label\_schema\_update\_request module +------------------------------------------------------------------------------------------------ + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_request + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_annotation\_queue\_request module ------------------------------------------------------------------------- @@ -16965,6 +17007,13 @@ datadog\_api\_client.v2.model.llm\_obs\_annotation\_queues\_response module :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_annotation\_schema module +----------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_annotation_schema + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_cursor\_meta module ----------------------------------------------------------- @@ -17553,6 +17602,20 @@ datadog\_api\_client.v2.model.llm\_obs\_interaction\_type module :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_label\_schema module +------------------------------------------------------------ + +.. automodule:: datadog_api_client.v2.model.llm_obs_label_schema + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_label\_schema\_type module +------------------------------------------------------------------ + +.. automodule:: datadog_api_client.v2.model.llm_obs_label_schema_type + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_metric\_assessment module ----------------------------------------------------------------- diff --git a/examples/v2/llm-observability/CreateLLMObsAnnotationQueue.py b/examples/v2/llm-observability/CreateLLMObsAnnotationQueue.py index 33d165615d..6498a1104d 100644 --- a/examples/v2/llm-observability/CreateLLMObsAnnotationQueue.py +++ b/examples/v2/llm-observability/CreateLLMObsAnnotationQueue.py @@ -10,10 +10,35 @@ from datadog_api_client.v2.model.llm_obs_annotation_queue_data_request import LLMObsAnnotationQueueDataRequest from datadog_api_client.v2.model.llm_obs_annotation_queue_request import LLMObsAnnotationQueueRequest from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType +from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema +from datadog_api_client.v2.model.llm_obs_label_schema import LLMObsLabelSchema +from datadog_api_client.v2.model.llm_obs_label_schema_type import LLMObsLabelSchemaType body = LLMObsAnnotationQueueRequest( data=LLMObsAnnotationQueueDataRequest( attributes=LLMObsAnnotationQueueDataAttributesRequest( + annotation_schema=LLMObsAnnotationSchema( + label_schemas=[ + LLMObsLabelSchema( + description="Rating of the response quality.", + has_assessment=False, + has_reasoning=False, + id="ab12cd34", + is_assessment=False, + is_integer=False, + is_required=True, + max=5.0, + min=0.0, + name="quality", + type=LLMObsLabelSchemaType.SCORE, + values=[ + "good", + "bad", + "neutral", + ], + ), + ], + ), description="Queue for annotating customer support traces", name="My annotation queue", project_id="a33671aa-24fd-4dcd-9b33-a8ec7dde7751", diff --git a/examples/v2/llm-observability/GetLLMObsAnnotationQueueLabelSchema.py b/examples/v2/llm-observability/GetLLMObsAnnotationQueueLabelSchema.py new file mode 100644 index 0000000000..029ba5628d --- /dev/null +++ b/examples/v2/llm-observability/GetLLMObsAnnotationQueueLabelSchema.py @@ -0,0 +1,16 @@ +""" +Get annotation queue label schema returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +configuration = Configuration() +configuration.unstable_operations["get_llm_obs_annotation_queue_label_schema"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.get_llm_obs_annotation_queue_label_schema( + queue_id="queue_id", + ) + + print(response) diff --git a/examples/v2/llm-observability/UpdateLLMObsAnnotationQueue.py b/examples/v2/llm-observability/UpdateLLMObsAnnotationQueue.py index 95e5b7a195..2308da73eb 100644 --- a/examples/v2/llm-observability/UpdateLLMObsAnnotationQueue.py +++ b/examples/v2/llm-observability/UpdateLLMObsAnnotationQueue.py @@ -12,10 +12,35 @@ LLMObsAnnotationQueueUpdateDataRequest, ) from datadog_api_client.v2.model.llm_obs_annotation_queue_update_request import LLMObsAnnotationQueueUpdateRequest +from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema +from datadog_api_client.v2.model.llm_obs_label_schema import LLMObsLabelSchema +from datadog_api_client.v2.model.llm_obs_label_schema_type import LLMObsLabelSchemaType body = LLMObsAnnotationQueueUpdateRequest( data=LLMObsAnnotationQueueUpdateDataRequest( attributes=LLMObsAnnotationQueueUpdateDataAttributesRequest( + annotation_schema=LLMObsAnnotationSchema( + label_schemas=[ + LLMObsLabelSchema( + description="Rating of the response quality.", + has_assessment=False, + has_reasoning=False, + id="ab12cd34", + is_assessment=False, + is_integer=False, + is_required=True, + max=5.0, + min=0.0, + name="quality", + type=LLMObsLabelSchemaType.SCORE, + values=[ + "good", + "bad", + "neutral", + ], + ), + ], + ), description="Updated description", name="Updated queue name", ), diff --git a/examples/v2/llm-observability/UpdateLLMObsAnnotationQueueLabelSchema.py b/examples/v2/llm-observability/UpdateLLMObsAnnotationQueueLabelSchema.py new file mode 100644 index 0000000000..b122dbbd96 --- /dev/null +++ b/examples/v2/llm-observability/UpdateLLMObsAnnotationQueueLabelSchema.py @@ -0,0 +1,57 @@ +""" +Update annotation queue label schema returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi +from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_attributes import ( + LLMObsAnnotationQueueLabelSchemaUpdateAttributes, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_data import ( + LLMObsAnnotationQueueLabelSchemaUpdateData, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_request import ( + LLMObsAnnotationQueueLabelSchemaUpdateRequest, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType +from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema +from datadog_api_client.v2.model.llm_obs_label_schema import LLMObsLabelSchema +from datadog_api_client.v2.model.llm_obs_label_schema_type import LLMObsLabelSchemaType + +body = LLMObsAnnotationQueueLabelSchemaUpdateRequest( + data=LLMObsAnnotationQueueLabelSchemaUpdateData( + attributes=LLMObsAnnotationQueueLabelSchemaUpdateAttributes( + annotation_schema=LLMObsAnnotationSchema( + label_schemas=[ + LLMObsLabelSchema( + description="Rating of the response quality.", + has_assessment=False, + has_reasoning=False, + id="ab12cd34", + is_assessment=False, + is_integer=False, + is_required=True, + max=5.0, + min=0.0, + name="quality", + type=LLMObsLabelSchemaType.SCORE, + values=[ + "good", + "bad", + "neutral", + ], + ), + ], + ), + ), + type=LLMObsAnnotationQueueType.QUEUES, + ), +) + +configuration = Configuration() +configuration.unstable_operations["update_llm_obs_annotation_queue_label_schema"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.update_llm_obs_annotation_queue_label_schema(queue_id="queue_id", body=body) + + print(response) diff --git a/src/datadog_api_client/configuration.py b/src/datadog_api_client/configuration.py index 104762de45..82b06f5fb3 100644 --- a/src/datadog_api_client/configuration.py +++ b/src/datadog_api_client/configuration.py @@ -280,6 +280,7 @@ def __init__( "v2.delete_llm_obs_experiments": False, "v2.delete_llm_obs_projects": False, "v2.get_llm_obs_annotated_interactions": False, + "v2.get_llm_obs_annotation_queue_label_schema": False, "v2.get_llm_obs_custom_eval_config": False, "v2.list_llm_obs_annotation_queues": False, "v2.list_llm_obs_dataset_records": False, @@ -287,6 +288,7 @@ def __init__( "v2.list_llm_obs_experiments": False, "v2.list_llm_obs_projects": False, "v2.update_llm_obs_annotation_queue": False, + "v2.update_llm_obs_annotation_queue_label_schema": False, "v2.update_llm_obs_custom_eval_config": False, "v2.update_llm_obs_dataset": False, "v2.update_llm_obs_dataset_records": False, diff --git a/src/datadog_api_client/v2/api/llm_observability_api.py b/src/datadog_api_client/v2/api/llm_observability_api.py index 2cb396a694..cb099e5635 100644 --- a/src/datadog_api_client/v2/api/llm_observability_api.py +++ b/src/datadog_api_client/v2/api/llm_observability_api.py @@ -27,6 +27,12 @@ from datadog_api_client.v2.model.llm_obs_delete_annotation_queue_interactions_request import ( LLMObsDeleteAnnotationQueueInteractionsRequest, ) +from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_response import ( + LLMObsAnnotationQueueLabelSchemaResponse, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_request import ( + LLMObsAnnotationQueueLabelSchemaUpdateRequest, +) from datadog_api_client.v2.model.llm_obs_experiments_response import LLMObsExperimentsResponse from datadog_api_client.v2.model.llm_obs_experiment_response import LLMObsExperimentResponse from datadog_api_client.v2.model.llm_obs_experiment_request import LLMObsExperimentRequest @@ -423,6 +429,29 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._get_llm_obs_annotation_queue_label_schema_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsAnnotationQueueLabelSchemaResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/annotation-queues/{queue_id}/label-schema", + "operation_id": "get_llm_obs_annotation_queue_label_schema", + "http_method": "GET", + "version": "v2", + }, + params_map={ + "queue_id": { + "required": True, + "openapi_types": (str,), + "attribute": "queue_id", + "location": "path", + }, + }, + headers_map={ + "accept": ["application/json"], + }, + api_client=api_client, + ) + self._get_llm_obs_custom_eval_config_endpoint = _Endpoint( settings={ "response_type": (LLMObsCustomEvalConfigResponse,), @@ -666,6 +695,32 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._update_llm_obs_annotation_queue_label_schema_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsAnnotationQueueLabelSchemaResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/annotation-queues/{queue_id}/label-schema", + "operation_id": "update_llm_obs_annotation_queue_label_schema", + "http_method": "PUT", + "version": "v2", + }, + params_map={ + "queue_id": { + "required": True, + "openapi_types": (str,), + "attribute": "queue_id", + "location": "path", + }, + "body": { + "required": True, + "openapi_types": (LLMObsAnnotationQueueLabelSchemaUpdateRequest,), + "location": "body", + }, + }, + headers_map={"accept": ["application/json"], "content_type": ["application/json"]}, + api_client=api_client, + ) + self._update_llm_obs_custom_eval_config_endpoint = _Endpoint( settings={ "response_type": None, @@ -814,8 +869,10 @@ def create_llm_obs_annotation_queue( ) -> LLMObsAnnotationQueueResponse: """Create an LLM Observability annotation queue. - Create a new annotation queue. Only ``name`` , ``project_id`` , and ``description`` are accepted. - Fields such as ``created_by`` , ``owned_by`` , ``created_at`` , ``modified_by`` , and ``modified_at`` are inferred by the backend. + Create an annotation queue. The ``name`` and ``project_id`` fields are required. + An optional ``annotation_schema`` can be provided to define the labels for the queue. + Fields such as ``created_by`` , ``owned_by`` , ``created_at`` , ``modified_by`` , + and ``modified_at`` are inferred by the backend. :param body: Create annotation queue payload. :type body: LLMObsAnnotationQueueRequest @@ -1110,6 +1167,23 @@ def get_llm_obs_annotated_interactions( return self._get_llm_obs_annotated_interactions_endpoint.call_with_http_info(**kwargs) + def get_llm_obs_annotation_queue_label_schema( + self, + queue_id: str, + ) -> LLMObsAnnotationQueueLabelSchemaResponse: + """Get annotation queue label schema. + + Retrieve the label schema for a given annotation queue. + + :param queue_id: The ID of the LLM Observability annotation queue. + :type queue_id: str + :rtype: LLMObsAnnotationQueueLabelSchemaResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["queue_id"] = queue_id + + return self._get_llm_obs_annotation_queue_label_schema_endpoint.call_with_http_info(**kwargs) + def get_llm_obs_custom_eval_config( self, eval_name: str, @@ -1323,7 +1397,7 @@ def update_llm_obs_annotation_queue( ) -> LLMObsAnnotationQueueResponse: """Update an LLM Observability annotation queue. - Partially update an annotation queue. Only ``name`` and ``description`` can be updated. + Partially update an annotation queue. The ``name`` , ``description`` , and ``annotation_schema`` fields can be updated. :param queue_id: The ID of the LLM Observability annotation queue. :type queue_id: str @@ -1338,6 +1412,31 @@ def update_llm_obs_annotation_queue( return self._update_llm_obs_annotation_queue_endpoint.call_with_http_info(**kwargs) + def update_llm_obs_annotation_queue_label_schema( + self, + queue_id: str, + body: LLMObsAnnotationQueueLabelSchemaUpdateRequest, + ) -> LLMObsAnnotationQueueLabelSchemaResponse: + """Update annotation queue label schema. + + Create or replace the label schema for a given annotation queue. + The label schema defines the labels annotators can apply to interactions in the queue. + Label names must be unique within the queue and match the pattern ``^[a-zA-Z0-9_-]+$``. + Each label must have a valid type: score, categorical, boolean, or text. + + :param queue_id: The ID of the LLM Observability annotation queue. + :type queue_id: str + :param body: Update label schema payload. + :type body: LLMObsAnnotationQueueLabelSchemaUpdateRequest + :rtype: LLMObsAnnotationQueueLabelSchemaResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["queue_id"] = queue_id + + kwargs["body"] = body + + return self._update_llm_obs_annotation_queue_label_schema_endpoint.call_with_http_info(**kwargs) + def update_llm_obs_custom_eval_config( self, eval_name: str, diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_request.py index 3edc63e44e..39ef4499ca 100644 --- a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_request.py +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_request.py @@ -3,7 +3,7 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations -from typing import Union +from typing import Union, TYPE_CHECKING from datadog_api_client.model_utils import ( ModelNormal, @@ -13,25 +13,43 @@ ) +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema + + class LLMObsAnnotationQueueDataAttributesRequest(ModelNormal): @cached_property def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema + return { + "annotation_schema": (LLMObsAnnotationSchema,), "description": (str,), "name": (str,), "project_id": (str,), } attribute_map = { + "annotation_schema": "annotation_schema", "description": "description", "name": "name", "project_id": "project_id", } - def __init__(self_, name: str, project_id: str, description: Union[str, UnsetType] = unset, **kwargs): + def __init__( + self_, + name: str, + project_id: str, + annotation_schema: Union[LLMObsAnnotationSchema, UnsetType] = unset, + description: Union[str, UnsetType] = unset, + **kwargs, + ): """ Attributes for creating an LLM Observability annotation queue. + :param annotation_schema: Schema defining the labels for an annotation queue. + :type annotation_schema: LLMObsAnnotationSchema, optional + :param description: Description of the annotation queue. :type description: str, optional @@ -41,6 +59,8 @@ def __init__(self_, name: str, project_id: str, description: Union[str, UnsetTyp :param project_id: Identifier of the project this queue belongs to. :type project_id: str """ + if annotation_schema is not unset: + kwargs["annotation_schema"] = annotation_schema if description is not unset: kwargs["description"] = description super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_response.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_response.py index 989cabc59d..7ae080317e 100644 --- a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_response.py +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_data_attributes_response.py @@ -3,18 +3,28 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations +from typing import Union, TYPE_CHECKING from datadog_api_client.model_utils import ( ModelNormal, cached_property, datetime, + unset, + UnsetType, ) +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema + + class LLMObsAnnotationQueueDataAttributesResponse(ModelNormal): @cached_property def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema + return { + "annotation_schema": (LLMObsAnnotationSchema,), "created_at": (datetime,), "created_by": (str,), "description": (str,), @@ -26,6 +36,7 @@ def openapi_types(_): } attribute_map = { + "annotation_schema": "annotation_schema", "created_at": "created_at", "created_by": "created_by", "description": "description", @@ -46,11 +57,15 @@ def __init__( name: str, owned_by: str, project_id: str, + annotation_schema: Union[LLMObsAnnotationSchema, UnsetType] = unset, **kwargs, ): """ Attributes of an LLM Observability annotation queue. + :param annotation_schema: Schema defining the labels for an annotation queue. + :type annotation_schema: LLMObsAnnotationSchema, optional + :param created_at: Timestamp when the queue was created. :type created_at: datetime @@ -75,6 +90,8 @@ def __init__( :param project_id: Identifier of the project this queue belongs to. :type project_id: str """ + if annotation_schema is not unset: + kwargs["annotation_schema"] = annotation_schema super().__init__(kwargs) self_.created_at = created_at diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_attributes.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_attributes.py new file mode 100644 index 0000000000..6dd35ed81e --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_attributes.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema + + +class LLMObsAnnotationQueueLabelSchemaAttributes(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema + + return { + "annotation_schema": (LLMObsAnnotationSchema,), + } + + attribute_map = { + "annotation_schema": "annotation_schema", + } + + def __init__(self_, annotation_schema: LLMObsAnnotationSchema, **kwargs): + """ + Attributes of an annotation queue label schema. + + :param annotation_schema: Schema defining the labels for an annotation queue. + :type annotation_schema: LLMObsAnnotationSchema + """ + super().__init__(kwargs) + + self_.annotation_schema = annotation_schema diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_data.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_data.py new file mode 100644 index 0000000000..8a34b4c99c --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_data.py @@ -0,0 +1,64 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_attributes import ( + LLMObsAnnotationQueueLabelSchemaAttributes, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType + + +class LLMObsAnnotationQueueLabelSchemaData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_attributes import ( + LLMObsAnnotationQueueLabelSchemaAttributes, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType + + return { + "attributes": (LLMObsAnnotationQueueLabelSchemaAttributes,), + "id": (str,), + "type": (LLMObsAnnotationQueueType,), + } + + attribute_map = { + "attributes": "attributes", + "id": "id", + "type": "type", + } + + def __init__( + self_, + attributes: LLMObsAnnotationQueueLabelSchemaAttributes, + id: str, + type: LLMObsAnnotationQueueType, + **kwargs, + ): + """ + Data object for an annotation queue label schema. + + :param attributes: Attributes of an annotation queue label schema. + :type attributes: LLMObsAnnotationQueueLabelSchemaAttributes + + :param id: Unique identifier of the annotation queue. + :type id: str + + :param type: Resource type of an LLM Observability annotation queue. + :type type: LLMObsAnnotationQueueType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_response.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_response.py new file mode 100644 index 0000000000..57437a2c9c --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_response.py @@ -0,0 +1,44 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_data import ( + LLMObsAnnotationQueueLabelSchemaData, + ) + + +class LLMObsAnnotationQueueLabelSchemaResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_data import ( + LLMObsAnnotationQueueLabelSchemaData, + ) + + return { + "data": (LLMObsAnnotationQueueLabelSchemaData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsAnnotationQueueLabelSchemaData, **kwargs): + """ + Response containing the label schema of an annotation queue. + + :param data: Data object for an annotation queue label schema. + :type data: LLMObsAnnotationQueueLabelSchemaData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_update_attributes.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_update_attributes.py new file mode 100644 index 0000000000..60b75c2a03 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_update_attributes.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema + + +class LLMObsAnnotationQueueLabelSchemaUpdateAttributes(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema + + return { + "annotation_schema": (LLMObsAnnotationSchema,), + } + + attribute_map = { + "annotation_schema": "annotation_schema", + } + + def __init__(self_, annotation_schema: LLMObsAnnotationSchema, **kwargs): + """ + Attributes for updating an annotation queue label schema. + + :param annotation_schema: Schema defining the labels for an annotation queue. + :type annotation_schema: LLMObsAnnotationSchema + """ + super().__init__(kwargs) + + self_.annotation_schema = annotation_schema diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_update_data.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_update_data.py new file mode 100644 index 0000000000..c1f66b0764 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_update_data.py @@ -0,0 +1,54 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_attributes import ( + LLMObsAnnotationQueueLabelSchemaUpdateAttributes, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType + + +class LLMObsAnnotationQueueLabelSchemaUpdateData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_attributes import ( + LLMObsAnnotationQueueLabelSchemaUpdateAttributes, + ) + from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType + + return { + "attributes": (LLMObsAnnotationQueueLabelSchemaUpdateAttributes,), + "type": (LLMObsAnnotationQueueType,), + } + + attribute_map = { + "attributes": "attributes", + "type": "type", + } + + def __init__( + self_, attributes: LLMObsAnnotationQueueLabelSchemaUpdateAttributes, type: LLMObsAnnotationQueueType, **kwargs + ): + """ + Data object for updating an annotation queue label schema. + + :param attributes: Attributes for updating an annotation queue label schema. + :type attributes: LLMObsAnnotationQueueLabelSchemaUpdateAttributes + + :param type: Resource type of an LLM Observability annotation queue. + :type type: LLMObsAnnotationQueueType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_update_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_update_request.py new file mode 100644 index 0000000000..dbd1f3a377 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_label_schema_update_request.py @@ -0,0 +1,44 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_data import ( + LLMObsAnnotationQueueLabelSchemaUpdateData, + ) + + +class LLMObsAnnotationQueueLabelSchemaUpdateRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_data import ( + LLMObsAnnotationQueueLabelSchemaUpdateData, + ) + + return { + "data": (LLMObsAnnotationQueueLabelSchemaUpdateData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsAnnotationQueueLabelSchemaUpdateData, **kwargs): + """ + Request to update the label schema of an annotation queue. + + :param data: Data object for updating an annotation queue label schema. + :type data: LLMObsAnnotationQueueLabelSchemaUpdateData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_attributes_request.py b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_attributes_request.py index 92dc720e76..04a146b6dc 100644 --- a/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_attributes_request.py +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_queue_update_data_attributes_request.py @@ -3,7 +3,7 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations -from typing import Union +from typing import Union, TYPE_CHECKING from datadog_api_client.model_utils import ( ModelNormal, @@ -13,29 +13,48 @@ ) +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema + + class LLMObsAnnotationQueueUpdateDataAttributesRequest(ModelNormal): @cached_property def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema + return { + "annotation_schema": (LLMObsAnnotationSchema,), "description": (str,), "name": (str,), } attribute_map = { + "annotation_schema": "annotation_schema", "description": "description", "name": "name", } - def __init__(self_, description: Union[str, UnsetType] = unset, name: Union[str, UnsetType] = unset, **kwargs): + def __init__( + self_, + annotation_schema: Union[LLMObsAnnotationSchema, UnsetType] = unset, + description: Union[str, UnsetType] = unset, + name: Union[str, UnsetType] = unset, + **kwargs, + ): """ Attributes for updating an LLM Observability annotation queue. All fields are optional. + :param annotation_schema: Schema defining the labels for an annotation queue. + :type annotation_schema: LLMObsAnnotationSchema, optional + :param description: Updated description of the annotation queue. :type description: str, optional :param name: Updated name of the annotation queue. :type name: str, optional """ + if annotation_schema is not unset: + kwargs["annotation_schema"] = annotation_schema if description is not unset: kwargs["description"] = description if name is not unset: diff --git a/src/datadog_api_client/v2/model/llm_obs_annotation_schema.py b/src/datadog_api_client/v2/model/llm_obs_annotation_schema.py new file mode 100644 index 0000000000..e40db77d0c --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_annotation_schema.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_label_schema import LLMObsLabelSchema + + +class LLMObsAnnotationSchema(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_label_schema import LLMObsLabelSchema + + return { + "label_schemas": ([LLMObsLabelSchema],), + } + + attribute_map = { + "label_schemas": "label_schemas", + } + + def __init__(self_, label_schemas: List[LLMObsLabelSchema], **kwargs): + """ + Schema defining the labels for an annotation queue. + + :param label_schemas: List of label schema definitions. + :type label_schemas: [LLMObsLabelSchema] + """ + super().__init__(kwargs) + + self_.label_schemas = label_schemas diff --git a/src/datadog_api_client/v2/model/llm_obs_label_schema.py b/src/datadog_api_client/v2/model/llm_obs_label_schema.py new file mode 100644 index 0000000000..cdb623bdc3 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_label_schema.py @@ -0,0 +1,133 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_label_schema_type import LLMObsLabelSchemaType + + +class LLMObsLabelSchema(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_label_schema_type import LLMObsLabelSchemaType + + return { + "description": (str,), + "has_assessment": (bool,), + "has_reasoning": (bool,), + "id": (str,), + "is_assessment": (bool,), + "is_integer": (bool,), + "is_required": (bool,), + "max": (float,), + "min": (float,), + "name": (str,), + "type": (LLMObsLabelSchemaType,), + "values": ([str],), + } + + attribute_map = { + "description": "description", + "has_assessment": "has_assessment", + "has_reasoning": "has_reasoning", + "id": "id", + "is_assessment": "is_assessment", + "is_integer": "is_integer", + "is_required": "is_required", + "max": "max", + "min": "min", + "name": "name", + "type": "type", + "values": "values", + } + + def __init__( + self_, + name: str, + type: LLMObsLabelSchemaType, + description: Union[str, UnsetType] = unset, + has_assessment: Union[bool, UnsetType] = unset, + has_reasoning: Union[bool, UnsetType] = unset, + id: Union[str, UnsetType] = unset, + is_assessment: Union[bool, UnsetType] = unset, + is_integer: Union[bool, UnsetType] = unset, + is_required: Union[bool, UnsetType] = unset, + max: Union[float, UnsetType] = unset, + min: Union[float, UnsetType] = unset, + values: Union[List[str], UnsetType] = unset, + **kwargs, + ): + """ + Schema definition for a single label in an annotation queue. + + :param description: Description of the label. + :type description: str, optional + + :param has_assessment: Whether this label includes an assessment field. + :type has_assessment: bool, optional + + :param has_reasoning: Whether this label includes a reasoning field. + :type has_reasoning: bool, optional + + :param id: Unique identifier of the label schema. Assigned by the server if not provided. + :type id: str, optional + + :param is_assessment: Whether the boolean label represents an assessment. Requires ``has_assessment`` to be true. + :type is_assessment: bool, optional + + :param is_integer: Whether score values must be integers. Applicable to score-type labels. + :type is_integer: bool, optional + + :param is_required: Whether this label is required for an annotation. + :type is_required: bool, optional + + :param max: Maximum value for score-type labels. + :type max: float, optional + + :param min: Minimum value for score-type labels. + :type min: float, optional + + :param name: Name of the label. Must match the pattern ``^[a-zA-Z0-9_-]+$`` and be unique within the queue. + :type name: str + + :param type: Type of a label in an annotation queue label schema. + :type type: LLMObsLabelSchemaType + + :param values: Allowed values for categorical-type labels. Must contain at least one non-empty, unique value. + :type values: [str], optional + """ + if description is not unset: + kwargs["description"] = description + if has_assessment is not unset: + kwargs["has_assessment"] = has_assessment + if has_reasoning is not unset: + kwargs["has_reasoning"] = has_reasoning + if id is not unset: + kwargs["id"] = id + if is_assessment is not unset: + kwargs["is_assessment"] = is_assessment + if is_integer is not unset: + kwargs["is_integer"] = is_integer + if is_required is not unset: + kwargs["is_required"] = is_required + if max is not unset: + kwargs["max"] = max + if min is not unset: + kwargs["min"] = min + if values is not unset: + kwargs["values"] = values + super().__init__(kwargs) + + self_.name = name + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_label_schema_type.py b/src/datadog_api_client/v2/model/llm_obs_label_schema_type.py new file mode 100644 index 0000000000..df9ae3680f --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_label_schema_type.py @@ -0,0 +1,44 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsLabelSchemaType(ModelSimple): + """ + Type of a label in an annotation queue label schema. + + :param value: Must be one of ["score", "categorical", "boolean", "text"]. + :type value: str + """ + + allowed_values = { + "score", + "categorical", + "boolean", + "text", + } + SCORE: ClassVar["LLMObsLabelSchemaType"] + CATEGORICAL: ClassVar["LLMObsLabelSchemaType"] + BOOLEAN: ClassVar["LLMObsLabelSchemaType"] + TEXT: ClassVar["LLMObsLabelSchemaType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsLabelSchemaType.SCORE = LLMObsLabelSchemaType("score") +LLMObsLabelSchemaType.CATEGORICAL = LLMObsLabelSchemaType("categorical") +LLMObsLabelSchemaType.BOOLEAN = LLMObsLabelSchemaType("boolean") +LLMObsLabelSchemaType.TEXT = LLMObsLabelSchemaType("text") diff --git a/src/datadog_api_client/v2/models/__init__.py b/src/datadog_api_client/v2/models/__init__.py index 122e284ae7..5ca8caf69a 100644 --- a/src/datadog_api_client/v2/models/__init__.py +++ b/src/datadog_api_client/v2/models/__init__.py @@ -3078,6 +3078,22 @@ LLMObsAnnotationQueueInteractionsResponse, ) from datadog_api_client.v2.model.llm_obs_annotation_queue_interactions_type import LLMObsAnnotationQueueInteractionsType +from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_attributes import ( + LLMObsAnnotationQueueLabelSchemaAttributes, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_data import LLMObsAnnotationQueueLabelSchemaData +from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_response import ( + LLMObsAnnotationQueueLabelSchemaResponse, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_attributes import ( + LLMObsAnnotationQueueLabelSchemaUpdateAttributes, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_data import ( + LLMObsAnnotationQueueLabelSchemaUpdateData, +) +from datadog_api_client.v2.model.llm_obs_annotation_queue_label_schema_update_request import ( + LLMObsAnnotationQueueLabelSchemaUpdateRequest, +) from datadog_api_client.v2.model.llm_obs_annotation_queue_request import LLMObsAnnotationQueueRequest from datadog_api_client.v2.model.llm_obs_annotation_queue_response import LLMObsAnnotationQueueResponse from datadog_api_client.v2.model.llm_obs_annotation_queue_type import LLMObsAnnotationQueueType @@ -3089,6 +3105,7 @@ ) from datadog_api_client.v2.model.llm_obs_annotation_queue_update_request import LLMObsAnnotationQueueUpdateRequest from datadog_api_client.v2.model.llm_obs_annotation_queues_response import LLMObsAnnotationQueuesResponse +from datadog_api_client.v2.model.llm_obs_annotation_schema import LLMObsAnnotationSchema from datadog_api_client.v2.model.llm_obs_cursor_meta import LLMObsCursorMeta from datadog_api_client.v2.model.llm_obs_custom_eval_config_assessment_criteria import ( LLMObsCustomEvalConfigAssessmentCriteria, @@ -3217,6 +3234,8 @@ from datadog_api_client.v2.model.llm_obs_experiment_update_request import LLMObsExperimentUpdateRequest from datadog_api_client.v2.model.llm_obs_experiments_response import LLMObsExperimentsResponse from datadog_api_client.v2.model.llm_obs_interaction_type import LLMObsInteractionType +from datadog_api_client.v2.model.llm_obs_label_schema import LLMObsLabelSchema +from datadog_api_client.v2.model.llm_obs_label_schema_type import LLMObsLabelSchemaType from datadog_api_client.v2.model.llm_obs_metric_assessment import LLMObsMetricAssessment from datadog_api_client.v2.model.llm_obs_metric_score_type import LLMObsMetricScoreType from datadog_api_client.v2.model.llm_obs_project_data_attributes_request import LLMObsProjectDataAttributesRequest @@ -9755,6 +9774,12 @@ "LLMObsAnnotationQueueInteractionsRequest", "LLMObsAnnotationQueueInteractionsResponse", "LLMObsAnnotationQueueInteractionsType", + "LLMObsAnnotationQueueLabelSchemaAttributes", + "LLMObsAnnotationQueueLabelSchemaData", + "LLMObsAnnotationQueueLabelSchemaResponse", + "LLMObsAnnotationQueueLabelSchemaUpdateAttributes", + "LLMObsAnnotationQueueLabelSchemaUpdateData", + "LLMObsAnnotationQueueLabelSchemaUpdateRequest", "LLMObsAnnotationQueueRequest", "LLMObsAnnotationQueueResponse", "LLMObsAnnotationQueueType", @@ -9762,6 +9787,7 @@ "LLMObsAnnotationQueueUpdateDataRequest", "LLMObsAnnotationQueueUpdateRequest", "LLMObsAnnotationQueuesResponse", + "LLMObsAnnotationSchema", "LLMObsCursorMeta", "LLMObsCustomEvalConfigAssessmentCriteria", "LLMObsCustomEvalConfigAttributes", @@ -9846,6 +9872,8 @@ "LLMObsExperimentUpdateRequest", "LLMObsExperimentsResponse", "LLMObsInteractionType", + "LLMObsLabelSchema", + "LLMObsLabelSchemaType", "LLMObsMetricAssessment", "LLMObsMetricScoreType", "LLMObsProjectDataAttributesRequest", diff --git a/tests/v2/features/llm_observability.feature b/tests/v2/features/llm_observability.feature index dc0a15364b..e9c054c736 100644 --- a/tests/v2/features/llm_observability.feature +++ b/tests/v2/features/llm_observability.feature @@ -79,7 +79,7 @@ Feature: LLM Observability Scenario: Create an LLM Observability annotation queue returns "Bad Request" response Given operation "CreateLLMObsAnnotationQueue" enabled And new "CreateLLMObsAnnotationQueue" request - And body with value {"data": {"attributes": {"description": "Queue for annotating customer support traces", "name": "My annotation queue", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "queues"}} + And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "ab12cd34", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}, "description": "Queue for annotating customer support traces", "name": "My annotation queue", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "queues"}} When the request is sent Then the response status is 400 Bad Request @@ -87,7 +87,7 @@ Feature: LLM Observability Scenario: Create an LLM Observability annotation queue returns "Created" response Given operation "CreateLLMObsAnnotationQueue" enabled And new "CreateLLMObsAnnotationQueue" request - And body with value {"data": {"attributes": {"description": "Queue for annotating customer support traces", "name": "My annotation queue", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "queues"}} + And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "ab12cd34", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}, "description": "Queue for annotating customer support traces", "name": "My annotation queue", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "queues"}} When the request is sent Then the response status is 201 Created @@ -415,6 +415,22 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: Get annotation queue label schema returns "Not Found" response + Given operation "GetLLMObsAnnotationQueueLabelSchema" enabled + And new "GetLLMObsAnnotationQueueLabelSchema" request + And request contains "queue_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Get annotation queue label schema returns "OK" response + Given operation "GetLLMObsAnnotationQueueLabelSchema" enabled + And new "GetLLMObsAnnotationQueueLabelSchema" request + And request contains "queue_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability annotation queues returns "Bad Request" response Given operation "ListLLMObsAnnotationQueues" enabled @@ -570,7 +586,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsAnnotationQueue" enabled And new "UpdateLLMObsAnnotationQueue" request And request contains "queue_id" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} + And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "ab12cd34", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}, "description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} When the request is sent Then the response status is 400 Bad Request @@ -579,7 +595,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsAnnotationQueue" enabled And new "UpdateLLMObsAnnotationQueue" request And request contains "queue_id" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} + And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "ab12cd34", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}, "description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} When the request is sent Then the response status is 404 Not Found @@ -588,7 +604,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsAnnotationQueue" enabled And new "UpdateLLMObsAnnotationQueue" request And request contains "queue_id" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} + And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "ab12cd34", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}, "description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} When the request is sent Then the response status is 200 OK @@ -675,3 +691,30 @@ Feature: LLM Observability And body with value {"data": {"attributes": {}, "type": "projects"}} When the request is sent Then the response status is 200 OK + + @generated @skip @team:DataDog/ml-observability + Scenario: Update annotation queue label schema returns "Bad Request" response + Given operation "UpdateLLMObsAnnotationQueueLabelSchema" enabled + And new "UpdateLLMObsAnnotationQueueLabelSchema" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "ab12cd34", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}}, "type": "queues"}} + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Update annotation queue label schema returns "Not Found" response + Given operation "UpdateLLMObsAnnotationQueueLabelSchema" enabled + And new "UpdateLLMObsAnnotationQueueLabelSchema" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "ab12cd34", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}}, "type": "queues"}} + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Update annotation queue label schema returns "OK" response + Given operation "UpdateLLMObsAnnotationQueueLabelSchema" enabled + And new "UpdateLLMObsAnnotationQueueLabelSchema" request + And request contains "queue_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "ab12cd34", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}}, "type": "queues"}} + When the request is sent + Then the response status is 200 OK diff --git a/tests/v2/features/undo.json b/tests/v2/features/undo.json index 8c591eca8c..f606e4a865 100644 --- a/tests/v2/features/undo.json +++ b/tests/v2/features/undo.json @@ -3267,6 +3267,18 @@ "type": "idempotent" } }, + "GetLLMObsAnnotationQueueLabelSchema": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, + "UpdateLLMObsAnnotationQueueLabelSchema": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, "ListLLMObsExperiments": { "tag": "LLM Observability", "undo": {