Skip to content

Commit 433cb9c

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add OpenAPI documentation for LLM Observability eval config CRUD endpoints (#3414)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent 7798a86 commit 433cb9c

33 files changed

Lines changed: 2759 additions & 24 deletions

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 651 additions & 0 deletions
Large diffs are not rendered by default.

‎docs/datadog_api_client.v2.model.rst‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16909,6 +16909,167 @@ datadog\_api\_client.v2.model.llm\_obs\_cursor\_meta module
1690916909
:members:
1691016910
:show-inheritance:
1691116911

16912+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_assessment\_criteria module
16913+
-----------------------------------------------------------------------------------------
16914+
16915+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_assessment_criteria
16916+
:members:
16917+
:show-inheritance:
16918+
16919+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_attributes module
16920+
-------------------------------------------------------------------------------
16921+
16922+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_attributes
16923+
:members:
16924+
:show-inheritance:
16925+
16926+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_bedrock\_options module
16927+
-------------------------------------------------------------------------------------
16928+
16929+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_bedrock_options
16930+
:members:
16931+
:show-inheritance:
16932+
16933+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_data module
16934+
-------------------------------------------------------------------------
16935+
16936+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_data
16937+
:members:
16938+
:show-inheritance:
16939+
16940+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_eval\_scope module
16941+
--------------------------------------------------------------------------------
16942+
16943+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_eval_scope
16944+
:members:
16945+
:show-inheritance:
16946+
16947+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_inference\_params module
16948+
--------------------------------------------------------------------------------------
16949+
16950+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_inference_params
16951+
:members:
16952+
:show-inheritance:
16953+
16954+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_integration\_provider module
16955+
------------------------------------------------------------------------------------------
16956+
16957+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_integration_provider
16958+
:members:
16959+
:show-inheritance:
16960+
16961+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_llm\_judge\_config module
16962+
---------------------------------------------------------------------------------------
16963+
16964+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_judge_config
16965+
:members:
16966+
:show-inheritance:
16967+
16968+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_llm\_provider module
16969+
----------------------------------------------------------------------------------
16970+
16971+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_provider
16972+
:members:
16973+
:show-inheritance:
16974+
16975+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_parsing\_type module
16976+
----------------------------------------------------------------------------------
16977+
16978+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_parsing_type
16979+
:members:
16980+
:show-inheritance:
16981+
16982+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_prompt\_content module
16983+
------------------------------------------------------------------------------------
16984+
16985+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content
16986+
:members:
16987+
:show-inheritance:
16988+
16989+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_prompt\_content\_value module
16990+
-------------------------------------------------------------------------------------------
16991+
16992+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content_value
16993+
:members:
16994+
:show-inheritance:
16995+
16996+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_prompt\_message module
16997+
------------------------------------------------------------------------------------
16998+
16999+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_message
17000+
:members:
17001+
:show-inheritance:
17002+
17003+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_prompt\_tool\_call module
17004+
---------------------------------------------------------------------------------------
17005+
17006+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_call
17007+
:members:
17008+
:show-inheritance:
17009+
17010+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_prompt\_tool\_result module
17011+
-----------------------------------------------------------------------------------------
17012+
17013+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_result
17014+
:members:
17015+
:show-inheritance:
17016+
17017+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_response module
17018+
-----------------------------------------------------------------------------
17019+
17020+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_response
17021+
:members:
17022+
:show-inheritance:
17023+
17024+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_target module
17025+
---------------------------------------------------------------------------
17026+
17027+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_target
17028+
:members:
17029+
:show-inheritance:
17030+
17031+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_type module
17032+
-------------------------------------------------------------------------
17033+
17034+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_type
17035+
:members:
17036+
:show-inheritance:
17037+
17038+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_update\_attributes module
17039+
---------------------------------------------------------------------------------------
17040+
17041+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_update_attributes
17042+
:members:
17043+
:show-inheritance:
17044+
17045+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_update\_data module
17046+
---------------------------------------------------------------------------------
17047+
17048+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_update_data
17049+
:members:
17050+
:show-inheritance:
17051+
17052+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_update\_request module
17053+
------------------------------------------------------------------------------------
17054+
17055+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_update_request
17056+
:members:
17057+
:show-inheritance:
17058+
17059+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_user module
17060+
-------------------------------------------------------------------------
17061+
17062+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_user
17063+
:members:
17064+
:show-inheritance:
17065+
17066+
datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_vertex\_ai\_options module
17067+
----------------------------------------------------------------------------------------
17068+
17069+
.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_vertex_ai_options
17070+
:members:
17071+
:show-inheritance:
17072+
1691217073
datadog\_api\_client.v2.model.llm\_obs\_dataset\_data\_attributes\_request module
1691317074
---------------------------------------------------------------------------------
1691417075

‎examples/v2/llm-observability/DeleteLLMObsCustomEvalConfig.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
"""
2+
Delete a custom evaluator configuration returns "No Content" response
3+
"""
4+
5+
from datadog_api_client import ApiClient, Configuration
6+
from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi
7+
8+
configuration = Configuration()
9+
configuration.unstable_operations["delete_llm_obs_custom_eval_config"] = True
10+
with ApiClient(configuration) as api_client:
11+
api_instance = LLMObservabilityApi(api_client)
12+
api_instance.delete_llm_obs_custom_eval_config(
13+
eval_name="eval_name",
14+
)

‎examples/v2/llm-observability/GetLLMObsCustomEvalConfig.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
"""
2+
Get a custom evaluator configuration returns "OK" response
3+
"""
4+
5+
from datadog_api_client import ApiClient, Configuration
6+
from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi
7+
8+
configuration = Configuration()
9+
configuration.unstable_operations["get_llm_obs_custom_eval_config"] = True
10+
with ApiClient(configuration) as api_client:
11+
api_instance = LLMObservabilityApi(api_client)
12+
response = api_instance.get_llm_obs_custom_eval_config(
13+
eval_name="eval_name",
14+
)
15+
16+
print(response)

‎examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.py‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
"""
2+
Create or update a custom evaluator configuration returns "OK" response
3+
"""
4+
5+
from datadog_api_client import ApiClient, Configuration
6+
from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi
7+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_assessment_criteria import (
8+
LLMObsCustomEvalConfigAssessmentCriteria,
9+
)
10+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_bedrock_options import LLMObsCustomEvalConfigBedrockOptions
11+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_eval_scope import LLMObsCustomEvalConfigEvalScope
12+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_inference_params import (
13+
LLMObsCustomEvalConfigInferenceParams,
14+
)
15+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_integration_provider import (
16+
LLMObsCustomEvalConfigIntegrationProvider,
17+
)
18+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_judge_config import LLMObsCustomEvalConfigLLMJudgeConfig
19+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_provider import LLMObsCustomEvalConfigLLMProvider
20+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_parsing_type import LLMObsCustomEvalConfigParsingType
21+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content import LLMObsCustomEvalConfigPromptContent
22+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content_value import (
23+
LLMObsCustomEvalConfigPromptContentValue,
24+
)
25+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_message import LLMObsCustomEvalConfigPromptMessage
26+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_call import LLMObsCustomEvalConfigPromptToolCall
27+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_result import (
28+
LLMObsCustomEvalConfigPromptToolResult,
29+
)
30+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_target import LLMObsCustomEvalConfigTarget
31+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_type import LLMObsCustomEvalConfigType
32+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_attributes import (
33+
LLMObsCustomEvalConfigUpdateAttributes,
34+
)
35+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_data import LLMObsCustomEvalConfigUpdateData
36+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_request import LLMObsCustomEvalConfigUpdateRequest
37+
from datadog_api_client.v2.model.llm_obs_custom_eval_config_vertex_ai_options import (
38+
LLMObsCustomEvalConfigVertexAIOptions,
39+
)
40+
41+
body = LLMObsCustomEvalConfigUpdateRequest(
42+
data=LLMObsCustomEvalConfigUpdateData(
43+
attributes=LLMObsCustomEvalConfigUpdateAttributes(
44+
category="Custom",
45+
eval_name="my-custom-evaluator",
46+
llm_judge_config=LLMObsCustomEvalConfigLLMJudgeConfig(
47+
assessment_criteria=LLMObsCustomEvalConfigAssessmentCriteria(
48+
max_threshold=1.0,
49+
min_threshold=0.7,
50+
pass_values=[
51+
"pass",
52+
"yes",
53+
],
54+
pass_when=True,
55+
),
56+
inference_params=LLMObsCustomEvalConfigInferenceParams(
57+
frequency_penalty=0.0,
58+
max_tokens=1024,
59+
presence_penalty=0.0,
60+
temperature=0.7,
61+
top_k=50,
62+
top_p=1.0,
63+
),
64+
last_used_library_prompt_template_name="sentiment-analysis-v1",
65+
modified_library_prompt_template=False,
66+
output_schema=None,
67+
parsing_type=LLMObsCustomEvalConfigParsingType.STRUCTURED_OUTPUT,
68+
prompt_template=[
69+
LLMObsCustomEvalConfigPromptMessage(
70+
content="Rate the quality of the following response:",
71+
contents=[
72+
LLMObsCustomEvalConfigPromptContent(
73+
type="text",
74+
value=LLMObsCustomEvalConfigPromptContentValue(
75+
text="What is the sentiment of this review?",
76+
tool_call=LLMObsCustomEvalConfigPromptToolCall(
77+
arguments='{"location": "San Francisco"}',
78+
id="call_abc123",
79+
name="get_weather",
80+
type="function",
81+
),
82+
tool_call_result=LLMObsCustomEvalConfigPromptToolResult(
83+
name="get_weather",
84+
result="sunny, 72F",
85+
tool_id="call_abc123",
86+
type="function",
87+
),
88+
),
89+
),
90+
],
91+
role="user",
92+
),
93+
],
94+
),
95+
llm_provider=LLMObsCustomEvalConfigLLMProvider(
96+
bedrock=LLMObsCustomEvalConfigBedrockOptions(
97+
region="us-east-1",
98+
),
99+
integration_account_id="my-account-id",
100+
integration_provider=LLMObsCustomEvalConfigIntegrationProvider.OPENAI,
101+
model_name="gpt-4o",
102+
vertex_ai=LLMObsCustomEvalConfigVertexAIOptions(
103+
location="us-central1",
104+
project="my-gcp-project",
105+
),
106+
),
107+
target=LLMObsCustomEvalConfigTarget(
108+
application_name="my-llm-app",
109+
enabled=True,
110+
eval_scope=LLMObsCustomEvalConfigEvalScope.SPAN,
111+
filter="@service:my-service",
112+
root_spans_only=True,
113+
sampling_percentage=50.0,
114+
),
115+
),
116+
id="my-custom-evaluator",
117+
type=LLMObsCustomEvalConfigType.EVALUATOR_CONFIG,
118+
),
119+
)
120+
121+
configuration = Configuration()
122+
configuration.unstable_operations["update_llm_obs_custom_eval_config"] = True
123+
with ApiClient(configuration) as api_client:
124+
api_instance = LLMObservabilityApi(api_client)
125+
api_instance.update_llm_obs_custom_eval_config(eval_name="eval_name", body=body)

‎src/datadog_api_client/configuration.py‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,33 @@ def __init__(
265265
"v2.list_fleet_tracers": False,
266266
"v2.trigger_fleet_schedule": False,
267267
"v2.update_fleet_schedule": False,
268+
"v2.create_llm_obs_annotation_queue": False,
269+
"v2.create_llm_obs_annotation_queue_interactions": False,
270+
"v2.create_llm_obs_dataset": False,
271+
"v2.create_llm_obs_dataset_records": False,
272+
"v2.create_llm_obs_experiment": False,
273+
"v2.create_llm_obs_experiment_events": False,
274+
"v2.create_llm_obs_project": False,
275+
"v2.delete_llm_obs_annotation_queue": False,
276+
"v2.delete_llm_obs_annotation_queue_interactions": False,
277+
"v2.delete_llm_obs_custom_eval_config": False,
278+
"v2.delete_llm_obs_dataset_records": False,
279+
"v2.delete_llm_obs_datasets": False,
280+
"v2.delete_llm_obs_experiments": False,
281+
"v2.delete_llm_obs_projects": False,
282+
"v2.get_llm_obs_annotated_interactions": False,
283+
"v2.get_llm_obs_custom_eval_config": False,
284+
"v2.list_llm_obs_annotation_queues": False,
285+
"v2.list_llm_obs_dataset_records": False,
286+
"v2.list_llm_obs_datasets": False,
287+
"v2.list_llm_obs_experiments": False,
288+
"v2.list_llm_obs_projects": False,
289+
"v2.update_llm_obs_annotation_queue": False,
290+
"v2.update_llm_obs_custom_eval_config": False,
291+
"v2.update_llm_obs_dataset": False,
292+
"v2.update_llm_obs_dataset_records": False,
293+
"v2.update_llm_obs_experiment": False,
294+
"v2.update_llm_obs_project": False,
268295
"v2.create_open_api": False,
269296
"v2.delete_open_api": False,
270297
"v2.get_open_api": False,
@@ -418,30 +445,6 @@ def __init__(
418445
"v2.update_jira_issue_template": False,
419446
"v2.create_tenancy_config": False,
420447
"v2.get_tenancy_configs": False,
421-
"v2.create_llm_obs_annotation_queue": False,
422-
"v2.create_llm_obs_annotation_queue_interactions": False,
423-
"v2.create_llm_obs_dataset": False,
424-
"v2.create_llm_obs_dataset_records": False,
425-
"v2.create_llm_obs_experiment": False,
426-
"v2.create_llm_obs_experiment_events": False,
427-
"v2.create_llm_obs_project": False,
428-
"v2.delete_llm_obs_annotation_queue": False,
429-
"v2.delete_llm_obs_annotation_queue_interactions": False,
430-
"v2.delete_llm_obs_dataset_records": False,
431-
"v2.delete_llm_obs_datasets": False,
432-
"v2.delete_llm_obs_experiments": False,
433-
"v2.delete_llm_obs_projects": False,
434-
"v2.get_llm_obs_annotated_interactions": False,
435-
"v2.list_llm_obs_annotation_queues": False,
436-
"v2.list_llm_obs_dataset_records": False,
437-
"v2.list_llm_obs_datasets": False,
438-
"v2.list_llm_obs_experiments": False,
439-
"v2.list_llm_obs_projects": False,
440-
"v2.update_llm_obs_annotation_queue": False,
441-
"v2.update_llm_obs_dataset": False,
442-
"v2.update_llm_obs_dataset_records": False,
443-
"v2.update_llm_obs_experiment": False,
444-
"v2.update_llm_obs_project": False,
445448
"v2.add_role_to_restriction_query": False,
446449
"v2.create_restriction_query": False,
447450
"v2.delete_restriction_query": False,

0 commit comments

Comments
 (0)