Skip to content

Commit fe01bf8

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Document Annotations label schema API for AI Observability (#3771)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent 0639ef6 commit fe01bf8

21 files changed

Lines changed: 2723 additions & 18 deletions

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 378 additions & 3 deletions
Large diffs are not rendered by default.

‎examples/v2/llm-observability/CreateLLMObsAnnotationQueue.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@
88
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueRequest;
99
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueResponse;
1010
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueType;
11+
import com.datadog.api.client.v2.model.LLMObsAnnotationSchema;
12+
import com.datadog.api.client.v2.model.LLMObsLabelSchema;
13+
import com.datadog.api.client.v2.model.LLMObsLabelSchemaType;
14+
import java.util.Arrays;
15+
import java.util.Collections;
1116

1217
public class Example {
1318
public static void main(String[] args) {
@@ -21,6 +26,22 @@ public static void main(String[] args) {
2126
new LLMObsAnnotationQueueDataRequest()
2227
.attributes(
2328
new LLMObsAnnotationQueueDataAttributesRequest()
29+
.annotationSchema(
30+
new LLMObsAnnotationSchema()
31+
.labelSchemas(
32+
Collections.singletonList(
33+
new LLMObsLabelSchema()
34+
.description("Rating of the response quality.")
35+
.hasAssessment(false)
36+
.hasReasoning(false)
37+
.id("ab12cd34")
38+
.isAssessment(false)
39+
.isInteger(false)
40+
.isRequired(true)
41+
.max(5.0)
42+
.name("quality")
43+
.type(LLMObsLabelSchemaType.SCORE)
44+
.values(Arrays.asList("good", "bad", "neutral")))))
2445
.description("Queue for annotating customer support traces")
2546
.name("My annotation queue")
2647
.projectId("a33671aa-24fd-4dcd-9b33-a8ec7dde7751"))

‎examples/v2/llm-observability/GetLLMObsAnnotationQueueLabelSchema.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Get annotation queue label schema returns "OK" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueLabelSchemaResponse;
7+
8+
public class Example {
9+
public static void main(String[] args) {
10+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
11+
defaultClient.setUnstableOperationEnabled("v2.getLLMObsAnnotationQueueLabelSchema", true);
12+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
13+
14+
try {
15+
LLMObsAnnotationQueueLabelSchemaResponse result =
16+
apiInstance.getLLMObsAnnotationQueueLabelSchema("b5e7f3a1-9c2d-4f8b-a1e6-3d7e9f0a2b4c");
17+
System.out.println(result);
18+
} catch (ApiException e) {
19+
System.err.println(
20+
"Exception when calling LlmObservabilityApi#getLLMObsAnnotationQueueLabelSchema");
21+
System.err.println("Status code: " + e.getCode());
22+
System.err.println("Reason: " + e.getResponseBody());
23+
System.err.println("Response headers: " + e.getResponseHeaders());
24+
e.printStackTrace();
25+
}
26+
}
27+
}

‎examples/v2/llm-observability/UpdateLLMObsAnnotationQueue.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@
88
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueUpdateDataAttributesRequest;
99
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueUpdateDataRequest;
1010
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueUpdateRequest;
11+
import com.datadog.api.client.v2.model.LLMObsAnnotationSchema;
12+
import com.datadog.api.client.v2.model.LLMObsLabelSchema;
13+
import com.datadog.api.client.v2.model.LLMObsLabelSchemaType;
14+
import java.util.Arrays;
15+
import java.util.Collections;
1116

1217
public class Example {
1318
public static void main(String[] args) {
@@ -21,6 +26,22 @@ public static void main(String[] args) {
2126
new LLMObsAnnotationQueueUpdateDataRequest()
2227
.attributes(
2328
new LLMObsAnnotationQueueUpdateDataAttributesRequest()
29+
.annotationSchema(
30+
new LLMObsAnnotationSchema()
31+
.labelSchemas(
32+
Collections.singletonList(
33+
new LLMObsLabelSchema()
34+
.description("Rating of the response quality.")
35+
.hasAssessment(false)
36+
.hasReasoning(false)
37+
.id("ab12cd34")
38+
.isAssessment(false)
39+
.isInteger(false)
40+
.isRequired(true)
41+
.max(5.0)
42+
.name("quality")
43+
.type(LLMObsLabelSchemaType.SCORE)
44+
.values(Arrays.asList("good", "bad", "neutral")))))
2445
.description("Updated description")
2546
.name("Updated queue name"))
2647
.type(LLMObsAnnotationQueueType.QUEUES));

‎examples/v2/llm-observability/UpdateLLMObsAnnotationQueueLabelSchema.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// Update annotation queue label schema returns "OK" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueLabelSchemaResponse;
7+
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueLabelSchemaUpdateAttributes;
8+
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueLabelSchemaUpdateData;
9+
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueLabelSchemaUpdateRequest;
10+
import com.datadog.api.client.v2.model.LLMObsAnnotationQueueType;
11+
import com.datadog.api.client.v2.model.LLMObsAnnotationSchema;
12+
import com.datadog.api.client.v2.model.LLMObsLabelSchema;
13+
import com.datadog.api.client.v2.model.LLMObsLabelSchemaType;
14+
import java.util.Arrays;
15+
import java.util.Collections;
16+
17+
public class Example {
18+
public static void main(String[] args) {
19+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
20+
defaultClient.setUnstableOperationEnabled("v2.updateLLMObsAnnotationQueueLabelSchema", true);
21+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
22+
23+
LLMObsAnnotationQueueLabelSchemaUpdateRequest body =
24+
new LLMObsAnnotationQueueLabelSchemaUpdateRequest()
25+
.data(
26+
new LLMObsAnnotationQueueLabelSchemaUpdateData()
27+
.attributes(
28+
new LLMObsAnnotationQueueLabelSchemaUpdateAttributes()
29+
.annotationSchema(
30+
new LLMObsAnnotationSchema()
31+
.labelSchemas(
32+
Collections.singletonList(
33+
new LLMObsLabelSchema()
34+
.description("Rating of the response quality.")
35+
.hasAssessment(false)
36+
.hasReasoning(false)
37+
.id("ab12cd34")
38+
.isAssessment(false)
39+
.isInteger(false)
40+
.isRequired(true)
41+
.max(5.0)
42+
.name("quality")
43+
.type(LLMObsLabelSchemaType.SCORE)
44+
.values(Arrays.asList("good", "bad", "neutral"))))))
45+
.type(LLMObsAnnotationQueueType.QUEUES));
46+
47+
try {
48+
LLMObsAnnotationQueueLabelSchemaResponse result =
49+
apiInstance.updateLLMObsAnnotationQueueLabelSchema(
50+
"b5e7f3a1-9c2d-4f8b-a1e6-3d7e9f0a2b4c", body);
51+
System.out.println(result);
52+
} catch (ApiException e) {
53+
System.err.println(
54+
"Exception when calling LlmObservabilityApi#updateLLMObsAnnotationQueueLabelSchema");
55+
System.err.println("Status code: " + e.getCode());
56+
System.err.println("Reason: " + e.getResponseBody());
57+
System.err.println("Response headers: " + e.getResponseHeaders());
58+
e.printStackTrace();
59+
}
60+
}
61+
}

‎src/main/java/com/datadog/api/client/ApiClient.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -788,13 +788,15 @@ public class ApiClient {
788788
put("v2.deleteLLMObsExperiments", false);
789789
put("v2.deleteLLMObsProjects", false);
790790
put("v2.getLLMObsAnnotatedInteractions", false);
791+
put("v2.getLLMObsAnnotationQueueLabelSchema", false);
791792
put("v2.getLLMObsCustomEvalConfig", false);
792793
put("v2.listLLMObsAnnotationQueues", false);
793794
put("v2.listLLMObsDatasetRecords", false);
794795
put("v2.listLLMObsDatasets", false);
795796
put("v2.listLLMObsExperiments", false);
796797
put("v2.listLLMObsProjects", false);
797798
put("v2.updateLLMObsAnnotationQueue", false);
799+
put("v2.updateLLMObsAnnotationQueueLabelSchema", false);
798800
put("v2.updateLLMObsCustomEvalConfig", false);
799801
put("v2.updateLLMObsDataset", false);
800802
put("v2.updateLLMObsDatasetRecords", false);

0 commit comments

Comments
 (0)