Add OpenAPI spec for LLM Observability Experiments API (DataDog#3042)

Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>

Author: api-clients-generation-pipeline[bot]

.generator/schemas/v2/openapi.yaml

@@ -0,0 +1,17 @@
+# Create an LLM Observability dataset returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.create_llm_obs_dataset".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsDatasetRequest.new({
+  data: DatadogAPIClient::V2::LLMObsDatasetDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsDatasetDataAttributesRequest.new({
+      name: "My LLM Dataset",
+    }),
+    type: DatadogAPIClient::V2::LLMObsDatasetType::DATASETS,
+  }),
+})
+p api_instance.create_llm_obs_dataset("project_id", body)

examples/v2/llm-observability/CreateLLMObsDataset.rb

@@ -0,0 +1,22 @@
+# Append records to an LLM Observability dataset returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.create_llm_obs_dataset_records".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsDatasetRecordsRequest.new({
+  data: DatadogAPIClient::V2::LLMObsDatasetRecordsDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsDatasetRecordsDataAttributesRequest.new({
+      records: [
+        DatadogAPIClient::V2::LLMObsDatasetRecordItem.new({
+          expected_output: nil,
+          input: nil,
+        }),
+      ],
+    }),
+    type: DatadogAPIClient::V2::LLMObsRecordType::RECORDS,
+  }),
+})
+p api_instance.create_llm_obs_dataset_records("project_id", "dataset_id", body)

examples/v2/llm-observability/CreateLLMObsDatasetRecords.rb

@@ -0,0 +1,19 @@
+# Create an LLM Observability experiment returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.create_llm_obs_experiment".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsExperimentRequest.new({
+  data: DatadogAPIClient::V2::LLMObsExperimentDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsExperimentDataAttributesRequest.new({
+      dataset_id: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d",
+      name: "My Experiment v1",
+      project_id: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751",
+    }),
+    type: DatadogAPIClient::V2::LLMObsExperimentType::EXPERIMENTS,
+  }),
+})
+p api_instance.create_llm_obs_experiment(body)

examples/v2/llm-observability/CreateLLMObsExperiment.rb

@@ -0,0 +1,49 @@
+# Push events for an LLM Observability experiment returns "Accepted" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.create_llm_obs_experiment_events".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsExperimentEventsRequest.new({
+  data: DatadogAPIClient::V2::LLMObsExperimentEventsDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsExperimentEventsDataAttributesRequest.new({
+      metrics: [
+        DatadogAPIClient::V2::LLMObsExperimentMetric.new({
+          assessment: DatadogAPIClient::V2::LLMObsMetricAssessment::PASS,
+          error: DatadogAPIClient::V2::LLMObsExperimentMetricError.new({}),
+          label: "faithfulness",
+          metric_type: DatadogAPIClient::V2::LLMObsMetricScoreType::SCORE,
+          span_id: "span-7a1b2c3d",
+          tags: [],
+          timestamp_ms: 1705314600000,
+        }),
+      ],
+      spans: [
+        DatadogAPIClient::V2::LLMObsExperimentSpan.new({
+          dataset_id: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d",
+          duration: 1500000000,
+          meta: DatadogAPIClient::V2::LLMObsExperimentSpanMeta.new({
+            error: DatadogAPIClient::V2::LLMObsExperimentSpanError.new({
+              message: "Model response timed out",
+              stack: 'Traceback (most recent call last):\n  File "main.py", line 10, in <module>\n    response = model.generate(input)\n  File "model.py", line 45, in generate\n    raise TimeoutError("Model response timed out")\nTimeoutError: Model response timed out',
+              type: "TimeoutError",
+            }),
+            input: nil,
+            output: nil,
+          }),
+          name: "llm_call",
+          project_id: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751",
+          span_id: "span-7a1b2c3d",
+          start_ns: 1705314600000000000,
+          status: DatadogAPIClient::V2::LLMObsExperimentSpanStatus::OK,
+          tags: [],
+          trace_id: "abc123def456",
+        }),
+      ],
+    }),
+    type: DatadogAPIClient::V2::LLMObsEventType::EVENTS,
+  }),
+})
+p api_instance.create_llm_obs_experiment_events("experiment_id", body)

examples/v2/llm-observability/CreateLLMObsExperimentEvents.rb

@@ -0,0 +1,17 @@
+# Create an LLM Observability project returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.create_llm_obs_project".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsProjectRequest.new({
+  data: DatadogAPIClient::V2::LLMObsProjectDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsProjectDataAttributesRequest.new({
+      name: "My LLM Project",
+    }),
+    type: DatadogAPIClient::V2::LLMObsProjectType::PROJECTS,
+  }),
+})
+p api_instance.create_llm_obs_project(body)

examples/v2/llm-observability/CreateLLMObsProject.rb

@@ -0,0 +1,19 @@
+# Delete LLM Observability dataset records returns "No Content" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.delete_llm_obs_dataset_records".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsDeleteDatasetRecordsRequest.new({
+  data: DatadogAPIClient::V2::LLMObsDeleteDatasetRecordsDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsDeleteDatasetRecordsDataAttributesRequest.new({
+      record_ids: [
+        "rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c",
+      ],
+    }),
+    type: DatadogAPIClient::V2::LLMObsRecordType::RECORDS,
+  }),
+})
+api_instance.delete_llm_obs_dataset_records("project_id", "dataset_id", body)

examples/v2/llm-observability/DeleteLLMObsDatasetRecords.rb

@@ -0,0 +1,19 @@
+# Delete LLM Observability datasets returns "No Content" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.delete_llm_obs_datasets".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsDeleteDatasetsRequest.new({
+  data: DatadogAPIClient::V2::LLMObsDeleteDatasetsDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsDeleteDatasetsDataAttributesRequest.new({
+      dataset_ids: [
+        "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d",
+      ],
+    }),
+    type: DatadogAPIClient::V2::LLMObsDatasetType::DATASETS,
+  }),
+})
+api_instance.delete_llm_obs_datasets("project_id", body)

examples/v2/llm-observability/DeleteLLMObsDatasets.rb

@@ -0,0 +1,19 @@
+# Delete LLM Observability experiments returns "No Content" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.delete_llm_obs_experiments".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsDeleteExperimentsRequest.new({
+  data: DatadogAPIClient::V2::LLMObsDeleteExperimentsDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsDeleteExperimentsDataAttributesRequest.new({
+      experiment_ids: [
+        "3fd6b5e0-8910-4b1c-a7d0-5b84de329012",
+      ],
+    }),
+    type: DatadogAPIClient::V2::LLMObsExperimentType::EXPERIMENTS,
+  }),
+})
+api_instance.delete_llm_obs_experiments(body)

examples/v2/llm-observability/DeleteLLMObsExperiments.rb

@@ -0,0 +1,19 @@
+# Delete LLM Observability projects returns "No Content" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.delete_llm_obs_projects".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsDeleteProjectsRequest.new({
+  data: DatadogAPIClient::V2::LLMObsDeleteProjectsDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsDeleteProjectsDataAttributesRequest.new({
+      project_ids: [
+        "a33671aa-24fd-4dcd-9b33-a8ec7dde7751",
+      ],
+    }),
+    type: DatadogAPIClient::V2::LLMObsProjectType::PROJECTS,
+  }),
+})
+api_instance.delete_llm_obs_projects(body)