diff --git a/content/docs/ingest-data/ai-agents/index.mdx b/content/docs/ingest-data/ai-agents/index.mdx index fff6425..67ae094 100644 --- a/content/docs/ingest-data/ai-agents/index.mdx +++ b/content/docs/ingest-data/ai-agents/index.mdx @@ -3,7 +3,7 @@ title: LLM Observability description: Monitor and debug LLM applications with Parseable --- -import { IconBrandOpenai, IconRobot, IconLink, IconBook, IconUsers, IconBrain, IconCode, IconSettingsAutomation, IconServer, IconApi, IconRoute, IconTimeline } from '@tabler/icons-react'; +import { IconBrandOpenai, IconRobot, IconLink, IconBook, IconUsers, IconBrain, IconCode, IconSettingsAutomation, IconServer, IconRoute, IconTimeline } from '@tabler/icons-react'; Monitor, debug, and optimize your LLM applications with Parseable. Track API calls, token usage, latency, and errors across all major LLM providers and frameworks. @@ -34,12 +34,19 @@ LLM applications present unique observability challenges: > Claude and Claude Instant models - } + } + > + Multi-provider LLM traces, logs, and metrics using OpenTelemetry + + } > - Unified API gateway for 100+ LLM providers + Agent, model, and tool-call traces using OpenTelemetry dict[str, str]: + return { + "Authorization": PARSEABLE_AUTHORIZATION, + "X-P-Stream": dataset, + "X-P-Log-Source": source, + } + + +resource = Resource.create( + { + "service.name": "litellm-sdk-app", + "deployment.environment": "production", + } +) + +# Traces +tracer_provider = TracerProvider(resource=resource) +tracer_provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter( + endpoint=f"{PARSEABLE_OTLP_ENDPOINT}/v1/traces", + headers=headers("litellm-sdk-traces", "otel-traces"), + ) + ) +) +trace.set_tracer_provider(tracer_provider) + +# Metrics +metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter( + endpoint=f"{PARSEABLE_OTLP_ENDPOINT}/v1/metrics", + headers=headers("litellm-sdk-metrics", "otel-metrics"), + ), + export_interval_millis=5000, +) +meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) +metrics.set_meter_provider(meter_provider) + +# Logs +logger_provider = LoggerProvider(resource=resource) +logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter( + endpoint=f"{PARSEABLE_OTLP_ENDPOINT}/v1/logs", + headers=headers("litellm-sdk-logs", "otel-logs"), + ) + ) +) +set_logger_provider(logger_provider) + +os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true" +os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS"] = "true" +os.environ["OTEL_SEMCONV_STABILITY_OPT_IN"] = "gen_ai_latest_experimental" +os.environ["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = "NO_CONTENT" +os.environ["USE_OTEL_LITELLM_REQUEST_SPAN"] = "true" + +litellm.callbacks = ["otel"] +litellm.turn_off_message_logging = True + +logger = logging.getLogger("llm-app") +logger.setLevel(logging.INFO) +logger.addHandler(LoggingHandler(logger_provider=logger_provider)) +tracer = trace.get_tracer("llm-app") + +with tracer.start_as_current_span("answer-question"): + logger.info("Sending model request") + response = litellm.completion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "Explain distributed tracing."}], + metadata={"feature": "documentation-example"}, + ) + logger.info("Model request completed") + +# Flush during application shutdown, not after every request. +tracer_provider.force_flush() +meter_provider.force_flush() +logger_provider.force_flush() +``` + +Set credentials and run the application: + +```bash +export OPENAI_API_KEY="" +export PARSEABLE_OTLP_ENDPOINT="https://:8000" +export PARSEABLE_AUTHORIZATION="Basic " +python app.py +``` + +`USE_OTEL_LITELLM_REQUEST_SPAN=true` creates a dedicated child span for each SDK +call. This preserves the application parent span and avoids delayed callback writes +to a parent span that has already ended. + +## Native metrics + +LiteLLM SDK emits these OpenTelemetry histograms: + +| Metric | Description | +| --- | --- | +| `gen_ai.client.operation.duration` | End-to-end SDK operation duration | +| `gen_ai.client.token.usage` | Input and output token usage | +| `gen_ai.client.token.cost` | Estimated request cost in USD | +| `gen_ai.client.response.time_to_first_token` | Streaming time to first token | +| `gen_ai.client.response.time_per_output_token` | Average generation time per output token | +| `gen_ai.client.response.duration` | Provider response-generation duration | + +These are OTLP histogram records, not Prometheus gauge values. Use +`data_point_sum` for totals, `data_point_count` for sample counts, and +`SUM(data_point_sum) / NULLIF(SUM(data_point_count), 0)` for averages. + +Example total-cost query: + +```sql +SELECT SUM(data_point_sum) AS total_cost +FROM "litellm-sdk-metrics" +WHERE metric_name = 'gen_ai.client.token.cost'; +``` + +Example average-latency query: + +```sql +SELECT + SUM(data_point_sum) / NULLIF(SUM(data_point_count), 0) AS avg_latency_seconds +FROM "litellm-sdk-metrics" +WHERE metric_name = 'gen_ai.client.operation.duration'; +``` + +Time-to-first-token and time-per-output-token appear only after the application +fully consumes a streamed response. + +## SDK versus Gateway telemetry + +The SDK runs inside your Python application and exports LLM-call telemetry. It does +not expose the LiteLLM Proxy `/metrics` endpoint or proxy metrics for authentication, +routing, budgets, rate limits, Redis, and PostgreSQL. + +Use [LiteLLM Gateway](/docs/ingest-data/gateways/litellm) when you need centralized +gateway telemetry and Prometheus scraping. Use this SDK integration when Python code +calls model providers through `litellm.completion()` or related SDK methods. + +## Protect prompt and response content + +The example uses `NO_CONTENT` and `turn_off_message_logging=True`, so prompts and +responses are not exported. Model, token, cost, latency, and error metadata remain +available. + +For sanitized development data only, set: + +```bash +export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="SPAN_AND_EVENT" +``` + +Never export API keys, authorization headers, personal data, or regulated content. + +## Troubleshooting + +### Traces appear, but metrics or logs do not + +Confirm each exporter uses the correct endpoint and source header: + +- `/v1/traces` with `X-P-Log-Source: otel-traces` +- `/v1/logs` with `X-P-Log-Source: otel-logs` +- `/v1/metrics` with `X-P-Log-Source: otel-metrics` + +Also confirm `LITELLM_OTEL_INTEGRATION_ENABLE_METRICS` and +`LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS` are set before the first SDK call. + +### Query reports `No field named data_point_value` + +LiteLLM metrics are histograms. Replace `data_point_value` with +`data_point_sum`, or calculate an average from `data_point_sum` and +`data_point_count`. + +### Streaming metrics are missing + +Consume the entire stream. LiteLLM records streaming latency after the final chunk. + +## See also + +- [LiteLLM Gateway](/docs/ingest-data/gateways/litellm) +- [Pydantic AI](/docs/ingest-data/ai-agents/pydantic-ai) +- [Traces](/docs/user-guide/traces) +- [Metrics](/docs/user-guide/metrics) diff --git a/content/docs/ingest-data/ai-agents/litellm.mdx b/content/docs/ingest-data/ai-agents/litellm.mdx deleted file mode 100644 index 94fd60d..0000000 --- a/content/docs/ingest-data/ai-agents/litellm.mdx +++ /dev/null @@ -1,276 +0,0 @@ ---- -title: LiteLLM -description: Send LiteLLM traces to Parseable for LLM observability ---- - -Send LiteLLM traces to Parseable through OpenTelemetry for complete LLM observability with SQL-queryable analytics. - -## Overview - -[LiteLLM](https://litellm.ai) is an open-source LLM gateway that provides a unified API for 100+ LLM providers. It acts as a proxy between your application and LLM APIs like OpenAI, Anthropic, Azure, Bedrock, and self-hosted models. - -Integrate LiteLLM with Parseable to: - -- **Track All LLM Calls** - Monitor requests across multiple providers -- **Analyze Latency** - Identify slow models and optimize routing -- **Monitor Token Usage** - Track consumption and estimate costs -- **Debug Errors** - Investigate failed requests with full context - -## Architecture - -``` -LiteLLM → OpenTelemetry Collector → Parseable - /v1/traces -``` - -## Prerequisites - -- Python 3.8+ -- LiteLLM installed -- OpenTelemetry Collector -- Parseable instance - -## Step 1: Install Dependencies - -```bash -pip install litellm opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp -``` - -## Step 2: Configure OpenTelemetry Collector - -Create `otel-collector-config.yaml`: - -```yaml -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 - -processors: - batch: - timeout: 5s - send_batch_size: 100 - -exporters: - otlphttp: - endpoint: http://localhost:8000 - headers: - Authorization: Basic YWRtaW46YWRtaW4= - X-P-Stream: litellm-traces - X-P-Log-Source: otel-traces - tls: - insecure: true - -service: - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [otlphttp] -``` - -| Setting | Description | -|---------|-------------| -| `receivers.otlp` | Accepts OTLP data on gRPC (4317) and HTTP (4318) | -| `processors.batch` | Batches spans before export for efficiency | -| `exporters.otlphttp.endpoint` | Parseable server URL | -| `X-P-Stream` | Target dataset in Parseable | -| `X-P-Log-Source` | Must be `otel-traces` for trace data | - -Start the collector: - -```bash -./otelcol --config ./otel-collector-config.yaml -``` - -## Step 3: Configure LiteLLM - -Set environment variables: - -```bash -export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318" -export OTEL_EXPORTER_OTLP_PROTOCOL="http/json" -``` - -Enable the OTEL callback in your Python code: - -```python -import litellm -from litellm import completion - -# Enable OpenTelemetry tracing -litellm.callbacks = ["otel"] - -# Make an LLM call -response = completion( - model="gpt-4", - messages=[{"role": "user", "content": "What is observability?"}] -) - -print(response.choices[0].message.content) -``` - -Every LLM call now emits a trace to Parseable. - -## Step 4: LiteLLM Proxy (Optional) - -If running LiteLLM as a proxy server, configure `litellm_config.yaml`: - -```yaml -model_list: - - model_name: gpt-4 - litellm_params: - model: openai/gpt-4 - api_key: sk-... - -litellm_settings: - callbacks: ["otel"] - -environment_variables: - OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318" - OTEL_EXPORTER_OTLP_PROTOCOL: "http/json" -``` - -Start the proxy: - -```bash -litellm --config litellm_config.yaml -``` - -## Trace Schema - -LiteLLM traces include rich metadata: - -| Field | Description | -|-------|-------------| -| `trace_id` | Unique trace identifier | -| `span_id` | Unique span identifier | -| `span_name` | Operation name (e.g., `litellm.completion`) | -| `span_duration_ms` | Span duration in milliseconds | -| `span_model` | Model used (e.g., `gpt-4`, `claude-3-opus`) | -| `span_input_tokens` | Input token count | -| `span_output_tokens` | Output token count | -| `span_total_tokens` | Total tokens used | -| `span_status_code` | HTTP status or error code | - -## Example Queries - -### Average Latency by Model - -```sql -SELECT - "span_model" AS model, - AVG("span_duration_ms") AS avg_latency_ms, - COUNT(*) AS call_count -FROM "litellm-traces" -WHERE "span_name" LIKE '%completion%' -GROUP BY model -ORDER BY avg_latency_ms DESC; -``` - -### Token Usage Over Time - -```sql -SELECT - DATE_TRUNC('hour', p_timestamp) AS hour, - SUM("span_input_tokens") AS input_tokens, - SUM("span_output_tokens") AS output_tokens, - SUM("span_total_tokens") AS total_tokens -FROM "litellm-traces" -GROUP BY hour -ORDER BY hour; -``` - -### Slowest Requests - -```sql -SELECT - trace_id, - "span_model", - "span_duration_ms", - "span_total_tokens", - p_timestamp -FROM "litellm-traces" -ORDER BY "span_duration_ms" DESC -LIMIT 20; -``` - -### Error Rate by Model - -```sql -SELECT - "span_model" AS model, - COUNT(*) AS total_calls, - SUM(CASE WHEN "span_status_code" >= 400 THEN 1 ELSE 0 END) AS errors, - ROUND(100.0 * SUM(CASE WHEN "span_status_code" >= 400 THEN 1 ELSE 0 END) / COUNT(*), 2) AS error_rate -FROM "litellm-traces" -GROUP BY model -ORDER BY error_rate DESC; -``` - -## Cost Tracking - -Calculate costs with token counts: - -```sql -SELECT - "span_model" AS model, - SUM("span_input_tokens") AS input_tokens, - SUM("span_output_tokens") AS output_tokens, - -- GPT-4 pricing: $0.03/1K input, $0.06/1K output - ROUND(SUM("span_input_tokens") * 0.00003 + SUM("span_output_tokens") * 0.00006, 2) AS estimated_cost_usd -FROM "litellm-traces" -WHERE "span_model" = 'gpt-4' - AND p_timestamp >= NOW() - INTERVAL '24 hours' -GROUP BY model; -``` - -## Alerting - -### High Latency Alert - -Create an alert in Parseable: -- **Stream**: `litellm-traces` -- **Column**: `span_duration_ms` -- **Aggregation**: `AVG` -- **Threshold**: `> 5000` - -### Token Spike Detection - -Use anomaly detection on: -- **Stream**: `litellm-traces` -- **Column**: `span_total_tokens` -- **Aggregation**: `SUM` - -## Privacy: Redacting Prompts - -LiteLLM can redact sensitive content from traces: - -**Redact all messages globally:** - -```python -litellm.turn_off_message_logging = True -``` - -**Redact per-request:** - -```python -response = completion( - model="gpt-4", - messages=[{"role": "user", "content": "Sensitive prompt"}], - metadata={ - "mask_input": True, - "mask_output": True - } -) -``` - -This keeps request metadata (model, tokens, latency) while hiding the actual content. - -## Resources - -- [LiteLLM Documentation](https://docs.litellm.ai/) -- [Blog Post](/blog/litellm-trace-analysis-parseable) diff --git a/content/docs/ingest-data/ai-agents/meta.json b/content/docs/ingest-data/ai-agents/meta.json index a54284f..3f5e48d 100644 --- a/content/docs/ingest-data/ai-agents/meta.json +++ b/content/docs/ingest-data/ai-agents/meta.json @@ -3,7 +3,8 @@ "pages": [ "openai", "anthropic", - "litellm", + "litellm-sdk", + "pydantic-ai", "openrouter", "vllm", "langchain", diff --git a/content/docs/ingest-data/ai-agents/pydantic-ai.mdx b/content/docs/ingest-data/ai-agents/pydantic-ai.mdx new file mode 100644 index 0000000..413abe5 --- /dev/null +++ b/content/docs/ingest-data/ai-agents/pydantic-ai.mdx @@ -0,0 +1,283 @@ +--- +title: Pydantic AI +description: Send Pydantic AI agent traces and metrics to Parseable +--- + +Pydantic AI has native OpenTelemetry instrumentation for agent runs, model requests, and tool execution. This guide sends those traces directly to Parseable using the OpenTelemetry GenAI semantic conventions. + +## What you get + +- `invoke_agent ` spans for complete agent runs +- `chat ` spans with model, token, and response metadata +- `execute_tool ` spans with tool arguments and results +- One parent-child trace across agent, model, and tool activity +- Compatibility with Parseable Agent Observability dashboards +- Token, cost, and streaming time-to-first-chunk metrics + +## Prerequisites + +- Python 3.10 or later +- A Parseable instance and credentials +- An OpenAI API key, or another model provider supported by Pydantic AI + +## Create a trace dataset + +Create a dataset and mark it as an Agent Observability trace source: + +```bash +curl -u "$PARSEABLE_USERNAME:$PARSEABLE_PASSWORD" \ + -X PUT "$PARSEABLE_URL/api/v1/logstream/pydantic-ai-traces" \ + -H 'X-P-Log-Source: otel-traces' \ + -H 'X-P-Telemetry-Type: traces' \ + -H 'X-P-Dataset-Tag: agent-observability' +``` + +`X-P-Dataset-Tag` tags the dataset itself. Do not use `X-P-Tag-*` here; those headers add fields to ingested records instead. + +Create a separate metrics dataset when you also want model usage and streaming metrics: + +```bash +curl -u "$PARSEABLE_USERNAME:$PARSEABLE_PASSWORD" \ + -X PUT "$PARSEABLE_URL/api/v1/logstream/pydantic-ai-metrics" \ + -H 'X-P-Log-Source: otel-metrics' \ + -H 'X-P-Telemetry-Type: metrics' +``` + +## Install dependencies + +```bash +pip install "pydantic-ai-slim[openai]" \ + opentelemetry-sdk \ + opentelemetry-exporter-otlp-proto-http +``` + +## Instrument the agent + +```python +import base64 +import os + +from opentelemetry import trace +from opentelemetry import metrics +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from pydantic_ai import Agent, InstrumentationSettings, RunContext +from pydantic_ai.models.openai import OpenAIChatModel +from pydantic_ai.providers.openai import OpenAIProvider + +parseable_url = os.environ["PARSEABLE_URL"].rstrip("/") +credentials = base64.b64encode( + f'{os.environ["PARSEABLE_USERNAME"]}:{os.environ["PARSEABLE_PASSWORD"]}'.encode() +).decode() + +resource = Resource.create({"service.name": "pydantic-ai-agent"}) + +tracer_provider = TracerProvider(resource=resource) +tracer_provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter( + endpoint=f"{parseable_url}/v1/traces", + headers={ + "Authorization": f"Basic {credentials}", + "X-P-Stream": "pydantic-ai-traces", + "X-P-Log-Source": "otel-traces", + }, + ) + ) +) +trace.set_tracer_provider(tracer_provider) + +metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter( + endpoint=f"{parseable_url}/v1/metrics", + headers={ + "Authorization": f"Basic {credentials}", + "X-P-Stream": "pydantic-ai-metrics", + "X-P-Log-Source": "otel-metrics", + }, + ) +) +meter_provider = MeterProvider(metric_readers=[metric_reader], resource=resource) +metrics.set_meter_provider(meter_provider) + +# Version 5 emits current GenAI semantic-convention fields. Content capture is +# required for tool arguments and results in Agent Observability. +Agent.instrument_all( + InstrumentationSettings( + version=5, + include_content=True, + tracer_provider=tracer_provider, + meter_provider=meter_provider, + ) +) + +model = OpenAIChatModel( + "gpt-4o-mini", + provider=OpenAIProvider(api_key=os.environ["OPENAI_API_KEY"]), +) +agent = Agent(model, name="weather_agent") + +@agent.tool +def get_weather(ctx: RunContext[None], city: str) -> str: + return f"The weather in {city} is sunny and 24 C." + +try: + result = agent.run_sync("What is the weather in Bengaluru?") + print(result.output) +finally: + tracer_provider.force_flush() + tracer_provider.shutdown() + meter_provider.force_flush() + meter_provider.shutdown() +``` + +Set the environment variables before running the application: + +```bash +export PARSEABLE_URL="https://your-parseable-host:8000" +export PARSEABLE_USERNAME="admin" +export PARSEABLE_PASSWORD="your-password" +export OPENAI_API_KEY="your-api-key" +python agent.py +``` + + + `include_content=True` can capture prompts, tool arguments, and tool results. Review this setting before using it with sensitive or regulated data. Setting it to `False` protects content, but the Agent Observability tool panels will not receive `gen_ai.tool.call.arguments` and `gen_ai.tool.call.result`. + + +## Instrumentation scope + +`Agent.instrument_all(...)` applies to every agent that does not provide its own instrumentation capability. To instrument only one agent, pass an `Instrumentation` capability: + +```python +from pydantic_ai import Agent, InstrumentationSettings +from pydantic_ai.capabilities import Instrumentation + +settings = InstrumentationSettings( + version=5, + include_content=False, + tracer_provider=tracer_provider, + meter_provider=meter_provider, +) +agent = Agent(model, capabilities=[Instrumentation(settings=settings)]) +``` + +Use per-agent settings when agents have different privacy requirements or telemetry destinations. + +## Metrics exported + +Pydantic AI records these histograms: + +| Metric | Unit | When emitted | +| --- | --- | --- | +| `gen_ai.client.token.usage` | tokens | Model or embedding requests; split by `gen_ai.token.type` | +| `operation.cost` | USD | When the model's price is known | +| `gen_ai.client.operation.time_to_first_chunk` | seconds | Streaming requests only | + +Metric points include provider, operation, requested model, and response model attributes. Time-to-first-chunk is still an evolving GenAI convention and its name or shape can change in future releases. + +Pydantic AI supplies recommended histogram boundaries for token usage and streaming latency. You can override them with OpenTelemetry `View` objects on the `MeterProvider` when your workload needs different buckets. + +## Token aggregation + +Model request spans use `gen_ai.usage.input_tokens` and `gen_ai.usage.output_tokens`. Agent run spans use `gen_ai.aggregated_usage.*` by default because they contain totals from their child model calls. This prevents dashboards from double-counting the same tokens at both levels. + +If your backend queries already distinguish parent agent spans from model spans, you can use the standard usage names everywhere: + +```python +InstrumentationSettings( + version=5, + use_aggregated_usage_attribute_names=False, +) +``` + +Update dashboard queries before changing this setting. + +## Privacy and payload size + +`InstrumentationSettings` provides independent controls: + +```python +InstrumentationSettings( + version=5, + include_content=False, + include_binary_content=False, + include_model_request_parameters=False, +) +``` + +- `include_content=False` removes prompts, completions, tool arguments, and tool results. +- `include_binary_content=False` prevents inline image, audio, video, and file content from entering telemetry. +- `include_model_request_parameters=False` omits the serialized model request configuration and large tool schemas. `gen_ai.tool.definitions` remains available. + +Agent Observability requires tool arguments and results, so disabling content trades complete tool panels for stronger data minimization. Consider separate sanitized instrumentation for production rather than exporting secrets or regulated content. + +## Async, streaming, retries, and errors + +The same instrumentation applies to `await agent.run(...)`, streamed runs, model retries, validation retries, and tool failures. Keep the provider alive for the full application lifetime; flush and shut it down during process termination rather than after every web request. + +For streaming code, consume or close the stream so Pydantic AI can finish spans and record time-to-first-chunk. Exceptions are recorded on the active span and should continue to propagate to application error handling. + +In instrumentation version 5, `CallDeferred` and `ApprovalRequired` are treated as agent control flow rather than failures. Their spans remain unset instead of being marked as errors. + +## Add application metadata + +Use the agent `metadata` parameter for stable application context such as environment, workflow, or tenant identifiers: + +```python +agent = Agent( + model, + name="weather_agent", + metadata={"environment": "production", "team": "support"}, +) +``` + +Do not attach unbounded or sensitive values unless they are required for investigation. + +## Required Agent Observability fields + +Tool spans should include: + +- `gen_ai.tool.name` +- `gen_ai.tool.call.id` +- `gen_ai.tool.call.arguments` +- `gen_ai.tool.call.result` + +If the Agent Observability setup page reports missing argument or result fields, confirm that instrumentation version 5 and `include_content=True` are configured, then run an agent that actually invokes a tool. + +## Verify in Parseable + +Open the `pydantic-ai-traces` dataset and confirm one trace contains a hierarchy similar to: + +```text +invoke_agent weather_agent +├── chat gpt-4o-mini +├── execute_tool get_weather +└── chat gpt-4o-mini +``` + +New datasets and fields can take a few minutes to appear in Agent Observability. + +For metrics, confirm Parseable lists the expected names: + +```bash +curl -u "$PARSEABLE_USERNAME:$PARSEABLE_PASSWORD" --get \ + "$PARSEABLE_URL/prometheus/api/v1/label/__name__/values" \ + --data-urlencode 'stream=pydantic-ai-metrics' +``` + +Cost appears only when Pydantic AI knows the model price. Streaming latency appears only after a streamed response is consumed. + +## Native instrumentation limits + +Pydantic AI's native integration emits spans and metrics. It does not by itself create a separate Parseable semantic-log dataset containing every message, choice, reasoning block, and tool event. If your retention or security design requires full content outside spans, add OpenTelemetry log records or a Collector pipeline and correlate them with `trace_id` and `span_id`. + +## Resources + +- [Pydantic AI instrumentation](https://ai.pydantic.dev/logfire/) +- [Parseable Agent Observability](/docs/user-guide/agent-observability) diff --git a/content/docs/ingest-data/gateways/index.mdx b/content/docs/ingest-data/gateways/index.mdx new file mode 100644 index 0000000..d262c03 --- /dev/null +++ b/content/docs/ingest-data/gateways/index.mdx @@ -0,0 +1,17 @@ +--- +title: AI Gateways +description: Observe LLM traffic routed through AI gateways +--- + +AI gateways provide one place to route model requests, apply policy, and collect telemetry across providers. + +## Supported gateways + + + + Export model requests, tokens, latency, cost, and errors with OpenTelemetry + + diff --git a/content/docs/ingest-data/gateways/litellm.mdx b/content/docs/ingest-data/gateways/litellm.mdx new file mode 100644 index 0000000..a3294d7 --- /dev/null +++ b/content/docs/ingest-data/gateways/litellm.mdx @@ -0,0 +1,313 @@ +--- +title: LiteLLM +description: Send LiteLLM Gateway traces and metrics to Parseable using OpenTelemetry +--- +import { Step, Steps } from 'fumadocs-ui/components/steps'; + +LiteLLM is an OpenAI-compatible gateway for routing model requests across providers such as OpenAI, Anthropic, Azure OpenAI, Vertex AI, Bedrock, and local model endpoints. Teams usually place it in front of applications when they want one place to manage model routing, keys, retries, budgets, fallbacks, and policy. + +Parseable helps you see what is happening inside that gateway. With OpenTelemetry enabled, LiteLLM can emit traces for model requests. With the Prometheus callback enabled, LiteLLM exposes gateway metrics at `/metrics`. This guide sends both signals to Parseable so you can inspect request paths, model latency, token usage, errors, and proxy-level behavior from one place. + +## How it works + +The cleanest production setup is to keep Parseable credentials in an OpenTelemetry Collector. LiteLLM sends OpenTelemetry traces to the Collector, the Collector scrapes LiteLLM's Prometheus metrics endpoint, and both signals are forwarded into Parseable datasets. + +```text +Application + | + | OpenAI-compatible request + v +LiteLLM Gateway + | | + | OTLP traces | Prometheus scrape + v v +OpenTelemetry Collector + | + +--> litellm-traces traces dataset in Parseable + +--> litellm-metrics metrics dataset in Parseable +``` + +Use separate datasets for traces and metrics. They use different telemetry sources and different `X-P-Log-Source` values. + +## Prerequisites + +Before you start, keep these ready: + +- A running Parseable instance +- A Parseable API key with ingest access +- Python 3.10 or later +- A model-provider API key, such as `OPENAI_API_KEY` +- LiteLLM Proxy or Gateway +- An OpenTelemetry Collector that LiteLLM can reach + +## Set up LiteLLM with Parseable + + + + +### Create Parseable datasets + +Create a trace dataset for LiteLLM spans: + +```bash +curl -X PUT "$PARSEABLE_URL/api/v1/logstream/litellm-traces" \ + -H "X-API-Key: ${PARSEABLE_API_KEY}" \ + -H "X-P-Log-Source: otel-traces" \ + -H "X-P-Telemetry-Type: traces" \ + -H "X-P-Dataset-Tag: agent-observability" +``` + +Create a metrics dataset for LiteLLM gateway and GenAI metrics: + +```bash +curl -X PUT "$PARSEABLE_URL/api/v1/logstream/litellm-metrics" \ + -H "X-API-Key: ${PARSEABLE_API_KEY}" \ + -H "X-P-Log-Source: otel-metrics" \ + -H "X-P-Telemetry-Type: metrics" +``` + +The trace dataset appears under traces and agent observability views. The metrics dataset appears in the Metrics page. + + + + +### Configure LiteLLM + +Install LiteLLM Proxy and the OpenTelemetry packages: + +```bash +pip install "litellm[proxy]" \ + opentelemetry-api \ + opentelemetry-sdk \ + opentelemetry-exporter-otlp-proto-http \ + opentelemetry-instrumentation-fastapi \ + prometheus-client==0.20.0 +``` + +Create `litellm-config.yaml`: + +```yaml +model_list: + - model_name: gpt-4o-mini + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + +litellm_settings: + callbacks: + - otel + - prometheus + +callback_settings: + otel: + attributes: + exclude_list: + - hidden_params + - metadata.requester_metadata + - metadata.requester_ip_address + - metadata.spend_logs_metadata + - metadata.mcp_tool_call_metadata + - metadata.vector_store_request_metadata + - metadata.prompt_management_metadata +``` + +The `otel` callback enables LiteLLM OpenTelemetry export. `LITELLM_OTEL_V2=true` turns on the newer trace shape, but the callback still needs to be present. The `prometheus` callback exposes gateway metrics at `/metrics`. + +The attribute filter matters in production. Some metadata fields are close to unique for every request. If those fields are attached to metrics, they can create too many time series. The example keeps traces useful while keeping metric cardinality more controlled. + + + + +### Configure the OpenTelemetry Collector + +Create `otel-collector-config.yaml`: + +```yaml +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + + prometheus/litellm: + config: + scrape_configs: + - job_name: litellm + scrape_interval: 15s + static_configs: + - targets: ["litellm:4000"] + authorization: + type: Bearer + credentials: "${env:LITELLM_MASTER_KEY}" + +processors: + batch: {} + +exporters: + otlphttp/parseable_traces: + endpoint: "${env:PARSEABLE_URL}" + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: litellm-traces + X-P-Log-Source: otel-traces + + otlphttp/parseable_metrics: + endpoint: "${env:PARSEABLE_URL}" + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: litellm-metrics + X-P-Log-Source: otel-metrics + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlphttp/parseable_traces] + + metrics: + receivers: [otlp, prometheus/litellm] + processors: [batch] + exporters: [otlphttp/parseable_metrics] +``` + +This Collector receives OpenTelemetry traces and optional GenAI metrics from LiteLLM. It also scrapes the LiteLLM `/metrics` endpoint, which gives you broader gateway metrics such as proxy requests, failures, routing behavior, spend, token usage, rate limits, queues, Redis, and PostgreSQL metrics. + +For local testing, if LiteLLM is running on your host instead of inside the same Docker network as the Collector, replace `litellm:4000` with `host.docker.internal:4000`. + + + + +### Start LiteLLM + +Set the LiteLLM and OpenTelemetry environment variables: + +```bash +export OPENAI_API_KEY="" +export LITELLM_MASTER_KEY="" + +export LITELLM_OTEL_V2="true" +export LITELLM_OTEL_INTEGRATION_ENABLE_METRICS="true" +export OTEL_EXPORTER="otlp_http" +export OTEL_ENDPOINT="http://localhost:4318" +export OTEL_SERVICE_NAME="litellm-gateway" +export OTEL_ENVIRONMENT_NAME="production" +export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="no_content" +``` + +Then start the gateway: + +```bash +litellm --config litellm-config.yaml --port 4000 +``` + +Send a request through LiteLLM: + +```bash +curl http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": "Explain observability in one sentence." + } + ] + }' +``` + +For streaming latency metrics, send a streaming request and consume the full response: + +```bash +curl -N http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "stream": true, + "messages": [ + { + "role": "user", + "content": "Explain tracing briefly." + } + ] + }' +``` + +Time-to-first-token and time-per-output-token metrics appear only after successful streaming responses with the required timing data. + + + + +## What you get in Parseable + +Open `litellm-traces` from the Traces page. A request can include the incoming HTTP call, routing work, guardrails, cache or database calls, and the LLM provider call. When an upstream application propagates W3C trace context, LiteLLM can continue that trace so the agent, gateway, and model call are connected. + +![LiteLLM traces in Parseable](./static/litellm-traces.png) + +Open `litellm-metrics` from the Metrics page. You should see LiteLLM proxy metrics from the Prometheus endpoint along with any GenAI metrics emitted by the OpenTelemetry callback. + +![LiteLLM metrics in Parseable](./static/litellm-metrics.png) + +Common GenAI fields on traces include: + +| Field | Meaning | +| --- | --- | +| `gen_ai.operation.name` | GenAI operation name | +| `gen_ai.provider.name` | Model provider | +| `gen_ai.request.model` | Requested model | +| `gen_ai.response.model` | Model returned by the provider | +| `gen_ai.usage.input_tokens` | Input token count | +| `gen_ai.usage.output_tokens` | Output token count | +| `litellm.cost.total` | Estimated request cost when available | + +Common GenAI metrics include: + +| Metric | Meaning | +| --- | --- | +| `gen_ai.client.operation.duration` | End-to-end model call duration | +| `gen_ai.client.response.duration` | Provider response duration | +| `gen_ai.client.response.time_to_first_token` | Time to first streamed token | +| `gen_ai.client.response.time_per_output_token` | Average generation time per output token | +| `gen_ai.client.token.usage` | Input and output token usage | +| `gen_ai.client.token.cost` | Estimated request cost when available | + +LiteLLM Prometheus metrics keep their Prometheus-style names, such as `litellm_proxy_total_requests_metric` and `litellm_deployment_failure_responses`. Prometheus labels become OpenTelemetry attributes after the Collector receives them. + +## Message content and privacy + +Keep `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="no_content"` unless you have reviewed the privacy and retention impact. Message capture can store prompts and completions in traces or events. + +LiteLLM also supports request-level masking with `metadata.mask_input=true` and `metadata.mask_output=true`. For a stronger global setting, use `litellm_settings.turn_off_message_logging=true` in the LiteLLM config. + +## Connect LiteLLM to agent traces + +If your application already creates agent spans, propagate the W3C `traceparent` header when it calls LiteLLM. Parseable can then show the agent run, tool work, LiteLLM gateway span, and model provider call in the same trace. + +For a Pydantic AI example, see [Pydantic AI](/docs/ingest-data/ai-agents/pydantic-ai). + +## Troubleshooting + +- **No traces arrive** + + Check that `litellm_settings.callbacks` includes `otel`, `LITELLM_OTEL_V2` is set to `true`, and `OTEL_ENDPOINT` points to the Collector OTLP HTTP receiver. + +- **No metrics arrive** + + Check that `litellm_settings.callbacks` includes `prometheus`, the `prometheus-client` package is installed, and the Collector can scrape `http://:4000/metrics` with the LiteLLM bearer token. + +- **Metrics show too many series** + + Review the `callback_settings.otel.attributes.exclude_list` in the LiteLLM config. For Prometheus metrics, avoid adding high-cardinality custom labels such as raw user identifiers, request IDs, or unique metadata values. + +- **Streaming metrics are missing** + + Confirm the request uses `"stream": true`, the client consumes the response to completion, and the provider returns timing and token usage information. + +- **The `/metrics` scrape returns unauthorized** + + LiteLLM requires bearer authentication for `/metrics` by default. Make sure the Collector Prometheus receiver uses the same `LITELLM_MASTER_KEY` that LiteLLM expects. diff --git a/content/docs/ingest-data/gateways/meta.json b/content/docs/ingest-data/gateways/meta.json new file mode 100644 index 0000000..83aa9e2 --- /dev/null +++ b/content/docs/ingest-data/gateways/meta.json @@ -0,0 +1,4 @@ +{ + "title": "Gateways", + "pages": ["index", "litellm"] +} diff --git a/content/docs/ingest-data/gateways/static/litellm-metrics.png b/content/docs/ingest-data/gateways/static/litellm-metrics.png new file mode 100644 index 0000000..fb2aad8 Binary files /dev/null and b/content/docs/ingest-data/gateways/static/litellm-metrics.png differ diff --git a/content/docs/ingest-data/gateways/static/litellm-traces.png b/content/docs/ingest-data/gateways/static/litellm-traces.png new file mode 100644 index 0000000..cbc3cc1 Binary files /dev/null and b/content/docs/ingest-data/gateways/static/litellm-traces.png differ diff --git a/content/docs/meta.json b/content/docs/meta.json index c83a5e2..24a261d 100644 --- a/content/docs/meta.json +++ b/content/docs/meta.json @@ -20,6 +20,7 @@ "ingest-data/otel", "ingest-data/zero-instrumentation", "ingest-data/ai-agents", + "ingest-data/gateways", "ingest-data/logging-agents", "ingest-data/databases", "ingest-data/infrastructure",