Merge pull request 'Capture LLM Observability to Open Telemetry compatible tools.' (#1) from opentelemetry-llm into main
Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
59
.env.example
59
.env.example
@@ -1,9 +1,52 @@
|
||||
# Enable debug logging
|
||||
# PostHog Configuration (existing)
|
||||
POSTHOG_API_KEY=your_posthog_api_key_here
|
||||
POSTHOG_HOST=https://app.posthog.com
|
||||
|
||||
# OpenTelemetry Configuration
|
||||
# ==========================
|
||||
|
||||
# Service Configuration
|
||||
OTEL_SERVICE_NAME=llm-observability-mcp
|
||||
OTEL_SERVICE_VERSION=1.0.0
|
||||
OTEL_ENVIRONMENT=development
|
||||
|
||||
# OTLP Endpoints
|
||||
# Uncomment and configure based on your backend
|
||||
|
||||
# Jaeger (local development)
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
|
||||
|
||||
# New Relic
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.nr-data.net:4318
|
||||
# OTEL_EXPORTER_OTLP_HEADERS=api-key=YOUR_NEW_RELIC_LICENSE_KEY
|
||||
|
||||
# Grafana Cloud
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-gateway-prod-us-central-0.grafana.net/otlp
|
||||
# OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic YOUR_BASE64_ENCODED_CREDENTIALS
|
||||
|
||||
# Datadog
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT=https://api.datadoghq.com/api/v2/series
|
||||
# OTEL_EXPORTER_OTLP_HEADERS=DD-API-KEY=YOUR_DD_API_KEY
|
||||
|
||||
# Honeycomb
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT=https://api.honeycomb.io/v1/traces
|
||||
# OTEL_EXPORTER_OTLP_HEADERS=x-honeycomb-team=YOUR_API_KEY
|
||||
|
||||
# Lightstep
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT=https://ingest.lightstep.com:443/api/v2/otel/trace
|
||||
# OTEL_EXPORTER_OTLP_HEADERS=lightstep-access-token=YOUR_ACCESS_TOKEN
|
||||
|
||||
# Separate endpoints (optional)
|
||||
# OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:4318/v1/metrics
|
||||
# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4318/v1/traces
|
||||
# OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://localhost:4318/v1/logs
|
||||
|
||||
# Export Configuration
|
||||
OTEL_METRIC_EXPORT_INTERVAL=10000
|
||||
OTEL_METRIC_EXPORT_TIMEOUT=5000
|
||||
|
||||
# Sampling Configuration
|
||||
OTEL_TRACES_SAMPLER_ARG=1.0
|
||||
|
||||
# Debug Mode
|
||||
DEBUG=false
|
||||
|
||||
# Transport mode
|
||||
TRANSPORT_MODE=<http | stdio>
|
||||
|
||||
# PostHog LLM Observability configuration
|
||||
POSTHOG_API_KEY=<YourPostHogAPIKeyGoesHere>
|
||||
POSTHOG_HOST=https://us.i.posthog.com
|
||||
|
||||
26
.github/dependabot.yml
vendored
26
.github/dependabot.yml
vendored
@@ -1,26 +0,0 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 10
|
||||
versioning-strategy: auto
|
||||
labels:
|
||||
- "dependencies"
|
||||
commit-message:
|
||||
prefix: "chore"
|
||||
include: "scope"
|
||||
allow:
|
||||
- dependency-type: "direct"
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
update-types: ["version-update:semver-patch"]
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 5
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "github-actions"
|
||||
45
.github/workflows/ci-dependabot-auto-merge.yml
vendored
45
.github/workflows/ci-dependabot-auto-merge.yml
vendored
@@ -1,45 +0,0 @@
|
||||
name: CI - Dependabot Auto-merge
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
checks: read
|
||||
|
||||
jobs:
|
||||
auto-merge-dependabot:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.actor == 'dependabot[bot]'
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
|
||||
- name: Run linting
|
||||
run: npm run lint
|
||||
|
||||
- name: Auto-approve PR
|
||||
uses: hmarr/auto-approve-action@v4
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Enable auto-merge
|
||||
if: success()
|
||||
run: gh pr merge --auto --merge "$PR_URL"
|
||||
env:
|
||||
PR_URL: ${{ github.event.pull_request.html_url }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
37
.github/workflows/ci-dependency-check.yml
vendored
37
.github/workflows/ci-dependency-check.yml
vendored
@@ -1,37 +0,0 @@
|
||||
name: CI - Dependency Check
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 5 * * 1' # Run at 5 AM UTC every Monday
|
||||
workflow_dispatch: # Allow manual triggering
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run npm audit
|
||||
run: npm audit
|
||||
|
||||
- name: Check for outdated dependencies
|
||||
run: npm outdated || true
|
||||
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
|
||||
- name: Run linting
|
||||
run: npm run lint
|
||||
|
||||
- name: Build project
|
||||
run: npm run build
|
||||
66
.github/workflows/ci-semantic-release.yml
vendored
66
.github/workflows/ci-semantic-release.yml
vendored
@@ -1,66 +0,0 @@
|
||||
name: CI - Semantic Release
|
||||
|
||||
# This workflow is triggered on every push to the main branch
|
||||
# It analyzes commits and automatically releases a new version when needed
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: Semantic Release
|
||||
runs-on: ubuntu-latest
|
||||
# Permissions needed for creating releases, updating issues, and publishing packages
|
||||
permissions:
|
||||
contents: write # Needed to create releases and tags
|
||||
issues: write # Needed to comment on issues
|
||||
pull-requests: write # Needed to comment on pull requests
|
||||
# packages permission removed as we're not using GitHub Packages
|
||||
steps:
|
||||
# Step 1: Check out the full Git history for proper versioning
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Fetches all history for all branches and tags
|
||||
|
||||
# Step 2: Setup Node.js environment
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22 # Using Node.js 22
|
||||
cache: 'npm' # Enable npm caching
|
||||
|
||||
# Step 3: Install dependencies with clean install
|
||||
- name: Install dependencies
|
||||
run: npm ci # Clean install preserving package-lock.json
|
||||
|
||||
# Step 4: Build the package
|
||||
- name: Build
|
||||
run: npm run build # Compiles TypeScript to JavaScript
|
||||
|
||||
# Step 5: Ensure executable permissions
|
||||
- name: Set executable permissions
|
||||
run: chmod +x dist/index.js
|
||||
|
||||
# Step 6: Run tests to ensure quality
|
||||
- name: Verify tests
|
||||
run: npm test # Runs Jest tests
|
||||
|
||||
# Step 7: Configure Git identity for releases
|
||||
- name: Configure Git User
|
||||
run: |
|
||||
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Step 8: Run semantic-release to analyze commits and publish to npm
|
||||
- name: Semantic Release
|
||||
id: semantic
|
||||
env:
|
||||
# Tokens needed for GitHub and npm authentication
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # For creating releases and commenting
|
||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }} # For publishing to npm
|
||||
run: |
|
||||
echo "Running semantic-release for version bump and npm publishing"
|
||||
npx semantic-release
|
||||
|
||||
# Note: GitHub Packages publishing has been removed
|
||||
182
README.md
182
README.md
@@ -1,21 +1,24 @@
|
||||
# LLM Observability MCP for PostHog
|
||||
# LLM Observability MCP Server
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
|
||||
A Model Context Protocol (MCP) server that provides a tool to capture LLM Observability events and send them to PostHog.
|
||||
A Model Context Protocol (MCP) server that provides comprehensive LLM observability tools supporting both PostHog and OpenTelemetry backends.
|
||||
|
||||
## Overview
|
||||
|
||||
This project is an MCP server designed to track and observe Large Language Model (LLM) interactions using [PostHog's LLM Observability](https://posthog.com/docs/llm-observability) features. It allows you to capture detailed information about LLM requests, responses, performance, and costs, providing valuable insights into your AI-powered applications.
|
||||
This project is an MCP server designed to track and observe Large Language Model (LLM) interactions using both [PostHog's LLM Observability](https://posthog.com/docs/llm-observability) and **OpenTelemetry** for universal observability across any backend that supports OpenTelemetry (Jaeger, New Relic, Grafana, Datadog, Honeycomb, etc.).
|
||||
|
||||
The server can be run as a local process communicating over `stdio` or as a remote `http` server, making it compatible with any MCP client, such as AI-powered IDEs (e.g., VS Code with an MCP extension, Cursor) or custom applications.
|
||||
|
||||
## Features
|
||||
|
||||
- **Capture LLM Metrics**: Log key details of LLM interactions, including model, provider, latency, token counts, and more.
|
||||
- **Flexible Transport**: Run as a local `stdio` process for tight IDE integration or as a standalone `http` server for remote access.
|
||||
- **Dynamic Configuration**: Configure the server easily using environment variables.
|
||||
- **Easy Integration**: Connect to MCP-compatible IDEs or use the programmatic client for use in any TypeScript/JavaScript application.
|
||||
- **Dual Backend Support**: Use PostHog, OpenTelemetry, or both in parallel
|
||||
- **Universal OpenTelemetry**: Works with any OpenTelemetry-compatible backend
|
||||
- **Comprehensive Metrics**: Request counts, token usage, latency, error rates
|
||||
- **Distributed Tracing**: Full request lifecycle tracking with spans
|
||||
- **Flexible Transport**: Run as local `stdio` process or standalone `http` server
|
||||
- **Dynamic Configuration**: Environment-based configuration for different backends
|
||||
- **Zero-Code Integration**: Easy integration with MCP-compatible clients
|
||||
|
||||
## Installation for Development
|
||||
|
||||
@@ -23,7 +26,7 @@ Follow these steps to set up the server for local development.
|
||||
|
||||
1. **Prerequisites**:
|
||||
- Node.js (>=18.x)
|
||||
- A [PostHog account](https://posthog.com/) with an API Key and Host URL.
|
||||
- A [PostHog account](https://posthog.com/) with an API Key and Host URL (if using PostHog).
|
||||
|
||||
2. **Clone and Install**:
|
||||
|
||||
@@ -40,16 +43,37 @@ Follow these steps to set up the server for local development.
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Then, edit the `.env` file with your PostHog credentials and desired transport mode.
|
||||
Then, edit the `.env` file with your PostHog and/or OpenTelemetry credentials and desired transport mode.
|
||||
|
||||
## Configuration
|
||||
|
||||
The server is configured via environment variables.
|
||||
The server is configured via environment variables. See `.env.example` for all options.
|
||||
|
||||
### PostHog Configuration
|
||||
|
||||
| Variable | Description | Default | Example |
|
||||
| ----------------- | --------------------------------------------------------------------------- | --------- | ------------------------------------- |
|
||||
| `POSTHOG_API_KEY` | Your PostHog Project API Key (required for PostHog tool) | - | `phc_...` |
|
||||
| `POSTHOG_HOST` | The URL of your PostHog instance | - | `https://us.i.posthog.com` |
|
||||
|
||||
### OpenTelemetry Configuration
|
||||
|
||||
See [OpenTelemetry Documentation](docs/opentelemetry.md) for full details and backend-specific setup.
|
||||
|
||||
| Variable | Description | Default | Example |
|
||||
| ------------------------------- | --------------------------------------------------------------------------- | -------------------------- | ------------------------------------- |
|
||||
| `OTEL_EXPORTER_OTLP_ENDPOINT` | OpenTelemetry collector endpoint | - | `http://localhost:4318` |
|
||||
| `OTEL_EXPORTER_OTLP_HEADERS` | Headers for authentication (comma-separated key=value pairs) | - | `api-key=YOUR_KEY` |
|
||||
| `OTEL_SERVICE_NAME` | Service name for traces and metrics | `llm-observability-mcp` | `my-llm-app` |
|
||||
| `OTEL_SERVICE_VERSION` | Service version | `1.0.0` | `2.1.0` |
|
||||
| `OTEL_ENVIRONMENT` | Environment name | `development` | `production` |
|
||||
| `OTEL_TRACES_SAMPLER_ARG` | Sampling ratio (0.0-1.0) | `1.0` | `0.1` |
|
||||
| `OTEL_METRIC_EXPORT_INTERVAL` | Metrics export interval in milliseconds | `10000` | `30000` |
|
||||
|
||||
### General Configuration
|
||||
|
||||
| Variable | Description | Default | Example |
|
||||
| ----------------- | --------------------------------------------------------------------------- | --------- | ------------------------------------- |
|
||||
| `POSTHOG_API_KEY` | **Required.** Your PostHog Project API Key. | - | `phc_...` |
|
||||
| `POSTHOG_HOST` | **Required.** The URL of your PostHog instance. | - | `https://us.i.posthog.com` |
|
||||
| `TRANSPORT_MODE` | The transport protocol to use. Can be `http` or `stdio`. | `http` | `stdio` |
|
||||
| `DEBUG` | Set to `true` to enable detailed debug logging. | `false` | `true` |
|
||||
|
||||
@@ -123,7 +147,7 @@ Use this if you prefer to run the server as a standalone process.
|
||||
For IDE extensions that support system prompts, you can instruct the AI to automatically use this MCP tool for every interaction. Add the following to your IDE's system prompt configuration:
|
||||
|
||||
```text
|
||||
Use `capture_llm_observability` MCP.
|
||||
Use `llm_observability_posthog` MCP.
|
||||
Make sure to include all parameters and for the `userId`, send `<my_username>`:
|
||||
userId - The distinct ID of the user
|
||||
traceId - The trace ID to group AI events
|
||||
@@ -140,6 +164,55 @@ baseUrl - The base URL of the LLM API
|
||||
|
||||
Replace `<my_username>` with a unique identifier for yourself. This ensures that all LLM activity is automatically logged in PostHog without needing to give the command each time.
|
||||
|
||||
#### Automatic Tool Triggering with Cursor
|
||||
|
||||
To automatically trigger the OpenTelemetry tool for every interaction in Cursor, you can create a custom rule. Create a file named `llm-observability.mdc` in your `.cursor/rules` directory with the following content:
|
||||
|
||||
```markdown
|
||||
---
|
||||
description: llm_observability_otel
|
||||
globs: *
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
## Rules
|
||||
|
||||
---
|
||||
description: llm_observability_otel
|
||||
globs: *
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
## Rules
|
||||
|
||||
SYSTEM PROMPT:
|
||||
- When the user asks anything, output a llm_observability_otel tool invocation in valid MCP JSON.
|
||||
- Repeat tool invocation until success invocation.
|
||||
- Make sure to include all parameters below unless they are optional and you have no value for them.
|
||||
- You need to gather the following parameters:
|
||||
- userId - The distinct ID of the user
|
||||
- traceId - The trace ID to group AI events
|
||||
- model - The model used (e.g., gpt-4, claude-3, etc.)
|
||||
- provider - The LLM provider (e.g., openai, anthropic, etc.)
|
||||
- input - The input to the LLM (messages, prompt, etc.)
|
||||
- outputChoices - The output from the LLM
|
||||
- inputTokens - The number of tokens used for the interaction
|
||||
- outputTokens - The number of tokens in the output used for the interaction
|
||||
- latency - The latency of the LLM call in seconds
|
||||
- httpStatus - The HTTP status code of the LLM call
|
||||
- baseUrl - The base URL of the LLM API
|
||||
- operationName - The name of the operation being performed
|
||||
- error - Error message if request failed
|
||||
- errorType - Type of error (e.g. rate_limit, timeout, etc.)
|
||||
- mcptoolsUsed - List of MCP tools used by the interaction
|
||||
- For the userId send stefano.fiorini.
|
||||
- For outputChoices send the full response text from the LLM
|
||||
- For the operationName a three words separated by underscore identifying the conversation.
|
||||
- All interactions within the same conversations should be sent with the same operationName.
|
||||
```
|
||||
|
||||
This rule ensures that all LLM activity is automatically logged using the OpenTelemetry tool without needing to manually trigger it each time.
|
||||
|
||||
### Programmatic Usage
|
||||
|
||||
You can use an MCP client library to interact with the server programmatically from your own applications.
|
||||
@@ -158,7 +231,7 @@ async function main() {
|
||||
|
||||
await client.connect();
|
||||
|
||||
const result = await client.useTool('capture_llm_observability', {
|
||||
const result = await client.useTool('llm_observability_posthog', {
|
||||
userId: 'user-123',
|
||||
model: 'gpt-4',
|
||||
provider: 'openai',
|
||||
@@ -177,25 +250,69 @@ async function main() {
|
||||
main().catch(console.error);
|
||||
```
|
||||
|
||||
## Tool Reference: `capture_llm_observability`
|
||||
## Available Tools
|
||||
|
||||
This is the core tool provided by the server. It captures LLM usage in PostHog for observability, including requests, responses, and performance metrics.
|
||||
### PostHog Tool: `llm_observability_posthog`
|
||||
|
||||
### Parameters
|
||||
Captures LLM usage in PostHog for observability, including requests, responses, and performance metrics.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
| --------------- | ------------------- | -------- | ----------------------------------------------- |
|
||||
| `userId` | `string` | Yes | The distinct ID of the user. |
|
||||
| `model` | `string` | Yes | The model used (e.g., `gpt-4`, `claude-3`). |
|
||||
| `provider` | `string` | Yes | The LLM provider (e.g., `openai`, `anthropic`). |
|
||||
| `traceId` | `string` | No | The trace ID to group related AI events. |
|
||||
| `input` | `any` | No | The input to the LLM (e.g., messages, prompt). |
|
||||
| `outputChoices` | `any` | No | The output choices from the LLM. |
|
||||
| `inputTokens` | `number` | No | The number of tokens in the input. |
|
||||
| `outputTokens` | `number` | No | The number of tokens in the output. |
|
||||
| `latency` | `number` | No | The latency of the LLM call in seconds. |
|
||||
| `httpStatus` | `number` | No | The HTTP status code of the LLM API call. |
|
||||
| `baseUrl` | `string` | No | The base URL of the LLM API. |
|
||||
#### Parameters for PostHog
|
||||
|
||||
- `userId` (string, required): The distinct ID of the user
|
||||
- `model` (string, required): The model used (e.g., `gpt-4`, `claude-3`)
|
||||
- `provider` (string, required): The LLM provider (e.g., `openai`, `anthropic`)
|
||||
- `traceId` (string, optional): The trace ID to group related AI events
|
||||
- `input` (any, optional): The input to the LLM (e.g., messages, prompt)
|
||||
- `outputChoices` (any, optional): The output choices from the LLM
|
||||
- `inputTokens` (number, optional): The number of tokens in the input
|
||||
- `outputTokens` (number, optional): The number of tokens in the output
|
||||
- `latency` (number, optional): The latency of the LLM call in seconds
|
||||
- `httpStatus` (number, optional): The HTTP status code of the LLM API call
|
||||
- `baseUrl` (string, optional): The base URL of the LLM API
|
||||
|
||||
### OpenTelemetry Tool: `llm_observability_otel`
|
||||
|
||||
Captures LLM usage using OpenTelemetry for universal observability across any OpenTelemetry-compatible backend.
|
||||
|
||||
See [OpenTelemetry Documentation](docs/opentelemetry.md) for full details, backend setup, advanced usage, and troubleshooting.
|
||||
|
||||
#### Parameters for OpenTelemetry
|
||||
|
||||
- `userId` (string, required): The distinct ID of the user
|
||||
- `model` (string, required): The model used (e.g., `gpt-4`, `claude-3`)
|
||||
- `provider` (string, required): The LLM provider (e.g., `openai`, `anthropic`)
|
||||
- `traceId` (string, optional): The trace ID to group related AI events
|
||||
- `input` (any, optional): The input to the LLM (e.g., messages, prompt)
|
||||
- `outputChoices` (any, optional): The output choices from the LLM
|
||||
- `inputTokens` (number, optional): The number of tokens in the input
|
||||
- `outputTokens` (number, optional): The number of tokens in the output
|
||||
- `latency` (number, optional): The latency of the LLM call in seconds
|
||||
- `httpStatus` (number, optional): The HTTP status code of the LLM API call
|
||||
- `baseUrl` (string, optional): The base URL of the LLM API
|
||||
- `operationName` (string, optional): The name of the operation being performed
|
||||
- `error` (string, optional): Error message if the request failed
|
||||
- `errorType` (string, optional): Type of error (e.g., rate_limit, timeout)
|
||||
- `mcpToolsUsed` (string[], optional): List of MCP tools used during the request
|
||||
|
||||
### Parameters Comparison
|
||||
|
||||
| Parameter | Type | Required | Description | PostHog | OpenTelemetry |
|
||||
| --------------- | ------------------- | -------- | ----------------------------------------------- | ------- | ------------- |
|
||||
| `userId` | `string` | Yes | The distinct ID of the user. | ✅ | ✅ |
|
||||
| `model` | `string` | Yes | The model used (e.g., `gpt-4`, `claude-3`). | ✅ | ✅ |
|
||||
| `provider` | `string` | Yes | The LLM provider (e.g., `openai`, `anthropic`). | ✅ | ✅ |
|
||||
| `traceId` | `string` | No | The trace ID to group related AI events. | ✅ | ✅ |
|
||||
| `input` | `any` | No | The input to the LLM (e.g., messages, prompt). | ✅ | ✅ |
|
||||
| `outputChoices` | `any` | No | The output choices from the LLM. | ✅ | ✅ |
|
||||
| `inputTokens` | `number` | No | The number of tokens in the input. | ✅ | ✅ |
|
||||
| `outputTokens` | `number` | No | The number of tokens in the output. | ✅ | ✅ |
|
||||
| `latency` | `number` | No | The latency of the LLM call in seconds. | ✅ | ✅ |
|
||||
| `httpStatus` | `number` | No | The HTTP status code of the LLM API call. | ✅ | ✅ |
|
||||
| `baseUrl` | `string` | No | The base URL of the LLM API. | ✅ | ✅ |
|
||||
| `operationName` | `string` | No | The name of the operation being performed. | ❌ | ✅ |
|
||||
| `error` | `string` | No | Error message if the request failed. | ❌ | ✅ |
|
||||
| `errorType` | `string` | No | Type of error (e.g., rate_limit, timeout). | ❌ | ✅ |
|
||||
| `mcpToolsUsed` | `string[]` | No | List of MCP tools used during the request. | ❌ | ✅ |
|
||||
|
||||
## Development
|
||||
|
||||
@@ -203,6 +320,11 @@ This is the core tool provided by the server. It captures LLM usage in PostHog f
|
||||
- **Run tests**: `npm test`
|
||||
- **Lint and format**: `npm run lint` and `npm run format`
|
||||
|
||||
## Documentation
|
||||
|
||||
- [OpenTelemetry Documentation](docs/opentelemetry.md) - Complete OpenTelemetry configuration, usage, and examples.
|
||||
- [Environment Configuration](.env.example) - All available configuration options.
|
||||
|
||||
## License
|
||||
|
||||
[MIT License](https://opensource.org/licenses/MIT)
|
||||
|
||||
332
docs/opentelemetry.md
Normal file
332
docs/opentelemetry.md
Normal file
@@ -0,0 +1,332 @@
|
||||
# OpenTelemetry LLM Observability
|
||||
|
||||
This document provides comprehensive guidance for using the OpenTelemetry LLM observability tool with the MCP server. It covers setup, configuration, usage, troubleshooting, and practical examples for all major OpenTelemetry-compatible backends.
|
||||
|
||||
## Features
|
||||
|
||||
- **Universal Compatibility**: Works with Jaeger, New Relic, Grafana, Datadog, Honeycomb, and more
|
||||
- **Comprehensive Metrics**: Request counts, token usage, latency, error rates
|
||||
- **Distributed Tracing**: Full request lifecycle tracking with spans
|
||||
- **Flexible Configuration**: Environment-based configuration for different backends
|
||||
- **Zero-Code Integration**: Drop-in replacement for existing observability tools
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Install Dependencies
|
||||
|
||||
OpenTelemetry dependencies are included in `package.json`:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
### 2. Configure Your Backend
|
||||
|
||||
#### Jaeger (Local Development)
|
||||
|
||||
```bash
|
||||
docker run -d --name jaeger \
|
||||
-e COLLECTOR_OTLP_ENABLED=true \
|
||||
-p 16686:16686 \
|
||||
-p 4317:4317 \
|
||||
-p 4318:4318 \
|
||||
jaegertracing/all-in-one:latest
|
||||
export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
|
||||
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||
```
|
||||
|
||||
#### New Relic
|
||||
|
||||
```bash
|
||||
export OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.nr-data.net:4318
|
||||
export OTEL_EXPORTER_OTLP_HEADERS="api-key=YOUR_NEW_RELIC_LICENSE_KEY"
|
||||
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||
```
|
||||
|
||||
#### Grafana Cloud
|
||||
|
||||
```bash
|
||||
export OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-gateway-prod-us-central-0.grafana.net/otlp
|
||||
export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic $(echo -n YOUR_INSTANCE_ID:YOUR_API_KEY | base64)"
|
||||
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||
```
|
||||
|
||||
#### Honeycomb
|
||||
|
||||
```bash
|
||||
export OTEL_EXPORTER_OTLP_ENDPOINT=https://api.honeycomb.io/v1/traces
|
||||
export OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=YOUR_API_KEY"
|
||||
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||
```
|
||||
|
||||
#### Datadog
|
||||
|
||||
```bash
|
||||
export OTEL_EXPORTER_OTLP_ENDPOINT=https://api.datadoghq.com/api/v2/series
|
||||
export OTEL_EXPORTER_OTLP_HEADERS="DD-API-KEY=YOUR_DD_API_KEY"
|
||||
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||
```
|
||||
|
||||
#### Lightstep
|
||||
|
||||
```bash
|
||||
export OTEL_EXPORTER_OTLP_ENDPOINT=https://ingest.lightstep.com:443/api/v2/otel/trace
|
||||
export OTEL_EXPORTER_OTLP_HEADERS="lightstep-access-token=YOUR_ACCESS_TOKEN"
|
||||
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||
```
|
||||
|
||||
#### Kubernetes Example
|
||||
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: llm-observability-mcp
|
||||
spec:
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: llm-observability-mcp
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: llm-observability-mcp
|
||||
spec:
|
||||
containers:
|
||||
- name: llm-observability-mcp
|
||||
image: llm-observability-mcp:latest
|
||||
ports:
|
||||
- containerPort: 3000
|
||||
env:
|
||||
- name: OTEL_SERVICE_NAME
|
||||
value: "llm-observability-mcp"
|
||||
- name: OTEL_SERVICE_VERSION
|
||||
value: "1.2.3"
|
||||
- name: OTEL_ENVIRONMENT
|
||||
value: "production"
|
||||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||
value: "https://your-backend.com:4318"
|
||||
- name: OTEL_EXPORTER_OTLP_HEADERS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: otel-credentials
|
||||
key: headers
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Running the MCP Server
|
||||
|
||||
```bash
|
||||
# Start with stdio transport
|
||||
npm run mcp:stdio
|
||||
# Start with HTTP transport
|
||||
npm run mcp:http
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
### OpenTelemetry Tool: `llm_observability_otel`
|
||||
|
||||
#### Required Parameters
|
||||
|
||||
- `userId`: The distinct ID of the user
|
||||
- `model`: The model used (e.g., "gpt-4", "claude-3")
|
||||
- `provider`: The LLM provider (e.g., "openai", "anthropic")
|
||||
|
||||
#### Optional Parameters
|
||||
|
||||
- `traceId`: Trace ID for grouping related events
|
||||
- `input`: The input to the LLM (messages, prompt, etc.)
|
||||
- `outputChoices`: The output from the LLM
|
||||
- `inputTokens`: Number of tokens in the input
|
||||
- `outputTokens`: Number of tokens in the output
|
||||
- `latency`: Latency of the LLM call in seconds
|
||||
- `httpStatus`: HTTP status code of the LLM call
|
||||
- `baseUrl`: Base URL of the LLM API
|
||||
- `operationName`: Name of the operation being performed
|
||||
- `error`: Error message if the request failed
|
||||
- `errorType`: Type of error (e.g., "rate_limit", "timeout")
|
||||
- `mcpToolsUsed`: List of MCP tools used during the request
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "llm_observability_otel",
|
||||
"arguments": {
|
||||
"userId": "user-12345",
|
||||
"model": "gpt-4",
|
||||
"provider": "openai",
|
||||
"inputTokens": 150,
|
||||
"outputTokens": 75,
|
||||
"latency": 2.3,
|
||||
"httpStatus": 200,
|
||||
"operationName": "chat-completion",
|
||||
"traceId": "trace-abc123",
|
||||
"input": "What is the weather like today?",
|
||||
"outputChoices": ["The weather is sunny and 75°F today."]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `OTEL_SERVICE_NAME` | Service name for OpenTelemetry | `llm-observability-mcp` |
|
||||
| `OTEL_SERVICE_VERSION` | Service version | `1.0.0` |
|
||||
| `OTEL_ENVIRONMENT` | Environment name | `development` |
|
||||
| `OTEL_EXPORTER_OTLP_ENDPOINT` | Default OTLP endpoint | - |
|
||||
| `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | Metrics endpoint | - |
|
||||
| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Traces endpoint | - |
|
||||
| `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` | Logs endpoint | - |
|
||||
| `OTEL_EXPORTER_OTLP_HEADERS` | Headers for authentication (format: "key1=value1,key2=value2") | - |
|
||||
| `OTEL_METRIC_EXPORT_INTERVAL` | Metrics export interval in ms | `10000` |
|
||||
| `OTEL_METRIC_EXPORT_TIMEOUT` | Metrics export timeout in ms | `5000` |
|
||||
| `OTEL_TRACES_SAMPLER_ARG` | Sampling ratio (0.0-1.0) | `1.0` |
|
||||
|
||||
---
|
||||
|
||||
## Metrics Collected
|
||||
|
||||
- `llm.requests.total`: Total number of LLM requests
|
||||
- `llm.tokens.total`: Total tokens used (input + output)
|
||||
- `llm.latency.duration`: Request latency in milliseconds
|
||||
- `llm.requests.active`: Number of active requests
|
||||
|
||||
### Trace Attributes
|
||||
|
||||
- `llm.model`, `llm.provider`, `llm.user_id`, `llm.operation`, `llm.input_tokens`, `llm.output_tokens`, `llm.total_tokens`, `llm.latency_ms`, `llm.http_status`, `llm.base_url`, `llm.error`, `llm.error_type`, `llm.input`, `llm.output`, `llm.mcp_tools_used`
|
||||
|
||||
---
|
||||
|
||||
## Practical Examples
|
||||
|
||||
### Jaeger: View Traces
|
||||
|
||||
Open <http://localhost:16686> to see your traces.
|
||||
|
||||
### Error Tracking Example
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "llm_observability_otel",
|
||||
"arguments": {
|
||||
"userId": "user-12345",
|
||||
"model": "gpt-4",
|
||||
"provider": "openai",
|
||||
"httpStatus": 429,
|
||||
"error": "Rate limit exceeded",
|
||||
"errorType": "rate_limit",
|
||||
"latency": 0.1,
|
||||
"operationName": "chat-completion"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Multi-Tool Usage Tracking Example
|
||||
|
||||
```json
|
||||
{
|
||||
"tool": "llm_observability_otel",
|
||||
"arguments": {
|
||||
"userId": "user-12345",
|
||||
"model": "gpt-4",
|
||||
"provider": "openai",
|
||||
"inputTokens": 500,
|
||||
"outputTokens": 200,
|
||||
"latency": 5.2,
|
||||
"httpStatus": 200,
|
||||
"operationName": "complex-workflow",
|
||||
"mcpToolsUsed": ["file_read", "web_search", "code_execution"],
|
||||
"traceId": "complex-workflow-123"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Testing Script
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# test-opentelemetry.sh
|
||||
|
||||
docker run -d --name jaeger-test \
|
||||
-e COLLECTOR_OTLP_ENABLED=true \
|
||||
-p 16686:16686 \
|
||||
-p 4318:4318 \
|
||||
jaegertracing/all-in-one:latest
|
||||
sleep 5
|
||||
export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
|
||||
export OTEL_SERVICE_NAME=llm-observability-test
|
||||
export OTEL_ENVIRONMENT=test
|
||||
npm run mcp:stdio &
|
||||
sleep 3
|
||||
curl -X POST http://localhost:3000/mcp \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"tool": "llm_observability_otel",
|
||||
"arguments": {
|
||||
"userId": "test-user",
|
||||
"model": "gpt-4",
|
||||
"provider": "openai",
|
||||
"inputTokens": 100,
|
||||
"outputTokens": 50,
|
||||
"latency": 1.5,
|
||||
"httpStatus": 200,
|
||||
"operationName": "test-completion"
|
||||
}
|
||||
}'
|
||||
echo "Test complete. View traces at http://localhost:16686"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Migration from PostHog
|
||||
|
||||
The OpenTelemetry tool is a drop-in replacement for the PostHog tool. Both can coexist for gradual migration:
|
||||
|
||||
- **PostHog Tool**: `llm_observability_posthog`
|
||||
- **OpenTelemetry Tool**: `llm_observability_otel`
|
||||
|
||||
Both accept the same parameters.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting & Performance
|
||||
|
||||
### Common Issues
|
||||
|
||||
- No data in backend: check endpoint URLs, authentication, network, server logs
|
||||
- High resource usage: lower sampling (`OTEL_TRACES_SAMPLER_ARG`), increase export intervals
|
||||
- Missing traces: verify OpenTelemetry is enabled, check logs, service name
|
||||
|
||||
### Debug Mode
|
||||
|
||||
```bash
|
||||
export DEBUG=true
|
||||
npm run mcp:stdio
|
||||
```
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
- Reduce sampling for high-volume: `OTEL_TRACES_SAMPLER_ARG=0.01`
|
||||
- Increase export intervals: `OTEL_METRIC_EXPORT_INTERVAL=60000`
|
||||
- Disable metrics/logs if not needed: `unset OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`, `unset OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`
|
||||
|
||||
---
|
||||
|
||||
## Support
|
||||
|
||||
For issues or questions:
|
||||
|
||||
1. Check this document and troubleshooting
|
||||
2. Review server logs with `DEBUG=true`
|
||||
3. Verify OpenTelemetry configuration
|
||||
4. Test with Jaeger locally first
|
||||
@@ -42,5 +42,16 @@ export default tseslint.config(
|
||||
'@typescript-eslint/no-unused-vars': 'off',
|
||||
},
|
||||
},
|
||||
// Rules for scripts
|
||||
{
|
||||
files: ['scripts/**/*.js'], // Target all JS files in the scripts directory
|
||||
languageOptions: {
|
||||
ecmaVersion: 'latest',
|
||||
sourceType: 'module',
|
||||
},
|
||||
env: {
|
||||
node: true, // Enable Node.js global variables and Node.js scoping
|
||||
},
|
||||
},
|
||||
eslintConfigPrettier,
|
||||
);
|
||||
|
||||
1587
package-lock.json
generated
1587
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
13
package.json
13
package.json
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@sfiorini/llm-observability-mcp",
|
||||
"version": "0.1.0",
|
||||
"description": "A Model Context Protocol (MCP) server that provides a tool to capture LLM Observability events and send them to PostHog.",
|
||||
"description": "A Model Context Protocol (MCP) server that provides comprehensive LLM observability tools supporting both PostHog and OpenTelemetry backends.",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"type": "commonjs",
|
||||
@@ -20,9 +20,7 @@
|
||||
"test": "jest",
|
||||
"test:coverage": "jest --coverage",
|
||||
"lint": "eslint src --ext .ts --config eslint.config.mjs",
|
||||
"update:deps": "npx npm-check-updates -u && npm install --legacy-peer-deps",
|
||||
"format": "prettier --write 'src/**/*.ts' 'scripts/**/*.js'",
|
||||
"cli": "npm run build && node dist/index.js",
|
||||
"mcp:stdio": "TRANSPORT_MODE=stdio npm run build && node dist/index.js",
|
||||
"mcp:http": "TRANSPORT_MODE=http npm run build && node dist/index.js",
|
||||
"mcp:inspect": "TRANSPORT_MODE=http npm run build && (node dist/index.js &) && sleep 2 && npx @modelcontextprotocol/inspector http://localhost:3000/mcp",
|
||||
@@ -76,6 +74,15 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@modelcontextprotocol/sdk": "^1.15.1",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/exporter-otlp-grpc": "^0.26.0",
|
||||
"@opentelemetry/exporter-otlp-http": "^0.26.0",
|
||||
"@opentelemetry/instrumentation": "^0.203.0",
|
||||
"@opentelemetry/resources": "^2.0.1",
|
||||
"@opentelemetry/sdk-metrics": "^2.0.1",
|
||||
"@opentelemetry/sdk-node": "^0.203.0",
|
||||
"@opentelemetry/sdk-trace-node": "^2.0.1",
|
||||
"@opentelemetry/semantic-conventions": "^1.36.0",
|
||||
"commander": "^14.0.0",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^17.2.0",
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
import { Command } from 'commander';
|
||||
import { Logger } from '../utils/logger.util.js';
|
||||
import { VERSION, CLI_NAME } from '../utils/constants.util.js';
|
||||
|
||||
import posthogLlmCli from './posthog-llm.cli.js';
|
||||
|
||||
/**
|
||||
* CLI entry point for the LLM Log MCP Server
|
||||
*
|
||||
* This file registers all CLI commands and handles command line parsing
|
||||
*/
|
||||
|
||||
// Package description
|
||||
const DESCRIPTION =
|
||||
'A LLM Log Model Context Protocol (MCP) server implementation using TypeScript';
|
||||
|
||||
/**
|
||||
* Run the CLI with the provided arguments
|
||||
*
|
||||
* @param args Command line arguments to process
|
||||
* @returns Promise that resolves when CLI command execution completes
|
||||
*/
|
||||
export async function runCli(args: string[]) {
|
||||
const cliLogger = Logger.forContext('cli/index.ts', 'runCli');
|
||||
cliLogger.debug('Initializing CLI with arguments', args);
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program.name(CLI_NAME).description(DESCRIPTION).version(VERSION);
|
||||
|
||||
// Register CLI commands
|
||||
cliLogger.debug('Registering CLI commands...');
|
||||
posthogLlmCli.register(program);
|
||||
cliLogger.debug('CLI commands registered successfully');
|
||||
|
||||
// Handle unknown commands
|
||||
program.on('command:*', (operands) => {
|
||||
cliLogger.error(`Unknown command: ${operands[0]}`);
|
||||
console.log('');
|
||||
program.help();
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Parse arguments; default to help if no command provided
|
||||
cliLogger.debug('Parsing CLI arguments');
|
||||
|
||||
// Special handling for top-level commands
|
||||
if (args.length === 1) {
|
||||
// Check if it's a known top-level command
|
||||
const command = program.commands.find((cmd) => cmd.name() === args[0]);
|
||||
if (command) {
|
||||
command.outputHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
await program.parseAsync(args.length ? args : ['--help'], { from: 'user' });
|
||||
cliLogger.debug('CLI command execution completed');
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
import { Command } from 'commander';
|
||||
import { Logger } from '../utils/logger.util.js';
|
||||
import { handleCliError } from '../utils/error.util.js';
|
||||
import postHogLlmController from '../controllers/posthog-llm.controller.js';
|
||||
|
||||
const logger = Logger.forContext('cli/posthog-llm.cli.ts');
|
||||
|
||||
/**
|
||||
* Register PostHog LLM CLI commands
|
||||
* @param program The Commander program instance
|
||||
*/
|
||||
function register(program: Command) {
|
||||
const methodLogger = logger.forMethod('register');
|
||||
methodLogger.debug('Registering PostHog LLM CLI commands...');
|
||||
|
||||
program
|
||||
.command('llm-observability-mcp')
|
||||
.description('Interact with the PostHog LLM API.')
|
||||
.argument('<eventName>', 'The name of the event to capture.')
|
||||
.argument('<distinctId>', 'The distinct ID of the user.')
|
||||
.option(
|
||||
'-p, --properties <properties...>',
|
||||
'JSON string of event properties. Can be a single quoted string or raw key-value pairs.',
|
||||
)
|
||||
.action(async (eventName, distinctId, options) => {
|
||||
const actionLogger = logger.forMethod('action:capture');
|
||||
try {
|
||||
actionLogger.debug(`CLI posthog-llm capture called`, {
|
||||
eventName,
|
||||
distinctId,
|
||||
options,
|
||||
});
|
||||
|
||||
// Handle JSON properties from command object
|
||||
let properties = {};
|
||||
if (options.properties && options.properties.length > 0) {
|
||||
const propertiesString = options.properties.join(' ');
|
||||
try {
|
||||
properties = JSON.parse(propertiesString);
|
||||
} catch {
|
||||
// Try to fix common issues with JSON formatting
|
||||
const fixedJson = propertiesString
|
||||
.replace(/(\w+):/g, '"$1":') // Add quotes around keys
|
||||
.replace(/'/g, '"'); // Replace single quotes with double quotes
|
||||
|
||||
properties = JSON.parse(fixedJson);
|
||||
}
|
||||
}
|
||||
|
||||
const args = { eventName, distinctId, properties };
|
||||
await postHogLlmController.capture(args);
|
||||
} catch (error) {
|
||||
handleCliError(error);
|
||||
}
|
||||
});
|
||||
|
||||
methodLogger.debug('PostHog LLM CLI commands registered successfully');
|
||||
}
|
||||
|
||||
export default { register };
|
||||
22
src/config/common.schema.ts
Normal file
22
src/config/common.schema.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* Common/shared configuration options schema (now includes global options)
|
||||
*/
|
||||
export const CommonConfigSchema = z.object({
|
||||
serviceName: z.string().default('llm-observability-mcp'),
|
||||
serviceVersion: z.string().default('1.0.0'),
|
||||
environment: z.string().default('development'),
|
||||
debug: z
|
||||
.preprocess((val) => {
|
||||
if (typeof val === 'string') {
|
||||
if (val.toLowerCase() === 'false') return false;
|
||||
if (val.toLowerCase() === 'true') return true;
|
||||
}
|
||||
return val;
|
||||
}, z.boolean())
|
||||
.default(false),
|
||||
logLevel: z.enum(['debug', 'info', 'warn', 'error']).default('info'),
|
||||
});
|
||||
|
||||
export type CommonConfigType = z.infer<typeof CommonConfigSchema>;
|
||||
202
src/config/config-loader.ts
Normal file
202
src/config/config-loader.ts
Normal file
@@ -0,0 +1,202 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { Logger } from '../utils/logger.util.js';
|
||||
import dotenv from 'dotenv';
|
||||
import os from 'os';
|
||||
import { CommonConfigSchema, CommonConfigType } from './common.schema.js';
|
||||
import { PostHogConfigSchema, PostHogConfigType } from './posthog.schema.js';
|
||||
import {
|
||||
OpenTelemetryConfigSchema,
|
||||
OpenTelemetryConfigType,
|
||||
} from './opentelemetry-llm.schema.js';
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* Configuration loader that handles multiple sources with priority:
|
||||
* 1. Direct ENV pass (process.env)
|
||||
* 2. .env file in project root
|
||||
* 3. Global config file at $HOME/.mcp/configs.json
|
||||
*/
|
||||
class ConfigLoader {
|
||||
private packageName: string;
|
||||
private configLoaded: boolean = false;
|
||||
|
||||
constructor(packageName: string) {
|
||||
this.packageName = packageName;
|
||||
}
|
||||
|
||||
load(): void {
|
||||
const logger = Logger.forContext('config-loader', 'load');
|
||||
|
||||
if (this.configLoaded) {
|
||||
logger.debug('Configuration already loaded, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug('Loading configuration...');
|
||||
|
||||
// Load from global config file
|
||||
this.loadFromGlobalConfig();
|
||||
|
||||
// Load from .env file
|
||||
this.loadFromEnvFile();
|
||||
|
||||
this.configLoaded = true;
|
||||
logger.debug('Configuration loaded successfully');
|
||||
}
|
||||
|
||||
private loadFromEnvFile(): void {
|
||||
const logger = Logger.forContext('config-loader', 'loadFromEnvFile');
|
||||
|
||||
try {
|
||||
const result = dotenv.config();
|
||||
if (result.error) {
|
||||
logger.debug('No .env file found or error reading it');
|
||||
return;
|
||||
}
|
||||
logger.debug('Loaded configuration from .env file');
|
||||
} catch (error) {
|
||||
logger.error('Error loading .env file', error);
|
||||
}
|
||||
}
|
||||
|
||||
private loadFromGlobalConfig(): void {
|
||||
const logger = Logger.forContext(
|
||||
'config-loader',
|
||||
'loadFromGlobalConfig',
|
||||
);
|
||||
|
||||
try {
|
||||
const homedir = os.homedir();
|
||||
const globalConfigPath = path.join(homedir, '.mcp', 'configs.json');
|
||||
|
||||
if (!fs.existsSync(globalConfigPath)) {
|
||||
logger.debug('Global config file not found');
|
||||
return;
|
||||
}
|
||||
|
||||
const configContent = fs.readFileSync(globalConfigPath, 'utf8');
|
||||
const config = JSON.parse(configContent);
|
||||
|
||||
const shortKey = 'llm-observability-mcp';
|
||||
const fullPackageName = this.packageName;
|
||||
const unscopedPackageName =
|
||||
fullPackageName.split('/')[1] || fullPackageName;
|
||||
|
||||
const potentialKeys = [
|
||||
shortKey,
|
||||
fullPackageName,
|
||||
unscopedPackageName,
|
||||
];
|
||||
let foundConfig = null;
|
||||
|
||||
for (const key of potentialKeys) {
|
||||
if (
|
||||
config[key] &&
|
||||
typeof config[key] === 'object' &&
|
||||
config[key].environments
|
||||
) {
|
||||
foundConfig = config[key];
|
||||
logger.debug(`Found configuration using key: ${key}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundConfig || !foundConfig.environments) {
|
||||
logger.debug(
|
||||
`No config found for ${this.packageName} using keys: ${potentialKeys.join(', ')}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(
|
||||
foundConfig.environments,
|
||||
)) {
|
||||
if (process.env[key] === undefined) {
|
||||
process.env[key] = String(value);
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug(`Loaded configuration from global config file`);
|
||||
} catch (error) {
|
||||
logger.error('Error loading global config file', error);
|
||||
}
|
||||
}
|
||||
|
||||
private getEnvObject(schema: z.ZodTypeAny): Record<string, unknown> {
|
||||
const shape =
|
||||
typeof schema._def.shape === 'function'
|
||||
? schema._def.shape()
|
||||
: schema._def.shape;
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const key in shape) {
|
||||
if (Object.prototype.hasOwnProperty.call(shape, key)) {
|
||||
const envKey = this.toEnvKey(key);
|
||||
const value = process.env[envKey];
|
||||
if (value !== undefined) {
|
||||
const type = shape[key]._def.typeName;
|
||||
if (type === 'ZodNumber') {
|
||||
result[key] = Number(value);
|
||||
} else if (type === 'ZodBoolean') {
|
||||
result[key] = value.toLowerCase() === 'true';
|
||||
} else {
|
||||
result[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private toEnvKey(key: string): string {
|
||||
// Map schema keys to environment variable names
|
||||
// e.g. serviceName -> OTEL_SERVICE_NAME, apiKey -> POSTHOG_API_KEY, etc.
|
||||
const map: Record<string, string> = {
|
||||
serviceName: 'OTEL_SERVICE_NAME',
|
||||
serviceVersion: 'OTEL_SERVICE_VERSION',
|
||||
environment: 'OTEL_ENVIRONMENT',
|
||||
metricsEndpoint: 'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT',
|
||||
tracesEndpoint: 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT',
|
||||
logsEndpoint: 'OTEL_EXPORTER_OTLP_LOGS_ENDPOINT',
|
||||
headers: 'OTEL_EXPORTER_OTLP_HEADERS',
|
||||
exportIntervalMillis: 'OTEL_METRIC_EXPORT_INTERVAL',
|
||||
exportTimeoutMillis: 'OTEL_METRIC_EXPORT_TIMEOUT',
|
||||
samplingRatio: 'OTEL_TRACES_SAMPLER_ARG',
|
||||
apiKey: 'POSTHOG_API_KEY',
|
||||
host: 'POSTHOG_HOST',
|
||||
debug: 'DEBUG',
|
||||
logLevel: 'LOG_LEVEL',
|
||||
};
|
||||
return map[key] || key;
|
||||
}
|
||||
|
||||
getCommonConfig(): CommonConfigType {
|
||||
return CommonConfigSchema.parse(this.getEnvObject(CommonConfigSchema));
|
||||
}
|
||||
|
||||
getPosthogConfig(): PostHogConfigType {
|
||||
return PostHogConfigSchema.parse(
|
||||
this.getEnvObject(PostHogConfigSchema),
|
||||
);
|
||||
}
|
||||
|
||||
getOpenTelemetryConfig(): OpenTelemetryConfigType {
|
||||
const raw = this.getEnvObject(OpenTelemetryConfigSchema);
|
||||
// Special handling for headers (parse comma-separated string to object)
|
||||
if (typeof raw.headers === 'string') {
|
||||
const headers: Record<string, string> = {};
|
||||
const pairs = raw.headers.split(',');
|
||||
for (const pair of pairs) {
|
||||
const [key, value] = pair.split('=');
|
||||
if (key && value) {
|
||||
headers[key.trim()] = value.trim();
|
||||
}
|
||||
}
|
||||
raw.headers = headers;
|
||||
}
|
||||
return OpenTelemetryConfigSchema.parse(raw);
|
||||
}
|
||||
}
|
||||
|
||||
const configLoader = new ConfigLoader('@sfiorini/llm-observability-mcp');
|
||||
export default configLoader;
|
||||
24
src/config/opentelemetry-llm.schema.ts
Normal file
24
src/config/opentelemetry-llm.schema.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
import { z } from 'zod';
|
||||
import { CommonConfigSchema } from './common.schema.js';
|
||||
|
||||
/**
|
||||
* Configuration schema for OpenTelemetry
|
||||
*/
|
||||
export const OpenTelemetryConfigSchema = CommonConfigSchema.extend({
|
||||
// OTLP endpoints
|
||||
metricsEndpoint: z.string().optional(),
|
||||
tracesEndpoint: z.string().optional(),
|
||||
logsEndpoint: z.string().optional(),
|
||||
|
||||
// Authentication headers
|
||||
headers: z.record(z.string()).optional(),
|
||||
|
||||
// Export configuration
|
||||
exportIntervalMillis: z.number().default(10000),
|
||||
exportTimeoutMillis: z.number().default(5000),
|
||||
|
||||
// Sampling configuration
|
||||
samplingRatio: z.number().min(0).max(1).default(1.0),
|
||||
});
|
||||
|
||||
export type OpenTelemetryConfigType = z.infer<typeof OpenTelemetryConfigSchema>;
|
||||
15
src/config/posthog.schema.ts
Normal file
15
src/config/posthog.schema.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import { z } from 'zod';
|
||||
import { CommonConfigSchema } from './common.schema.js';
|
||||
|
||||
/**
|
||||
* PostHog configuration options schema
|
||||
*/
|
||||
export const PostHogConfigSchema = CommonConfigSchema.extend({
|
||||
apiKey: z.string().describe('PostHog API key'),
|
||||
host: z
|
||||
.string()
|
||||
.default('https://app.posthog.com')
|
||||
.describe('PostHog host URL'),
|
||||
});
|
||||
|
||||
export type PostHogConfigType = z.infer<typeof PostHogConfigSchema>;
|
||||
8
src/controllers/base.controller.ts
Normal file
8
src/controllers/base.controller.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
import { ControllerResponse } from '../types/common.types.js';
|
||||
|
||||
export abstract class BaseController {
|
||||
// Instance method (must be implemented by subclasses)
|
||||
static capture(data: unknown): Promise<ControllerResponse> {
|
||||
throw new Error(`Not implemented: capture: ${data}`);
|
||||
}
|
||||
}
|
||||
65
src/controllers/opentelemetry-llm.controller.ts
Normal file
65
src/controllers/opentelemetry-llm.controller.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
import { Logger } from '../utils/logger.util.js';
|
||||
import { BaseController } from './base.controller.js';
|
||||
import { OpenTelemetryService } from '../services/opentelemetry-llm.service.js';
|
||||
import { OpenTelemetryLlmInputSchemaType } from '../types/opentelemetry-llm.types.js';
|
||||
import { ControllerResponse } from '../types/common.types.js';
|
||||
|
||||
/**
|
||||
* Controller for OpenTelemetry LLM observability
|
||||
*/
|
||||
export class OpenTelemetryController extends BaseController {
|
||||
/**
|
||||
* Capture LLM observability data
|
||||
*/
|
||||
static async capture(
|
||||
data: OpenTelemetryLlmInputSchemaType,
|
||||
): Promise<ControllerResponse> {
|
||||
const logger = Logger.forContext(
|
||||
'opentelemetry.controller',
|
||||
'captureLlmObservability',
|
||||
);
|
||||
logger.debug('Capturing LLM observability data', data);
|
||||
|
||||
try {
|
||||
// Get and initialize the service
|
||||
const service = OpenTelemetryService.getInstance();
|
||||
service.initialize();
|
||||
|
||||
// Record the LLM request
|
||||
service.recordLlmRequest(data);
|
||||
|
||||
const content = `## OpenTelemetry LLM Observability Captured
|
||||
|
||||
Successfully recorded LLM observability data:
|
||||
|
||||
- **Model**: ${data.model}
|
||||
- **Provider**: ${data.provider}
|
||||
- **User ID**: ${data.userId}
|
||||
- **Operation**: ${data.operationName || 'generate'}
|
||||
- **Input Tokens**: ${data.inputTokens || 'N/A'}
|
||||
- **Output Tokens**: ${data.outputTokens || 'N/A'}
|
||||
- **Total Tokens**: ${(data.inputTokens || 0) + (data.outputTokens || 0)}
|
||||
- **Latency**: ${data.latency ? `${data.latency}s` : 'N/A'}
|
||||
- **Status**: ${data.error ? 'Error' : 'Success'}
|
||||
${data.error ? `- **Error**: ${data.error}` : ''}
|
||||
|
||||
### Metrics Recorded
|
||||
- ✅ Request counter incremented
|
||||
- ✅ Token usage recorded
|
||||
- ✅ Latency histogram updated
|
||||
- ✅ Distributed trace created
|
||||
|
||||
### Trace Details
|
||||
- **Trace ID**: ${data.traceId || 'auto-generated'}
|
||||
- **Span Name**: ${data.operationName || 'llm.generate'}
|
||||
- **Attributes**: model, provider, user_id, tokens, latency, error details
|
||||
|
||||
The data has been sent to your configured OpenTelemetry collector and is now available in your observability platform (Jaeger, New Relic, Grafana, etc.).`;
|
||||
|
||||
return { content };
|
||||
} catch (error) {
|
||||
logger.error('Error capturing LLM observability', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,32 +1,35 @@
|
||||
import { Logger } from '../utils/logger.util.js';
|
||||
import { ControllerResponse } from '../types/common.types.js';
|
||||
import postHogLlmService from '../services/posthog-llm.service.js';
|
||||
import { PostHogLlmInputSchemaType } from '../types/posthog-llm.types.js';
|
||||
|
||||
const logger = Logger.forContext('controllers/posthog-llm.controller.ts');
|
||||
/**
|
||||
* Controller for PostHog LLM observability
|
||||
*/
|
||||
export class PostHogController {
|
||||
/**
|
||||
* Capture LLM observability data
|
||||
*/
|
||||
|
||||
async function capture(args: {
|
||||
eventName: string;
|
||||
distinctId: string;
|
||||
properties: Record<string, unknown>;
|
||||
}): Promise<ControllerResponse> {
|
||||
const methodLogger = logger.forMethod('capture');
|
||||
methodLogger.debug('Capturing PostHog event...');
|
||||
methodLogger.debug('Arguments:', args);
|
||||
static async capture(
|
||||
data: PostHogLlmInputSchemaType,
|
||||
): Promise<ControllerResponse> {
|
||||
const logger = Logger.forContext('posthog-llm.controller', 'capture');
|
||||
logger.debug('Capturing LLM observability data', data);
|
||||
|
||||
try {
|
||||
await postHogLlmService.capture(args);
|
||||
return {
|
||||
content: 'Event captured successfully.',
|
||||
};
|
||||
} catch (error) {
|
||||
methodLogger.error('Error capturing event:', error);
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : 'Unknown error';
|
||||
methodLogger.error('Error capturing event:', errorMessage);
|
||||
return {
|
||||
content: `Failed to capture event: ${errorMessage}`,
|
||||
};
|
||||
try {
|
||||
await postHogLlmService.capture(data);
|
||||
return {
|
||||
content: 'Event captured successfully.',
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Error capturing event:', error);
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : 'Unknown error';
|
||||
logger.error('Error capturing event:', errorMessage);
|
||||
return {
|
||||
content: `Failed to capture event: ${errorMessage}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default { capture };
|
||||
|
||||
14
src/index.ts
14
src/index.ts
@@ -1,7 +1,5 @@
|
||||
#!/usr/bin/env node
|
||||
import { Logger } from './utils/logger.util.js';
|
||||
import { config } from './utils/config.util.js';
|
||||
import { runCli } from './cli/index.js';
|
||||
import { stdioTransport } from './server/stdio.js';
|
||||
import { streamableHttpTransport } from './server/streamableHttp.js';
|
||||
|
||||
@@ -43,18 +41,6 @@ export async function startServer(): Promise<void> {
|
||||
* Main entry point
|
||||
*/
|
||||
async function main() {
|
||||
const mainLogger = Logger.forContext('index.ts', 'main');
|
||||
|
||||
// Load configuration
|
||||
config.load();
|
||||
|
||||
// CLI mode - if any arguments are provided
|
||||
if (process.argv.length > 2) {
|
||||
mainLogger.info('CLI mode detected');
|
||||
await runCli(process.argv.slice(2));
|
||||
return;
|
||||
}
|
||||
|
||||
// Server mode - determine transport and start server
|
||||
await startServer();
|
||||
}
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { Logger } from '../utils/logger.util.js';
|
||||
import posthogLlmController from '../controllers/posthog-llm.controller.js';
|
||||
import { formatErrorForMcpResource } from '../utils/error.util.js';
|
||||
|
||||
const logger = Logger.forContext('resources/posthog-llm.resource.ts');
|
||||
|
||||
/**
|
||||
* Register PostHog LLM resources with the MCP server
|
||||
*
|
||||
* @param server The MCP server instance
|
||||
*/
|
||||
function registerResources(server: McpServer) {
|
||||
const registerLogger = logger.forMethod('registerResources');
|
||||
registerLogger.debug('Registering PostHog LLM resources...');
|
||||
|
||||
// Register the PostHog event capture resource
|
||||
server.resource(
|
||||
'capture_llm_observability',
|
||||
'Capture a LLM event in PostHog',
|
||||
async (uri: URL) => {
|
||||
const methodLogger = logger.forMethod('posthogEventResource');
|
||||
try {
|
||||
methodLogger.debug('PostHog event resource called', {
|
||||
uri: uri.toString(),
|
||||
});
|
||||
|
||||
// Extract parameters from the URI
|
||||
// Format: posthog://event_name/distinct_id?properties=JSON
|
||||
const pathParts = uri.pathname.split('/').filter(Boolean);
|
||||
const eventName = pathParts[0] || 'event';
|
||||
const distinctId = pathParts[1] || 'anonymous';
|
||||
|
||||
// Parse properties from query parameters if present
|
||||
let properties: Record<string, unknown> = {};
|
||||
if (uri.searchParams.has('properties')) {
|
||||
try {
|
||||
properties = JSON.parse(
|
||||
uri.searchParams.get('properties') || '{}',
|
||||
);
|
||||
} catch (e) {
|
||||
methodLogger.warn('Failed to parse properties JSON', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Call the controller to capture the event
|
||||
const result = await posthogLlmController.capture({
|
||||
eventName,
|
||||
distinctId,
|
||||
properties,
|
||||
});
|
||||
|
||||
// Return the content as a text resource
|
||||
return {
|
||||
contents: [
|
||||
{
|
||||
uri: uri.toString(),
|
||||
text: result.content,
|
||||
mimeType: 'text/plain',
|
||||
description: `PostHog event: ${eventName}`,
|
||||
},
|
||||
],
|
||||
};
|
||||
} catch (error) {
|
||||
methodLogger.error('Resource error', error);
|
||||
return formatErrorForMcpResource(error, uri.toString());
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
registerLogger.debug('PostHog LLM resources registered successfully');
|
||||
}
|
||||
|
||||
export default { registerResources };
|
||||
@@ -1,21 +1,21 @@
|
||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { SSEServerTransport } from '@modelcontextprotocol/sdk/server/sse.js';
|
||||
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
||||
|
||||
import { Logger } from '../utils/logger.util';
|
||||
import { config } from '../utils/config.util';
|
||||
import configLoader from '../config/config-loader';
|
||||
import { PACKAGE_NAME, VERSION } from '../utils/constants.util';
|
||||
import posthogLlmResources from '../resources/posthog-llm.resource.js';
|
||||
import posthogLlmTools from '../tools/posthog-llm.tool.js';
|
||||
import openTelemetryTools from '../tools/opentelemetry-llm.tool.js';
|
||||
|
||||
export function createServer() {
|
||||
const serverLogger = Logger.forContext('utils/server.util.ts', 'getServer');
|
||||
|
||||
// Load configuration
|
||||
serverLogger.info('Starting MCP server initialization...');
|
||||
config.load();
|
||||
configLoader.load();
|
||||
|
||||
if (config.getBoolean('DEBUG')) {
|
||||
const commonConfig = configLoader.getCommonConfig();
|
||||
if (commonConfig.debug) {
|
||||
serverLogger.debug('Debug mode enabled');
|
||||
}
|
||||
|
||||
@@ -25,11 +25,11 @@ export function createServer() {
|
||||
version: VERSION,
|
||||
});
|
||||
|
||||
// Register tools and resources
|
||||
serverLogger.info('Registering MCP tools and resources...');
|
||||
// Register tools
|
||||
serverLogger.info('Registering MCP tools...');
|
||||
posthogLlmTools.registerTools(server);
|
||||
posthogLlmResources.registerResources(server);
|
||||
serverLogger.debug('All tools and resources registered');
|
||||
openTelemetryTools.registerTools(server);
|
||||
serverLogger.debug('All tools registered');
|
||||
|
||||
return server;
|
||||
}
|
||||
|
||||
290
src/services/opentelemetry-llm.service.ts
Normal file
290
src/services/opentelemetry-llm.service.ts
Normal file
@@ -0,0 +1,290 @@
|
||||
import {
|
||||
MeterProvider,
|
||||
PeriodicExportingMetricReader,
|
||||
} from '@opentelemetry/sdk-metrics';
|
||||
import { NodeSDK } from '@opentelemetry/sdk-node';
|
||||
import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-node';
|
||||
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
||||
import { resourceFromAttributes } from '@opentelemetry/resources';
|
||||
import {
|
||||
ATTR_SERVICE_NAME,
|
||||
ATTR_SERVICE_VERSION,
|
||||
SEMRESATTRS_DEPLOYMENT_ENVIRONMENT,
|
||||
} from '@opentelemetry/semantic-conventions';
|
||||
import { Logger } from '../utils/logger.util.js';
|
||||
import type { OpenTelemetryConfigType } from '../config/opentelemetry-llm.schema.js';
|
||||
import { OpenTelemetryLlmInputSchemaType } from '../types/opentelemetry-llm.types.js';
|
||||
import {
|
||||
Meter,
|
||||
Counter,
|
||||
Histogram,
|
||||
UpDownCounter,
|
||||
metrics,
|
||||
} from '@opentelemetry/api';
|
||||
import { trace, Tracer, SpanStatusCode } from '@opentelemetry/api';
|
||||
import configLoader from '../config/config-loader';
|
||||
|
||||
/**
|
||||
* OpenTelemetry service for LLM observability
|
||||
*/
|
||||
export class OpenTelemetryService {
|
||||
private static instance: OpenTelemetryService;
|
||||
private sdk: NodeSDK | null = null;
|
||||
private meterProvider: MeterProvider | null = null;
|
||||
private tracer: Tracer;
|
||||
private meter: Meter;
|
||||
private config: OpenTelemetryConfigType;
|
||||
|
||||
// Metrics
|
||||
private requestCounter: Counter;
|
||||
private tokenCounter: Counter;
|
||||
private latencyHistogram: Histogram;
|
||||
private activeRequests: UpDownCounter;
|
||||
|
||||
private constructor() {
|
||||
this.config = configLoader.getOpenTelemetryConfig();
|
||||
this.tracer = trace.getTracer('llm-observability-mcp');
|
||||
this.meter = this.initializeMetrics();
|
||||
|
||||
// Initialize metrics
|
||||
this.requestCounter = this.meter.createCounter('llm.requests.total', {
|
||||
description: 'Total number of LLM requests',
|
||||
});
|
||||
|
||||
this.tokenCounter = this.meter.createCounter('llm.tokens.total', {
|
||||
description: 'Total number of tokens processed',
|
||||
});
|
||||
|
||||
this.latencyHistogram = this.meter.createHistogram(
|
||||
'llm.latency.duration',
|
||||
{
|
||||
description: 'Duration of LLM requests in milliseconds',
|
||||
unit: 'ms',
|
||||
},
|
||||
);
|
||||
|
||||
this.activeRequests = this.meter.createUpDownCounter(
|
||||
'llm.requests.active',
|
||||
{
|
||||
description: 'Number of active LLM requests',
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get singleton instance
|
||||
*/
|
||||
public static getInstance(): OpenTelemetryService {
|
||||
if (!OpenTelemetryService.instance) {
|
||||
OpenTelemetryService.instance = new OpenTelemetryService();
|
||||
}
|
||||
return OpenTelemetryService.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize OpenTelemetry SDK
|
||||
*/
|
||||
public initialize(): void {
|
||||
const logger = Logger.forContext('opentelemetry.service', 'initialize');
|
||||
|
||||
if (this.sdk) {
|
||||
logger.warn('OpenTelemetry SDK already initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const resource = resourceFromAttributes({
|
||||
[ATTR_SERVICE_NAME]: this.config.serviceName,
|
||||
[ATTR_SERVICE_VERSION]: this.config.serviceVersion,
|
||||
[SEMRESATTRS_DEPLOYMENT_ENVIRONMENT]: this.config.environment,
|
||||
debug: this.config.debug,
|
||||
logLevel: this.config.logLevel,
|
||||
});
|
||||
|
||||
// Configure metric exporter
|
||||
const metricExporter = this.config.metricsEndpoint
|
||||
? new OTLPMetricExporter({
|
||||
url: this.config.metricsEndpoint,
|
||||
headers: this.config.headers,
|
||||
})
|
||||
: undefined;
|
||||
|
||||
// Configure trace exporter
|
||||
const traceExporter = this.config.tracesEndpoint
|
||||
? new OTLPTraceExporter({
|
||||
url: this.config.tracesEndpoint,
|
||||
headers: this.config.headers,
|
||||
})
|
||||
: undefined;
|
||||
|
||||
// Initialize meter provider
|
||||
if (metricExporter) {
|
||||
this.meterProvider = new MeterProvider({
|
||||
resource,
|
||||
readers: [
|
||||
new PeriodicExportingMetricReader({
|
||||
exporter: metricExporter,
|
||||
exportIntervalMillis:
|
||||
this.config.exportIntervalMillis,
|
||||
exportTimeoutMillis:
|
||||
this.config.exportTimeoutMillis,
|
||||
}),
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
// Initialize SDK
|
||||
this.sdk = new NodeSDK({
|
||||
resource,
|
||||
spanProcessor: traceExporter
|
||||
? new BatchSpanProcessor(traceExporter)
|
||||
: undefined,
|
||||
});
|
||||
|
||||
this.sdk.start();
|
||||
logger.info('OpenTelemetry SDK initialized successfully');
|
||||
} catch (error) {
|
||||
logger.error('Failed to initialize OpenTelemetry SDK', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize metrics
|
||||
*/
|
||||
private initializeMetrics(): Meter {
|
||||
if (this.meterProvider) {
|
||||
return this.meterProvider.getMeter('llm-observability-mcp');
|
||||
}
|
||||
return metrics.getMeter('llm-observability-mcp');
|
||||
}
|
||||
|
||||
/**
|
||||
* Record LLM request metrics and traces
|
||||
*/
|
||||
public recordLlmRequest(data: OpenTelemetryLlmInputSchemaType): void {
|
||||
const labels = {
|
||||
model: data.model,
|
||||
provider: data.provider,
|
||||
userId: data.userId,
|
||||
operationName: data.operationName || 'generate',
|
||||
status: data.error ? 'error' : 'success',
|
||||
errorType: data.errorType,
|
||||
mcpToolsUsed: data.mcpToolsUsed?.join(',') || '',
|
||||
};
|
||||
|
||||
// Record metrics
|
||||
this.requestCounter.add(1, labels);
|
||||
|
||||
if (data.inputTokens || data.outputTokens) {
|
||||
const totalTokens =
|
||||
(data.inputTokens || 0) + (data.outputTokens || 0);
|
||||
this.tokenCounter.add(totalTokens, {
|
||||
...labels,
|
||||
type: 'total',
|
||||
});
|
||||
|
||||
if (data.inputTokens) {
|
||||
this.tokenCounter.add(data.inputTokens, {
|
||||
...labels,
|
||||
type: 'input',
|
||||
});
|
||||
}
|
||||
|
||||
if (data.outputTokens) {
|
||||
this.tokenCounter.add(data.outputTokens, {
|
||||
...labels,
|
||||
type: 'output',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (data.latency) {
|
||||
this.latencyHistogram.record(data.latency * 1000, labels);
|
||||
}
|
||||
|
||||
// Create trace
|
||||
this.createLlmTrace(data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create distributed trace for LLM request
|
||||
*/
|
||||
private createLlmTrace(data: OpenTelemetryLlmInputSchemaType): void {
|
||||
const spanName = data.operationName || 'llm.generate';
|
||||
const span = this.tracer.startSpan(spanName, {
|
||||
attributes: {
|
||||
'llm.model': data.model,
|
||||
'llm.provider': data.provider,
|
||||
'llm.user_id': data.userId,
|
||||
'llm.operation': data.operationName || 'generate',
|
||||
'llm.input_tokens': data.inputTokens,
|
||||
'llm.output_tokens': data.outputTokens,
|
||||
'llm.total_tokens':
|
||||
(data.inputTokens || 0) + (data.outputTokens || 0),
|
||||
'llm.latency_ms': data.latency
|
||||
? data.latency * 1000
|
||||
: undefined,
|
||||
'llm.http_status': data.httpStatus,
|
||||
'llm.base_url': data.baseUrl,
|
||||
'llm.error': data.error,
|
||||
'llm.error_type': data.errorType,
|
||||
'llm.input':
|
||||
typeof data.input === 'string'
|
||||
? data.input
|
||||
: JSON.stringify(data.input),
|
||||
'llm.output':
|
||||
typeof data.outputChoices === 'string'
|
||||
? data.outputChoices
|
||||
: JSON.stringify(data.outputChoices),
|
||||
'llm.mcp_tools_used': data.mcpToolsUsed?.join(','),
|
||||
},
|
||||
});
|
||||
|
||||
if (data.traceId) {
|
||||
span.setAttribute('trace.id', data.traceId);
|
||||
}
|
||||
|
||||
if (data.error) {
|
||||
span.setStatus({
|
||||
code: SpanStatusCode.ERROR,
|
||||
message: data.error,
|
||||
});
|
||||
} else {
|
||||
span.setStatus({ code: SpanStatusCode.OK });
|
||||
}
|
||||
|
||||
span.end();
|
||||
}
|
||||
|
||||
/**
|
||||
* Record active request count
|
||||
*/
|
||||
public incrementActiveRequests(): void {
|
||||
this.activeRequests.add(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decrement active request count
|
||||
*/
|
||||
public decrementActiveRequests(): void {
|
||||
this.activeRequests.add(-1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown OpenTelemetry SDK
|
||||
*/
|
||||
public async shutdown(): Promise<void> {
|
||||
const logger = Logger.forContext('opentelemetry.service', 'shutdown');
|
||||
|
||||
if (this.sdk) {
|
||||
try {
|
||||
await this.sdk.shutdown();
|
||||
logger.info('OpenTelemetry SDK shutdown successfully');
|
||||
} catch (error) {
|
||||
logger.error('Error shutting down OpenTelemetry SDK', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,18 +1,18 @@
|
||||
import { Logger } from '../utils/logger.util.js';
|
||||
import { Logger } from '../utils/logger.util';
|
||||
import { PostHog } from 'posthog-node';
|
||||
import { config } from '../utils/config.util.js';
|
||||
import configLoader from '../config/config-loader';
|
||||
|
||||
// Ensure configuration is loaded before accessing environment variables
|
||||
config.load();
|
||||
configLoader.load();
|
||||
|
||||
const logger = Logger.forContext('services/posthog-llm.service.ts');
|
||||
|
||||
const posthogApiKey = config.get('POSTHOG_API_KEY');
|
||||
const posthogConfig = configLoader.getPosthogConfig();
|
||||
let posthogClient: PostHog | null = null;
|
||||
|
||||
if (posthogApiKey) {
|
||||
posthogClient = new PostHog(posthogApiKey, {
|
||||
host: config.get('POSTHOG_HOST') || 'https://app.posthog.com',
|
||||
if (posthogConfig.apiKey) {
|
||||
posthogClient = new PostHog(posthogConfig.apiKey, {
|
||||
host: posthogConfig.host,
|
||||
});
|
||||
} else {
|
||||
logger.warn('POSTHOG_API_KEY is not set. PostHog client not initialized.');
|
||||
|
||||
77
src/tools/opentelemetry-llm.tool.ts
Normal file
77
src/tools/opentelemetry-llm.tool.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { Logger } from '../utils/logger.util.js';
|
||||
import { formatErrorForMcpTool } from '../utils/error.util.js';
|
||||
import { OpenTelemetryController } from '../controllers/opentelemetry-llm.controller.js';
|
||||
import { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
||||
import {
|
||||
OpenTelemetryLlmInputSchema,
|
||||
OpenTelemetryLlmInputSchemaType,
|
||||
} from '../types/opentelemetry-llm.types.js';
|
||||
|
||||
/**
|
||||
* @function captureOpenTelemetryLlmObservability
|
||||
* @description MCP Tool handler to capture LLM observability events using OpenTelemetry.
|
||||
* It records metrics, traces, and spans for LLM requests that can be sent to any OpenTelemetry-compatible backend.
|
||||
* @param {OpenTelemetryLlmInputSchemaType} args - Arguments provided to the tool.
|
||||
* @returns {Promise<CallToolResult>} Formatted response for the MCP.
|
||||
* @throws {McpError} Formatted error if the controller or service layer encounters an issue.
|
||||
*/
|
||||
async function captureOpenTelemetryLlmObservability(
|
||||
args: OpenTelemetryLlmInputSchemaType,
|
||||
): Promise<CallToolResult> {
|
||||
const methodLogger = Logger.forContext(
|
||||
'opentelemetry.tool',
|
||||
'captureOpenTelemetryLlmObservability',
|
||||
);
|
||||
methodLogger.debug('Capture LLM Observability with OpenTelemetry...', args);
|
||||
|
||||
try {
|
||||
// Parse and validate arguments
|
||||
const validatedArgs = OpenTelemetryLlmInputSchema.parse(args);
|
||||
|
||||
// Pass validated args to the controller
|
||||
const result = await OpenTelemetryController.capture(validatedArgs);
|
||||
methodLogger.debug('Got response from controller', result);
|
||||
|
||||
// Format the response for the MCP tool
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text' as const,
|
||||
text: result.content,
|
||||
},
|
||||
],
|
||||
};
|
||||
} catch (error) {
|
||||
methodLogger.error(
|
||||
'Error tracking LLM generation with OpenTelemetry',
|
||||
error,
|
||||
);
|
||||
return formatErrorForMcpTool(error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @function registerTools
|
||||
* @description Registers the OpenTelemetry LLM observability tool with the MCP server.
|
||||
*
|
||||
* @param {McpServer} server - The MCP server instance.
|
||||
*/
|
||||
function registerTools(server: McpServer) {
|
||||
const methodLogger = Logger.forContext(
|
||||
'opentelemetry.tool',
|
||||
'registerTools',
|
||||
);
|
||||
methodLogger.debug('Registering OpenTelemetry LLM observability tools...');
|
||||
|
||||
server.tool(
|
||||
'llm_observability_otel',
|
||||
`Captures LLM usage using OpenTelemetry for observability, including requests, responses, and performance metrics. Works with any OpenTelemetry-compatible backend like Jaeger, New Relic, Grafana, etc.`,
|
||||
OpenTelemetryLlmInputSchema.shape,
|
||||
captureOpenTelemetryLlmObservability,
|
||||
);
|
||||
|
||||
methodLogger.debug('Successfully registered llm_observability_otel tool.');
|
||||
}
|
||||
|
||||
export default { registerTools };
|
||||
@@ -1,23 +1,25 @@
|
||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { Logger } from '../utils/logger.util.js';
|
||||
import { formatErrorForMcpTool } from '../utils/error.util.js';
|
||||
import posthogLlmController from '../controllers/posthog-llm.controller.js';
|
||||
import { PostHogController } from '../controllers/posthog-llm.controller.js';
|
||||
import { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
||||
import {
|
||||
GetToolInputSchema,
|
||||
GetToolInputSchemaType,
|
||||
} from './posthog-llm.types.js';
|
||||
PostHogLlmInputSchemaType,
|
||||
PostHogLlmPropertiesApiSchemaType,
|
||||
PostHogLlmPropertiesPayloadSchema,
|
||||
PostHogLlmPropertiesPayloadSchemaType,
|
||||
} from '../types/posthog-llm.types.js';
|
||||
|
||||
/**
|
||||
* @function capturePosthogLlmObservability
|
||||
* @description MCP Tool handler to capture LLM observability events in PostHog.
|
||||
* It calls the posthogLlmController to track the data and formats the response for the MCP.
|
||||
* @param {GetToolInputSchemaType} args - Arguments provided to the tool.
|
||||
* @param {PostHogLlmInputSchemaType} args - Arguments provided to the tool.
|
||||
* @returns {Promise<CallToolResult>} Formatted response for the MCP.
|
||||
* @throws {McpError} Formatted error if the controller or service layer encounters an issue.
|
||||
*/
|
||||
async function capturePosthogLlmObservability(
|
||||
args: GetToolInputSchemaType,
|
||||
args: PostHogLlmPropertiesPayloadSchemaType,
|
||||
): Promise<CallToolResult> {
|
||||
const methodLogger = Logger.forContext(
|
||||
'tools/posthog-llm.tool.ts',
|
||||
@@ -26,15 +28,16 @@ async function capturePosthogLlmObservability(
|
||||
methodLogger.debug(`Capture LLM Observability in PostHog...`, args);
|
||||
|
||||
try {
|
||||
const trackArgs = GetToolInputSchema.parse(args);
|
||||
const trackArgs = PostHogLlmPropertiesPayloadSchema.parse(args);
|
||||
|
||||
const posthogProperties: Record<string, unknown> = {
|
||||
const posthogProperties: PostHogLlmPropertiesApiSchemaType = {
|
||||
userId: trackArgs.userId,
|
||||
$ai_model: trackArgs.model,
|
||||
$ai_provider: trackArgs.provider,
|
||||
};
|
||||
|
||||
const toPostHogKey: Partial<
|
||||
Record<keyof GetToolInputSchemaType, string>
|
||||
Record<keyof PostHogLlmPropertiesPayloadSchemaType, string>
|
||||
> = {
|
||||
input: '$ai_input',
|
||||
outputChoices: '$ai_output_choices',
|
||||
@@ -47,22 +50,27 @@ async function capturePosthogLlmObservability(
|
||||
};
|
||||
|
||||
for (const key of Object.keys(toPostHogKey) as Array<
|
||||
keyof GetToolInputSchemaType
|
||||
keyof PostHogLlmPropertiesPayloadSchemaType
|
||||
>) {
|
||||
if (trackArgs[key] !== undefined) {
|
||||
const posthogKey = toPostHogKey[key];
|
||||
if (posthogKey) {
|
||||
posthogProperties[posthogKey] = trackArgs[key];
|
||||
// Type assertion to satisfy TS: posthogKey is a key of posthogProperties
|
||||
(posthogProperties as Record<string, unknown>)[posthogKey] =
|
||||
trackArgs[key];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pass validated args to the controller
|
||||
const result = await posthogLlmController.capture({
|
||||
// validated input for the controller
|
||||
const posthogInput: PostHogLlmInputSchemaType = {
|
||||
eventName: '$ai_generation',
|
||||
distinctId: trackArgs.userId,
|
||||
properties: posthogProperties,
|
||||
});
|
||||
};
|
||||
|
||||
// Pass validated args to the controller
|
||||
const result = await PostHogController.capture(posthogInput);
|
||||
methodLogger.debug(`Got the response from the controller`, result);
|
||||
|
||||
// Format the response for the MCP tool
|
||||
@@ -82,7 +90,7 @@ async function capturePosthogLlmObservability(
|
||||
|
||||
/**
|
||||
* @function registerTools
|
||||
* @description Registers the PostHog LLM observability tool ('capture_llm_observability') with the MCP server.
|
||||
* @description Registers the PostHog LLM observability tool ('llm_observability_posthog') with the MCP server.
|
||||
*
|
||||
* @param {McpServer} server - The MCP server instance.
|
||||
*/
|
||||
@@ -94,14 +102,14 @@ function registerTools(server: McpServer) {
|
||||
methodLogger.debug(`Registering PostHog LLM observability tools...`);
|
||||
|
||||
server.tool(
|
||||
'capture_llm_observability',
|
||||
'llm_observability_posthog',
|
||||
`Captures LLM usage in PostHog for observability, including requests, responses, and performance metrics`,
|
||||
GetToolInputSchema.shape,
|
||||
PostHogLlmPropertiesPayloadSchema.shape,
|
||||
capturePosthogLlmObservability,
|
||||
);
|
||||
|
||||
methodLogger.debug(
|
||||
'Successfully registered capture_llm_observability tool.',
|
||||
'Successfully registered llm_observability_posthog tool.',
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* Zod schema for the PostHog LLM observability tool arguments.
|
||||
*/
|
||||
export const GetToolInputSchema = z.object({
|
||||
userId: z.string().describe('The distinct ID of the user'),
|
||||
traceId: z.string().optional().describe('The trace ID to group AI events'),
|
||||
model: z.string().describe('The model used (e.g., gpt-4, claude-3, etc.)'),
|
||||
provider: z
|
||||
.string()
|
||||
.describe('The LLM provider (e.g., openai, anthropic, etc.)'),
|
||||
input: z
|
||||
.any()
|
||||
.optional()
|
||||
.describe('The input to the LLM (messages, prompt, etc.)'),
|
||||
outputChoices: z.any().optional().describe('The output from the LLM'),
|
||||
inputTokens: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The number of tokens in the input'),
|
||||
outputTokens: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The number of tokens in the output'),
|
||||
latency: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The latency of the LLM call in seconds'),
|
||||
httpStatus: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The HTTP status code of the LLM call'),
|
||||
baseUrl: z.string().optional().describe('The base URL of the LLM API'),
|
||||
});
|
||||
|
||||
/**
|
||||
* TypeScript type inferred from the GetToolInputSchema Zod schema.
|
||||
* This represents the optional arguments passed to the tool handler and controller.
|
||||
*/
|
||||
export type GetToolInputSchemaType = z.infer<typeof GetToolInputSchema>;
|
||||
168
src/types/opentelemetry-llm.types.ts
Normal file
168
src/types/opentelemetry-llm.types.ts
Normal file
@@ -0,0 +1,168 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* Zod schema for the OpenTelemetry LLM observability tool arguments
|
||||
*/
|
||||
export const OpenTelemetryLlmInputSchema = z.object({
|
||||
// Required fields
|
||||
userId: z.string().describe('The distinct ID of the user'),
|
||||
model: z.string().describe('The model used (e.g., gpt-4, claude-3, etc.)'),
|
||||
provider: z
|
||||
.string()
|
||||
.describe('The LLM provider (e.g., openai, anthropic, etc.)'),
|
||||
|
||||
// Optional fields
|
||||
traceId: z.string().optional().describe('The trace ID to group AI events'),
|
||||
input: z
|
||||
.any()
|
||||
.optional()
|
||||
.describe('The input to the LLM (messages, prompt, etc.)'),
|
||||
outputChoices: z.any().optional().describe('The output from the LLM'),
|
||||
inputTokens: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The number of tokens in the input'),
|
||||
outputTokens: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The number of tokens in the output'),
|
||||
latency: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The latency of the LLM call in seconds'),
|
||||
httpStatus: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The HTTP status code of the LLM call'),
|
||||
baseUrl: z.string().optional().describe('The base URL of the LLM API'),
|
||||
|
||||
// OpenTelemetry specific fields
|
||||
operationName: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe('The name of the operation being performed'),
|
||||
error: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe('Error message if the request failed'),
|
||||
errorType: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe('Type of error (e.g., rate_limit, timeout, etc.)'),
|
||||
mcpToolsUsed: z
|
||||
.array(z.string())
|
||||
.optional()
|
||||
.describe('List of MCP tools used during the request'),
|
||||
});
|
||||
|
||||
/**
|
||||
* TypeScript type inferred from the OpenTelemetryLlmInputSchema Zod schema
|
||||
*/
|
||||
export type OpenTelemetryLlmInputSchemaType = z.infer<
|
||||
typeof OpenTelemetryLlmInputSchema
|
||||
>;
|
||||
|
||||
/**
|
||||
* Configuration for OpenTelemetry exporters
|
||||
*/
|
||||
export interface OpenTelemetryExporterConfig {
|
||||
url: string;
|
||||
headers?: Record<string, string>;
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Supported OpenTelemetry backends
|
||||
*/
|
||||
export type OpenTelemetryBackend =
|
||||
| 'jaeger'
|
||||
| 'newrelic'
|
||||
| 'grafana'
|
||||
| 'datadog'
|
||||
| 'honeycomb'
|
||||
| 'lightstep'
|
||||
| 'custom';
|
||||
|
||||
/**
|
||||
* Pre-configured settings for popular OpenTelemetry backends
|
||||
*/
|
||||
export const OpenTelemetryBackendConfigs: Record<
|
||||
OpenTelemetryBackend,
|
||||
Partial<OpenTelemetryExporterConfig>
|
||||
> = {
|
||||
jaeger: {
|
||||
url: 'http://localhost:4318/v1/traces',
|
||||
headers: {},
|
||||
},
|
||||
newrelic: {
|
||||
url: 'https://otlp.nr-data.net:4318/v1/traces',
|
||||
headers: {
|
||||
'api-key': process.env.NEW_RELIC_LICENSE_KEY || '',
|
||||
},
|
||||
},
|
||||
grafana: {
|
||||
url: 'https://otlp-gateway-prod-us-central-0.grafana.net/otlp/v1/traces',
|
||||
headers: {
|
||||
Authorization:
|
||||
process.env.GRAFANA_INSTANCE_ID && process.env.GRAFANA_API_KEY
|
||||
? `Basic ${Buffer.from(
|
||||
`${process.env.GRAFANA_INSTANCE_ID}:${process.env.GRAFANA_API_KEY}`,
|
||||
).toString('base64')}`
|
||||
: '',
|
||||
},
|
||||
},
|
||||
datadog: {
|
||||
url: 'https://api.datadoghq.com/api/v2/series',
|
||||
headers: {
|
||||
'DD-API-KEY': process.env.DD_API_KEY || '',
|
||||
},
|
||||
},
|
||||
honeycomb: {
|
||||
url: 'https://api.honeycomb.io/v1/traces',
|
||||
headers: {
|
||||
'x-honeycomb-team': process.env.HONEYCOMB_API_KEY || '',
|
||||
},
|
||||
},
|
||||
lightstep: {
|
||||
url: 'https://ingest.lightstep.com:443/api/v2/otel/trace',
|
||||
headers: {
|
||||
'lightstep-access-token': process.env.LIGHTSTEP_ACCESS_TOKEN || '',
|
||||
},
|
||||
},
|
||||
custom: {
|
||||
url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318',
|
||||
headers: {},
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Metric names used by the OpenTelemetry service
|
||||
*/
|
||||
export const OpenTelemetryMetrics = {
|
||||
REQUESTS_TOTAL: 'llm.requests.total',
|
||||
TOKENS_TOTAL: 'llm.tokens.total',
|
||||
LATENCY_DURATION: 'llm.latency.duration',
|
||||
REQUESTS_ACTIVE: 'llm.requests.active',
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Trace attribute names used by the OpenTelemetry service
|
||||
*/
|
||||
export const OpenTelemetryAttributes = {
|
||||
LLM_MODEL: 'llm.model',
|
||||
LLM_PROVIDER: 'llm.provider',
|
||||
LLM_USER_ID: 'llm.user_id',
|
||||
LLM_OPERATION: 'llm.operation',
|
||||
LLM_INPUT_TOKENS: 'llm.input_tokens',
|
||||
LLM_OUTPUT_TOKENS: 'llm.output_tokens',
|
||||
LLM_TOTAL_TOKENS: 'llm.total_tokens',
|
||||
LLM_LATENCY_MS: 'llm.latency_ms',
|
||||
LLM_HTTP_STATUS: 'llm.http_status',
|
||||
LLM_BASE_URL: 'llm.base_url',
|
||||
LLM_ERROR: 'llm.error',
|
||||
LLM_ERROR_TYPE: 'llm.error_type',
|
||||
LLM_INPUT: 'llm.input',
|
||||
LLM_OUTPUT: 'llm.output',
|
||||
LLM_MCP_TOOLS_USED: 'llm.mcp_tools_used',
|
||||
TRACE_ID: 'trace.id',
|
||||
} as const;
|
||||
93
src/types/posthog-llm.types.ts
Normal file
93
src/types/posthog-llm.types.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* Zod schema for the PostHog LLM observability tool properties (payload).
|
||||
*/
|
||||
export const PostHogLlmPropertiesPayloadSchema = z.object({
|
||||
userId: z.string().describe('The distinct ID of the user'),
|
||||
traceId: z.string().optional().describe('The trace ID to group AI events'),
|
||||
model: z.string().describe('The model used (e.g., gpt-4, claude-3, etc.)'),
|
||||
provider: z
|
||||
.string()
|
||||
.describe('The LLM provider (e.g., openai, anthropic, etc.)'),
|
||||
input: z
|
||||
.any()
|
||||
.optional()
|
||||
.describe('The input to the LLM (messages, prompt, etc.)'),
|
||||
outputChoices: z.any().optional().describe('The output from the LLM'),
|
||||
inputTokens: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The number of tokens in the input'),
|
||||
outputTokens: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The number of tokens in the output'),
|
||||
latency: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The latency of the LLM call in seconds'),
|
||||
httpStatus: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The HTTP status code of the LLM call'),
|
||||
baseUrl: z.string().optional().describe('The base URL of the LLM API'),
|
||||
});
|
||||
|
||||
export type PostHogLlmPropertiesPayloadSchemaType = z.infer<
|
||||
typeof PostHogLlmPropertiesPayloadSchema
|
||||
>;
|
||||
|
||||
/**
|
||||
* Zod schema for the PostHog LLM observability tool properties (api payload).
|
||||
*/
|
||||
export const PostHogLlmPropertiesApiSchema = z.object({
|
||||
userId: z.string().describe('The distinct ID of the user'),
|
||||
$ai_trace_id: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe('The trace ID to group AI events'),
|
||||
$ai_model: z
|
||||
.string()
|
||||
.describe('The model used (e.g., gpt-4, claude-3, etc.)'),
|
||||
$ai_provider: z
|
||||
.string()
|
||||
.describe('The LLM provider (e.g., openai, anthropic, etc.)'),
|
||||
$ai_input: z
|
||||
.any()
|
||||
.optional()
|
||||
.describe('The input to the LLM (messages, prompt, etc.)'),
|
||||
$ai_output_choices: z.any().optional().describe('The output from the LLM'),
|
||||
$ai_input_tokens: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The number of tokens in the input'),
|
||||
$ai_output_tokens: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The number of tokens in the output'),
|
||||
$ai_latency: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The latency of the LLM call in seconds'),
|
||||
$ai_http_status: z
|
||||
.number()
|
||||
.optional()
|
||||
.describe('The HTTP status code of the LLM call'),
|
||||
$ai_base_url: z.string().optional().describe('The base URL of the LLM API'),
|
||||
});
|
||||
|
||||
export type PostHogLlmPropertiesApiSchemaType = z.infer<
|
||||
typeof PostHogLlmPropertiesApiSchema
|
||||
>;
|
||||
|
||||
/**
|
||||
* Zod schema for the PostHog LLM observability tool arguments (top-level).
|
||||
*/
|
||||
export const PostHogLlmInputSchema = z.object({
|
||||
eventName: z.string().describe('The name of the event to capture.'),
|
||||
distinctId: z.string().describe('The distinct ID of the user.'),
|
||||
properties: PostHogLlmPropertiesApiSchema.describe('The event properties.'),
|
||||
});
|
||||
|
||||
export type PostHogLlmInputSchemaType = z.infer<typeof PostHogLlmInputSchema>;
|
||||
@@ -1,157 +0,0 @@
|
||||
import { spawn } from 'child_process';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Utility for testing CLI commands with real execution
|
||||
*/
|
||||
export class CliTestUtil {
|
||||
/**
|
||||
* Executes a CLI command and returns the result
|
||||
*
|
||||
* @param args - CLI arguments to pass to the command
|
||||
* @param options - Test options
|
||||
* @returns Promise with stdout, stderr, and exit code
|
||||
*/
|
||||
static async runCommand(
|
||||
args: string[],
|
||||
options: {
|
||||
timeoutMs?: number;
|
||||
env?: Record<string, string>;
|
||||
} = {},
|
||||
): Promise<{
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
exitCode: number;
|
||||
}> {
|
||||
// Default timeout of 30 seconds
|
||||
const timeoutMs = options.timeoutMs || 30000;
|
||||
|
||||
// CLI execution path - points to the built CLI script
|
||||
const cliPath = join(process.cwd(), 'dist', 'index.js');
|
||||
|
||||
// Log what command we're about to run
|
||||
console.log(`Running CLI command: node ${cliPath} ${args.join(' ')}`);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
// Set up timeout handler
|
||||
const timeoutId = setTimeout(() => {
|
||||
child.kill();
|
||||
reject(new Error(`CLI command timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
|
||||
// Capture stdout and stderr
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
||||
// Spawn the process with given arguments and enhanced environment
|
||||
const child = spawn('node', [cliPath, ...args], {
|
||||
env: {
|
||||
...process.env,
|
||||
...options.env,
|
||||
DEBUG: 'true', // Enable debug logging
|
||||
NODE_ENV: 'test', // Ensure tests are detected
|
||||
},
|
||||
});
|
||||
|
||||
// Collect stdout data
|
||||
child.stdout.on('data', (data) => {
|
||||
const chunk = data.toString();
|
||||
stdout += chunk;
|
||||
console.log(`STDOUT chunk: ${chunk.substring(0, 50)}...`);
|
||||
});
|
||||
|
||||
// Collect stderr data
|
||||
child.stderr.on('data', (data) => {
|
||||
const chunk = data.toString();
|
||||
stderr += chunk;
|
||||
console.log(`STDERR chunk: ${chunk.substring(0, 50)}...`);
|
||||
});
|
||||
|
||||
// Handle process completion
|
||||
child.on('close', (exitCode) => {
|
||||
clearTimeout(timeoutId);
|
||||
console.log(`Command completed with exit code: ${exitCode}`);
|
||||
console.log(`Total STDOUT length: ${stdout.length} chars`);
|
||||
|
||||
// Get the non-debug output for debugging purposes
|
||||
const nonDebugOutput = stdout
|
||||
.split('\n')
|
||||
.filter((line) => !line.match(/^\[\d{2}:\d{2}:\d{2}\]/))
|
||||
.join('\n');
|
||||
|
||||
console.log(
|
||||
`Non-debug output length: ${nonDebugOutput.length} chars`,
|
||||
);
|
||||
console.log(`STDOUT excerpt: ${stdout.substring(0, 100)}...`);
|
||||
console.log(
|
||||
`Filtered excerpt: ${nonDebugOutput.substring(0, 100)}...`,
|
||||
);
|
||||
|
||||
resolve({
|
||||
stdout,
|
||||
stderr,
|
||||
exitCode: exitCode ?? 0,
|
||||
});
|
||||
});
|
||||
|
||||
// Handle process errors
|
||||
child.on('error', (err) => {
|
||||
clearTimeout(timeoutId);
|
||||
console.error(`Command error: ${err.message}`);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that stdout contains expected strings/patterns
|
||||
*/
|
||||
static validateOutputContains(
|
||||
output: string,
|
||||
expectedPatterns: (string | RegExp)[],
|
||||
): void {
|
||||
// Filter out debug log lines for cleaner validation
|
||||
const cleanOutput = output
|
||||
.split('\n')
|
||||
.filter((line) => !line.match(/^\[\d{2}:\d{2}:\d{2}\]/))
|
||||
.join('\n');
|
||||
|
||||
console.log('==== Cleaned output for validation ====');
|
||||
console.log(cleanOutput);
|
||||
console.log('=======================================');
|
||||
|
||||
for (const pattern of expectedPatterns) {
|
||||
if (typeof pattern === 'string') {
|
||||
expect(cleanOutput).toContain(pattern);
|
||||
} else {
|
||||
expect(cleanOutput).toMatch(pattern);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates Markdown output format
|
||||
*/
|
||||
static validateMarkdownOutput(output: string): void {
|
||||
// Filter out debug log lines for cleaner validation
|
||||
const cleanOutput = output
|
||||
.split('\n')
|
||||
.filter((line) => !line.match(/^\[\d{2}:\d{2}:\d{2}\]/))
|
||||
.join('\n');
|
||||
|
||||
// Check for Markdown heading
|
||||
expect(cleanOutput).toMatch(/^#\s.+/m);
|
||||
|
||||
// Check for markdown formatting elements like bold text, lists, etc.
|
||||
const markdownElements = [
|
||||
/\*\*.+\*\*/, // Bold text
|
||||
/-\s.+/, // List items
|
||||
/\|.+\|.+\|/, // Table rows
|
||||
/\[.+\]\(.+\)/, // Links
|
||||
];
|
||||
|
||||
expect(
|
||||
markdownElements.some((pattern) => pattern.test(cleanOutput)),
|
||||
).toBe(true);
|
||||
}
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
import {
|
||||
ErrorType,
|
||||
McpError,
|
||||
createApiError,
|
||||
createAuthMissingError,
|
||||
createAuthInvalidError,
|
||||
createUnexpectedError,
|
||||
ensureMcpError,
|
||||
formatErrorForMcpTool,
|
||||
formatErrorForMcpResource,
|
||||
} from './error.util.js';
|
||||
|
||||
describe('Error Utility', () => {
|
||||
describe('McpError', () => {
|
||||
it('should create an error with the correct properties', () => {
|
||||
const error = new McpError('Test error', ErrorType.API_ERROR, 404);
|
||||
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect(error).toBeInstanceOf(McpError);
|
||||
expect(error.message).toBe('Test error');
|
||||
expect(error.type).toBe(ErrorType.API_ERROR);
|
||||
expect(error.statusCode).toBe(404);
|
||||
expect(error.name).toBe('McpError');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Factory Functions', () => {
|
||||
it('should create auth missing error', () => {
|
||||
const error = createAuthMissingError();
|
||||
|
||||
expect(error).toBeInstanceOf(McpError);
|
||||
expect(error.type).toBe(ErrorType.AUTH_MISSING);
|
||||
expect(error.message).toBe(
|
||||
'Authentication credentials are missing',
|
||||
);
|
||||
});
|
||||
|
||||
it('should create auth invalid error', () => {
|
||||
const error = createAuthInvalidError('Invalid token');
|
||||
|
||||
expect(error).toBeInstanceOf(McpError);
|
||||
expect(error.type).toBe(ErrorType.AUTH_INVALID);
|
||||
expect(error.statusCode).toBe(401);
|
||||
expect(error.message).toBe('Invalid token');
|
||||
});
|
||||
|
||||
it('should create API error', () => {
|
||||
const originalError = new Error('Original error');
|
||||
const error = createApiError('API failed', 500, originalError);
|
||||
|
||||
expect(error).toBeInstanceOf(McpError);
|
||||
expect(error.type).toBe(ErrorType.API_ERROR);
|
||||
expect(error.statusCode).toBe(500);
|
||||
expect(error.message).toBe('API failed');
|
||||
expect(error.originalError).toBe(originalError);
|
||||
});
|
||||
|
||||
it('should create unexpected error', () => {
|
||||
const error = createUnexpectedError();
|
||||
|
||||
expect(error).toBeInstanceOf(McpError);
|
||||
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
||||
expect(error.message).toBe('An unexpected error occurred');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ensureMcpError', () => {
|
||||
it('should return the same error if it is already an McpError', () => {
|
||||
const originalError = createApiError('Original error');
|
||||
const error = ensureMcpError(originalError);
|
||||
|
||||
expect(error).toBe(originalError);
|
||||
});
|
||||
|
||||
it('should wrap a standard Error', () => {
|
||||
const originalError = new Error('Standard error');
|
||||
const error = ensureMcpError(originalError);
|
||||
|
||||
expect(error).toBeInstanceOf(McpError);
|
||||
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
||||
expect(error.message).toBe('Standard error');
|
||||
expect(error.originalError).toBe(originalError);
|
||||
});
|
||||
|
||||
it('should handle non-Error objects', () => {
|
||||
const error = ensureMcpError('String error');
|
||||
|
||||
expect(error).toBeInstanceOf(McpError);
|
||||
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
||||
expect(error.message).toBe('String error');
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatErrorForMcpTool', () => {
|
||||
it('should format an error for MCP tool response', () => {
|
||||
const error = createApiError('API error');
|
||||
const response = formatErrorForMcpTool(error);
|
||||
|
||||
expect(response).toHaveProperty('content');
|
||||
expect(response.content).toHaveLength(1);
|
||||
expect(response.content[0]).toHaveProperty('type', 'text');
|
||||
expect(response.content[0]).toHaveProperty(
|
||||
'text',
|
||||
'Error: API error',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatErrorForMcpResource', () => {
|
||||
it('should format an error for MCP resource response', () => {
|
||||
const error = createApiError('API error');
|
||||
const response = formatErrorForMcpResource(error, 'test://uri');
|
||||
|
||||
expect(response).toHaveProperty('contents');
|
||||
expect(response.contents).toHaveLength(1);
|
||||
expect(response.contents[0]).toHaveProperty('uri', 'test://uri');
|
||||
expect(response.contents[0]).toHaveProperty(
|
||||
'text',
|
||||
'Error: API error',
|
||||
);
|
||||
expect(response.contents[0]).toHaveProperty(
|
||||
'mimeType',
|
||||
'text/plain',
|
||||
);
|
||||
expect(response.contents[0]).toHaveProperty(
|
||||
'description',
|
||||
'Error: API_ERROR',
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,173 +0,0 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { Logger } from './logger.util.js';
|
||||
import dotenv from 'dotenv';
|
||||
import os from 'os';
|
||||
|
||||
/**
|
||||
* Configuration loader that handles multiple sources with priority:
|
||||
* 1. Direct ENV pass (process.env)
|
||||
* 2. .env file in project root
|
||||
* 3. Global config file at $HOME/.mcp/configs.json
|
||||
*/
|
||||
class ConfigLoader {
|
||||
private packageName: string;
|
||||
private configLoaded: boolean = false;
|
||||
|
||||
/**
|
||||
* Create a new ConfigLoader instance
|
||||
* @param packageName The package name to use for global config lookup
|
||||
*/
|
||||
constructor(packageName: string) {
|
||||
this.packageName = packageName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load configuration from all sources with proper priority
|
||||
*/
|
||||
load(): void {
|
||||
const methodLogger = Logger.forContext('utils/config.util.ts', 'load');
|
||||
|
||||
if (this.configLoaded) {
|
||||
methodLogger.debug('Configuration already loaded, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
methodLogger.debug('Loading configuration...');
|
||||
|
||||
// Priority 3: Load from global config file
|
||||
this.loadFromGlobalConfig();
|
||||
|
||||
// Priority 2: Load from .env file
|
||||
this.loadFromEnvFile();
|
||||
|
||||
// Priority 1: Direct ENV pass is already in process.env
|
||||
// No need to do anything as it already has highest priority
|
||||
|
||||
this.configLoaded = true;
|
||||
methodLogger.debug('Configuration loaded successfully');
|
||||
}
|
||||
|
||||
/**
|
||||
* Load configuration from .env file in project root
|
||||
*/
|
||||
private loadFromEnvFile(): void {
|
||||
const methodLogger = Logger.forContext(
|
||||
'utils/config.util.ts',
|
||||
'loadFromEnvFile',
|
||||
);
|
||||
|
||||
try {
|
||||
const result = dotenv.config();
|
||||
if (result.error) {
|
||||
methodLogger.debug('No .env file found or error reading it');
|
||||
return;
|
||||
}
|
||||
methodLogger.debug('Loaded configuration from .env file');
|
||||
} catch (error) {
|
||||
methodLogger.error('Error loading .env file', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load configuration from global config file at $HOME/.mcp/configs.json
|
||||
*/
|
||||
private loadFromGlobalConfig(): void {
|
||||
const methodLogger = Logger.forContext(
|
||||
'utils/config.util.ts',
|
||||
'loadFromGlobalConfig',
|
||||
);
|
||||
|
||||
try {
|
||||
const homedir = os.homedir();
|
||||
const globalConfigPath = path.join(homedir, '.mcp', 'configs.json');
|
||||
|
||||
if (!fs.existsSync(globalConfigPath)) {
|
||||
methodLogger.debug('Global config file not found');
|
||||
return;
|
||||
}
|
||||
|
||||
const configContent = fs.readFileSync(globalConfigPath, 'utf8');
|
||||
const config = JSON.parse(configContent);
|
||||
|
||||
// Determine the potential keys for the current package
|
||||
const shortKey = 'llm-observability-mcp'; // Project-specific short key
|
||||
const fullPackageName = this.packageName; // e.g., '@sfiorini/llm-observability-mcp'
|
||||
const unscopedPackageName =
|
||||
fullPackageName.split('/')[1] || fullPackageName; // e.g., 'llm-observability-mcp'
|
||||
|
||||
const potentialKeys = [
|
||||
shortKey,
|
||||
fullPackageName,
|
||||
unscopedPackageName,
|
||||
];
|
||||
let foundConfigSection: {
|
||||
environments?: Record<string, unknown>;
|
||||
} | null = null;
|
||||
let usedKey: string | null = null;
|
||||
|
||||
for (const key of potentialKeys) {
|
||||
if (
|
||||
config[key] &&
|
||||
typeof config[key] === 'object' &&
|
||||
config[key].environments
|
||||
) {
|
||||
foundConfigSection = config[key];
|
||||
usedKey = key;
|
||||
methodLogger.debug(`Found configuration using key: ${key}`);
|
||||
break; // Stop once found
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundConfigSection || !foundConfigSection.environments) {
|
||||
methodLogger.debug(
|
||||
`No configuration found for ${
|
||||
this.packageName
|
||||
} using keys: ${potentialKeys.join(', ')}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const environments = foundConfigSection.environments;
|
||||
for (const [key, value] of Object.entries(environments)) {
|
||||
// Only set if not already defined in process.env
|
||||
if (process.env[key] === undefined) {
|
||||
process.env[key] = String(value);
|
||||
}
|
||||
}
|
||||
|
||||
methodLogger.debug(
|
||||
`Loaded configuration from global config file using key: ${usedKey}`,
|
||||
);
|
||||
} catch (error) {
|
||||
methodLogger.error('Error loading global config file', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a configuration value
|
||||
* @param key The configuration key
|
||||
* @param defaultValue The default value if the key is not found
|
||||
* @returns The configuration value or the default value
|
||||
*/
|
||||
get(key: string, defaultValue?: string): string | undefined {
|
||||
return process.env[key] || defaultValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a boolean configuration value
|
||||
* @param key The configuration key
|
||||
* @param defaultValue The default value if the key is not found
|
||||
* @returns The boolean configuration value or the default value
|
||||
*/
|
||||
getBoolean(key: string, defaultValue: boolean = false): boolean {
|
||||
const value = this.get(key);
|
||||
if (value === undefined) {
|
||||
return defaultValue;
|
||||
}
|
||||
return value.toLowerCase() === 'true';
|
||||
}
|
||||
}
|
||||
|
||||
// Create and export a singleton instance with the package name from package.json
|
||||
export const config = new ConfigLoader('@sfiorini/llm-observability-mcp');
|
||||
@@ -1,128 +0,0 @@
|
||||
import {
|
||||
ErrorCode,
|
||||
detectErrorType,
|
||||
buildErrorContext,
|
||||
} from './error-handler.util.js';
|
||||
import { createApiError, McpError, ErrorType } from './error.util.js';
|
||||
|
||||
describe('error-handler.util', () => {
|
||||
describe('detectErrorType', () => {
|
||||
it('should detect network errors', () => {
|
||||
const networkErrors = [
|
||||
'network error occurred',
|
||||
'fetch failed with error',
|
||||
'ECONNREFUSED on 127.0.0.1:8080',
|
||||
'ENOTFOUND api.example.com',
|
||||
'Failed to fetch data from server',
|
||||
'Network request failed',
|
||||
];
|
||||
|
||||
networkErrors.forEach((msg) => {
|
||||
const { code, statusCode } = detectErrorType(new Error(msg));
|
||||
expect(code).toBe(ErrorCode.NETWORK_ERROR);
|
||||
expect(statusCode).toBe(500);
|
||||
});
|
||||
});
|
||||
|
||||
it('should detect rate limit errors', () => {
|
||||
const rateLimitErrors = [
|
||||
'rate limit exceeded',
|
||||
'too many requests',
|
||||
new McpError('API error', ErrorType.API_ERROR, 429),
|
||||
];
|
||||
|
||||
rateLimitErrors.forEach((error) => {
|
||||
const { code, statusCode } = detectErrorType(error);
|
||||
expect(code).toBe(ErrorCode.RATE_LIMIT_ERROR);
|
||||
expect(statusCode).toBe(429);
|
||||
});
|
||||
});
|
||||
|
||||
it('should detect not found errors', () => {
|
||||
const notFoundErrors = [
|
||||
'resource not found',
|
||||
'entity does not exist',
|
||||
new McpError('Not found', ErrorType.API_ERROR, 404),
|
||||
];
|
||||
|
||||
notFoundErrors.forEach((error) => {
|
||||
const { code } = detectErrorType(error);
|
||||
expect(code).toBe(ErrorCode.NOT_FOUND);
|
||||
});
|
||||
});
|
||||
|
||||
it('should detect access denied errors', () => {
|
||||
const accessDeniedErrors = [
|
||||
'access denied',
|
||||
'permission denied',
|
||||
'not authorized to access',
|
||||
'authentication required',
|
||||
new McpError('Forbidden', ErrorType.API_ERROR, 403),
|
||||
new McpError('Unauthorized', ErrorType.API_ERROR, 401),
|
||||
];
|
||||
|
||||
accessDeniedErrors.forEach((error) => {
|
||||
const { code } = detectErrorType(error);
|
||||
expect(code).toBe(ErrorCode.ACCESS_DENIED);
|
||||
});
|
||||
});
|
||||
|
||||
it('should default to unexpected error when no patterns match', () => {
|
||||
const { code, statusCode } = detectErrorType(
|
||||
new Error('some random error'),
|
||||
);
|
||||
expect(code).toBe(ErrorCode.UNEXPECTED_ERROR);
|
||||
expect(statusCode).toBe(500);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildErrorContext', () => {
|
||||
it('should build a context object with all parameters', () => {
|
||||
const context = buildErrorContext(
|
||||
'User',
|
||||
'create',
|
||||
'controllers/user.controller.ts@create',
|
||||
'user123',
|
||||
{ requestBody: { name: 'Test User' } },
|
||||
);
|
||||
|
||||
expect(context).toEqual({
|
||||
entityType: 'User',
|
||||
operation: 'create',
|
||||
source: 'controllers/user.controller.ts@create',
|
||||
entityId: 'user123',
|
||||
additionalInfo: { requestBody: { name: 'Test User' } },
|
||||
});
|
||||
});
|
||||
|
||||
it('should build a context object with only required parameters', () => {
|
||||
const context = buildErrorContext(
|
||||
'User',
|
||||
'list',
|
||||
'controllers/user.controller.ts@list',
|
||||
);
|
||||
|
||||
expect(context).toEqual({
|
||||
entityType: 'User',
|
||||
operation: 'list',
|
||||
source: 'controllers/user.controller.ts@list',
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle object entityId', () => {
|
||||
const context = buildErrorContext(
|
||||
'Document',
|
||||
'get',
|
||||
'controllers/document.controller.ts@get',
|
||||
{ project: 'project1', id: 'doc123' },
|
||||
);
|
||||
|
||||
expect(context).toEqual({
|
||||
entityType: 'Document',
|
||||
operation: 'get',
|
||||
source: 'controllers/document.controller.ts@get',
|
||||
entityId: { project: 'project1', id: 'doc123' },
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,307 +0,0 @@
|
||||
import { createApiError } from './error.util.js';
|
||||
import { Logger } from './logger.util.js';
|
||||
|
||||
/**
|
||||
* Standard error codes for consistent handling
|
||||
*/
|
||||
export enum ErrorCode {
|
||||
NOT_FOUND = 'NOT_FOUND',
|
||||
INVALID_CURSOR = 'INVALID_CURSOR',
|
||||
ACCESS_DENIED = 'ACCESS_DENIED',
|
||||
VALIDATION_ERROR = 'VALIDATION_ERROR',
|
||||
UNEXPECTED_ERROR = 'UNEXPECTED_ERROR',
|
||||
NETWORK_ERROR = 'NETWORK_ERROR',
|
||||
RATE_LIMIT_ERROR = 'RATE_LIMIT_ERROR',
|
||||
}
|
||||
|
||||
/**
|
||||
* Context information for error handling
|
||||
*/
|
||||
export interface ErrorContext {
|
||||
/**
|
||||
* Source of the error (e.g., file path and function)
|
||||
*/
|
||||
source?: string;
|
||||
|
||||
/**
|
||||
* Type of entity being processed (e.g., 'User')
|
||||
*/
|
||||
entityType?: string;
|
||||
|
||||
/**
|
||||
* Identifier of the entity being processed
|
||||
*/
|
||||
entityId?: string | Record<string, string>;
|
||||
|
||||
/**
|
||||
* Operation being performed (e.g., 'retrieving', 'searching')
|
||||
*/
|
||||
operation?: string;
|
||||
|
||||
/**
|
||||
* Additional information for debugging
|
||||
*/
|
||||
additionalInfo?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to create a consistent error context object
|
||||
* @param entityType Type of entity being processed
|
||||
* @param operation Operation being performed
|
||||
* @param source Source of the error (typically file path and function)
|
||||
* @param entityId Optional identifier of the entity
|
||||
* @param additionalInfo Optional additional information for debugging
|
||||
* @returns A formatted ErrorContext object
|
||||
*/
|
||||
export function buildErrorContext(
|
||||
entityType: string,
|
||||
operation: string,
|
||||
source: string,
|
||||
entityId?: string | Record<string, string>,
|
||||
additionalInfo?: Record<string, unknown>,
|
||||
): ErrorContext {
|
||||
return {
|
||||
entityType,
|
||||
operation,
|
||||
source,
|
||||
...(entityId && { entityId }),
|
||||
...(additionalInfo && { additionalInfo }),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect specific error types from raw errors
|
||||
* @param error The error to analyze
|
||||
* @param context Context information for better error detection
|
||||
* @returns Object containing the error code and status code
|
||||
*/
|
||||
export function detectErrorType(
|
||||
error: unknown,
|
||||
context: ErrorContext = {},
|
||||
): { code: ErrorCode; statusCode: number } {
|
||||
const methodLogger = Logger.forContext(
|
||||
'utils/error-handler.util.ts',
|
||||
'detectErrorType',
|
||||
);
|
||||
methodLogger.debug(`Detecting error type`, { error, context });
|
||||
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
const statusCode =
|
||||
error instanceof Error && 'statusCode' in error
|
||||
? (error as { statusCode: number }).statusCode
|
||||
: undefined;
|
||||
|
||||
// Network error detection
|
||||
if (
|
||||
errorMessage.includes('network error') ||
|
||||
errorMessage.includes('fetch failed') ||
|
||||
errorMessage.includes('ECONNREFUSED') ||
|
||||
errorMessage.includes('ENOTFOUND') ||
|
||||
errorMessage.includes('Failed to fetch') ||
|
||||
errorMessage.includes('Network request failed')
|
||||
) {
|
||||
return { code: ErrorCode.NETWORK_ERROR, statusCode: 500 };
|
||||
}
|
||||
|
||||
// Rate limiting detection
|
||||
if (
|
||||
errorMessage.includes('rate limit') ||
|
||||
errorMessage.includes('too many requests') ||
|
||||
statusCode === 429
|
||||
) {
|
||||
return { code: ErrorCode.RATE_LIMIT_ERROR, statusCode: 429 };
|
||||
}
|
||||
|
||||
// Not Found detection
|
||||
if (
|
||||
errorMessage.includes('not found') ||
|
||||
errorMessage.includes('does not exist') ||
|
||||
statusCode === 404
|
||||
) {
|
||||
return { code: ErrorCode.NOT_FOUND, statusCode: 404 };
|
||||
}
|
||||
|
||||
// Access Denied detection
|
||||
if (
|
||||
errorMessage.includes('access') ||
|
||||
errorMessage.includes('permission') ||
|
||||
errorMessage.includes('authorize') ||
|
||||
errorMessage.includes('authentication') ||
|
||||
statusCode === 401 ||
|
||||
statusCode === 403
|
||||
) {
|
||||
return { code: ErrorCode.ACCESS_DENIED, statusCode: statusCode || 403 };
|
||||
}
|
||||
|
||||
// Invalid Cursor detection
|
||||
if (
|
||||
(errorMessage.includes('cursor') ||
|
||||
errorMessage.includes('startAt') ||
|
||||
errorMessage.includes('page')) &&
|
||||
(errorMessage.includes('invalid') || errorMessage.includes('not valid'))
|
||||
) {
|
||||
return { code: ErrorCode.INVALID_CURSOR, statusCode: 400 };
|
||||
}
|
||||
|
||||
// Validation Error detection
|
||||
if (
|
||||
errorMessage.includes('validation') ||
|
||||
errorMessage.includes('invalid') ||
|
||||
errorMessage.includes('required') ||
|
||||
statusCode === 400 ||
|
||||
statusCode === 422
|
||||
) {
|
||||
return {
|
||||
code: ErrorCode.VALIDATION_ERROR,
|
||||
statusCode: statusCode || 400,
|
||||
};
|
||||
}
|
||||
|
||||
// Default to unexpected error
|
||||
return {
|
||||
code: ErrorCode.UNEXPECTED_ERROR,
|
||||
statusCode: statusCode || 500,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create user-friendly error messages based on error type and context
|
||||
* @param code The error code
|
||||
* @param context Context information for better error messages
|
||||
* @param originalMessage The original error message
|
||||
* @returns User-friendly error message
|
||||
*/
|
||||
export function createUserFriendlyErrorMessage(
|
||||
code: ErrorCode,
|
||||
context: ErrorContext = {},
|
||||
originalMessage?: string,
|
||||
): string {
|
||||
const methodLogger = Logger.forContext(
|
||||
'utils/error-handler.util.ts',
|
||||
'createUserFriendlyErrorMessage',
|
||||
);
|
||||
const { entityType, entityId, operation } = context;
|
||||
|
||||
// Format entity ID for display
|
||||
let entityIdStr = '';
|
||||
if (entityId) {
|
||||
if (typeof entityId === 'string') {
|
||||
entityIdStr = entityId;
|
||||
} else {
|
||||
// Handle object entityId
|
||||
entityIdStr = Object.values(entityId).join('/');
|
||||
}
|
||||
}
|
||||
|
||||
// Determine entity display name
|
||||
const entity = entityType
|
||||
? `${entityType}${entityIdStr ? ` ${entityIdStr}` : ''}`
|
||||
: 'Resource';
|
||||
|
||||
let message = '';
|
||||
|
||||
switch (code) {
|
||||
case ErrorCode.NOT_FOUND:
|
||||
message = `${entity} not found${entityIdStr ? `: ${entityIdStr}` : ''}. Verify the ID is correct and that you have access to this ${entityType?.toLowerCase() || 'resource'}.`;
|
||||
break;
|
||||
|
||||
case ErrorCode.ACCESS_DENIED:
|
||||
message = `Access denied for ${entity.toLowerCase()}${entityIdStr ? ` ${entityIdStr}` : ''}. Verify your credentials and permissions.`;
|
||||
break;
|
||||
|
||||
case ErrorCode.INVALID_CURSOR:
|
||||
message = `Invalid pagination cursor. Use the exact cursor string returned from previous results.`;
|
||||
break;
|
||||
|
||||
case ErrorCode.VALIDATION_ERROR:
|
||||
message =
|
||||
originalMessage ||
|
||||
`Invalid data provided for ${operation || 'operation'} ${entity.toLowerCase()}.`;
|
||||
break;
|
||||
|
||||
case ErrorCode.NETWORK_ERROR:
|
||||
message = `Network error while ${operation || 'connecting to'} the service. Please check your internet connection and try again.`;
|
||||
break;
|
||||
|
||||
case ErrorCode.RATE_LIMIT_ERROR:
|
||||
message = `Rate limit exceeded. Please wait a moment and try again, or reduce the frequency of requests.`;
|
||||
break;
|
||||
|
||||
default:
|
||||
message = `An unexpected error occurred while ${operation || 'processing'} ${entity.toLowerCase()}.`;
|
||||
}
|
||||
|
||||
// Include original message details if available and appropriate
|
||||
if (
|
||||
originalMessage &&
|
||||
code !== ErrorCode.NOT_FOUND &&
|
||||
code !== ErrorCode.ACCESS_DENIED
|
||||
) {
|
||||
message += ` Error details: ${originalMessage}`;
|
||||
}
|
||||
|
||||
methodLogger.debug(`Created user-friendly message: ${message}`, {
|
||||
code,
|
||||
context,
|
||||
});
|
||||
return message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle controller errors consistently
|
||||
* @param error The error to handle
|
||||
* @param context Context information for better error messages
|
||||
* @returns Never returns, always throws an error
|
||||
*/
|
||||
export function handleControllerError(
|
||||
error: unknown,
|
||||
context: ErrorContext = {},
|
||||
): never {
|
||||
const methodLogger = Logger.forContext(
|
||||
'utils/error-handler.util.ts',
|
||||
'handleControllerError',
|
||||
);
|
||||
|
||||
// Extract error details
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
const statusCode =
|
||||
error instanceof Error && 'statusCode' in error
|
||||
? (error as { statusCode: number }).statusCode
|
||||
: undefined;
|
||||
|
||||
// Detect error type using utility
|
||||
const { code, statusCode: detectedStatus } = detectErrorType(
|
||||
error,
|
||||
context,
|
||||
);
|
||||
|
||||
// Combine detected status with explicit status
|
||||
const finalStatusCode = statusCode || detectedStatus;
|
||||
|
||||
// Format entity information for logging
|
||||
const { entityType, entityId, operation } = context;
|
||||
const entity = entityType || 'resource';
|
||||
const entityIdStr = entityId
|
||||
? typeof entityId === 'string'
|
||||
? entityId
|
||||
: JSON.stringify(entityId)
|
||||
: '';
|
||||
const actionStr = operation || 'processing';
|
||||
|
||||
// Log detailed error information
|
||||
methodLogger.error(
|
||||
`Error ${actionStr} ${entity}${
|
||||
entityIdStr ? `: ${entityIdStr}` : ''
|
||||
}: ${errorMessage}`,
|
||||
error,
|
||||
);
|
||||
|
||||
// Create user-friendly error message for the response
|
||||
const message =
|
||||
code === ErrorCode.VALIDATION_ERROR
|
||||
? errorMessage
|
||||
: createUserFriendlyErrorMessage(code, context, errorMessage);
|
||||
|
||||
// Throw an appropriate API error with the user-friendly message
|
||||
throw createApiError(message, finalStatusCode, error);
|
||||
}
|
||||
@@ -1,29 +1,35 @@
|
||||
import {
|
||||
McpError,
|
||||
ErrorType,
|
||||
createApiError,
|
||||
createUnexpectedError,
|
||||
getDeepOriginalError,
|
||||
formatErrorForMcpTool,
|
||||
createUnexpectedError,
|
||||
ensureMcpError,
|
||||
} from './error.util.js';
|
||||
|
||||
describe('error.util', () => {
|
||||
describe('getDeepOriginalError', () => {
|
||||
it('should return the deepest original error in a chain', () => {
|
||||
// Create a nested chain of errors
|
||||
const deepestError = new Error('Root cause');
|
||||
const middleError = createApiError(
|
||||
'Middle error',
|
||||
500,
|
||||
deepestError,
|
||||
);
|
||||
const topError = createUnexpectedError('Top error', middleError);
|
||||
describe('McpError', () => {
|
||||
it('should create an error with the correct properties', () => {
|
||||
const error = new McpError('Test error', ErrorType.API_ERROR, 404);
|
||||
|
||||
// Should extract the deepest error
|
||||
const result = getDeepOriginalError(topError);
|
||||
expect(result).toBe(deepestError);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect(error).toBeInstanceOf(McpError);
|
||||
expect(error.message).toBe('Test error');
|
||||
expect(error.type).toBe(ErrorType.API_ERROR);
|
||||
expect(error.statusCode).toBe(404);
|
||||
expect(error.name).toBe('McpError');
|
||||
});
|
||||
});
|
||||
describe('createUnexpectedError', () => {
|
||||
it('should create unexpected error', () => {
|
||||
const error = createUnexpectedError();
|
||||
|
||||
expect(error).toBeInstanceOf(McpError);
|
||||
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
||||
expect(error.message).toBe('An unexpected error occurred');
|
||||
});
|
||||
});
|
||||
describe('getDeepOriginalError', () => {
|
||||
it('should handle null/undefined input', () => {
|
||||
expect(getDeepOriginalError(null)).toBeNull();
|
||||
expect(getDeepOriginalError(undefined)).toBeUndefined();
|
||||
@@ -57,30 +63,37 @@ describe('error.util', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatErrorForMcpTool', () => {
|
||||
it('should format McpError with metadata', () => {
|
||||
const error = createApiError('Test error', 404, {
|
||||
detail: 'Not found',
|
||||
});
|
||||
const result = formatErrorForMcpTool(error);
|
||||
describe('ensureMcpError', () => {
|
||||
it('should wrap a standard Error', () => {
|
||||
const originalError = new Error('Standard error');
|
||||
const error = ensureMcpError(originalError);
|
||||
|
||||
// Check the content
|
||||
expect(result.content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text: 'Error: Test error',
|
||||
},
|
||||
]);
|
||||
|
||||
// Check the metadata
|
||||
expect(result.metadata).toBeDefined();
|
||||
expect(result.metadata?.errorType).toBe(ErrorType.API_ERROR);
|
||||
expect(result.metadata?.statusCode).toBe(404);
|
||||
expect(result.metadata?.errorDetails).toEqual({
|
||||
detail: 'Not found',
|
||||
});
|
||||
expect(error).toBeInstanceOf(McpError);
|
||||
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
||||
expect(error.message).toBe('Standard error');
|
||||
expect(error.originalError).toBe(originalError);
|
||||
});
|
||||
|
||||
it('should handle non-Error objects', () => {
|
||||
const error = ensureMcpError('String error');
|
||||
|
||||
expect(error).toBeInstanceOf(McpError);
|
||||
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
||||
expect(error.message).toBe('String error');
|
||||
});
|
||||
|
||||
it('should return the same McpError instance if passed in', () => {
|
||||
const mcpError = new McpError(
|
||||
'Already MCP',
|
||||
ErrorType.API_ERROR,
|
||||
400,
|
||||
);
|
||||
const result = ensureMcpError(mcpError);
|
||||
expect(result).toBe(mcpError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatErrorForMcpTool', () => {
|
||||
it('should wrap non-McpError with metadata', () => {
|
||||
const error = new Error('Regular error');
|
||||
const result = formatErrorForMcpTool(error);
|
||||
@@ -101,15 +114,5 @@ describe('error.util', () => {
|
||||
expect(result.content[0].text).toBe('Error: String error');
|
||||
expect(result.metadata?.errorType).toBe(ErrorType.UNEXPECTED_ERROR);
|
||||
});
|
||||
|
||||
it('should extract deep original error details', () => {
|
||||
const deepError = { code: 'DEEP_ERROR', message: 'Deep cause' };
|
||||
const middleError = createApiError('Middle layer', 500, deepError);
|
||||
const topError = createUnexpectedError('Top error', middleError);
|
||||
|
||||
const result = formatErrorForMcpTool(topError);
|
||||
|
||||
expect(result.metadata?.errorDetails).toEqual(deepError);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -32,40 +32,6 @@ export class McpError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an authentication missing error
|
||||
*/
|
||||
export function createAuthMissingError(
|
||||
message: string = 'Authentication credentials are missing',
|
||||
): McpError {
|
||||
return new McpError(message, ErrorType.AUTH_MISSING);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an authentication invalid error
|
||||
*/
|
||||
export function createAuthInvalidError(
|
||||
message: string = 'Authentication credentials are invalid',
|
||||
): McpError {
|
||||
return new McpError(message, ErrorType.AUTH_INVALID, 401);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an API error
|
||||
*/
|
||||
export function createApiError(
|
||||
message: string,
|
||||
statusCode?: number,
|
||||
originalError?: unknown,
|
||||
): McpError {
|
||||
return new McpError(
|
||||
message,
|
||||
ErrorType.API_ERROR,
|
||||
statusCode,
|
||||
originalError,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an unexpected error
|
||||
*/
|
||||
@@ -164,77 +130,3 @@ export function formatErrorForMcpTool(error: unknown): {
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format error for MCP resource response
|
||||
*/
|
||||
export function formatErrorForMcpResource(
|
||||
error: unknown,
|
||||
uri: string,
|
||||
): {
|
||||
contents: Array<{
|
||||
uri: string;
|
||||
text: string;
|
||||
mimeType: string;
|
||||
description?: string;
|
||||
}>;
|
||||
} {
|
||||
const methodLogger = Logger.forContext(
|
||||
'utils/error.util.ts',
|
||||
'formatErrorForMcpResource',
|
||||
);
|
||||
const mcpError = ensureMcpError(error);
|
||||
methodLogger.error(`${mcpError.type} error`, mcpError);
|
||||
|
||||
return {
|
||||
contents: [
|
||||
{
|
||||
uri,
|
||||
text: `Error: ${mcpError.message}`,
|
||||
mimeType: 'text/plain',
|
||||
description: `Error: ${mcpError.type}`,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle error in CLI context with improved user feedback
|
||||
*/
|
||||
export function handleCliError(error: unknown): never {
|
||||
const methodLogger = Logger.forContext(
|
||||
'utils/error.util.ts',
|
||||
'handleCliError',
|
||||
);
|
||||
const mcpError = ensureMcpError(error);
|
||||
methodLogger.error(`${mcpError.type} error`, mcpError);
|
||||
|
||||
// Print the error message
|
||||
console.error(`Error: ${mcpError.message}`);
|
||||
|
||||
// Provide helpful context based on error type
|
||||
if (mcpError.type === ErrorType.AUTH_MISSING) {
|
||||
console.error(
|
||||
'\nTip: Make sure to set up your API token in the configuration file or environment variables.',
|
||||
);
|
||||
} else if (mcpError.type === ErrorType.AUTH_INVALID) {
|
||||
console.error(
|
||||
'\nTip: Check that your API token is correct and has not expired.',
|
||||
);
|
||||
} else if (mcpError.type === ErrorType.API_ERROR) {
|
||||
if (mcpError.statusCode === 429) {
|
||||
console.error(
|
||||
'\nTip: You may have exceeded your API rate limits. Try again later or upgrade your API plan.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Display DEBUG tip
|
||||
if (process.env.DEBUG !== 'mcp:*') {
|
||||
console.error(
|
||||
'\nFor more detailed error information, run with DEBUG=mcp:* environment variable.',
|
||||
);
|
||||
}
|
||||
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
/**
|
||||
* Standardized formatting utilities for consistent output across all CLI and Tool interfaces.
|
||||
* These functions should be used by all formatters to ensure consistent formatting.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Format a date in a standardized way: YYYY-MM-DD HH:MM:SS UTC
|
||||
* @param dateString - ISO date string or Date object
|
||||
* @returns Formatted date string
|
||||
*/
|
||||
export function formatDate(dateString?: string | Date): string {
|
||||
if (!dateString) {
|
||||
return 'Not available';
|
||||
}
|
||||
|
||||
try {
|
||||
const date =
|
||||
typeof dateString === 'string' ? new Date(dateString) : dateString;
|
||||
|
||||
// Format: YYYY-MM-DD HH:MM:SS UTC
|
||||
return date
|
||||
.toISOString()
|
||||
.replace('T', ' ')
|
||||
.replace(/\.\d+Z$/, ' UTC');
|
||||
} catch {
|
||||
return 'Invalid date';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a URL as a markdown link
|
||||
* @param url - URL to format
|
||||
* @param title - Link title
|
||||
* @returns Formatted markdown link
|
||||
*/
|
||||
export function formatUrl(url?: string, title?: string): string {
|
||||
if (!url) {
|
||||
return 'Not available';
|
||||
}
|
||||
|
||||
const linkTitle = title || url;
|
||||
return `[${linkTitle}](${url})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a heading with consistent style
|
||||
* @param text - Heading text
|
||||
* @param level - Heading level (1-6)
|
||||
* @returns Formatted heading
|
||||
*/
|
||||
export function formatHeading(text: string, level: number = 1): string {
|
||||
const validLevel = Math.min(Math.max(level, 1), 6);
|
||||
const prefix = '#'.repeat(validLevel);
|
||||
return `${prefix} ${text}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a list of key-value pairs as a bullet list
|
||||
* @param items - Object with key-value pairs
|
||||
* @param keyFormatter - Optional function to format keys
|
||||
* @returns Formatted bullet list
|
||||
*/
|
||||
export function formatBulletList(
|
||||
items: Record<string, unknown>,
|
||||
keyFormatter?: (key: string) => string,
|
||||
): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
for (const [key, value] of Object.entries(items)) {
|
||||
if (value === undefined || value === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const formattedKey = keyFormatter ? keyFormatter(key) : key;
|
||||
const formattedValue = formatValue(value);
|
||||
lines.push(`- **${formattedKey}**: ${formattedValue}`);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a value based on its type
|
||||
* @param value - Value to format
|
||||
* @returns Formatted value
|
||||
*/
|
||||
function formatValue(value: unknown): string {
|
||||
if (value === undefined || value === null) {
|
||||
return 'Not available';
|
||||
}
|
||||
|
||||
if (value instanceof Date) {
|
||||
return formatDate(value);
|
||||
}
|
||||
|
||||
// Handle URL objects with url and title properties
|
||||
if (typeof value === 'object' && value !== null && 'url' in value) {
|
||||
const urlObj = value as { url: string; title?: string };
|
||||
if (typeof urlObj.url === 'string') {
|
||||
return formatUrl(urlObj.url, urlObj.title);
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof value === 'string') {
|
||||
// Check if it's a URL
|
||||
if (value.startsWith('http://') || value.startsWith('https://')) {
|
||||
return formatUrl(value);
|
||||
}
|
||||
|
||||
// Check if it might be a date
|
||||
if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/.test(value)) {
|
||||
return formatDate(value);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
if (typeof value === 'boolean') {
|
||||
return value ? 'Yes' : 'No';
|
||||
}
|
||||
|
||||
return String(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a separator line
|
||||
* @returns Separator line
|
||||
*/
|
||||
export function formatSeparator(): string {
|
||||
return '---';
|
||||
}
|
||||
@@ -194,7 +194,7 @@ let isLoggerInitialized = false;
|
||||
* 5. Set DEBUG environment variable to control which modules show debug logs:
|
||||
* - DEBUG=true (enable all debug logs)
|
||||
* - DEBUG=controllers/*,services/* (enable for specific module groups)
|
||||
* - DEBUG=transport,utils/formatter* (enable specific modules, supports wildcards)
|
||||
* - DEBUG=transport,utils/error* (enable specific modules, supports wildcards)
|
||||
*/
|
||||
class Logger {
|
||||
private context?: string;
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
import { Logger } from './logger.util.js';
|
||||
import {
|
||||
createApiError,
|
||||
createAuthInvalidError,
|
||||
createUnexpectedError,
|
||||
McpError,
|
||||
} from './error.util.js';
|
||||
|
||||
// Create a contextualized logger for this file
|
||||
const transportLogger = Logger.forContext('utils/transport.util.ts');
|
||||
|
||||
// Log transport utility initialization
|
||||
transportLogger.debug('Transport utility initialized');
|
||||
|
||||
/**
|
||||
* Interface for HTTP request options
|
||||
*/
|
||||
export interface RequestOptions {
|
||||
method?: 'GET' | 'POST' | 'PUT' | 'DELETE';
|
||||
headers?: Record<string, string>;
|
||||
body?: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic and reusable function to fetch data from any API endpoint.
|
||||
* Handles standard HTTP request setup, response checking, basic error handling, and logging.
|
||||
*
|
||||
* @param url The full URL to fetch data from.
|
||||
* @param options Request options including method, headers, and body.
|
||||
* @returns The response data parsed as type T.
|
||||
* @throws {McpError} If the request fails, including network errors, non-OK HTTP status, or JSON parsing issues.
|
||||
*/
|
||||
export async function fetchApi<T>(
|
||||
url: string,
|
||||
options: RequestOptions = {},
|
||||
): Promise<T> {
|
||||
const methodLogger = Logger.forContext(
|
||||
'utils/transport.util.ts',
|
||||
'fetchApi',
|
||||
);
|
||||
|
||||
// Prepare standard request options
|
||||
const requestOptions: RequestInit = {
|
||||
method: options.method || 'GET',
|
||||
headers: {
|
||||
// Standard headers, allow overrides via options.headers
|
||||
'Content-Type': 'application/json',
|
||||
Accept: 'application/json',
|
||||
...options.headers,
|
||||
},
|
||||
body: options.body ? JSON.stringify(options.body) : undefined,
|
||||
};
|
||||
|
||||
methodLogger.debug(`Executing API call: ${requestOptions.method} ${url}`);
|
||||
const startTime = performance.now(); // Track performance
|
||||
|
||||
try {
|
||||
const response = await fetch(url, requestOptions);
|
||||
const endTime = performance.now();
|
||||
const duration = (endTime - startTime).toFixed(2);
|
||||
|
||||
methodLogger.debug(
|
||||
`API call completed in ${duration}ms with status: ${response.status} ${response.statusText}`,
|
||||
{ url, status: response.status },
|
||||
);
|
||||
|
||||
// Check if the response status is OK (2xx)
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text(); // Get error body for context
|
||||
methodLogger.error(
|
||||
`API error response (${response.status}):`,
|
||||
errorText,
|
||||
);
|
||||
|
||||
// Classify standard HTTP errors
|
||||
if (response.status === 401) {
|
||||
throw createAuthInvalidError(
|
||||
'Authentication failed. Check API token if required.',
|
||||
);
|
||||
} else if (response.status === 403) {
|
||||
throw createAuthInvalidError(
|
||||
'Permission denied for the requested resource.',
|
||||
);
|
||||
} else if (response.status === 404) {
|
||||
throw createApiError(
|
||||
'Resource not found at the specified URL.',
|
||||
response.status,
|
||||
errorText,
|
||||
);
|
||||
} else {
|
||||
// Generic API error for other non-2xx statuses
|
||||
throw createApiError(
|
||||
`API request failed with status ${response.status}: ${response.statusText}`,
|
||||
response.status,
|
||||
errorText,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to parse the response body as JSON
|
||||
try {
|
||||
const responseData = await response.json();
|
||||
methodLogger.debug('Response body successfully parsed as JSON.');
|
||||
// methodLogger.debug('Response Data:', responseData); // Uncomment for full response logging
|
||||
return responseData as T;
|
||||
} catch (parseError) {
|
||||
methodLogger.error(
|
||||
'Failed to parse API response JSON:',
|
||||
parseError,
|
||||
);
|
||||
// Throw a specific error for JSON parsing failure
|
||||
throw createApiError(
|
||||
`Failed to parse API response JSON: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
|
||||
response.status, // Include original status for context
|
||||
parseError,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
const endTime = performance.now();
|
||||
const duration = (endTime - startTime).toFixed(2);
|
||||
methodLogger.error(
|
||||
`API call failed after ${duration}ms for ${url}:`,
|
||||
error,
|
||||
);
|
||||
|
||||
// Rethrow if it's already an McpError (e.g., from status checks or parsing)
|
||||
if (error instanceof McpError) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Handle potential network errors (TypeError in fetch)
|
||||
if (error instanceof TypeError) {
|
||||
throw createApiError(
|
||||
`Network error during API call: ${error.message}`,
|
||||
undefined, // No specific HTTP status for network errors
|
||||
error,
|
||||
);
|
||||
}
|
||||
|
||||
// Wrap any other unexpected errors
|
||||
throw createUnexpectedError('Unexpected error during API call', error);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user