Merge pull request 'Capture LLM Observability to Open Telemetry compatible tools.' (#1) from opentelemetry-llm into main
Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
59
.env.example
59
.env.example
@@ -1,9 +1,52 @@
|
|||||||
# Enable debug logging
|
# PostHog Configuration (existing)
|
||||||
|
POSTHOG_API_KEY=your_posthog_api_key_here
|
||||||
|
POSTHOG_HOST=https://app.posthog.com
|
||||||
|
|
||||||
|
# OpenTelemetry Configuration
|
||||||
|
# ==========================
|
||||||
|
|
||||||
|
# Service Configuration
|
||||||
|
OTEL_SERVICE_NAME=llm-observability-mcp
|
||||||
|
OTEL_SERVICE_VERSION=1.0.0
|
||||||
|
OTEL_ENVIRONMENT=development
|
||||||
|
|
||||||
|
# OTLP Endpoints
|
||||||
|
# Uncomment and configure based on your backend
|
||||||
|
|
||||||
|
# Jaeger (local development)
|
||||||
|
# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
|
||||||
|
|
||||||
|
# New Relic
|
||||||
|
# OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.nr-data.net:4318
|
||||||
|
# OTEL_EXPORTER_OTLP_HEADERS=api-key=YOUR_NEW_RELIC_LICENSE_KEY
|
||||||
|
|
||||||
|
# Grafana Cloud
|
||||||
|
# OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-gateway-prod-us-central-0.grafana.net/otlp
|
||||||
|
# OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic YOUR_BASE64_ENCODED_CREDENTIALS
|
||||||
|
|
||||||
|
# Datadog
|
||||||
|
# OTEL_EXPORTER_OTLP_ENDPOINT=https://api.datadoghq.com/api/v2/series
|
||||||
|
# OTEL_EXPORTER_OTLP_HEADERS=DD-API-KEY=YOUR_DD_API_KEY
|
||||||
|
|
||||||
|
# Honeycomb
|
||||||
|
# OTEL_EXPORTER_OTLP_ENDPOINT=https://api.honeycomb.io/v1/traces
|
||||||
|
# OTEL_EXPORTER_OTLP_HEADERS=x-honeycomb-team=YOUR_API_KEY
|
||||||
|
|
||||||
|
# Lightstep
|
||||||
|
# OTEL_EXPORTER_OTLP_ENDPOINT=https://ingest.lightstep.com:443/api/v2/otel/trace
|
||||||
|
# OTEL_EXPORTER_OTLP_HEADERS=lightstep-access-token=YOUR_ACCESS_TOKEN
|
||||||
|
|
||||||
|
# Separate endpoints (optional)
|
||||||
|
# OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:4318/v1/metrics
|
||||||
|
# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4318/v1/traces
|
||||||
|
# OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://localhost:4318/v1/logs
|
||||||
|
|
||||||
|
# Export Configuration
|
||||||
|
OTEL_METRIC_EXPORT_INTERVAL=10000
|
||||||
|
OTEL_METRIC_EXPORT_TIMEOUT=5000
|
||||||
|
|
||||||
|
# Sampling Configuration
|
||||||
|
OTEL_TRACES_SAMPLER_ARG=1.0
|
||||||
|
|
||||||
|
# Debug Mode
|
||||||
DEBUG=false
|
DEBUG=false
|
||||||
|
|
||||||
# Transport mode
|
|
||||||
TRANSPORT_MODE=<http | stdio>
|
|
||||||
|
|
||||||
# PostHog LLM Observability configuration
|
|
||||||
POSTHOG_API_KEY=<YourPostHogAPIKeyGoesHere>
|
|
||||||
POSTHOG_HOST=https://us.i.posthog.com
|
|
||||||
|
|||||||
26
.github/dependabot.yml
vendored
26
.github/dependabot.yml
vendored
@@ -1,26 +0,0 @@
|
|||||||
version: 2
|
|
||||||
updates:
|
|
||||||
- package-ecosystem: "npm"
|
|
||||||
directory: "/"
|
|
||||||
schedule:
|
|
||||||
interval: "weekly"
|
|
||||||
open-pull-requests-limit: 10
|
|
||||||
versioning-strategy: auto
|
|
||||||
labels:
|
|
||||||
- "dependencies"
|
|
||||||
commit-message:
|
|
||||||
prefix: "chore"
|
|
||||||
include: "scope"
|
|
||||||
allow:
|
|
||||||
- dependency-type: "direct"
|
|
||||||
ignore:
|
|
||||||
- dependency-name: "*"
|
|
||||||
update-types: ["version-update:semver-patch"]
|
|
||||||
- package-ecosystem: "github-actions"
|
|
||||||
directory: "/"
|
|
||||||
schedule:
|
|
||||||
interval: "weekly"
|
|
||||||
open-pull-requests-limit: 5
|
|
||||||
labels:
|
|
||||||
- "dependencies"
|
|
||||||
- "github-actions"
|
|
||||||
45
.github/workflows/ci-dependabot-auto-merge.yml
vendored
45
.github/workflows/ci-dependabot-auto-merge.yml
vendored
@@ -1,45 +0,0 @@
|
|||||||
name: CI - Dependabot Auto-merge
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
branches: [main]
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
pull-requests: write
|
|
||||||
checks: read
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
auto-merge-dependabot:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
if: github.actor == 'dependabot[bot]'
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Setup Node.js
|
|
||||||
uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: '22'
|
|
||||||
cache: 'npm'
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
run: npm ci
|
|
||||||
|
|
||||||
- name: Run tests
|
|
||||||
run: npm test
|
|
||||||
|
|
||||||
- name: Run linting
|
|
||||||
run: npm run lint
|
|
||||||
|
|
||||||
- name: Auto-approve PR
|
|
||||||
uses: hmarr/auto-approve-action@v4
|
|
||||||
with:
|
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Enable auto-merge
|
|
||||||
if: success()
|
|
||||||
run: gh pr merge --auto --merge "$PR_URL"
|
|
||||||
env:
|
|
||||||
PR_URL: ${{ github.event.pull_request.html_url }}
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
37
.github/workflows/ci-dependency-check.yml
vendored
37
.github/workflows/ci-dependency-check.yml
vendored
@@ -1,37 +0,0 @@
|
|||||||
name: CI - Dependency Check
|
|
||||||
|
|
||||||
on:
|
|
||||||
schedule:
|
|
||||||
- cron: '0 5 * * 1' # Run at 5 AM UTC every Monday
|
|
||||||
workflow_dispatch: # Allow manual triggering
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
validate:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Setup Node.js
|
|
||||||
uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: '22'
|
|
||||||
cache: 'npm'
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
run: npm ci
|
|
||||||
|
|
||||||
- name: Run npm audit
|
|
||||||
run: npm audit
|
|
||||||
|
|
||||||
- name: Check for outdated dependencies
|
|
||||||
run: npm outdated || true
|
|
||||||
|
|
||||||
- name: Run tests
|
|
||||||
run: npm test
|
|
||||||
|
|
||||||
- name: Run linting
|
|
||||||
run: npm run lint
|
|
||||||
|
|
||||||
- name: Build project
|
|
||||||
run: npm run build
|
|
||||||
66
.github/workflows/ci-semantic-release.yml
vendored
66
.github/workflows/ci-semantic-release.yml
vendored
@@ -1,66 +0,0 @@
|
|||||||
name: CI - Semantic Release
|
|
||||||
|
|
||||||
# This workflow is triggered on every push to the main branch
|
|
||||||
# It analyzes commits and automatically releases a new version when needed
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
release:
|
|
||||||
name: Semantic Release
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
# Permissions needed for creating releases, updating issues, and publishing packages
|
|
||||||
permissions:
|
|
||||||
contents: write # Needed to create releases and tags
|
|
||||||
issues: write # Needed to comment on issues
|
|
||||||
pull-requests: write # Needed to comment on pull requests
|
|
||||||
# packages permission removed as we're not using GitHub Packages
|
|
||||||
steps:
|
|
||||||
# Step 1: Check out the full Git history for proper versioning
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
fetch-depth: 0 # Fetches all history for all branches and tags
|
|
||||||
|
|
||||||
# Step 2: Setup Node.js environment
|
|
||||||
- name: Setup Node.js
|
|
||||||
uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: 22 # Using Node.js 22
|
|
||||||
cache: 'npm' # Enable npm caching
|
|
||||||
|
|
||||||
# Step 3: Install dependencies with clean install
|
|
||||||
- name: Install dependencies
|
|
||||||
run: npm ci # Clean install preserving package-lock.json
|
|
||||||
|
|
||||||
# Step 4: Build the package
|
|
||||||
- name: Build
|
|
||||||
run: npm run build # Compiles TypeScript to JavaScript
|
|
||||||
|
|
||||||
# Step 5: Ensure executable permissions
|
|
||||||
- name: Set executable permissions
|
|
||||||
run: chmod +x dist/index.js
|
|
||||||
|
|
||||||
# Step 6: Run tests to ensure quality
|
|
||||||
- name: Verify tests
|
|
||||||
run: npm test # Runs Jest tests
|
|
||||||
|
|
||||||
# Step 7: Configure Git identity for releases
|
|
||||||
- name: Configure Git User
|
|
||||||
run: |
|
|
||||||
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
|
||||||
git config --global user.name "github-actions[bot]"
|
|
||||||
|
|
||||||
# Step 8: Run semantic-release to analyze commits and publish to npm
|
|
||||||
- name: Semantic Release
|
|
||||||
id: semantic
|
|
||||||
env:
|
|
||||||
# Tokens needed for GitHub and npm authentication
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # For creating releases and commenting
|
|
||||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }} # For publishing to npm
|
|
||||||
run: |
|
|
||||||
echo "Running semantic-release for version bump and npm publishing"
|
|
||||||
npx semantic-release
|
|
||||||
|
|
||||||
# Note: GitHub Packages publishing has been removed
|
|
||||||
182
README.md
182
README.md
@@ -1,21 +1,24 @@
|
|||||||
# LLM Observability MCP for PostHog
|
# LLM Observability MCP Server
|
||||||
|
|
||||||
[](https://opensource.org/licenses/MIT)
|
[](https://opensource.org/licenses/MIT)
|
||||||
|
|
||||||
A Model Context Protocol (MCP) server that provides a tool to capture LLM Observability events and send them to PostHog.
|
A Model Context Protocol (MCP) server that provides comprehensive LLM observability tools supporting both PostHog and OpenTelemetry backends.
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
This project is an MCP server designed to track and observe Large Language Model (LLM) interactions using [PostHog's LLM Observability](https://posthog.com/docs/llm-observability) features. It allows you to capture detailed information about LLM requests, responses, performance, and costs, providing valuable insights into your AI-powered applications.
|
This project is an MCP server designed to track and observe Large Language Model (LLM) interactions using both [PostHog's LLM Observability](https://posthog.com/docs/llm-observability) and **OpenTelemetry** for universal observability across any backend that supports OpenTelemetry (Jaeger, New Relic, Grafana, Datadog, Honeycomb, etc.).
|
||||||
|
|
||||||
The server can be run as a local process communicating over `stdio` or as a remote `http` server, making it compatible with any MCP client, such as AI-powered IDEs (e.g., VS Code with an MCP extension, Cursor) or custom applications.
|
The server can be run as a local process communicating over `stdio` or as a remote `http` server, making it compatible with any MCP client, such as AI-powered IDEs (e.g., VS Code with an MCP extension, Cursor) or custom applications.
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
- **Capture LLM Metrics**: Log key details of LLM interactions, including model, provider, latency, token counts, and more.
|
- **Dual Backend Support**: Use PostHog, OpenTelemetry, or both in parallel
|
||||||
- **Flexible Transport**: Run as a local `stdio` process for tight IDE integration or as a standalone `http` server for remote access.
|
- **Universal OpenTelemetry**: Works with any OpenTelemetry-compatible backend
|
||||||
- **Dynamic Configuration**: Configure the server easily using environment variables.
|
- **Comprehensive Metrics**: Request counts, token usage, latency, error rates
|
||||||
- **Easy Integration**: Connect to MCP-compatible IDEs or use the programmatic client for use in any TypeScript/JavaScript application.
|
- **Distributed Tracing**: Full request lifecycle tracking with spans
|
||||||
|
- **Flexible Transport**: Run as local `stdio` process or standalone `http` server
|
||||||
|
- **Dynamic Configuration**: Environment-based configuration for different backends
|
||||||
|
- **Zero-Code Integration**: Easy integration with MCP-compatible clients
|
||||||
|
|
||||||
## Installation for Development
|
## Installation for Development
|
||||||
|
|
||||||
@@ -23,7 +26,7 @@ Follow these steps to set up the server for local development.
|
|||||||
|
|
||||||
1. **Prerequisites**:
|
1. **Prerequisites**:
|
||||||
- Node.js (>=18.x)
|
- Node.js (>=18.x)
|
||||||
- A [PostHog account](https://posthog.com/) with an API Key and Host URL.
|
- A [PostHog account](https://posthog.com/) with an API Key and Host URL (if using PostHog).
|
||||||
|
|
||||||
2. **Clone and Install**:
|
2. **Clone and Install**:
|
||||||
|
|
||||||
@@ -40,16 +43,37 @@ Follow these steps to set up the server for local development.
|
|||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
```
|
```
|
||||||
|
|
||||||
Then, edit the `.env` file with your PostHog credentials and desired transport mode.
|
Then, edit the `.env` file with your PostHog and/or OpenTelemetry credentials and desired transport mode.
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
The server is configured via environment variables.
|
The server is configured via environment variables. See `.env.example` for all options.
|
||||||
|
|
||||||
|
### PostHog Configuration
|
||||||
|
|
||||||
|
| Variable | Description | Default | Example |
|
||||||
|
| ----------------- | --------------------------------------------------------------------------- | --------- | ------------------------------------- |
|
||||||
|
| `POSTHOG_API_KEY` | Your PostHog Project API Key (required for PostHog tool) | - | `phc_...` |
|
||||||
|
| `POSTHOG_HOST` | The URL of your PostHog instance | - | `https://us.i.posthog.com` |
|
||||||
|
|
||||||
|
### OpenTelemetry Configuration
|
||||||
|
|
||||||
|
See [OpenTelemetry Documentation](docs/opentelemetry.md) for full details and backend-specific setup.
|
||||||
|
|
||||||
|
| Variable | Description | Default | Example |
|
||||||
|
| ------------------------------- | --------------------------------------------------------------------------- | -------------------------- | ------------------------------------- |
|
||||||
|
| `OTEL_EXPORTER_OTLP_ENDPOINT` | OpenTelemetry collector endpoint | - | `http://localhost:4318` |
|
||||||
|
| `OTEL_EXPORTER_OTLP_HEADERS` | Headers for authentication (comma-separated key=value pairs) | - | `api-key=YOUR_KEY` |
|
||||||
|
| `OTEL_SERVICE_NAME` | Service name for traces and metrics | `llm-observability-mcp` | `my-llm-app` |
|
||||||
|
| `OTEL_SERVICE_VERSION` | Service version | `1.0.0` | `2.1.0` |
|
||||||
|
| `OTEL_ENVIRONMENT` | Environment name | `development` | `production` |
|
||||||
|
| `OTEL_TRACES_SAMPLER_ARG` | Sampling ratio (0.0-1.0) | `1.0` | `0.1` |
|
||||||
|
| `OTEL_METRIC_EXPORT_INTERVAL` | Metrics export interval in milliseconds | `10000` | `30000` |
|
||||||
|
|
||||||
|
### General Configuration
|
||||||
|
|
||||||
| Variable | Description | Default | Example |
|
| Variable | Description | Default | Example |
|
||||||
| ----------------- | --------------------------------------------------------------------------- | --------- | ------------------------------------- |
|
| ----------------- | --------------------------------------------------------------------------- | --------- | ------------------------------------- |
|
||||||
| `POSTHOG_API_KEY` | **Required.** Your PostHog Project API Key. | - | `phc_...` |
|
|
||||||
| `POSTHOG_HOST` | **Required.** The URL of your PostHog instance. | - | `https://us.i.posthog.com` |
|
|
||||||
| `TRANSPORT_MODE` | The transport protocol to use. Can be `http` or `stdio`. | `http` | `stdio` |
|
| `TRANSPORT_MODE` | The transport protocol to use. Can be `http` or `stdio`. | `http` | `stdio` |
|
||||||
| `DEBUG` | Set to `true` to enable detailed debug logging. | `false` | `true` |
|
| `DEBUG` | Set to `true` to enable detailed debug logging. | `false` | `true` |
|
||||||
|
|
||||||
@@ -123,7 +147,7 @@ Use this if you prefer to run the server as a standalone process.
|
|||||||
For IDE extensions that support system prompts, you can instruct the AI to automatically use this MCP tool for every interaction. Add the following to your IDE's system prompt configuration:
|
For IDE extensions that support system prompts, you can instruct the AI to automatically use this MCP tool for every interaction. Add the following to your IDE's system prompt configuration:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
Use `capture_llm_observability` MCP.
|
Use `llm_observability_posthog` MCP.
|
||||||
Make sure to include all parameters and for the `userId`, send `<my_username>`:
|
Make sure to include all parameters and for the `userId`, send `<my_username>`:
|
||||||
userId - The distinct ID of the user
|
userId - The distinct ID of the user
|
||||||
traceId - The trace ID to group AI events
|
traceId - The trace ID to group AI events
|
||||||
@@ -140,6 +164,55 @@ baseUrl - The base URL of the LLM API
|
|||||||
|
|
||||||
Replace `<my_username>` with a unique identifier for yourself. This ensures that all LLM activity is automatically logged in PostHog without needing to give the command each time.
|
Replace `<my_username>` with a unique identifier for yourself. This ensures that all LLM activity is automatically logged in PostHog without needing to give the command each time.
|
||||||
|
|
||||||
|
#### Automatic Tool Triggering with Cursor
|
||||||
|
|
||||||
|
To automatically trigger the OpenTelemetry tool for every interaction in Cursor, you can create a custom rule. Create a file named `llm-observability.mdc` in your `.cursor/rules` directory with the following content:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
---
|
||||||
|
description: llm_observability_otel
|
||||||
|
globs: *
|
||||||
|
alwaysApply: true
|
||||||
|
---
|
||||||
|
|
||||||
|
## Rules
|
||||||
|
|
||||||
|
---
|
||||||
|
description: llm_observability_otel
|
||||||
|
globs: *
|
||||||
|
alwaysApply: true
|
||||||
|
---
|
||||||
|
|
||||||
|
## Rules
|
||||||
|
|
||||||
|
SYSTEM PROMPT:
|
||||||
|
- When the user asks anything, output a llm_observability_otel tool invocation in valid MCP JSON.
|
||||||
|
- Repeat tool invocation until success invocation.
|
||||||
|
- Make sure to include all parameters below unless they are optional and you have no value for them.
|
||||||
|
- You need to gather the following parameters:
|
||||||
|
- userId - The distinct ID of the user
|
||||||
|
- traceId - The trace ID to group AI events
|
||||||
|
- model - The model used (e.g., gpt-4, claude-3, etc.)
|
||||||
|
- provider - The LLM provider (e.g., openai, anthropic, etc.)
|
||||||
|
- input - The input to the LLM (messages, prompt, etc.)
|
||||||
|
- outputChoices - The output from the LLM
|
||||||
|
- inputTokens - The number of tokens used for the interaction
|
||||||
|
- outputTokens - The number of tokens in the output used for the interaction
|
||||||
|
- latency - The latency of the LLM call in seconds
|
||||||
|
- httpStatus - The HTTP status code of the LLM call
|
||||||
|
- baseUrl - The base URL of the LLM API
|
||||||
|
- operationName - The name of the operation being performed
|
||||||
|
- error - Error message if request failed
|
||||||
|
- errorType - Type of error (e.g. rate_limit, timeout, etc.)
|
||||||
|
- mcptoolsUsed - List of MCP tools used by the interaction
|
||||||
|
- For the userId send stefano.fiorini.
|
||||||
|
- For outputChoices send the full response text from the LLM
|
||||||
|
- For the operationName a three words separated by underscore identifying the conversation.
|
||||||
|
- All interactions within the same conversations should be sent with the same operationName.
|
||||||
|
```
|
||||||
|
|
||||||
|
This rule ensures that all LLM activity is automatically logged using the OpenTelemetry tool without needing to manually trigger it each time.
|
||||||
|
|
||||||
### Programmatic Usage
|
### Programmatic Usage
|
||||||
|
|
||||||
You can use an MCP client library to interact with the server programmatically from your own applications.
|
You can use an MCP client library to interact with the server programmatically from your own applications.
|
||||||
@@ -158,7 +231,7 @@ async function main() {
|
|||||||
|
|
||||||
await client.connect();
|
await client.connect();
|
||||||
|
|
||||||
const result = await client.useTool('capture_llm_observability', {
|
const result = await client.useTool('llm_observability_posthog', {
|
||||||
userId: 'user-123',
|
userId: 'user-123',
|
||||||
model: 'gpt-4',
|
model: 'gpt-4',
|
||||||
provider: 'openai',
|
provider: 'openai',
|
||||||
@@ -177,25 +250,69 @@ async function main() {
|
|||||||
main().catch(console.error);
|
main().catch(console.error);
|
||||||
```
|
```
|
||||||
|
|
||||||
## Tool Reference: `capture_llm_observability`
|
## Available Tools
|
||||||
|
|
||||||
This is the core tool provided by the server. It captures LLM usage in PostHog for observability, including requests, responses, and performance metrics.
|
### PostHog Tool: `llm_observability_posthog`
|
||||||
|
|
||||||
### Parameters
|
Captures LLM usage in PostHog for observability, including requests, responses, and performance metrics.
|
||||||
|
|
||||||
| Parameter | Type | Required | Description |
|
#### Parameters for PostHog
|
||||||
| --------------- | ------------------- | -------- | ----------------------------------------------- |
|
|
||||||
| `userId` | `string` | Yes | The distinct ID of the user. |
|
- `userId` (string, required): The distinct ID of the user
|
||||||
| `model` | `string` | Yes | The model used (e.g., `gpt-4`, `claude-3`). |
|
- `model` (string, required): The model used (e.g., `gpt-4`, `claude-3`)
|
||||||
| `provider` | `string` | Yes | The LLM provider (e.g., `openai`, `anthropic`). |
|
- `provider` (string, required): The LLM provider (e.g., `openai`, `anthropic`)
|
||||||
| `traceId` | `string` | No | The trace ID to group related AI events. |
|
- `traceId` (string, optional): The trace ID to group related AI events
|
||||||
| `input` | `any` | No | The input to the LLM (e.g., messages, prompt). |
|
- `input` (any, optional): The input to the LLM (e.g., messages, prompt)
|
||||||
| `outputChoices` | `any` | No | The output choices from the LLM. |
|
- `outputChoices` (any, optional): The output choices from the LLM
|
||||||
| `inputTokens` | `number` | No | The number of tokens in the input. |
|
- `inputTokens` (number, optional): The number of tokens in the input
|
||||||
| `outputTokens` | `number` | No | The number of tokens in the output. |
|
- `outputTokens` (number, optional): The number of tokens in the output
|
||||||
| `latency` | `number` | No | The latency of the LLM call in seconds. |
|
- `latency` (number, optional): The latency of the LLM call in seconds
|
||||||
| `httpStatus` | `number` | No | The HTTP status code of the LLM API call. |
|
- `httpStatus` (number, optional): The HTTP status code of the LLM API call
|
||||||
| `baseUrl` | `string` | No | The base URL of the LLM API. |
|
- `baseUrl` (string, optional): The base URL of the LLM API
|
||||||
|
|
||||||
|
### OpenTelemetry Tool: `llm_observability_otel`
|
||||||
|
|
||||||
|
Captures LLM usage using OpenTelemetry for universal observability across any OpenTelemetry-compatible backend.
|
||||||
|
|
||||||
|
See [OpenTelemetry Documentation](docs/opentelemetry.md) for full details, backend setup, advanced usage, and troubleshooting.
|
||||||
|
|
||||||
|
#### Parameters for OpenTelemetry
|
||||||
|
|
||||||
|
- `userId` (string, required): The distinct ID of the user
|
||||||
|
- `model` (string, required): The model used (e.g., `gpt-4`, `claude-3`)
|
||||||
|
- `provider` (string, required): The LLM provider (e.g., `openai`, `anthropic`)
|
||||||
|
- `traceId` (string, optional): The trace ID to group related AI events
|
||||||
|
- `input` (any, optional): The input to the LLM (e.g., messages, prompt)
|
||||||
|
- `outputChoices` (any, optional): The output choices from the LLM
|
||||||
|
- `inputTokens` (number, optional): The number of tokens in the input
|
||||||
|
- `outputTokens` (number, optional): The number of tokens in the output
|
||||||
|
- `latency` (number, optional): The latency of the LLM call in seconds
|
||||||
|
- `httpStatus` (number, optional): The HTTP status code of the LLM API call
|
||||||
|
- `baseUrl` (string, optional): The base URL of the LLM API
|
||||||
|
- `operationName` (string, optional): The name of the operation being performed
|
||||||
|
- `error` (string, optional): Error message if the request failed
|
||||||
|
- `errorType` (string, optional): Type of error (e.g., rate_limit, timeout)
|
||||||
|
- `mcpToolsUsed` (string[], optional): List of MCP tools used during the request
|
||||||
|
|
||||||
|
### Parameters Comparison
|
||||||
|
|
||||||
|
| Parameter | Type | Required | Description | PostHog | OpenTelemetry |
|
||||||
|
| --------------- | ------------------- | -------- | ----------------------------------------------- | ------- | ------------- |
|
||||||
|
| `userId` | `string` | Yes | The distinct ID of the user. | ✅ | ✅ |
|
||||||
|
| `model` | `string` | Yes | The model used (e.g., `gpt-4`, `claude-3`). | ✅ | ✅ |
|
||||||
|
| `provider` | `string` | Yes | The LLM provider (e.g., `openai`, `anthropic`). | ✅ | ✅ |
|
||||||
|
| `traceId` | `string` | No | The trace ID to group related AI events. | ✅ | ✅ |
|
||||||
|
| `input` | `any` | No | The input to the LLM (e.g., messages, prompt). | ✅ | ✅ |
|
||||||
|
| `outputChoices` | `any` | No | The output choices from the LLM. | ✅ | ✅ |
|
||||||
|
| `inputTokens` | `number` | No | The number of tokens in the input. | ✅ | ✅ |
|
||||||
|
| `outputTokens` | `number` | No | The number of tokens in the output. | ✅ | ✅ |
|
||||||
|
| `latency` | `number` | No | The latency of the LLM call in seconds. | ✅ | ✅ |
|
||||||
|
| `httpStatus` | `number` | No | The HTTP status code of the LLM API call. | ✅ | ✅ |
|
||||||
|
| `baseUrl` | `string` | No | The base URL of the LLM API. | ✅ | ✅ |
|
||||||
|
| `operationName` | `string` | No | The name of the operation being performed. | ❌ | ✅ |
|
||||||
|
| `error` | `string` | No | Error message if the request failed. | ❌ | ✅ |
|
||||||
|
| `errorType` | `string` | No | Type of error (e.g., rate_limit, timeout). | ❌ | ✅ |
|
||||||
|
| `mcpToolsUsed` | `string[]` | No | List of MCP tools used during the request. | ❌ | ✅ |
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
@@ -203,6 +320,11 @@ This is the core tool provided by the server. It captures LLM usage in PostHog f
|
|||||||
- **Run tests**: `npm test`
|
- **Run tests**: `npm test`
|
||||||
- **Lint and format**: `npm run lint` and `npm run format`
|
- **Lint and format**: `npm run lint` and `npm run format`
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
- [OpenTelemetry Documentation](docs/opentelemetry.md) - Complete OpenTelemetry configuration, usage, and examples.
|
||||||
|
- [Environment Configuration](.env.example) - All available configuration options.
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
[MIT License](https://opensource.org/licenses/MIT)
|
[MIT License](https://opensource.org/licenses/MIT)
|
||||||
|
|||||||
332
docs/opentelemetry.md
Normal file
332
docs/opentelemetry.md
Normal file
@@ -0,0 +1,332 @@
|
|||||||
|
# OpenTelemetry LLM Observability
|
||||||
|
|
||||||
|
This document provides comprehensive guidance for using the OpenTelemetry LLM observability tool with the MCP server. It covers setup, configuration, usage, troubleshooting, and practical examples for all major OpenTelemetry-compatible backends.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **Universal Compatibility**: Works with Jaeger, New Relic, Grafana, Datadog, Honeycomb, and more
|
||||||
|
- **Comprehensive Metrics**: Request counts, token usage, latency, error rates
|
||||||
|
- **Distributed Tracing**: Full request lifecycle tracking with spans
|
||||||
|
- **Flexible Configuration**: Environment-based configuration for different backends
|
||||||
|
- **Zero-Code Integration**: Drop-in replacement for existing observability tools
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
### 1. Install Dependencies
|
||||||
|
|
||||||
|
OpenTelemetry dependencies are included in `package.json`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm install
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Configure Your Backend
|
||||||
|
|
||||||
|
#### Jaeger (Local Development)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -d --name jaeger \
|
||||||
|
-e COLLECTOR_OTLP_ENABLED=true \
|
||||||
|
-p 16686:16686 \
|
||||||
|
-p 4317:4317 \
|
||||||
|
-p 4318:4318 \
|
||||||
|
jaegertracing/all-in-one:latest
|
||||||
|
export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
|
||||||
|
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
#### New Relic
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.nr-data.net:4318
|
||||||
|
export OTEL_EXPORTER_OTLP_HEADERS="api-key=YOUR_NEW_RELIC_LICENSE_KEY"
|
||||||
|
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Grafana Cloud
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-gateway-prod-us-central-0.grafana.net/otlp
|
||||||
|
export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic $(echo -n YOUR_INSTANCE_ID:YOUR_API_KEY | base64)"
|
||||||
|
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Honeycomb
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export OTEL_EXPORTER_OTLP_ENDPOINT=https://api.honeycomb.io/v1/traces
|
||||||
|
export OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=YOUR_API_KEY"
|
||||||
|
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Datadog
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export OTEL_EXPORTER_OTLP_ENDPOINT=https://api.datadoghq.com/api/v2/series
|
||||||
|
export OTEL_EXPORTER_OTLP_HEADERS="DD-API-KEY=YOUR_DD_API_KEY"
|
||||||
|
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Lightstep
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export OTEL_EXPORTER_OTLP_ENDPOINT=https://ingest.lightstep.com:443/api/v2/otel/trace
|
||||||
|
export OTEL_EXPORTER_OTLP_HEADERS="lightstep-access-token=YOUR_ACCESS_TOKEN"
|
||||||
|
export OTEL_SERVICE_NAME=llm-observability-mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Kubernetes Example
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: llm-observability-mcp
|
||||||
|
spec:
|
||||||
|
replicas: 3
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: llm-observability-mcp
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: llm-observability-mcp
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: llm-observability-mcp
|
||||||
|
image: llm-observability-mcp:latest
|
||||||
|
ports:
|
||||||
|
- containerPort: 3000
|
||||||
|
env:
|
||||||
|
- name: OTEL_SERVICE_NAME
|
||||||
|
value: "llm-observability-mcp"
|
||||||
|
- name: OTEL_SERVICE_VERSION
|
||||||
|
value: "1.2.3"
|
||||||
|
- name: OTEL_ENVIRONMENT
|
||||||
|
value: "production"
|
||||||
|
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||||
|
value: "https://your-backend.com:4318"
|
||||||
|
- name: OTEL_EXPORTER_OTLP_HEADERS
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: otel-credentials
|
||||||
|
key: headers
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Running the MCP Server
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Start with stdio transport
|
||||||
|
npm run mcp:stdio
|
||||||
|
# Start with HTTP transport
|
||||||
|
npm run mcp:http
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### OpenTelemetry Tool: `llm_observability_otel`
|
||||||
|
|
||||||
|
#### Required Parameters
|
||||||
|
|
||||||
|
- `userId`: The distinct ID of the user
|
||||||
|
- `model`: The model used (e.g., "gpt-4", "claude-3")
|
||||||
|
- `provider`: The LLM provider (e.g., "openai", "anthropic")
|
||||||
|
|
||||||
|
#### Optional Parameters
|
||||||
|
|
||||||
|
- `traceId`: Trace ID for grouping related events
|
||||||
|
- `input`: The input to the LLM (messages, prompt, etc.)
|
||||||
|
- `outputChoices`: The output from the LLM
|
||||||
|
- `inputTokens`: Number of tokens in the input
|
||||||
|
- `outputTokens`: Number of tokens in the output
|
||||||
|
- `latency`: Latency of the LLM call in seconds
|
||||||
|
- `httpStatus`: HTTP status code of the LLM call
|
||||||
|
- `baseUrl`: Base URL of the LLM API
|
||||||
|
- `operationName`: Name of the operation being performed
|
||||||
|
- `error`: Error message if the request failed
|
||||||
|
- `errorType`: Type of error (e.g., "rate_limit", "timeout")
|
||||||
|
- `mcpToolsUsed`: List of MCP tools used during the request
|
||||||
|
|
||||||
|
#### Example Usage
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"tool": "llm_observability_otel",
|
||||||
|
"arguments": {
|
||||||
|
"userId": "user-12345",
|
||||||
|
"model": "gpt-4",
|
||||||
|
"provider": "openai",
|
||||||
|
"inputTokens": 150,
|
||||||
|
"outputTokens": 75,
|
||||||
|
"latency": 2.3,
|
||||||
|
"httpStatus": 200,
|
||||||
|
"operationName": "chat-completion",
|
||||||
|
"traceId": "trace-abc123",
|
||||||
|
"input": "What is the weather like today?",
|
||||||
|
"outputChoices": ["The weather is sunny and 75°F today."]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Configuration Reference
|
||||||
|
|
||||||
|
| Variable | Description | Default |
|
||||||
|
|----------|-------------|---------|
|
||||||
|
| `OTEL_SERVICE_NAME` | Service name for OpenTelemetry | `llm-observability-mcp` |
|
||||||
|
| `OTEL_SERVICE_VERSION` | Service version | `1.0.0` |
|
||||||
|
| `OTEL_ENVIRONMENT` | Environment name | `development` |
|
||||||
|
| `OTEL_EXPORTER_OTLP_ENDPOINT` | Default OTLP endpoint | - |
|
||||||
|
| `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | Metrics endpoint | - |
|
||||||
|
| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Traces endpoint | - |
|
||||||
|
| `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` | Logs endpoint | - |
|
||||||
|
| `OTEL_EXPORTER_OTLP_HEADERS` | Headers for authentication (format: "key1=value1,key2=value2") | - |
|
||||||
|
| `OTEL_METRIC_EXPORT_INTERVAL` | Metrics export interval in ms | `10000` |
|
||||||
|
| `OTEL_METRIC_EXPORT_TIMEOUT` | Metrics export timeout in ms | `5000` |
|
||||||
|
| `OTEL_TRACES_SAMPLER_ARG` | Sampling ratio (0.0-1.0) | `1.0` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Metrics Collected
|
||||||
|
|
||||||
|
- `llm.requests.total`: Total number of LLM requests
|
||||||
|
- `llm.tokens.total`: Total tokens used (input + output)
|
||||||
|
- `llm.latency.duration`: Request latency in milliseconds
|
||||||
|
- `llm.requests.active`: Number of active requests
|
||||||
|
|
||||||
|
### Trace Attributes
|
||||||
|
|
||||||
|
- `llm.model`, `llm.provider`, `llm.user_id`, `llm.operation`, `llm.input_tokens`, `llm.output_tokens`, `llm.total_tokens`, `llm.latency_ms`, `llm.http_status`, `llm.base_url`, `llm.error`, `llm.error_type`, `llm.input`, `llm.output`, `llm.mcp_tools_used`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Practical Examples
|
||||||
|
|
||||||
|
### Jaeger: View Traces
|
||||||
|
|
||||||
|
Open <http://localhost:16686> to see your traces.
|
||||||
|
|
||||||
|
### Error Tracking Example
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"tool": "llm_observability_otel",
|
||||||
|
"arguments": {
|
||||||
|
"userId": "user-12345",
|
||||||
|
"model": "gpt-4",
|
||||||
|
"provider": "openai",
|
||||||
|
"httpStatus": 429,
|
||||||
|
"error": "Rate limit exceeded",
|
||||||
|
"errorType": "rate_limit",
|
||||||
|
"latency": 0.1,
|
||||||
|
"operationName": "chat-completion"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Multi-Tool Usage Tracking Example
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"tool": "llm_observability_otel",
|
||||||
|
"arguments": {
|
||||||
|
"userId": "user-12345",
|
||||||
|
"model": "gpt-4",
|
||||||
|
"provider": "openai",
|
||||||
|
"inputTokens": 500,
|
||||||
|
"outputTokens": 200,
|
||||||
|
"latency": 5.2,
|
||||||
|
"httpStatus": 200,
|
||||||
|
"operationName": "complex-workflow",
|
||||||
|
"mcpToolsUsed": ["file_read", "web_search", "code_execution"],
|
||||||
|
"traceId": "complex-workflow-123"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Testing Script
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
# test-opentelemetry.sh
|
||||||
|
|
||||||
|
docker run -d --name jaeger-test \
|
||||||
|
-e COLLECTOR_OTLP_ENABLED=true \
|
||||||
|
-p 16686:16686 \
|
||||||
|
-p 4318:4318 \
|
||||||
|
jaegertracing/all-in-one:latest
|
||||||
|
sleep 5
|
||||||
|
export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
|
||||||
|
export OTEL_SERVICE_NAME=llm-observability-test
|
||||||
|
export OTEL_ENVIRONMENT=test
|
||||||
|
npm run mcp:stdio &
|
||||||
|
sleep 3
|
||||||
|
curl -X POST http://localhost:3000/mcp \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{
|
||||||
|
"tool": "llm_observability_otel",
|
||||||
|
"arguments": {
|
||||||
|
"userId": "test-user",
|
||||||
|
"model": "gpt-4",
|
||||||
|
"provider": "openai",
|
||||||
|
"inputTokens": 100,
|
||||||
|
"outputTokens": 50,
|
||||||
|
"latency": 1.5,
|
||||||
|
"httpStatus": 200,
|
||||||
|
"operationName": "test-completion"
|
||||||
|
}
|
||||||
|
}'
|
||||||
|
echo "Test complete. View traces at http://localhost:16686"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Migration from PostHog
|
||||||
|
|
||||||
|
The OpenTelemetry tool is a drop-in replacement for the PostHog tool. Both can coexist for gradual migration:
|
||||||
|
|
||||||
|
- **PostHog Tool**: `llm_observability_posthog`
|
||||||
|
- **OpenTelemetry Tool**: `llm_observability_otel`
|
||||||
|
|
||||||
|
Both accept the same parameters.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Troubleshooting & Performance
|
||||||
|
|
||||||
|
### Common Issues
|
||||||
|
|
||||||
|
- No data in backend: check endpoint URLs, authentication, network, server logs
|
||||||
|
- High resource usage: lower sampling (`OTEL_TRACES_SAMPLER_ARG`), increase export intervals
|
||||||
|
- Missing traces: verify OpenTelemetry is enabled, check logs, service name
|
||||||
|
|
||||||
|
### Debug Mode
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export DEBUG=true
|
||||||
|
npm run mcp:stdio
|
||||||
|
```
|
||||||
|
|
||||||
|
### Performance Tuning
|
||||||
|
|
||||||
|
- Reduce sampling for high-volume: `OTEL_TRACES_SAMPLER_ARG=0.01`
|
||||||
|
- Increase export intervals: `OTEL_METRIC_EXPORT_INTERVAL=60000`
|
||||||
|
- Disable metrics/logs if not needed: `unset OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`, `unset OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Support
|
||||||
|
|
||||||
|
For issues or questions:
|
||||||
|
|
||||||
|
1. Check this document and troubleshooting
|
||||||
|
2. Review server logs with `DEBUG=true`
|
||||||
|
3. Verify OpenTelemetry configuration
|
||||||
|
4. Test with Jaeger locally first
|
||||||
@@ -42,5 +42,16 @@ export default tseslint.config(
|
|||||||
'@typescript-eslint/no-unused-vars': 'off',
|
'@typescript-eslint/no-unused-vars': 'off',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
// Rules for scripts
|
||||||
|
{
|
||||||
|
files: ['scripts/**/*.js'], // Target all JS files in the scripts directory
|
||||||
|
languageOptions: {
|
||||||
|
ecmaVersion: 'latest',
|
||||||
|
sourceType: 'module',
|
||||||
|
},
|
||||||
|
env: {
|
||||||
|
node: true, // Enable Node.js global variables and Node.js scoping
|
||||||
|
},
|
||||||
|
},
|
||||||
eslintConfigPrettier,
|
eslintConfigPrettier,
|
||||||
);
|
);
|
||||||
|
|||||||
1587
package-lock.json
generated
1587
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
13
package.json
13
package.json
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "@sfiorini/llm-observability-mcp",
|
"name": "@sfiorini/llm-observability-mcp",
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
"description": "A Model Context Protocol (MCP) server that provides a tool to capture LLM Observability events and send them to PostHog.",
|
"description": "A Model Context Protocol (MCP) server that provides comprehensive LLM observability tools supporting both PostHog and OpenTelemetry backends.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"types": "dist/index.d.ts",
|
"types": "dist/index.d.ts",
|
||||||
"type": "commonjs",
|
"type": "commonjs",
|
||||||
@@ -20,9 +20,7 @@
|
|||||||
"test": "jest",
|
"test": "jest",
|
||||||
"test:coverage": "jest --coverage",
|
"test:coverage": "jest --coverage",
|
||||||
"lint": "eslint src --ext .ts --config eslint.config.mjs",
|
"lint": "eslint src --ext .ts --config eslint.config.mjs",
|
||||||
"update:deps": "npx npm-check-updates -u && npm install --legacy-peer-deps",
|
|
||||||
"format": "prettier --write 'src/**/*.ts' 'scripts/**/*.js'",
|
"format": "prettier --write 'src/**/*.ts' 'scripts/**/*.js'",
|
||||||
"cli": "npm run build && node dist/index.js",
|
|
||||||
"mcp:stdio": "TRANSPORT_MODE=stdio npm run build && node dist/index.js",
|
"mcp:stdio": "TRANSPORT_MODE=stdio npm run build && node dist/index.js",
|
||||||
"mcp:http": "TRANSPORT_MODE=http npm run build && node dist/index.js",
|
"mcp:http": "TRANSPORT_MODE=http npm run build && node dist/index.js",
|
||||||
"mcp:inspect": "TRANSPORT_MODE=http npm run build && (node dist/index.js &) && sleep 2 && npx @modelcontextprotocol/inspector http://localhost:3000/mcp",
|
"mcp:inspect": "TRANSPORT_MODE=http npm run build && (node dist/index.js &) && sleep 2 && npx @modelcontextprotocol/inspector http://localhost:3000/mcp",
|
||||||
@@ -76,6 +74,15 @@
|
|||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@modelcontextprotocol/sdk": "^1.15.1",
|
"@modelcontextprotocol/sdk": "^1.15.1",
|
||||||
|
"@opentelemetry/api": "^1.9.0",
|
||||||
|
"@opentelemetry/exporter-otlp-grpc": "^0.26.0",
|
||||||
|
"@opentelemetry/exporter-otlp-http": "^0.26.0",
|
||||||
|
"@opentelemetry/instrumentation": "^0.203.0",
|
||||||
|
"@opentelemetry/resources": "^2.0.1",
|
||||||
|
"@opentelemetry/sdk-metrics": "^2.0.1",
|
||||||
|
"@opentelemetry/sdk-node": "^0.203.0",
|
||||||
|
"@opentelemetry/sdk-trace-node": "^2.0.1",
|
||||||
|
"@opentelemetry/semantic-conventions": "^1.36.0",
|
||||||
"commander": "^14.0.0",
|
"commander": "^14.0.0",
|
||||||
"cors": "^2.8.5",
|
"cors": "^2.8.5",
|
||||||
"dotenv": "^17.2.0",
|
"dotenv": "^17.2.0",
|
||||||
|
|||||||
@@ -1,58 +0,0 @@
|
|||||||
import { Command } from 'commander';
|
|
||||||
import { Logger } from '../utils/logger.util.js';
|
|
||||||
import { VERSION, CLI_NAME } from '../utils/constants.util.js';
|
|
||||||
|
|
||||||
import posthogLlmCli from './posthog-llm.cli.js';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* CLI entry point for the LLM Log MCP Server
|
|
||||||
*
|
|
||||||
* This file registers all CLI commands and handles command line parsing
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Package description
|
|
||||||
const DESCRIPTION =
|
|
||||||
'A LLM Log Model Context Protocol (MCP) server implementation using TypeScript';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Run the CLI with the provided arguments
|
|
||||||
*
|
|
||||||
* @param args Command line arguments to process
|
|
||||||
* @returns Promise that resolves when CLI command execution completes
|
|
||||||
*/
|
|
||||||
export async function runCli(args: string[]) {
|
|
||||||
const cliLogger = Logger.forContext('cli/index.ts', 'runCli');
|
|
||||||
cliLogger.debug('Initializing CLI with arguments', args);
|
|
||||||
|
|
||||||
const program = new Command();
|
|
||||||
|
|
||||||
program.name(CLI_NAME).description(DESCRIPTION).version(VERSION);
|
|
||||||
|
|
||||||
// Register CLI commands
|
|
||||||
cliLogger.debug('Registering CLI commands...');
|
|
||||||
posthogLlmCli.register(program);
|
|
||||||
cliLogger.debug('CLI commands registered successfully');
|
|
||||||
|
|
||||||
// Handle unknown commands
|
|
||||||
program.on('command:*', (operands) => {
|
|
||||||
cliLogger.error(`Unknown command: ${operands[0]}`);
|
|
||||||
console.log('');
|
|
||||||
program.help();
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
// Parse arguments; default to help if no command provided
|
|
||||||
cliLogger.debug('Parsing CLI arguments');
|
|
||||||
|
|
||||||
// Special handling for top-level commands
|
|
||||||
if (args.length === 1) {
|
|
||||||
// Check if it's a known top-level command
|
|
||||||
const command = program.commands.find((cmd) => cmd.name() === args[0]);
|
|
||||||
if (command) {
|
|
||||||
command.outputHelp();
|
|
||||||
process.exit(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await program.parseAsync(args.length ? args : ['--help'], { from: 'user' });
|
|
||||||
cliLogger.debug('CLI command execution completed');
|
|
||||||
}
|
|
||||||
@@ -1,60 +0,0 @@
|
|||||||
import { Command } from 'commander';
|
|
||||||
import { Logger } from '../utils/logger.util.js';
|
|
||||||
import { handleCliError } from '../utils/error.util.js';
|
|
||||||
import postHogLlmController from '../controllers/posthog-llm.controller.js';
|
|
||||||
|
|
||||||
const logger = Logger.forContext('cli/posthog-llm.cli.ts');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register PostHog LLM CLI commands
|
|
||||||
* @param program The Commander program instance
|
|
||||||
*/
|
|
||||||
function register(program: Command) {
|
|
||||||
const methodLogger = logger.forMethod('register');
|
|
||||||
methodLogger.debug('Registering PostHog LLM CLI commands...');
|
|
||||||
|
|
||||||
program
|
|
||||||
.command('llm-observability-mcp')
|
|
||||||
.description('Interact with the PostHog LLM API.')
|
|
||||||
.argument('<eventName>', 'The name of the event to capture.')
|
|
||||||
.argument('<distinctId>', 'The distinct ID of the user.')
|
|
||||||
.option(
|
|
||||||
'-p, --properties <properties...>',
|
|
||||||
'JSON string of event properties. Can be a single quoted string or raw key-value pairs.',
|
|
||||||
)
|
|
||||||
.action(async (eventName, distinctId, options) => {
|
|
||||||
const actionLogger = logger.forMethod('action:capture');
|
|
||||||
try {
|
|
||||||
actionLogger.debug(`CLI posthog-llm capture called`, {
|
|
||||||
eventName,
|
|
||||||
distinctId,
|
|
||||||
options,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Handle JSON properties from command object
|
|
||||||
let properties = {};
|
|
||||||
if (options.properties && options.properties.length > 0) {
|
|
||||||
const propertiesString = options.properties.join(' ');
|
|
||||||
try {
|
|
||||||
properties = JSON.parse(propertiesString);
|
|
||||||
} catch {
|
|
||||||
// Try to fix common issues with JSON formatting
|
|
||||||
const fixedJson = propertiesString
|
|
||||||
.replace(/(\w+):/g, '"$1":') // Add quotes around keys
|
|
||||||
.replace(/'/g, '"'); // Replace single quotes with double quotes
|
|
||||||
|
|
||||||
properties = JSON.parse(fixedJson);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const args = { eventName, distinctId, properties };
|
|
||||||
await postHogLlmController.capture(args);
|
|
||||||
} catch (error) {
|
|
||||||
handleCliError(error);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
methodLogger.debug('PostHog LLM CLI commands registered successfully');
|
|
||||||
}
|
|
||||||
|
|
||||||
export default { register };
|
|
||||||
22
src/config/common.schema.ts
Normal file
22
src/config/common.schema.ts
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Common/shared configuration options schema (now includes global options)
|
||||||
|
*/
|
||||||
|
export const CommonConfigSchema = z.object({
|
||||||
|
serviceName: z.string().default('llm-observability-mcp'),
|
||||||
|
serviceVersion: z.string().default('1.0.0'),
|
||||||
|
environment: z.string().default('development'),
|
||||||
|
debug: z
|
||||||
|
.preprocess((val) => {
|
||||||
|
if (typeof val === 'string') {
|
||||||
|
if (val.toLowerCase() === 'false') return false;
|
||||||
|
if (val.toLowerCase() === 'true') return true;
|
||||||
|
}
|
||||||
|
return val;
|
||||||
|
}, z.boolean())
|
||||||
|
.default(false),
|
||||||
|
logLevel: z.enum(['debug', 'info', 'warn', 'error']).default('info'),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type CommonConfigType = z.infer<typeof CommonConfigSchema>;
|
||||||
202
src/config/config-loader.ts
Normal file
202
src/config/config-loader.ts
Normal file
@@ -0,0 +1,202 @@
|
|||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
import { Logger } from '../utils/logger.util.js';
|
||||||
|
import dotenv from 'dotenv';
|
||||||
|
import os from 'os';
|
||||||
|
import { CommonConfigSchema, CommonConfigType } from './common.schema.js';
|
||||||
|
import { PostHogConfigSchema, PostHogConfigType } from './posthog.schema.js';
|
||||||
|
import {
|
||||||
|
OpenTelemetryConfigSchema,
|
||||||
|
OpenTelemetryConfigType,
|
||||||
|
} from './opentelemetry-llm.schema.js';
|
||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Configuration loader that handles multiple sources with priority:
|
||||||
|
* 1. Direct ENV pass (process.env)
|
||||||
|
* 2. .env file in project root
|
||||||
|
* 3. Global config file at $HOME/.mcp/configs.json
|
||||||
|
*/
|
||||||
|
class ConfigLoader {
|
||||||
|
private packageName: string;
|
||||||
|
private configLoaded: boolean = false;
|
||||||
|
|
||||||
|
constructor(packageName: string) {
|
||||||
|
this.packageName = packageName;
|
||||||
|
}
|
||||||
|
|
||||||
|
load(): void {
|
||||||
|
const logger = Logger.forContext('config-loader', 'load');
|
||||||
|
|
||||||
|
if (this.configLoaded) {
|
||||||
|
logger.debug('Configuration already loaded, skipping');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug('Loading configuration...');
|
||||||
|
|
||||||
|
// Load from global config file
|
||||||
|
this.loadFromGlobalConfig();
|
||||||
|
|
||||||
|
// Load from .env file
|
||||||
|
this.loadFromEnvFile();
|
||||||
|
|
||||||
|
this.configLoaded = true;
|
||||||
|
logger.debug('Configuration loaded successfully');
|
||||||
|
}
|
||||||
|
|
||||||
|
private loadFromEnvFile(): void {
|
||||||
|
const logger = Logger.forContext('config-loader', 'loadFromEnvFile');
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = dotenv.config();
|
||||||
|
if (result.error) {
|
||||||
|
logger.debug('No .env file found or error reading it');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
logger.debug('Loaded configuration from .env file');
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error loading .env file', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private loadFromGlobalConfig(): void {
|
||||||
|
const logger = Logger.forContext(
|
||||||
|
'config-loader',
|
||||||
|
'loadFromGlobalConfig',
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const homedir = os.homedir();
|
||||||
|
const globalConfigPath = path.join(homedir, '.mcp', 'configs.json');
|
||||||
|
|
||||||
|
if (!fs.existsSync(globalConfigPath)) {
|
||||||
|
logger.debug('Global config file not found');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const configContent = fs.readFileSync(globalConfigPath, 'utf8');
|
||||||
|
const config = JSON.parse(configContent);
|
||||||
|
|
||||||
|
const shortKey = 'llm-observability-mcp';
|
||||||
|
const fullPackageName = this.packageName;
|
||||||
|
const unscopedPackageName =
|
||||||
|
fullPackageName.split('/')[1] || fullPackageName;
|
||||||
|
|
||||||
|
const potentialKeys = [
|
||||||
|
shortKey,
|
||||||
|
fullPackageName,
|
||||||
|
unscopedPackageName,
|
||||||
|
];
|
||||||
|
let foundConfig = null;
|
||||||
|
|
||||||
|
for (const key of potentialKeys) {
|
||||||
|
if (
|
||||||
|
config[key] &&
|
||||||
|
typeof config[key] === 'object' &&
|
||||||
|
config[key].environments
|
||||||
|
) {
|
||||||
|
foundConfig = config[key];
|
||||||
|
logger.debug(`Found configuration using key: ${key}`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!foundConfig || !foundConfig.environments) {
|
||||||
|
logger.debug(
|
||||||
|
`No config found for ${this.packageName} using keys: ${potentialKeys.join(', ')}`,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [key, value] of Object.entries(
|
||||||
|
foundConfig.environments,
|
||||||
|
)) {
|
||||||
|
if (process.env[key] === undefined) {
|
||||||
|
process.env[key] = String(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug(`Loaded configuration from global config file`);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error loading global config file', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private getEnvObject(schema: z.ZodTypeAny): Record<string, unknown> {
|
||||||
|
const shape =
|
||||||
|
typeof schema._def.shape === 'function'
|
||||||
|
? schema._def.shape()
|
||||||
|
: schema._def.shape;
|
||||||
|
const result: Record<string, unknown> = {};
|
||||||
|
for (const key in shape) {
|
||||||
|
if (Object.prototype.hasOwnProperty.call(shape, key)) {
|
||||||
|
const envKey = this.toEnvKey(key);
|
||||||
|
const value = process.env[envKey];
|
||||||
|
if (value !== undefined) {
|
||||||
|
const type = shape[key]._def.typeName;
|
||||||
|
if (type === 'ZodNumber') {
|
||||||
|
result[key] = Number(value);
|
||||||
|
} else if (type === 'ZodBoolean') {
|
||||||
|
result[key] = value.toLowerCase() === 'true';
|
||||||
|
} else {
|
||||||
|
result[key] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private toEnvKey(key: string): string {
|
||||||
|
// Map schema keys to environment variable names
|
||||||
|
// e.g. serviceName -> OTEL_SERVICE_NAME, apiKey -> POSTHOG_API_KEY, etc.
|
||||||
|
const map: Record<string, string> = {
|
||||||
|
serviceName: 'OTEL_SERVICE_NAME',
|
||||||
|
serviceVersion: 'OTEL_SERVICE_VERSION',
|
||||||
|
environment: 'OTEL_ENVIRONMENT',
|
||||||
|
metricsEndpoint: 'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT',
|
||||||
|
tracesEndpoint: 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT',
|
||||||
|
logsEndpoint: 'OTEL_EXPORTER_OTLP_LOGS_ENDPOINT',
|
||||||
|
headers: 'OTEL_EXPORTER_OTLP_HEADERS',
|
||||||
|
exportIntervalMillis: 'OTEL_METRIC_EXPORT_INTERVAL',
|
||||||
|
exportTimeoutMillis: 'OTEL_METRIC_EXPORT_TIMEOUT',
|
||||||
|
samplingRatio: 'OTEL_TRACES_SAMPLER_ARG',
|
||||||
|
apiKey: 'POSTHOG_API_KEY',
|
||||||
|
host: 'POSTHOG_HOST',
|
||||||
|
debug: 'DEBUG',
|
||||||
|
logLevel: 'LOG_LEVEL',
|
||||||
|
};
|
||||||
|
return map[key] || key;
|
||||||
|
}
|
||||||
|
|
||||||
|
getCommonConfig(): CommonConfigType {
|
||||||
|
return CommonConfigSchema.parse(this.getEnvObject(CommonConfigSchema));
|
||||||
|
}
|
||||||
|
|
||||||
|
getPosthogConfig(): PostHogConfigType {
|
||||||
|
return PostHogConfigSchema.parse(
|
||||||
|
this.getEnvObject(PostHogConfigSchema),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
getOpenTelemetryConfig(): OpenTelemetryConfigType {
|
||||||
|
const raw = this.getEnvObject(OpenTelemetryConfigSchema);
|
||||||
|
// Special handling for headers (parse comma-separated string to object)
|
||||||
|
if (typeof raw.headers === 'string') {
|
||||||
|
const headers: Record<string, string> = {};
|
||||||
|
const pairs = raw.headers.split(',');
|
||||||
|
for (const pair of pairs) {
|
||||||
|
const [key, value] = pair.split('=');
|
||||||
|
if (key && value) {
|
||||||
|
headers[key.trim()] = value.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
raw.headers = headers;
|
||||||
|
}
|
||||||
|
return OpenTelemetryConfigSchema.parse(raw);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const configLoader = new ConfigLoader('@sfiorini/llm-observability-mcp');
|
||||||
|
export default configLoader;
|
||||||
24
src/config/opentelemetry-llm.schema.ts
Normal file
24
src/config/opentelemetry-llm.schema.ts
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
import { z } from 'zod';
|
||||||
|
import { CommonConfigSchema } from './common.schema.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Configuration schema for OpenTelemetry
|
||||||
|
*/
|
||||||
|
export const OpenTelemetryConfigSchema = CommonConfigSchema.extend({
|
||||||
|
// OTLP endpoints
|
||||||
|
metricsEndpoint: z.string().optional(),
|
||||||
|
tracesEndpoint: z.string().optional(),
|
||||||
|
logsEndpoint: z.string().optional(),
|
||||||
|
|
||||||
|
// Authentication headers
|
||||||
|
headers: z.record(z.string()).optional(),
|
||||||
|
|
||||||
|
// Export configuration
|
||||||
|
exportIntervalMillis: z.number().default(10000),
|
||||||
|
exportTimeoutMillis: z.number().default(5000),
|
||||||
|
|
||||||
|
// Sampling configuration
|
||||||
|
samplingRatio: z.number().min(0).max(1).default(1.0),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type OpenTelemetryConfigType = z.infer<typeof OpenTelemetryConfigSchema>;
|
||||||
15
src/config/posthog.schema.ts
Normal file
15
src/config/posthog.schema.ts
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
import { z } from 'zod';
|
||||||
|
import { CommonConfigSchema } from './common.schema.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PostHog configuration options schema
|
||||||
|
*/
|
||||||
|
export const PostHogConfigSchema = CommonConfigSchema.extend({
|
||||||
|
apiKey: z.string().describe('PostHog API key'),
|
||||||
|
host: z
|
||||||
|
.string()
|
||||||
|
.default('https://app.posthog.com')
|
||||||
|
.describe('PostHog host URL'),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type PostHogConfigType = z.infer<typeof PostHogConfigSchema>;
|
||||||
8
src/controllers/base.controller.ts
Normal file
8
src/controllers/base.controller.ts
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
import { ControllerResponse } from '../types/common.types.js';
|
||||||
|
|
||||||
|
export abstract class BaseController {
|
||||||
|
// Instance method (must be implemented by subclasses)
|
||||||
|
static capture(data: unknown): Promise<ControllerResponse> {
|
||||||
|
throw new Error(`Not implemented: capture: ${data}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
65
src/controllers/opentelemetry-llm.controller.ts
Normal file
65
src/controllers/opentelemetry-llm.controller.ts
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
import { Logger } from '../utils/logger.util.js';
|
||||||
|
import { BaseController } from './base.controller.js';
|
||||||
|
import { OpenTelemetryService } from '../services/opentelemetry-llm.service.js';
|
||||||
|
import { OpenTelemetryLlmInputSchemaType } from '../types/opentelemetry-llm.types.js';
|
||||||
|
import { ControllerResponse } from '../types/common.types.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Controller for OpenTelemetry LLM observability
|
||||||
|
*/
|
||||||
|
export class OpenTelemetryController extends BaseController {
|
||||||
|
/**
|
||||||
|
* Capture LLM observability data
|
||||||
|
*/
|
||||||
|
static async capture(
|
||||||
|
data: OpenTelemetryLlmInputSchemaType,
|
||||||
|
): Promise<ControllerResponse> {
|
||||||
|
const logger = Logger.forContext(
|
||||||
|
'opentelemetry.controller',
|
||||||
|
'captureLlmObservability',
|
||||||
|
);
|
||||||
|
logger.debug('Capturing LLM observability data', data);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Get and initialize the service
|
||||||
|
const service = OpenTelemetryService.getInstance();
|
||||||
|
service.initialize();
|
||||||
|
|
||||||
|
// Record the LLM request
|
||||||
|
service.recordLlmRequest(data);
|
||||||
|
|
||||||
|
const content = `## OpenTelemetry LLM Observability Captured
|
||||||
|
|
||||||
|
Successfully recorded LLM observability data:
|
||||||
|
|
||||||
|
- **Model**: ${data.model}
|
||||||
|
- **Provider**: ${data.provider}
|
||||||
|
- **User ID**: ${data.userId}
|
||||||
|
- **Operation**: ${data.operationName || 'generate'}
|
||||||
|
- **Input Tokens**: ${data.inputTokens || 'N/A'}
|
||||||
|
- **Output Tokens**: ${data.outputTokens || 'N/A'}
|
||||||
|
- **Total Tokens**: ${(data.inputTokens || 0) + (data.outputTokens || 0)}
|
||||||
|
- **Latency**: ${data.latency ? `${data.latency}s` : 'N/A'}
|
||||||
|
- **Status**: ${data.error ? 'Error' : 'Success'}
|
||||||
|
${data.error ? `- **Error**: ${data.error}` : ''}
|
||||||
|
|
||||||
|
### Metrics Recorded
|
||||||
|
- ✅ Request counter incremented
|
||||||
|
- ✅ Token usage recorded
|
||||||
|
- ✅ Latency histogram updated
|
||||||
|
- ✅ Distributed trace created
|
||||||
|
|
||||||
|
### Trace Details
|
||||||
|
- **Trace ID**: ${data.traceId || 'auto-generated'}
|
||||||
|
- **Span Name**: ${data.operationName || 'llm.generate'}
|
||||||
|
- **Attributes**: model, provider, user_id, tokens, latency, error details
|
||||||
|
|
||||||
|
The data has been sent to your configured OpenTelemetry collector and is now available in your observability platform (Jaeger, New Relic, Grafana, etc.).`;
|
||||||
|
|
||||||
|
return { content };
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error capturing LLM observability', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,32 +1,35 @@
|
|||||||
import { Logger } from '../utils/logger.util.js';
|
import { Logger } from '../utils/logger.util.js';
|
||||||
import { ControllerResponse } from '../types/common.types.js';
|
import { ControllerResponse } from '../types/common.types.js';
|
||||||
import postHogLlmService from '../services/posthog-llm.service.js';
|
import postHogLlmService from '../services/posthog-llm.service.js';
|
||||||
|
import { PostHogLlmInputSchemaType } from '../types/posthog-llm.types.js';
|
||||||
|
|
||||||
const logger = Logger.forContext('controllers/posthog-llm.controller.ts');
|
/**
|
||||||
|
* Controller for PostHog LLM observability
|
||||||
|
*/
|
||||||
|
export class PostHogController {
|
||||||
|
/**
|
||||||
|
* Capture LLM observability data
|
||||||
|
*/
|
||||||
|
|
||||||
async function capture(args: {
|
static async capture(
|
||||||
eventName: string;
|
data: PostHogLlmInputSchemaType,
|
||||||
distinctId: string;
|
): Promise<ControllerResponse> {
|
||||||
properties: Record<string, unknown>;
|
const logger = Logger.forContext('posthog-llm.controller', 'capture');
|
||||||
}): Promise<ControllerResponse> {
|
logger.debug('Capturing LLM observability data', data);
|
||||||
const methodLogger = logger.forMethod('capture');
|
|
||||||
methodLogger.debug('Capturing PostHog event...');
|
|
||||||
methodLogger.debug('Arguments:', args);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await postHogLlmService.capture(args);
|
await postHogLlmService.capture(data);
|
||||||
return {
|
return {
|
||||||
content: 'Event captured successfully.',
|
content: 'Event captured successfully.',
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
methodLogger.error('Error capturing event:', error);
|
logger.error('Error capturing event:', error);
|
||||||
const errorMessage =
|
const errorMessage =
|
||||||
error instanceof Error ? error.message : 'Unknown error';
|
error instanceof Error ? error.message : 'Unknown error';
|
||||||
methodLogger.error('Error capturing event:', errorMessage);
|
logger.error('Error capturing event:', errorMessage);
|
||||||
return {
|
return {
|
||||||
content: `Failed to capture event: ${errorMessage}`,
|
content: `Failed to capture event: ${errorMessage}`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
export default { capture };
|
|
||||||
|
|||||||
14
src/index.ts
14
src/index.ts
@@ -1,7 +1,5 @@
|
|||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
import { Logger } from './utils/logger.util.js';
|
import { Logger } from './utils/logger.util.js';
|
||||||
import { config } from './utils/config.util.js';
|
|
||||||
import { runCli } from './cli/index.js';
|
|
||||||
import { stdioTransport } from './server/stdio.js';
|
import { stdioTransport } from './server/stdio.js';
|
||||||
import { streamableHttpTransport } from './server/streamableHttp.js';
|
import { streamableHttpTransport } from './server/streamableHttp.js';
|
||||||
|
|
||||||
@@ -43,18 +41,6 @@ export async function startServer(): Promise<void> {
|
|||||||
* Main entry point
|
* Main entry point
|
||||||
*/
|
*/
|
||||||
async function main() {
|
async function main() {
|
||||||
const mainLogger = Logger.forContext('index.ts', 'main');
|
|
||||||
|
|
||||||
// Load configuration
|
|
||||||
config.load();
|
|
||||||
|
|
||||||
// CLI mode - if any arguments are provided
|
|
||||||
if (process.argv.length > 2) {
|
|
||||||
mainLogger.info('CLI mode detected');
|
|
||||||
await runCli(process.argv.slice(2));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Server mode - determine transport and start server
|
// Server mode - determine transport and start server
|
||||||
await startServer();
|
await startServer();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,74 +0,0 @@
|
|||||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
||||||
import { Logger } from '../utils/logger.util.js';
|
|
||||||
import posthogLlmController from '../controllers/posthog-llm.controller.js';
|
|
||||||
import { formatErrorForMcpResource } from '../utils/error.util.js';
|
|
||||||
|
|
||||||
const logger = Logger.forContext('resources/posthog-llm.resource.ts');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Register PostHog LLM resources with the MCP server
|
|
||||||
*
|
|
||||||
* @param server The MCP server instance
|
|
||||||
*/
|
|
||||||
function registerResources(server: McpServer) {
|
|
||||||
const registerLogger = logger.forMethod('registerResources');
|
|
||||||
registerLogger.debug('Registering PostHog LLM resources...');
|
|
||||||
|
|
||||||
// Register the PostHog event capture resource
|
|
||||||
server.resource(
|
|
||||||
'capture_llm_observability',
|
|
||||||
'Capture a LLM event in PostHog',
|
|
||||||
async (uri: URL) => {
|
|
||||||
const methodLogger = logger.forMethod('posthogEventResource');
|
|
||||||
try {
|
|
||||||
methodLogger.debug('PostHog event resource called', {
|
|
||||||
uri: uri.toString(),
|
|
||||||
});
|
|
||||||
|
|
||||||
// Extract parameters from the URI
|
|
||||||
// Format: posthog://event_name/distinct_id?properties=JSON
|
|
||||||
const pathParts = uri.pathname.split('/').filter(Boolean);
|
|
||||||
const eventName = pathParts[0] || 'event';
|
|
||||||
const distinctId = pathParts[1] || 'anonymous';
|
|
||||||
|
|
||||||
// Parse properties from query parameters if present
|
|
||||||
let properties: Record<string, unknown> = {};
|
|
||||||
if (uri.searchParams.has('properties')) {
|
|
||||||
try {
|
|
||||||
properties = JSON.parse(
|
|
||||||
uri.searchParams.get('properties') || '{}',
|
|
||||||
);
|
|
||||||
} catch (e) {
|
|
||||||
methodLogger.warn('Failed to parse properties JSON', e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Call the controller to capture the event
|
|
||||||
const result = await posthogLlmController.capture({
|
|
||||||
eventName,
|
|
||||||
distinctId,
|
|
||||||
properties,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Return the content as a text resource
|
|
||||||
return {
|
|
||||||
contents: [
|
|
||||||
{
|
|
||||||
uri: uri.toString(),
|
|
||||||
text: result.content,
|
|
||||||
mimeType: 'text/plain',
|
|
||||||
description: `PostHog event: ${eventName}`,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
methodLogger.error('Resource error', error);
|
|
||||||
return formatErrorForMcpResource(error, uri.toString());
|
|
||||||
}
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
registerLogger.debug('PostHog LLM resources registered successfully');
|
|
||||||
}
|
|
||||||
|
|
||||||
export default { registerResources };
|
|
||||||
@@ -1,21 +1,21 @@
|
|||||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||||
import { SSEServerTransport } from '@modelcontextprotocol/sdk/server/sse.js';
|
import { SSEServerTransport } from '@modelcontextprotocol/sdk/server/sse.js';
|
||||||
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
||||||
|
|
||||||
import { Logger } from '../utils/logger.util';
|
import { Logger } from '../utils/logger.util';
|
||||||
import { config } from '../utils/config.util';
|
import configLoader from '../config/config-loader';
|
||||||
import { PACKAGE_NAME, VERSION } from '../utils/constants.util';
|
import { PACKAGE_NAME, VERSION } from '../utils/constants.util';
|
||||||
import posthogLlmResources from '../resources/posthog-llm.resource.js';
|
|
||||||
import posthogLlmTools from '../tools/posthog-llm.tool.js';
|
import posthogLlmTools from '../tools/posthog-llm.tool.js';
|
||||||
|
import openTelemetryTools from '../tools/opentelemetry-llm.tool.js';
|
||||||
|
|
||||||
export function createServer() {
|
export function createServer() {
|
||||||
const serverLogger = Logger.forContext('utils/server.util.ts', 'getServer');
|
const serverLogger = Logger.forContext('utils/server.util.ts', 'getServer');
|
||||||
|
|
||||||
// Load configuration
|
// Load configuration
|
||||||
serverLogger.info('Starting MCP server initialization...');
|
serverLogger.info('Starting MCP server initialization...');
|
||||||
config.load();
|
configLoader.load();
|
||||||
|
|
||||||
if (config.getBoolean('DEBUG')) {
|
const commonConfig = configLoader.getCommonConfig();
|
||||||
|
if (commonConfig.debug) {
|
||||||
serverLogger.debug('Debug mode enabled');
|
serverLogger.debug('Debug mode enabled');
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -25,11 +25,11 @@ export function createServer() {
|
|||||||
version: VERSION,
|
version: VERSION,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Register tools and resources
|
// Register tools
|
||||||
serverLogger.info('Registering MCP tools and resources...');
|
serverLogger.info('Registering MCP tools...');
|
||||||
posthogLlmTools.registerTools(server);
|
posthogLlmTools.registerTools(server);
|
||||||
posthogLlmResources.registerResources(server);
|
openTelemetryTools.registerTools(server);
|
||||||
serverLogger.debug('All tools and resources registered');
|
serverLogger.debug('All tools registered');
|
||||||
|
|
||||||
return server;
|
return server;
|
||||||
}
|
}
|
||||||
|
|||||||
290
src/services/opentelemetry-llm.service.ts
Normal file
290
src/services/opentelemetry-llm.service.ts
Normal file
@@ -0,0 +1,290 @@
|
|||||||
|
import {
|
||||||
|
MeterProvider,
|
||||||
|
PeriodicExportingMetricReader,
|
||||||
|
} from '@opentelemetry/sdk-metrics';
|
||||||
|
import { NodeSDK } from '@opentelemetry/sdk-node';
|
||||||
|
import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-node';
|
||||||
|
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
|
||||||
|
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
||||||
|
import { resourceFromAttributes } from '@opentelemetry/resources';
|
||||||
|
import {
|
||||||
|
ATTR_SERVICE_NAME,
|
||||||
|
ATTR_SERVICE_VERSION,
|
||||||
|
SEMRESATTRS_DEPLOYMENT_ENVIRONMENT,
|
||||||
|
} from '@opentelemetry/semantic-conventions';
|
||||||
|
import { Logger } from '../utils/logger.util.js';
|
||||||
|
import type { OpenTelemetryConfigType } from '../config/opentelemetry-llm.schema.js';
|
||||||
|
import { OpenTelemetryLlmInputSchemaType } from '../types/opentelemetry-llm.types.js';
|
||||||
|
import {
|
||||||
|
Meter,
|
||||||
|
Counter,
|
||||||
|
Histogram,
|
||||||
|
UpDownCounter,
|
||||||
|
metrics,
|
||||||
|
} from '@opentelemetry/api';
|
||||||
|
import { trace, Tracer, SpanStatusCode } from '@opentelemetry/api';
|
||||||
|
import configLoader from '../config/config-loader';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* OpenTelemetry service for LLM observability
|
||||||
|
*/
|
||||||
|
export class OpenTelemetryService {
|
||||||
|
private static instance: OpenTelemetryService;
|
||||||
|
private sdk: NodeSDK | null = null;
|
||||||
|
private meterProvider: MeterProvider | null = null;
|
||||||
|
private tracer: Tracer;
|
||||||
|
private meter: Meter;
|
||||||
|
private config: OpenTelemetryConfigType;
|
||||||
|
|
||||||
|
// Metrics
|
||||||
|
private requestCounter: Counter;
|
||||||
|
private tokenCounter: Counter;
|
||||||
|
private latencyHistogram: Histogram;
|
||||||
|
private activeRequests: UpDownCounter;
|
||||||
|
|
||||||
|
private constructor() {
|
||||||
|
this.config = configLoader.getOpenTelemetryConfig();
|
||||||
|
this.tracer = trace.getTracer('llm-observability-mcp');
|
||||||
|
this.meter = this.initializeMetrics();
|
||||||
|
|
||||||
|
// Initialize metrics
|
||||||
|
this.requestCounter = this.meter.createCounter('llm.requests.total', {
|
||||||
|
description: 'Total number of LLM requests',
|
||||||
|
});
|
||||||
|
|
||||||
|
this.tokenCounter = this.meter.createCounter('llm.tokens.total', {
|
||||||
|
description: 'Total number of tokens processed',
|
||||||
|
});
|
||||||
|
|
||||||
|
this.latencyHistogram = this.meter.createHistogram(
|
||||||
|
'llm.latency.duration',
|
||||||
|
{
|
||||||
|
description: 'Duration of LLM requests in milliseconds',
|
||||||
|
unit: 'ms',
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
this.activeRequests = this.meter.createUpDownCounter(
|
||||||
|
'llm.requests.active',
|
||||||
|
{
|
||||||
|
description: 'Number of active LLM requests',
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get singleton instance
|
||||||
|
*/
|
||||||
|
public static getInstance(): OpenTelemetryService {
|
||||||
|
if (!OpenTelemetryService.instance) {
|
||||||
|
OpenTelemetryService.instance = new OpenTelemetryService();
|
||||||
|
}
|
||||||
|
return OpenTelemetryService.instance;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize OpenTelemetry SDK
|
||||||
|
*/
|
||||||
|
public initialize(): void {
|
||||||
|
const logger = Logger.forContext('opentelemetry.service', 'initialize');
|
||||||
|
|
||||||
|
if (this.sdk) {
|
||||||
|
logger.warn('OpenTelemetry SDK already initialized');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const resource = resourceFromAttributes({
|
||||||
|
[ATTR_SERVICE_NAME]: this.config.serviceName,
|
||||||
|
[ATTR_SERVICE_VERSION]: this.config.serviceVersion,
|
||||||
|
[SEMRESATTRS_DEPLOYMENT_ENVIRONMENT]: this.config.environment,
|
||||||
|
debug: this.config.debug,
|
||||||
|
logLevel: this.config.logLevel,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Configure metric exporter
|
||||||
|
const metricExporter = this.config.metricsEndpoint
|
||||||
|
? new OTLPMetricExporter({
|
||||||
|
url: this.config.metricsEndpoint,
|
||||||
|
headers: this.config.headers,
|
||||||
|
})
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
// Configure trace exporter
|
||||||
|
const traceExporter = this.config.tracesEndpoint
|
||||||
|
? new OTLPTraceExporter({
|
||||||
|
url: this.config.tracesEndpoint,
|
||||||
|
headers: this.config.headers,
|
||||||
|
})
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
// Initialize meter provider
|
||||||
|
if (metricExporter) {
|
||||||
|
this.meterProvider = new MeterProvider({
|
||||||
|
resource,
|
||||||
|
readers: [
|
||||||
|
new PeriodicExportingMetricReader({
|
||||||
|
exporter: metricExporter,
|
||||||
|
exportIntervalMillis:
|
||||||
|
this.config.exportIntervalMillis,
|
||||||
|
exportTimeoutMillis:
|
||||||
|
this.config.exportTimeoutMillis,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize SDK
|
||||||
|
this.sdk = new NodeSDK({
|
||||||
|
resource,
|
||||||
|
spanProcessor: traceExporter
|
||||||
|
? new BatchSpanProcessor(traceExporter)
|
||||||
|
: undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
this.sdk.start();
|
||||||
|
logger.info('OpenTelemetry SDK initialized successfully');
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to initialize OpenTelemetry SDK', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize metrics
|
||||||
|
*/
|
||||||
|
private initializeMetrics(): Meter {
|
||||||
|
if (this.meterProvider) {
|
||||||
|
return this.meterProvider.getMeter('llm-observability-mcp');
|
||||||
|
}
|
||||||
|
return metrics.getMeter('llm-observability-mcp');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record LLM request metrics and traces
|
||||||
|
*/
|
||||||
|
public recordLlmRequest(data: OpenTelemetryLlmInputSchemaType): void {
|
||||||
|
const labels = {
|
||||||
|
model: data.model,
|
||||||
|
provider: data.provider,
|
||||||
|
userId: data.userId,
|
||||||
|
operationName: data.operationName || 'generate',
|
||||||
|
status: data.error ? 'error' : 'success',
|
||||||
|
errorType: data.errorType,
|
||||||
|
mcpToolsUsed: data.mcpToolsUsed?.join(',') || '',
|
||||||
|
};
|
||||||
|
|
||||||
|
// Record metrics
|
||||||
|
this.requestCounter.add(1, labels);
|
||||||
|
|
||||||
|
if (data.inputTokens || data.outputTokens) {
|
||||||
|
const totalTokens =
|
||||||
|
(data.inputTokens || 0) + (data.outputTokens || 0);
|
||||||
|
this.tokenCounter.add(totalTokens, {
|
||||||
|
...labels,
|
||||||
|
type: 'total',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (data.inputTokens) {
|
||||||
|
this.tokenCounter.add(data.inputTokens, {
|
||||||
|
...labels,
|
||||||
|
type: 'input',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data.outputTokens) {
|
||||||
|
this.tokenCounter.add(data.outputTokens, {
|
||||||
|
...labels,
|
||||||
|
type: 'output',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data.latency) {
|
||||||
|
this.latencyHistogram.record(data.latency * 1000, labels);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create trace
|
||||||
|
this.createLlmTrace(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create distributed trace for LLM request
|
||||||
|
*/
|
||||||
|
private createLlmTrace(data: OpenTelemetryLlmInputSchemaType): void {
|
||||||
|
const spanName = data.operationName || 'llm.generate';
|
||||||
|
const span = this.tracer.startSpan(spanName, {
|
||||||
|
attributes: {
|
||||||
|
'llm.model': data.model,
|
||||||
|
'llm.provider': data.provider,
|
||||||
|
'llm.user_id': data.userId,
|
||||||
|
'llm.operation': data.operationName || 'generate',
|
||||||
|
'llm.input_tokens': data.inputTokens,
|
||||||
|
'llm.output_tokens': data.outputTokens,
|
||||||
|
'llm.total_tokens':
|
||||||
|
(data.inputTokens || 0) + (data.outputTokens || 0),
|
||||||
|
'llm.latency_ms': data.latency
|
||||||
|
? data.latency * 1000
|
||||||
|
: undefined,
|
||||||
|
'llm.http_status': data.httpStatus,
|
||||||
|
'llm.base_url': data.baseUrl,
|
||||||
|
'llm.error': data.error,
|
||||||
|
'llm.error_type': data.errorType,
|
||||||
|
'llm.input':
|
||||||
|
typeof data.input === 'string'
|
||||||
|
? data.input
|
||||||
|
: JSON.stringify(data.input),
|
||||||
|
'llm.output':
|
||||||
|
typeof data.outputChoices === 'string'
|
||||||
|
? data.outputChoices
|
||||||
|
: JSON.stringify(data.outputChoices),
|
||||||
|
'llm.mcp_tools_used': data.mcpToolsUsed?.join(','),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (data.traceId) {
|
||||||
|
span.setAttribute('trace.id', data.traceId);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data.error) {
|
||||||
|
span.setStatus({
|
||||||
|
code: SpanStatusCode.ERROR,
|
||||||
|
message: data.error,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
span.setStatus({ code: SpanStatusCode.OK });
|
||||||
|
}
|
||||||
|
|
||||||
|
span.end();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record active request count
|
||||||
|
*/
|
||||||
|
public incrementActiveRequests(): void {
|
||||||
|
this.activeRequests.add(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decrement active request count
|
||||||
|
*/
|
||||||
|
public decrementActiveRequests(): void {
|
||||||
|
this.activeRequests.add(-1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shutdown OpenTelemetry SDK
|
||||||
|
*/
|
||||||
|
public async shutdown(): Promise<void> {
|
||||||
|
const logger = Logger.forContext('opentelemetry.service', 'shutdown');
|
||||||
|
|
||||||
|
if (this.sdk) {
|
||||||
|
try {
|
||||||
|
await this.sdk.shutdown();
|
||||||
|
logger.info('OpenTelemetry SDK shutdown successfully');
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error shutting down OpenTelemetry SDK', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,18 +1,18 @@
|
|||||||
import { Logger } from '../utils/logger.util.js';
|
import { Logger } from '../utils/logger.util';
|
||||||
import { PostHog } from 'posthog-node';
|
import { PostHog } from 'posthog-node';
|
||||||
import { config } from '../utils/config.util.js';
|
import configLoader from '../config/config-loader';
|
||||||
|
|
||||||
// Ensure configuration is loaded before accessing environment variables
|
// Ensure configuration is loaded before accessing environment variables
|
||||||
config.load();
|
configLoader.load();
|
||||||
|
|
||||||
const logger = Logger.forContext('services/posthog-llm.service.ts');
|
const logger = Logger.forContext('services/posthog-llm.service.ts');
|
||||||
|
|
||||||
const posthogApiKey = config.get('POSTHOG_API_KEY');
|
const posthogConfig = configLoader.getPosthogConfig();
|
||||||
let posthogClient: PostHog | null = null;
|
let posthogClient: PostHog | null = null;
|
||||||
|
|
||||||
if (posthogApiKey) {
|
if (posthogConfig.apiKey) {
|
||||||
posthogClient = new PostHog(posthogApiKey, {
|
posthogClient = new PostHog(posthogConfig.apiKey, {
|
||||||
host: config.get('POSTHOG_HOST') || 'https://app.posthog.com',
|
host: posthogConfig.host,
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
logger.warn('POSTHOG_API_KEY is not set. PostHog client not initialized.');
|
logger.warn('POSTHOG_API_KEY is not set. PostHog client not initialized.');
|
||||||
|
|||||||
77
src/tools/opentelemetry-llm.tool.ts
Normal file
77
src/tools/opentelemetry-llm.tool.ts
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||||
|
import { Logger } from '../utils/logger.util.js';
|
||||||
|
import { formatErrorForMcpTool } from '../utils/error.util.js';
|
||||||
|
import { OpenTelemetryController } from '../controllers/opentelemetry-llm.controller.js';
|
||||||
|
import { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
||||||
|
import {
|
||||||
|
OpenTelemetryLlmInputSchema,
|
||||||
|
OpenTelemetryLlmInputSchemaType,
|
||||||
|
} from '../types/opentelemetry-llm.types.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @function captureOpenTelemetryLlmObservability
|
||||||
|
* @description MCP Tool handler to capture LLM observability events using OpenTelemetry.
|
||||||
|
* It records metrics, traces, and spans for LLM requests that can be sent to any OpenTelemetry-compatible backend.
|
||||||
|
* @param {OpenTelemetryLlmInputSchemaType} args - Arguments provided to the tool.
|
||||||
|
* @returns {Promise<CallToolResult>} Formatted response for the MCP.
|
||||||
|
* @throws {McpError} Formatted error if the controller or service layer encounters an issue.
|
||||||
|
*/
|
||||||
|
async function captureOpenTelemetryLlmObservability(
|
||||||
|
args: OpenTelemetryLlmInputSchemaType,
|
||||||
|
): Promise<CallToolResult> {
|
||||||
|
const methodLogger = Logger.forContext(
|
||||||
|
'opentelemetry.tool',
|
||||||
|
'captureOpenTelemetryLlmObservability',
|
||||||
|
);
|
||||||
|
methodLogger.debug('Capture LLM Observability with OpenTelemetry...', args);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Parse and validate arguments
|
||||||
|
const validatedArgs = OpenTelemetryLlmInputSchema.parse(args);
|
||||||
|
|
||||||
|
// Pass validated args to the controller
|
||||||
|
const result = await OpenTelemetryController.capture(validatedArgs);
|
||||||
|
methodLogger.debug('Got response from controller', result);
|
||||||
|
|
||||||
|
// Format the response for the MCP tool
|
||||||
|
return {
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: 'text' as const,
|
||||||
|
text: result.content,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
methodLogger.error(
|
||||||
|
'Error tracking LLM generation with OpenTelemetry',
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
return formatErrorForMcpTool(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @function registerTools
|
||||||
|
* @description Registers the OpenTelemetry LLM observability tool with the MCP server.
|
||||||
|
*
|
||||||
|
* @param {McpServer} server - The MCP server instance.
|
||||||
|
*/
|
||||||
|
function registerTools(server: McpServer) {
|
||||||
|
const methodLogger = Logger.forContext(
|
||||||
|
'opentelemetry.tool',
|
||||||
|
'registerTools',
|
||||||
|
);
|
||||||
|
methodLogger.debug('Registering OpenTelemetry LLM observability tools...');
|
||||||
|
|
||||||
|
server.tool(
|
||||||
|
'llm_observability_otel',
|
||||||
|
`Captures LLM usage using OpenTelemetry for observability, including requests, responses, and performance metrics. Works with any OpenTelemetry-compatible backend like Jaeger, New Relic, Grafana, etc.`,
|
||||||
|
OpenTelemetryLlmInputSchema.shape,
|
||||||
|
captureOpenTelemetryLlmObservability,
|
||||||
|
);
|
||||||
|
|
||||||
|
methodLogger.debug('Successfully registered llm_observability_otel tool.');
|
||||||
|
}
|
||||||
|
|
||||||
|
export default { registerTools };
|
||||||
@@ -1,23 +1,25 @@
|
|||||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||||
import { Logger } from '../utils/logger.util.js';
|
import { Logger } from '../utils/logger.util.js';
|
||||||
import { formatErrorForMcpTool } from '../utils/error.util.js';
|
import { formatErrorForMcpTool } from '../utils/error.util.js';
|
||||||
import posthogLlmController from '../controllers/posthog-llm.controller.js';
|
import { PostHogController } from '../controllers/posthog-llm.controller.js';
|
||||||
import { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
import { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
||||||
import {
|
import {
|
||||||
GetToolInputSchema,
|
PostHogLlmInputSchemaType,
|
||||||
GetToolInputSchemaType,
|
PostHogLlmPropertiesApiSchemaType,
|
||||||
} from './posthog-llm.types.js';
|
PostHogLlmPropertiesPayloadSchema,
|
||||||
|
PostHogLlmPropertiesPayloadSchemaType,
|
||||||
|
} from '../types/posthog-llm.types.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @function capturePosthogLlmObservability
|
* @function capturePosthogLlmObservability
|
||||||
* @description MCP Tool handler to capture LLM observability events in PostHog.
|
* @description MCP Tool handler to capture LLM observability events in PostHog.
|
||||||
* It calls the posthogLlmController to track the data and formats the response for the MCP.
|
* It calls the posthogLlmController to track the data and formats the response for the MCP.
|
||||||
* @param {GetToolInputSchemaType} args - Arguments provided to the tool.
|
* @param {PostHogLlmInputSchemaType} args - Arguments provided to the tool.
|
||||||
* @returns {Promise<CallToolResult>} Formatted response for the MCP.
|
* @returns {Promise<CallToolResult>} Formatted response for the MCP.
|
||||||
* @throws {McpError} Formatted error if the controller or service layer encounters an issue.
|
* @throws {McpError} Formatted error if the controller or service layer encounters an issue.
|
||||||
*/
|
*/
|
||||||
async function capturePosthogLlmObservability(
|
async function capturePosthogLlmObservability(
|
||||||
args: GetToolInputSchemaType,
|
args: PostHogLlmPropertiesPayloadSchemaType,
|
||||||
): Promise<CallToolResult> {
|
): Promise<CallToolResult> {
|
||||||
const methodLogger = Logger.forContext(
|
const methodLogger = Logger.forContext(
|
||||||
'tools/posthog-llm.tool.ts',
|
'tools/posthog-llm.tool.ts',
|
||||||
@@ -26,15 +28,16 @@ async function capturePosthogLlmObservability(
|
|||||||
methodLogger.debug(`Capture LLM Observability in PostHog...`, args);
|
methodLogger.debug(`Capture LLM Observability in PostHog...`, args);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const trackArgs = GetToolInputSchema.parse(args);
|
const trackArgs = PostHogLlmPropertiesPayloadSchema.parse(args);
|
||||||
|
|
||||||
const posthogProperties: Record<string, unknown> = {
|
const posthogProperties: PostHogLlmPropertiesApiSchemaType = {
|
||||||
|
userId: trackArgs.userId,
|
||||||
$ai_model: trackArgs.model,
|
$ai_model: trackArgs.model,
|
||||||
$ai_provider: trackArgs.provider,
|
$ai_provider: trackArgs.provider,
|
||||||
};
|
};
|
||||||
|
|
||||||
const toPostHogKey: Partial<
|
const toPostHogKey: Partial<
|
||||||
Record<keyof GetToolInputSchemaType, string>
|
Record<keyof PostHogLlmPropertiesPayloadSchemaType, string>
|
||||||
> = {
|
> = {
|
||||||
input: '$ai_input',
|
input: '$ai_input',
|
||||||
outputChoices: '$ai_output_choices',
|
outputChoices: '$ai_output_choices',
|
||||||
@@ -47,22 +50,27 @@ async function capturePosthogLlmObservability(
|
|||||||
};
|
};
|
||||||
|
|
||||||
for (const key of Object.keys(toPostHogKey) as Array<
|
for (const key of Object.keys(toPostHogKey) as Array<
|
||||||
keyof GetToolInputSchemaType
|
keyof PostHogLlmPropertiesPayloadSchemaType
|
||||||
>) {
|
>) {
|
||||||
if (trackArgs[key] !== undefined) {
|
if (trackArgs[key] !== undefined) {
|
||||||
const posthogKey = toPostHogKey[key];
|
const posthogKey = toPostHogKey[key];
|
||||||
if (posthogKey) {
|
if (posthogKey) {
|
||||||
posthogProperties[posthogKey] = trackArgs[key];
|
// Type assertion to satisfy TS: posthogKey is a key of posthogProperties
|
||||||
|
(posthogProperties as Record<string, unknown>)[posthogKey] =
|
||||||
|
trackArgs[key];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Pass validated args to the controller
|
// validated input for the controller
|
||||||
const result = await posthogLlmController.capture({
|
const posthogInput: PostHogLlmInputSchemaType = {
|
||||||
eventName: '$ai_generation',
|
eventName: '$ai_generation',
|
||||||
distinctId: trackArgs.userId,
|
distinctId: trackArgs.userId,
|
||||||
properties: posthogProperties,
|
properties: posthogProperties,
|
||||||
});
|
};
|
||||||
|
|
||||||
|
// Pass validated args to the controller
|
||||||
|
const result = await PostHogController.capture(posthogInput);
|
||||||
methodLogger.debug(`Got the response from the controller`, result);
|
methodLogger.debug(`Got the response from the controller`, result);
|
||||||
|
|
||||||
// Format the response for the MCP tool
|
// Format the response for the MCP tool
|
||||||
@@ -82,7 +90,7 @@ async function capturePosthogLlmObservability(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* @function registerTools
|
* @function registerTools
|
||||||
* @description Registers the PostHog LLM observability tool ('capture_llm_observability') with the MCP server.
|
* @description Registers the PostHog LLM observability tool ('llm_observability_posthog') with the MCP server.
|
||||||
*
|
*
|
||||||
* @param {McpServer} server - The MCP server instance.
|
* @param {McpServer} server - The MCP server instance.
|
||||||
*/
|
*/
|
||||||
@@ -94,14 +102,14 @@ function registerTools(server: McpServer) {
|
|||||||
methodLogger.debug(`Registering PostHog LLM observability tools...`);
|
methodLogger.debug(`Registering PostHog LLM observability tools...`);
|
||||||
|
|
||||||
server.tool(
|
server.tool(
|
||||||
'capture_llm_observability',
|
'llm_observability_posthog',
|
||||||
`Captures LLM usage in PostHog for observability, including requests, responses, and performance metrics`,
|
`Captures LLM usage in PostHog for observability, including requests, responses, and performance metrics`,
|
||||||
GetToolInputSchema.shape,
|
PostHogLlmPropertiesPayloadSchema.shape,
|
||||||
capturePosthogLlmObservability,
|
capturePosthogLlmObservability,
|
||||||
);
|
);
|
||||||
|
|
||||||
methodLogger.debug(
|
methodLogger.debug(
|
||||||
'Successfully registered capture_llm_observability tool.',
|
'Successfully registered llm_observability_posthog tool.',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,41 +0,0 @@
|
|||||||
import { z } from 'zod';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Zod schema for the PostHog LLM observability tool arguments.
|
|
||||||
*/
|
|
||||||
export const GetToolInputSchema = z.object({
|
|
||||||
userId: z.string().describe('The distinct ID of the user'),
|
|
||||||
traceId: z.string().optional().describe('The trace ID to group AI events'),
|
|
||||||
model: z.string().describe('The model used (e.g., gpt-4, claude-3, etc.)'),
|
|
||||||
provider: z
|
|
||||||
.string()
|
|
||||||
.describe('The LLM provider (e.g., openai, anthropic, etc.)'),
|
|
||||||
input: z
|
|
||||||
.any()
|
|
||||||
.optional()
|
|
||||||
.describe('The input to the LLM (messages, prompt, etc.)'),
|
|
||||||
outputChoices: z.any().optional().describe('The output from the LLM'),
|
|
||||||
inputTokens: z
|
|
||||||
.number()
|
|
||||||
.optional()
|
|
||||||
.describe('The number of tokens in the input'),
|
|
||||||
outputTokens: z
|
|
||||||
.number()
|
|
||||||
.optional()
|
|
||||||
.describe('The number of tokens in the output'),
|
|
||||||
latency: z
|
|
||||||
.number()
|
|
||||||
.optional()
|
|
||||||
.describe('The latency of the LLM call in seconds'),
|
|
||||||
httpStatus: z
|
|
||||||
.number()
|
|
||||||
.optional()
|
|
||||||
.describe('The HTTP status code of the LLM call'),
|
|
||||||
baseUrl: z.string().optional().describe('The base URL of the LLM API'),
|
|
||||||
});
|
|
||||||
|
|
||||||
/**
|
|
||||||
* TypeScript type inferred from the GetToolInputSchema Zod schema.
|
|
||||||
* This represents the optional arguments passed to the tool handler and controller.
|
|
||||||
*/
|
|
||||||
export type GetToolInputSchemaType = z.infer<typeof GetToolInputSchema>;
|
|
||||||
168
src/types/opentelemetry-llm.types.ts
Normal file
168
src/types/opentelemetry-llm.types.ts
Normal file
@@ -0,0 +1,168 @@
|
|||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Zod schema for the OpenTelemetry LLM observability tool arguments
|
||||||
|
*/
|
||||||
|
export const OpenTelemetryLlmInputSchema = z.object({
|
||||||
|
// Required fields
|
||||||
|
userId: z.string().describe('The distinct ID of the user'),
|
||||||
|
model: z.string().describe('The model used (e.g., gpt-4, claude-3, etc.)'),
|
||||||
|
provider: z
|
||||||
|
.string()
|
||||||
|
.describe('The LLM provider (e.g., openai, anthropic, etc.)'),
|
||||||
|
|
||||||
|
// Optional fields
|
||||||
|
traceId: z.string().optional().describe('The trace ID to group AI events'),
|
||||||
|
input: z
|
||||||
|
.any()
|
||||||
|
.optional()
|
||||||
|
.describe('The input to the LLM (messages, prompt, etc.)'),
|
||||||
|
outputChoices: z.any().optional().describe('The output from the LLM'),
|
||||||
|
inputTokens: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The number of tokens in the input'),
|
||||||
|
outputTokens: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The number of tokens in the output'),
|
||||||
|
latency: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The latency of the LLM call in seconds'),
|
||||||
|
httpStatus: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The HTTP status code of the LLM call'),
|
||||||
|
baseUrl: z.string().optional().describe('The base URL of the LLM API'),
|
||||||
|
|
||||||
|
// OpenTelemetry specific fields
|
||||||
|
operationName: z
|
||||||
|
.string()
|
||||||
|
.optional()
|
||||||
|
.describe('The name of the operation being performed'),
|
||||||
|
error: z
|
||||||
|
.string()
|
||||||
|
.optional()
|
||||||
|
.describe('Error message if the request failed'),
|
||||||
|
errorType: z
|
||||||
|
.string()
|
||||||
|
.optional()
|
||||||
|
.describe('Type of error (e.g., rate_limit, timeout, etc.)'),
|
||||||
|
mcpToolsUsed: z
|
||||||
|
.array(z.string())
|
||||||
|
.optional()
|
||||||
|
.describe('List of MCP tools used during the request'),
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TypeScript type inferred from the OpenTelemetryLlmInputSchema Zod schema
|
||||||
|
*/
|
||||||
|
export type OpenTelemetryLlmInputSchemaType = z.infer<
|
||||||
|
typeof OpenTelemetryLlmInputSchema
|
||||||
|
>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Configuration for OpenTelemetry exporters
|
||||||
|
*/
|
||||||
|
export interface OpenTelemetryExporterConfig {
|
||||||
|
url: string;
|
||||||
|
headers?: Record<string, string>;
|
||||||
|
timeout?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Supported OpenTelemetry backends
|
||||||
|
*/
|
||||||
|
export type OpenTelemetryBackend =
|
||||||
|
| 'jaeger'
|
||||||
|
| 'newrelic'
|
||||||
|
| 'grafana'
|
||||||
|
| 'datadog'
|
||||||
|
| 'honeycomb'
|
||||||
|
| 'lightstep'
|
||||||
|
| 'custom';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pre-configured settings for popular OpenTelemetry backends
|
||||||
|
*/
|
||||||
|
export const OpenTelemetryBackendConfigs: Record<
|
||||||
|
OpenTelemetryBackend,
|
||||||
|
Partial<OpenTelemetryExporterConfig>
|
||||||
|
> = {
|
||||||
|
jaeger: {
|
||||||
|
url: 'http://localhost:4318/v1/traces',
|
||||||
|
headers: {},
|
||||||
|
},
|
||||||
|
newrelic: {
|
||||||
|
url: 'https://otlp.nr-data.net:4318/v1/traces',
|
||||||
|
headers: {
|
||||||
|
'api-key': process.env.NEW_RELIC_LICENSE_KEY || '',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
grafana: {
|
||||||
|
url: 'https://otlp-gateway-prod-us-central-0.grafana.net/otlp/v1/traces',
|
||||||
|
headers: {
|
||||||
|
Authorization:
|
||||||
|
process.env.GRAFANA_INSTANCE_ID && process.env.GRAFANA_API_KEY
|
||||||
|
? `Basic ${Buffer.from(
|
||||||
|
`${process.env.GRAFANA_INSTANCE_ID}:${process.env.GRAFANA_API_KEY}`,
|
||||||
|
).toString('base64')}`
|
||||||
|
: '',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
datadog: {
|
||||||
|
url: 'https://api.datadoghq.com/api/v2/series',
|
||||||
|
headers: {
|
||||||
|
'DD-API-KEY': process.env.DD_API_KEY || '',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
honeycomb: {
|
||||||
|
url: 'https://api.honeycomb.io/v1/traces',
|
||||||
|
headers: {
|
||||||
|
'x-honeycomb-team': process.env.HONEYCOMB_API_KEY || '',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
lightstep: {
|
||||||
|
url: 'https://ingest.lightstep.com:443/api/v2/otel/trace',
|
||||||
|
headers: {
|
||||||
|
'lightstep-access-token': process.env.LIGHTSTEP_ACCESS_TOKEN || '',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
custom: {
|
||||||
|
url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318',
|
||||||
|
headers: {},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Metric names used by the OpenTelemetry service
|
||||||
|
*/
|
||||||
|
export const OpenTelemetryMetrics = {
|
||||||
|
REQUESTS_TOTAL: 'llm.requests.total',
|
||||||
|
TOKENS_TOTAL: 'llm.tokens.total',
|
||||||
|
LATENCY_DURATION: 'llm.latency.duration',
|
||||||
|
REQUESTS_ACTIVE: 'llm.requests.active',
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Trace attribute names used by the OpenTelemetry service
|
||||||
|
*/
|
||||||
|
export const OpenTelemetryAttributes = {
|
||||||
|
LLM_MODEL: 'llm.model',
|
||||||
|
LLM_PROVIDER: 'llm.provider',
|
||||||
|
LLM_USER_ID: 'llm.user_id',
|
||||||
|
LLM_OPERATION: 'llm.operation',
|
||||||
|
LLM_INPUT_TOKENS: 'llm.input_tokens',
|
||||||
|
LLM_OUTPUT_TOKENS: 'llm.output_tokens',
|
||||||
|
LLM_TOTAL_TOKENS: 'llm.total_tokens',
|
||||||
|
LLM_LATENCY_MS: 'llm.latency_ms',
|
||||||
|
LLM_HTTP_STATUS: 'llm.http_status',
|
||||||
|
LLM_BASE_URL: 'llm.base_url',
|
||||||
|
LLM_ERROR: 'llm.error',
|
||||||
|
LLM_ERROR_TYPE: 'llm.error_type',
|
||||||
|
LLM_INPUT: 'llm.input',
|
||||||
|
LLM_OUTPUT: 'llm.output',
|
||||||
|
LLM_MCP_TOOLS_USED: 'llm.mcp_tools_used',
|
||||||
|
TRACE_ID: 'trace.id',
|
||||||
|
} as const;
|
||||||
93
src/types/posthog-llm.types.ts
Normal file
93
src/types/posthog-llm.types.ts
Normal file
@@ -0,0 +1,93 @@
|
|||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Zod schema for the PostHog LLM observability tool properties (payload).
|
||||||
|
*/
|
||||||
|
export const PostHogLlmPropertiesPayloadSchema = z.object({
|
||||||
|
userId: z.string().describe('The distinct ID of the user'),
|
||||||
|
traceId: z.string().optional().describe('The trace ID to group AI events'),
|
||||||
|
model: z.string().describe('The model used (e.g., gpt-4, claude-3, etc.)'),
|
||||||
|
provider: z
|
||||||
|
.string()
|
||||||
|
.describe('The LLM provider (e.g., openai, anthropic, etc.)'),
|
||||||
|
input: z
|
||||||
|
.any()
|
||||||
|
.optional()
|
||||||
|
.describe('The input to the LLM (messages, prompt, etc.)'),
|
||||||
|
outputChoices: z.any().optional().describe('The output from the LLM'),
|
||||||
|
inputTokens: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The number of tokens in the input'),
|
||||||
|
outputTokens: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The number of tokens in the output'),
|
||||||
|
latency: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The latency of the LLM call in seconds'),
|
||||||
|
httpStatus: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The HTTP status code of the LLM call'),
|
||||||
|
baseUrl: z.string().optional().describe('The base URL of the LLM API'),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type PostHogLlmPropertiesPayloadSchemaType = z.infer<
|
||||||
|
typeof PostHogLlmPropertiesPayloadSchema
|
||||||
|
>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Zod schema for the PostHog LLM observability tool properties (api payload).
|
||||||
|
*/
|
||||||
|
export const PostHogLlmPropertiesApiSchema = z.object({
|
||||||
|
userId: z.string().describe('The distinct ID of the user'),
|
||||||
|
$ai_trace_id: z
|
||||||
|
.string()
|
||||||
|
.optional()
|
||||||
|
.describe('The trace ID to group AI events'),
|
||||||
|
$ai_model: z
|
||||||
|
.string()
|
||||||
|
.describe('The model used (e.g., gpt-4, claude-3, etc.)'),
|
||||||
|
$ai_provider: z
|
||||||
|
.string()
|
||||||
|
.describe('The LLM provider (e.g., openai, anthropic, etc.)'),
|
||||||
|
$ai_input: z
|
||||||
|
.any()
|
||||||
|
.optional()
|
||||||
|
.describe('The input to the LLM (messages, prompt, etc.)'),
|
||||||
|
$ai_output_choices: z.any().optional().describe('The output from the LLM'),
|
||||||
|
$ai_input_tokens: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The number of tokens in the input'),
|
||||||
|
$ai_output_tokens: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The number of tokens in the output'),
|
||||||
|
$ai_latency: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The latency of the LLM call in seconds'),
|
||||||
|
$ai_http_status: z
|
||||||
|
.number()
|
||||||
|
.optional()
|
||||||
|
.describe('The HTTP status code of the LLM call'),
|
||||||
|
$ai_base_url: z.string().optional().describe('The base URL of the LLM API'),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type PostHogLlmPropertiesApiSchemaType = z.infer<
|
||||||
|
typeof PostHogLlmPropertiesApiSchema
|
||||||
|
>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Zod schema for the PostHog LLM observability tool arguments (top-level).
|
||||||
|
*/
|
||||||
|
export const PostHogLlmInputSchema = z.object({
|
||||||
|
eventName: z.string().describe('The name of the event to capture.'),
|
||||||
|
distinctId: z.string().describe('The distinct ID of the user.'),
|
||||||
|
properties: PostHogLlmPropertiesApiSchema.describe('The event properties.'),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type PostHogLlmInputSchemaType = z.infer<typeof PostHogLlmInputSchema>;
|
||||||
@@ -1,157 +0,0 @@
|
|||||||
import { spawn } from 'child_process';
|
|
||||||
import { join } from 'path';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Utility for testing CLI commands with real execution
|
|
||||||
*/
|
|
||||||
export class CliTestUtil {
|
|
||||||
/**
|
|
||||||
* Executes a CLI command and returns the result
|
|
||||||
*
|
|
||||||
* @param args - CLI arguments to pass to the command
|
|
||||||
* @param options - Test options
|
|
||||||
* @returns Promise with stdout, stderr, and exit code
|
|
||||||
*/
|
|
||||||
static async runCommand(
|
|
||||||
args: string[],
|
|
||||||
options: {
|
|
||||||
timeoutMs?: number;
|
|
||||||
env?: Record<string, string>;
|
|
||||||
} = {},
|
|
||||||
): Promise<{
|
|
||||||
stdout: string;
|
|
||||||
stderr: string;
|
|
||||||
exitCode: number;
|
|
||||||
}> {
|
|
||||||
// Default timeout of 30 seconds
|
|
||||||
const timeoutMs = options.timeoutMs || 30000;
|
|
||||||
|
|
||||||
// CLI execution path - points to the built CLI script
|
|
||||||
const cliPath = join(process.cwd(), 'dist', 'index.js');
|
|
||||||
|
|
||||||
// Log what command we're about to run
|
|
||||||
console.log(`Running CLI command: node ${cliPath} ${args.join(' ')}`);
|
|
||||||
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
// Set up timeout handler
|
|
||||||
const timeoutId = setTimeout(() => {
|
|
||||||
child.kill();
|
|
||||||
reject(new Error(`CLI command timed out after ${timeoutMs}ms`));
|
|
||||||
}, timeoutMs);
|
|
||||||
|
|
||||||
// Capture stdout and stderr
|
|
||||||
let stdout = '';
|
|
||||||
let stderr = '';
|
|
||||||
|
|
||||||
// Spawn the process with given arguments and enhanced environment
|
|
||||||
const child = spawn('node', [cliPath, ...args], {
|
|
||||||
env: {
|
|
||||||
...process.env,
|
|
||||||
...options.env,
|
|
||||||
DEBUG: 'true', // Enable debug logging
|
|
||||||
NODE_ENV: 'test', // Ensure tests are detected
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
// Collect stdout data
|
|
||||||
child.stdout.on('data', (data) => {
|
|
||||||
const chunk = data.toString();
|
|
||||||
stdout += chunk;
|
|
||||||
console.log(`STDOUT chunk: ${chunk.substring(0, 50)}...`);
|
|
||||||
});
|
|
||||||
|
|
||||||
// Collect stderr data
|
|
||||||
child.stderr.on('data', (data) => {
|
|
||||||
const chunk = data.toString();
|
|
||||||
stderr += chunk;
|
|
||||||
console.log(`STDERR chunk: ${chunk.substring(0, 50)}...`);
|
|
||||||
});
|
|
||||||
|
|
||||||
// Handle process completion
|
|
||||||
child.on('close', (exitCode) => {
|
|
||||||
clearTimeout(timeoutId);
|
|
||||||
console.log(`Command completed with exit code: ${exitCode}`);
|
|
||||||
console.log(`Total STDOUT length: ${stdout.length} chars`);
|
|
||||||
|
|
||||||
// Get the non-debug output for debugging purposes
|
|
||||||
const nonDebugOutput = stdout
|
|
||||||
.split('\n')
|
|
||||||
.filter((line) => !line.match(/^\[\d{2}:\d{2}:\d{2}\]/))
|
|
||||||
.join('\n');
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
`Non-debug output length: ${nonDebugOutput.length} chars`,
|
|
||||||
);
|
|
||||||
console.log(`STDOUT excerpt: ${stdout.substring(0, 100)}...`);
|
|
||||||
console.log(
|
|
||||||
`Filtered excerpt: ${nonDebugOutput.substring(0, 100)}...`,
|
|
||||||
);
|
|
||||||
|
|
||||||
resolve({
|
|
||||||
stdout,
|
|
||||||
stderr,
|
|
||||||
exitCode: exitCode ?? 0,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// Handle process errors
|
|
||||||
child.on('error', (err) => {
|
|
||||||
clearTimeout(timeoutId);
|
|
||||||
console.error(`Command error: ${err.message}`);
|
|
||||||
reject(err);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validates that stdout contains expected strings/patterns
|
|
||||||
*/
|
|
||||||
static validateOutputContains(
|
|
||||||
output: string,
|
|
||||||
expectedPatterns: (string | RegExp)[],
|
|
||||||
): void {
|
|
||||||
// Filter out debug log lines for cleaner validation
|
|
||||||
const cleanOutput = output
|
|
||||||
.split('\n')
|
|
||||||
.filter((line) => !line.match(/^\[\d{2}:\d{2}:\d{2}\]/))
|
|
||||||
.join('\n');
|
|
||||||
|
|
||||||
console.log('==== Cleaned output for validation ====');
|
|
||||||
console.log(cleanOutput);
|
|
||||||
console.log('=======================================');
|
|
||||||
|
|
||||||
for (const pattern of expectedPatterns) {
|
|
||||||
if (typeof pattern === 'string') {
|
|
||||||
expect(cleanOutput).toContain(pattern);
|
|
||||||
} else {
|
|
||||||
expect(cleanOutput).toMatch(pattern);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validates Markdown output format
|
|
||||||
*/
|
|
||||||
static validateMarkdownOutput(output: string): void {
|
|
||||||
// Filter out debug log lines for cleaner validation
|
|
||||||
const cleanOutput = output
|
|
||||||
.split('\n')
|
|
||||||
.filter((line) => !line.match(/^\[\d{2}:\d{2}:\d{2}\]/))
|
|
||||||
.join('\n');
|
|
||||||
|
|
||||||
// Check for Markdown heading
|
|
||||||
expect(cleanOutput).toMatch(/^#\s.+/m);
|
|
||||||
|
|
||||||
// Check for markdown formatting elements like bold text, lists, etc.
|
|
||||||
const markdownElements = [
|
|
||||||
/\*\*.+\*\*/, // Bold text
|
|
||||||
/-\s.+/, // List items
|
|
||||||
/\|.+\|.+\|/, // Table rows
|
|
||||||
/\[.+\]\(.+\)/, // Links
|
|
||||||
];
|
|
||||||
|
|
||||||
expect(
|
|
||||||
markdownElements.some((pattern) => pattern.test(cleanOutput)),
|
|
||||||
).toBe(true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,131 +0,0 @@
|
|||||||
import {
|
|
||||||
ErrorType,
|
|
||||||
McpError,
|
|
||||||
createApiError,
|
|
||||||
createAuthMissingError,
|
|
||||||
createAuthInvalidError,
|
|
||||||
createUnexpectedError,
|
|
||||||
ensureMcpError,
|
|
||||||
formatErrorForMcpTool,
|
|
||||||
formatErrorForMcpResource,
|
|
||||||
} from './error.util.js';
|
|
||||||
|
|
||||||
describe('Error Utility', () => {
|
|
||||||
describe('McpError', () => {
|
|
||||||
it('should create an error with the correct properties', () => {
|
|
||||||
const error = new McpError('Test error', ErrorType.API_ERROR, 404);
|
|
||||||
|
|
||||||
expect(error).toBeInstanceOf(Error);
|
|
||||||
expect(error).toBeInstanceOf(McpError);
|
|
||||||
expect(error.message).toBe('Test error');
|
|
||||||
expect(error.type).toBe(ErrorType.API_ERROR);
|
|
||||||
expect(error.statusCode).toBe(404);
|
|
||||||
expect(error.name).toBe('McpError');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('Error Factory Functions', () => {
|
|
||||||
it('should create auth missing error', () => {
|
|
||||||
const error = createAuthMissingError();
|
|
||||||
|
|
||||||
expect(error).toBeInstanceOf(McpError);
|
|
||||||
expect(error.type).toBe(ErrorType.AUTH_MISSING);
|
|
||||||
expect(error.message).toBe(
|
|
||||||
'Authentication credentials are missing',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should create auth invalid error', () => {
|
|
||||||
const error = createAuthInvalidError('Invalid token');
|
|
||||||
|
|
||||||
expect(error).toBeInstanceOf(McpError);
|
|
||||||
expect(error.type).toBe(ErrorType.AUTH_INVALID);
|
|
||||||
expect(error.statusCode).toBe(401);
|
|
||||||
expect(error.message).toBe('Invalid token');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should create API error', () => {
|
|
||||||
const originalError = new Error('Original error');
|
|
||||||
const error = createApiError('API failed', 500, originalError);
|
|
||||||
|
|
||||||
expect(error).toBeInstanceOf(McpError);
|
|
||||||
expect(error.type).toBe(ErrorType.API_ERROR);
|
|
||||||
expect(error.statusCode).toBe(500);
|
|
||||||
expect(error.message).toBe('API failed');
|
|
||||||
expect(error.originalError).toBe(originalError);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should create unexpected error', () => {
|
|
||||||
const error = createUnexpectedError();
|
|
||||||
|
|
||||||
expect(error).toBeInstanceOf(McpError);
|
|
||||||
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
|
||||||
expect(error.message).toBe('An unexpected error occurred');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('ensureMcpError', () => {
|
|
||||||
it('should return the same error if it is already an McpError', () => {
|
|
||||||
const originalError = createApiError('Original error');
|
|
||||||
const error = ensureMcpError(originalError);
|
|
||||||
|
|
||||||
expect(error).toBe(originalError);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should wrap a standard Error', () => {
|
|
||||||
const originalError = new Error('Standard error');
|
|
||||||
const error = ensureMcpError(originalError);
|
|
||||||
|
|
||||||
expect(error).toBeInstanceOf(McpError);
|
|
||||||
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
|
||||||
expect(error.message).toBe('Standard error');
|
|
||||||
expect(error.originalError).toBe(originalError);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should handle non-Error objects', () => {
|
|
||||||
const error = ensureMcpError('String error');
|
|
||||||
|
|
||||||
expect(error).toBeInstanceOf(McpError);
|
|
||||||
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
|
||||||
expect(error.message).toBe('String error');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('formatErrorForMcpTool', () => {
|
|
||||||
it('should format an error for MCP tool response', () => {
|
|
||||||
const error = createApiError('API error');
|
|
||||||
const response = formatErrorForMcpTool(error);
|
|
||||||
|
|
||||||
expect(response).toHaveProperty('content');
|
|
||||||
expect(response.content).toHaveLength(1);
|
|
||||||
expect(response.content[0]).toHaveProperty('type', 'text');
|
|
||||||
expect(response.content[0]).toHaveProperty(
|
|
||||||
'text',
|
|
||||||
'Error: API error',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('formatErrorForMcpResource', () => {
|
|
||||||
it('should format an error for MCP resource response', () => {
|
|
||||||
const error = createApiError('API error');
|
|
||||||
const response = formatErrorForMcpResource(error, 'test://uri');
|
|
||||||
|
|
||||||
expect(response).toHaveProperty('contents');
|
|
||||||
expect(response.contents).toHaveLength(1);
|
|
||||||
expect(response.contents[0]).toHaveProperty('uri', 'test://uri');
|
|
||||||
expect(response.contents[0]).toHaveProperty(
|
|
||||||
'text',
|
|
||||||
'Error: API error',
|
|
||||||
);
|
|
||||||
expect(response.contents[0]).toHaveProperty(
|
|
||||||
'mimeType',
|
|
||||||
'text/plain',
|
|
||||||
);
|
|
||||||
expect(response.contents[0]).toHaveProperty(
|
|
||||||
'description',
|
|
||||||
'Error: API_ERROR',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,173 +0,0 @@
|
|||||||
import fs from 'fs';
|
|
||||||
import path from 'path';
|
|
||||||
import { Logger } from './logger.util.js';
|
|
||||||
import dotenv from 'dotenv';
|
|
||||||
import os from 'os';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configuration loader that handles multiple sources with priority:
|
|
||||||
* 1. Direct ENV pass (process.env)
|
|
||||||
* 2. .env file in project root
|
|
||||||
* 3. Global config file at $HOME/.mcp/configs.json
|
|
||||||
*/
|
|
||||||
class ConfigLoader {
|
|
||||||
private packageName: string;
|
|
||||||
private configLoaded: boolean = false;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new ConfigLoader instance
|
|
||||||
* @param packageName The package name to use for global config lookup
|
|
||||||
*/
|
|
||||||
constructor(packageName: string) {
|
|
||||||
this.packageName = packageName;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load configuration from all sources with proper priority
|
|
||||||
*/
|
|
||||||
load(): void {
|
|
||||||
const methodLogger = Logger.forContext('utils/config.util.ts', 'load');
|
|
||||||
|
|
||||||
if (this.configLoaded) {
|
|
||||||
methodLogger.debug('Configuration already loaded, skipping');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
methodLogger.debug('Loading configuration...');
|
|
||||||
|
|
||||||
// Priority 3: Load from global config file
|
|
||||||
this.loadFromGlobalConfig();
|
|
||||||
|
|
||||||
// Priority 2: Load from .env file
|
|
||||||
this.loadFromEnvFile();
|
|
||||||
|
|
||||||
// Priority 1: Direct ENV pass is already in process.env
|
|
||||||
// No need to do anything as it already has highest priority
|
|
||||||
|
|
||||||
this.configLoaded = true;
|
|
||||||
methodLogger.debug('Configuration loaded successfully');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load configuration from .env file in project root
|
|
||||||
*/
|
|
||||||
private loadFromEnvFile(): void {
|
|
||||||
const methodLogger = Logger.forContext(
|
|
||||||
'utils/config.util.ts',
|
|
||||||
'loadFromEnvFile',
|
|
||||||
);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const result = dotenv.config();
|
|
||||||
if (result.error) {
|
|
||||||
methodLogger.debug('No .env file found or error reading it');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
methodLogger.debug('Loaded configuration from .env file');
|
|
||||||
} catch (error) {
|
|
||||||
methodLogger.error('Error loading .env file', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load configuration from global config file at $HOME/.mcp/configs.json
|
|
||||||
*/
|
|
||||||
private loadFromGlobalConfig(): void {
|
|
||||||
const methodLogger = Logger.forContext(
|
|
||||||
'utils/config.util.ts',
|
|
||||||
'loadFromGlobalConfig',
|
|
||||||
);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const homedir = os.homedir();
|
|
||||||
const globalConfigPath = path.join(homedir, '.mcp', 'configs.json');
|
|
||||||
|
|
||||||
if (!fs.existsSync(globalConfigPath)) {
|
|
||||||
methodLogger.debug('Global config file not found');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const configContent = fs.readFileSync(globalConfigPath, 'utf8');
|
|
||||||
const config = JSON.parse(configContent);
|
|
||||||
|
|
||||||
// Determine the potential keys for the current package
|
|
||||||
const shortKey = 'llm-observability-mcp'; // Project-specific short key
|
|
||||||
const fullPackageName = this.packageName; // e.g., '@sfiorini/llm-observability-mcp'
|
|
||||||
const unscopedPackageName =
|
|
||||||
fullPackageName.split('/')[1] || fullPackageName; // e.g., 'llm-observability-mcp'
|
|
||||||
|
|
||||||
const potentialKeys = [
|
|
||||||
shortKey,
|
|
||||||
fullPackageName,
|
|
||||||
unscopedPackageName,
|
|
||||||
];
|
|
||||||
let foundConfigSection: {
|
|
||||||
environments?: Record<string, unknown>;
|
|
||||||
} | null = null;
|
|
||||||
let usedKey: string | null = null;
|
|
||||||
|
|
||||||
for (const key of potentialKeys) {
|
|
||||||
if (
|
|
||||||
config[key] &&
|
|
||||||
typeof config[key] === 'object' &&
|
|
||||||
config[key].environments
|
|
||||||
) {
|
|
||||||
foundConfigSection = config[key];
|
|
||||||
usedKey = key;
|
|
||||||
methodLogger.debug(`Found configuration using key: ${key}`);
|
|
||||||
break; // Stop once found
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!foundConfigSection || !foundConfigSection.environments) {
|
|
||||||
methodLogger.debug(
|
|
||||||
`No configuration found for ${
|
|
||||||
this.packageName
|
|
||||||
} using keys: ${potentialKeys.join(', ')}`,
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const environments = foundConfigSection.environments;
|
|
||||||
for (const [key, value] of Object.entries(environments)) {
|
|
||||||
// Only set if not already defined in process.env
|
|
||||||
if (process.env[key] === undefined) {
|
|
||||||
process.env[key] = String(value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
methodLogger.debug(
|
|
||||||
`Loaded configuration from global config file using key: ${usedKey}`,
|
|
||||||
);
|
|
||||||
} catch (error) {
|
|
||||||
methodLogger.error('Error loading global config file', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get a configuration value
|
|
||||||
* @param key The configuration key
|
|
||||||
* @param defaultValue The default value if the key is not found
|
|
||||||
* @returns The configuration value or the default value
|
|
||||||
*/
|
|
||||||
get(key: string, defaultValue?: string): string | undefined {
|
|
||||||
return process.env[key] || defaultValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get a boolean configuration value
|
|
||||||
* @param key The configuration key
|
|
||||||
* @param defaultValue The default value if the key is not found
|
|
||||||
* @returns The boolean configuration value or the default value
|
|
||||||
*/
|
|
||||||
getBoolean(key: string, defaultValue: boolean = false): boolean {
|
|
||||||
const value = this.get(key);
|
|
||||||
if (value === undefined) {
|
|
||||||
return defaultValue;
|
|
||||||
}
|
|
||||||
return value.toLowerCase() === 'true';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create and export a singleton instance with the package name from package.json
|
|
||||||
export const config = new ConfigLoader('@sfiorini/llm-observability-mcp');
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
import {
|
|
||||||
ErrorCode,
|
|
||||||
detectErrorType,
|
|
||||||
buildErrorContext,
|
|
||||||
} from './error-handler.util.js';
|
|
||||||
import { createApiError, McpError, ErrorType } from './error.util.js';
|
|
||||||
|
|
||||||
describe('error-handler.util', () => {
|
|
||||||
describe('detectErrorType', () => {
|
|
||||||
it('should detect network errors', () => {
|
|
||||||
const networkErrors = [
|
|
||||||
'network error occurred',
|
|
||||||
'fetch failed with error',
|
|
||||||
'ECONNREFUSED on 127.0.0.1:8080',
|
|
||||||
'ENOTFOUND api.example.com',
|
|
||||||
'Failed to fetch data from server',
|
|
||||||
'Network request failed',
|
|
||||||
];
|
|
||||||
|
|
||||||
networkErrors.forEach((msg) => {
|
|
||||||
const { code, statusCode } = detectErrorType(new Error(msg));
|
|
||||||
expect(code).toBe(ErrorCode.NETWORK_ERROR);
|
|
||||||
expect(statusCode).toBe(500);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should detect rate limit errors', () => {
|
|
||||||
const rateLimitErrors = [
|
|
||||||
'rate limit exceeded',
|
|
||||||
'too many requests',
|
|
||||||
new McpError('API error', ErrorType.API_ERROR, 429),
|
|
||||||
];
|
|
||||||
|
|
||||||
rateLimitErrors.forEach((error) => {
|
|
||||||
const { code, statusCode } = detectErrorType(error);
|
|
||||||
expect(code).toBe(ErrorCode.RATE_LIMIT_ERROR);
|
|
||||||
expect(statusCode).toBe(429);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should detect not found errors', () => {
|
|
||||||
const notFoundErrors = [
|
|
||||||
'resource not found',
|
|
||||||
'entity does not exist',
|
|
||||||
new McpError('Not found', ErrorType.API_ERROR, 404),
|
|
||||||
];
|
|
||||||
|
|
||||||
notFoundErrors.forEach((error) => {
|
|
||||||
const { code } = detectErrorType(error);
|
|
||||||
expect(code).toBe(ErrorCode.NOT_FOUND);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should detect access denied errors', () => {
|
|
||||||
const accessDeniedErrors = [
|
|
||||||
'access denied',
|
|
||||||
'permission denied',
|
|
||||||
'not authorized to access',
|
|
||||||
'authentication required',
|
|
||||||
new McpError('Forbidden', ErrorType.API_ERROR, 403),
|
|
||||||
new McpError('Unauthorized', ErrorType.API_ERROR, 401),
|
|
||||||
];
|
|
||||||
|
|
||||||
accessDeniedErrors.forEach((error) => {
|
|
||||||
const { code } = detectErrorType(error);
|
|
||||||
expect(code).toBe(ErrorCode.ACCESS_DENIED);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should default to unexpected error when no patterns match', () => {
|
|
||||||
const { code, statusCode } = detectErrorType(
|
|
||||||
new Error('some random error'),
|
|
||||||
);
|
|
||||||
expect(code).toBe(ErrorCode.UNEXPECTED_ERROR);
|
|
||||||
expect(statusCode).toBe(500);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('buildErrorContext', () => {
|
|
||||||
it('should build a context object with all parameters', () => {
|
|
||||||
const context = buildErrorContext(
|
|
||||||
'User',
|
|
||||||
'create',
|
|
||||||
'controllers/user.controller.ts@create',
|
|
||||||
'user123',
|
|
||||||
{ requestBody: { name: 'Test User' } },
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(context).toEqual({
|
|
||||||
entityType: 'User',
|
|
||||||
operation: 'create',
|
|
||||||
source: 'controllers/user.controller.ts@create',
|
|
||||||
entityId: 'user123',
|
|
||||||
additionalInfo: { requestBody: { name: 'Test User' } },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should build a context object with only required parameters', () => {
|
|
||||||
const context = buildErrorContext(
|
|
||||||
'User',
|
|
||||||
'list',
|
|
||||||
'controllers/user.controller.ts@list',
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(context).toEqual({
|
|
||||||
entityType: 'User',
|
|
||||||
operation: 'list',
|
|
||||||
source: 'controllers/user.controller.ts@list',
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should handle object entityId', () => {
|
|
||||||
const context = buildErrorContext(
|
|
||||||
'Document',
|
|
||||||
'get',
|
|
||||||
'controllers/document.controller.ts@get',
|
|
||||||
{ project: 'project1', id: 'doc123' },
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(context).toEqual({
|
|
||||||
entityType: 'Document',
|
|
||||||
operation: 'get',
|
|
||||||
source: 'controllers/document.controller.ts@get',
|
|
||||||
entityId: { project: 'project1', id: 'doc123' },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,307 +0,0 @@
|
|||||||
import { createApiError } from './error.util.js';
|
|
||||||
import { Logger } from './logger.util.js';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Standard error codes for consistent handling
|
|
||||||
*/
|
|
||||||
export enum ErrorCode {
|
|
||||||
NOT_FOUND = 'NOT_FOUND',
|
|
||||||
INVALID_CURSOR = 'INVALID_CURSOR',
|
|
||||||
ACCESS_DENIED = 'ACCESS_DENIED',
|
|
||||||
VALIDATION_ERROR = 'VALIDATION_ERROR',
|
|
||||||
UNEXPECTED_ERROR = 'UNEXPECTED_ERROR',
|
|
||||||
NETWORK_ERROR = 'NETWORK_ERROR',
|
|
||||||
RATE_LIMIT_ERROR = 'RATE_LIMIT_ERROR',
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Context information for error handling
|
|
||||||
*/
|
|
||||||
export interface ErrorContext {
|
|
||||||
/**
|
|
||||||
* Source of the error (e.g., file path and function)
|
|
||||||
*/
|
|
||||||
source?: string;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Type of entity being processed (e.g., 'User')
|
|
||||||
*/
|
|
||||||
entityType?: string;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Identifier of the entity being processed
|
|
||||||
*/
|
|
||||||
entityId?: string | Record<string, string>;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Operation being performed (e.g., 'retrieving', 'searching')
|
|
||||||
*/
|
|
||||||
operation?: string;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Additional information for debugging
|
|
||||||
*/
|
|
||||||
additionalInfo?: Record<string, unknown>;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper function to create a consistent error context object
|
|
||||||
* @param entityType Type of entity being processed
|
|
||||||
* @param operation Operation being performed
|
|
||||||
* @param source Source of the error (typically file path and function)
|
|
||||||
* @param entityId Optional identifier of the entity
|
|
||||||
* @param additionalInfo Optional additional information for debugging
|
|
||||||
* @returns A formatted ErrorContext object
|
|
||||||
*/
|
|
||||||
export function buildErrorContext(
|
|
||||||
entityType: string,
|
|
||||||
operation: string,
|
|
||||||
source: string,
|
|
||||||
entityId?: string | Record<string, string>,
|
|
||||||
additionalInfo?: Record<string, unknown>,
|
|
||||||
): ErrorContext {
|
|
||||||
return {
|
|
||||||
entityType,
|
|
||||||
operation,
|
|
||||||
source,
|
|
||||||
...(entityId && { entityId }),
|
|
||||||
...(additionalInfo && { additionalInfo }),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Detect specific error types from raw errors
|
|
||||||
* @param error The error to analyze
|
|
||||||
* @param context Context information for better error detection
|
|
||||||
* @returns Object containing the error code and status code
|
|
||||||
*/
|
|
||||||
export function detectErrorType(
|
|
||||||
error: unknown,
|
|
||||||
context: ErrorContext = {},
|
|
||||||
): { code: ErrorCode; statusCode: number } {
|
|
||||||
const methodLogger = Logger.forContext(
|
|
||||||
'utils/error-handler.util.ts',
|
|
||||||
'detectErrorType',
|
|
||||||
);
|
|
||||||
methodLogger.debug(`Detecting error type`, { error, context });
|
|
||||||
|
|
||||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
||||||
const statusCode =
|
|
||||||
error instanceof Error && 'statusCode' in error
|
|
||||||
? (error as { statusCode: number }).statusCode
|
|
||||||
: undefined;
|
|
||||||
|
|
||||||
// Network error detection
|
|
||||||
if (
|
|
||||||
errorMessage.includes('network error') ||
|
|
||||||
errorMessage.includes('fetch failed') ||
|
|
||||||
errorMessage.includes('ECONNREFUSED') ||
|
|
||||||
errorMessage.includes('ENOTFOUND') ||
|
|
||||||
errorMessage.includes('Failed to fetch') ||
|
|
||||||
errorMessage.includes('Network request failed')
|
|
||||||
) {
|
|
||||||
return { code: ErrorCode.NETWORK_ERROR, statusCode: 500 };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rate limiting detection
|
|
||||||
if (
|
|
||||||
errorMessage.includes('rate limit') ||
|
|
||||||
errorMessage.includes('too many requests') ||
|
|
||||||
statusCode === 429
|
|
||||||
) {
|
|
||||||
return { code: ErrorCode.RATE_LIMIT_ERROR, statusCode: 429 };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Not Found detection
|
|
||||||
if (
|
|
||||||
errorMessage.includes('not found') ||
|
|
||||||
errorMessage.includes('does not exist') ||
|
|
||||||
statusCode === 404
|
|
||||||
) {
|
|
||||||
return { code: ErrorCode.NOT_FOUND, statusCode: 404 };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Access Denied detection
|
|
||||||
if (
|
|
||||||
errorMessage.includes('access') ||
|
|
||||||
errorMessage.includes('permission') ||
|
|
||||||
errorMessage.includes('authorize') ||
|
|
||||||
errorMessage.includes('authentication') ||
|
|
||||||
statusCode === 401 ||
|
|
||||||
statusCode === 403
|
|
||||||
) {
|
|
||||||
return { code: ErrorCode.ACCESS_DENIED, statusCode: statusCode || 403 };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Invalid Cursor detection
|
|
||||||
if (
|
|
||||||
(errorMessage.includes('cursor') ||
|
|
||||||
errorMessage.includes('startAt') ||
|
|
||||||
errorMessage.includes('page')) &&
|
|
||||||
(errorMessage.includes('invalid') || errorMessage.includes('not valid'))
|
|
||||||
) {
|
|
||||||
return { code: ErrorCode.INVALID_CURSOR, statusCode: 400 };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validation Error detection
|
|
||||||
if (
|
|
||||||
errorMessage.includes('validation') ||
|
|
||||||
errorMessage.includes('invalid') ||
|
|
||||||
errorMessage.includes('required') ||
|
|
||||||
statusCode === 400 ||
|
|
||||||
statusCode === 422
|
|
||||||
) {
|
|
||||||
return {
|
|
||||||
code: ErrorCode.VALIDATION_ERROR,
|
|
||||||
statusCode: statusCode || 400,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Default to unexpected error
|
|
||||||
return {
|
|
||||||
code: ErrorCode.UNEXPECTED_ERROR,
|
|
||||||
statusCode: statusCode || 500,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create user-friendly error messages based on error type and context
|
|
||||||
* @param code The error code
|
|
||||||
* @param context Context information for better error messages
|
|
||||||
* @param originalMessage The original error message
|
|
||||||
* @returns User-friendly error message
|
|
||||||
*/
|
|
||||||
export function createUserFriendlyErrorMessage(
|
|
||||||
code: ErrorCode,
|
|
||||||
context: ErrorContext = {},
|
|
||||||
originalMessage?: string,
|
|
||||||
): string {
|
|
||||||
const methodLogger = Logger.forContext(
|
|
||||||
'utils/error-handler.util.ts',
|
|
||||||
'createUserFriendlyErrorMessage',
|
|
||||||
);
|
|
||||||
const { entityType, entityId, operation } = context;
|
|
||||||
|
|
||||||
// Format entity ID for display
|
|
||||||
let entityIdStr = '';
|
|
||||||
if (entityId) {
|
|
||||||
if (typeof entityId === 'string') {
|
|
||||||
entityIdStr = entityId;
|
|
||||||
} else {
|
|
||||||
// Handle object entityId
|
|
||||||
entityIdStr = Object.values(entityId).join('/');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine entity display name
|
|
||||||
const entity = entityType
|
|
||||||
? `${entityType}${entityIdStr ? ` ${entityIdStr}` : ''}`
|
|
||||||
: 'Resource';
|
|
||||||
|
|
||||||
let message = '';
|
|
||||||
|
|
||||||
switch (code) {
|
|
||||||
case ErrorCode.NOT_FOUND:
|
|
||||||
message = `${entity} not found${entityIdStr ? `: ${entityIdStr}` : ''}. Verify the ID is correct and that you have access to this ${entityType?.toLowerCase() || 'resource'}.`;
|
|
||||||
break;
|
|
||||||
|
|
||||||
case ErrorCode.ACCESS_DENIED:
|
|
||||||
message = `Access denied for ${entity.toLowerCase()}${entityIdStr ? ` ${entityIdStr}` : ''}. Verify your credentials and permissions.`;
|
|
||||||
break;
|
|
||||||
|
|
||||||
case ErrorCode.INVALID_CURSOR:
|
|
||||||
message = `Invalid pagination cursor. Use the exact cursor string returned from previous results.`;
|
|
||||||
break;
|
|
||||||
|
|
||||||
case ErrorCode.VALIDATION_ERROR:
|
|
||||||
message =
|
|
||||||
originalMessage ||
|
|
||||||
`Invalid data provided for ${operation || 'operation'} ${entity.toLowerCase()}.`;
|
|
||||||
break;
|
|
||||||
|
|
||||||
case ErrorCode.NETWORK_ERROR:
|
|
||||||
message = `Network error while ${operation || 'connecting to'} the service. Please check your internet connection and try again.`;
|
|
||||||
break;
|
|
||||||
|
|
||||||
case ErrorCode.RATE_LIMIT_ERROR:
|
|
||||||
message = `Rate limit exceeded. Please wait a moment and try again, or reduce the frequency of requests.`;
|
|
||||||
break;
|
|
||||||
|
|
||||||
default:
|
|
||||||
message = `An unexpected error occurred while ${operation || 'processing'} ${entity.toLowerCase()}.`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Include original message details if available and appropriate
|
|
||||||
if (
|
|
||||||
originalMessage &&
|
|
||||||
code !== ErrorCode.NOT_FOUND &&
|
|
||||||
code !== ErrorCode.ACCESS_DENIED
|
|
||||||
) {
|
|
||||||
message += ` Error details: ${originalMessage}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
methodLogger.debug(`Created user-friendly message: ${message}`, {
|
|
||||||
code,
|
|
||||||
context,
|
|
||||||
});
|
|
||||||
return message;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle controller errors consistently
|
|
||||||
* @param error The error to handle
|
|
||||||
* @param context Context information for better error messages
|
|
||||||
* @returns Never returns, always throws an error
|
|
||||||
*/
|
|
||||||
export function handleControllerError(
|
|
||||||
error: unknown,
|
|
||||||
context: ErrorContext = {},
|
|
||||||
): never {
|
|
||||||
const methodLogger = Logger.forContext(
|
|
||||||
'utils/error-handler.util.ts',
|
|
||||||
'handleControllerError',
|
|
||||||
);
|
|
||||||
|
|
||||||
// Extract error details
|
|
||||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
||||||
const statusCode =
|
|
||||||
error instanceof Error && 'statusCode' in error
|
|
||||||
? (error as { statusCode: number }).statusCode
|
|
||||||
: undefined;
|
|
||||||
|
|
||||||
// Detect error type using utility
|
|
||||||
const { code, statusCode: detectedStatus } = detectErrorType(
|
|
||||||
error,
|
|
||||||
context,
|
|
||||||
);
|
|
||||||
|
|
||||||
// Combine detected status with explicit status
|
|
||||||
const finalStatusCode = statusCode || detectedStatus;
|
|
||||||
|
|
||||||
// Format entity information for logging
|
|
||||||
const { entityType, entityId, operation } = context;
|
|
||||||
const entity = entityType || 'resource';
|
|
||||||
const entityIdStr = entityId
|
|
||||||
? typeof entityId === 'string'
|
|
||||||
? entityId
|
|
||||||
: JSON.stringify(entityId)
|
|
||||||
: '';
|
|
||||||
const actionStr = operation || 'processing';
|
|
||||||
|
|
||||||
// Log detailed error information
|
|
||||||
methodLogger.error(
|
|
||||||
`Error ${actionStr} ${entity}${
|
|
||||||
entityIdStr ? `: ${entityIdStr}` : ''
|
|
||||||
}: ${errorMessage}`,
|
|
||||||
error,
|
|
||||||
);
|
|
||||||
|
|
||||||
// Create user-friendly error message for the response
|
|
||||||
const message =
|
|
||||||
code === ErrorCode.VALIDATION_ERROR
|
|
||||||
? errorMessage
|
|
||||||
: createUserFriendlyErrorMessage(code, context, errorMessage);
|
|
||||||
|
|
||||||
// Throw an appropriate API error with the user-friendly message
|
|
||||||
throw createApiError(message, finalStatusCode, error);
|
|
||||||
}
|
|
||||||
@@ -1,29 +1,35 @@
|
|||||||
import {
|
import {
|
||||||
McpError,
|
McpError,
|
||||||
ErrorType,
|
ErrorType,
|
||||||
createApiError,
|
|
||||||
createUnexpectedError,
|
|
||||||
getDeepOriginalError,
|
getDeepOriginalError,
|
||||||
formatErrorForMcpTool,
|
formatErrorForMcpTool,
|
||||||
|
createUnexpectedError,
|
||||||
|
ensureMcpError,
|
||||||
} from './error.util.js';
|
} from './error.util.js';
|
||||||
|
|
||||||
describe('error.util', () => {
|
describe('error.util', () => {
|
||||||
describe('getDeepOriginalError', () => {
|
describe('McpError', () => {
|
||||||
it('should return the deepest original error in a chain', () => {
|
it('should create an error with the correct properties', () => {
|
||||||
// Create a nested chain of errors
|
const error = new McpError('Test error', ErrorType.API_ERROR, 404);
|
||||||
const deepestError = new Error('Root cause');
|
|
||||||
const middleError = createApiError(
|
|
||||||
'Middle error',
|
|
||||||
500,
|
|
||||||
deepestError,
|
|
||||||
);
|
|
||||||
const topError = createUnexpectedError('Top error', middleError);
|
|
||||||
|
|
||||||
// Should extract the deepest error
|
expect(error).toBeInstanceOf(Error);
|
||||||
const result = getDeepOriginalError(topError);
|
expect(error).toBeInstanceOf(McpError);
|
||||||
expect(result).toBe(deepestError);
|
expect(error.message).toBe('Test error');
|
||||||
|
expect(error.type).toBe(ErrorType.API_ERROR);
|
||||||
|
expect(error.statusCode).toBe(404);
|
||||||
|
expect(error.name).toBe('McpError');
|
||||||
});
|
});
|
||||||
|
});
|
||||||
|
describe('createUnexpectedError', () => {
|
||||||
|
it('should create unexpected error', () => {
|
||||||
|
const error = createUnexpectedError();
|
||||||
|
|
||||||
|
expect(error).toBeInstanceOf(McpError);
|
||||||
|
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
||||||
|
expect(error.message).toBe('An unexpected error occurred');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
describe('getDeepOriginalError', () => {
|
||||||
it('should handle null/undefined input', () => {
|
it('should handle null/undefined input', () => {
|
||||||
expect(getDeepOriginalError(null)).toBeNull();
|
expect(getDeepOriginalError(null)).toBeNull();
|
||||||
expect(getDeepOriginalError(undefined)).toBeUndefined();
|
expect(getDeepOriginalError(undefined)).toBeUndefined();
|
||||||
@@ -57,30 +63,37 @@ describe('error.util', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('ensureMcpError', () => {
|
||||||
|
it('should wrap a standard Error', () => {
|
||||||
|
const originalError = new Error('Standard error');
|
||||||
|
const error = ensureMcpError(originalError);
|
||||||
|
|
||||||
|
expect(error).toBeInstanceOf(McpError);
|
||||||
|
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
||||||
|
expect(error.message).toBe('Standard error');
|
||||||
|
expect(error.originalError).toBe(originalError);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle non-Error objects', () => {
|
||||||
|
const error = ensureMcpError('String error');
|
||||||
|
|
||||||
|
expect(error).toBeInstanceOf(McpError);
|
||||||
|
expect(error.type).toBe(ErrorType.UNEXPECTED_ERROR);
|
||||||
|
expect(error.message).toBe('String error');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return the same McpError instance if passed in', () => {
|
||||||
|
const mcpError = new McpError(
|
||||||
|
'Already MCP',
|
||||||
|
ErrorType.API_ERROR,
|
||||||
|
400,
|
||||||
|
);
|
||||||
|
const result = ensureMcpError(mcpError);
|
||||||
|
expect(result).toBe(mcpError);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('formatErrorForMcpTool', () => {
|
describe('formatErrorForMcpTool', () => {
|
||||||
it('should format McpError with metadata', () => {
|
|
||||||
const error = createApiError('Test error', 404, {
|
|
||||||
detail: 'Not found',
|
|
||||||
});
|
|
||||||
const result = formatErrorForMcpTool(error);
|
|
||||||
|
|
||||||
// Check the content
|
|
||||||
expect(result.content).toEqual([
|
|
||||||
{
|
|
||||||
type: 'text',
|
|
||||||
text: 'Error: Test error',
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
|
|
||||||
// Check the metadata
|
|
||||||
expect(result.metadata).toBeDefined();
|
|
||||||
expect(result.metadata?.errorType).toBe(ErrorType.API_ERROR);
|
|
||||||
expect(result.metadata?.statusCode).toBe(404);
|
|
||||||
expect(result.metadata?.errorDetails).toEqual({
|
|
||||||
detail: 'Not found',
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should wrap non-McpError with metadata', () => {
|
it('should wrap non-McpError with metadata', () => {
|
||||||
const error = new Error('Regular error');
|
const error = new Error('Regular error');
|
||||||
const result = formatErrorForMcpTool(error);
|
const result = formatErrorForMcpTool(error);
|
||||||
@@ -101,15 +114,5 @@ describe('error.util', () => {
|
|||||||
expect(result.content[0].text).toBe('Error: String error');
|
expect(result.content[0].text).toBe('Error: String error');
|
||||||
expect(result.metadata?.errorType).toBe(ErrorType.UNEXPECTED_ERROR);
|
expect(result.metadata?.errorType).toBe(ErrorType.UNEXPECTED_ERROR);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should extract deep original error details', () => {
|
|
||||||
const deepError = { code: 'DEEP_ERROR', message: 'Deep cause' };
|
|
||||||
const middleError = createApiError('Middle layer', 500, deepError);
|
|
||||||
const topError = createUnexpectedError('Top error', middleError);
|
|
||||||
|
|
||||||
const result = formatErrorForMcpTool(topError);
|
|
||||||
|
|
||||||
expect(result.metadata?.errorDetails).toEqual(deepError);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -32,40 +32,6 @@ export class McpError extends Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Create an authentication missing error
|
|
||||||
*/
|
|
||||||
export function createAuthMissingError(
|
|
||||||
message: string = 'Authentication credentials are missing',
|
|
||||||
): McpError {
|
|
||||||
return new McpError(message, ErrorType.AUTH_MISSING);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create an authentication invalid error
|
|
||||||
*/
|
|
||||||
export function createAuthInvalidError(
|
|
||||||
message: string = 'Authentication credentials are invalid',
|
|
||||||
): McpError {
|
|
||||||
return new McpError(message, ErrorType.AUTH_INVALID, 401);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create an API error
|
|
||||||
*/
|
|
||||||
export function createApiError(
|
|
||||||
message: string,
|
|
||||||
statusCode?: number,
|
|
||||||
originalError?: unknown,
|
|
||||||
): McpError {
|
|
||||||
return new McpError(
|
|
||||||
message,
|
|
||||||
ErrorType.API_ERROR,
|
|
||||||
statusCode,
|
|
||||||
originalError,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create an unexpected error
|
* Create an unexpected error
|
||||||
*/
|
*/
|
||||||
@@ -164,77 +130,3 @@ export function formatErrorForMcpTool(error: unknown): {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Format error for MCP resource response
|
|
||||||
*/
|
|
||||||
export function formatErrorForMcpResource(
|
|
||||||
error: unknown,
|
|
||||||
uri: string,
|
|
||||||
): {
|
|
||||||
contents: Array<{
|
|
||||||
uri: string;
|
|
||||||
text: string;
|
|
||||||
mimeType: string;
|
|
||||||
description?: string;
|
|
||||||
}>;
|
|
||||||
} {
|
|
||||||
const methodLogger = Logger.forContext(
|
|
||||||
'utils/error.util.ts',
|
|
||||||
'formatErrorForMcpResource',
|
|
||||||
);
|
|
||||||
const mcpError = ensureMcpError(error);
|
|
||||||
methodLogger.error(`${mcpError.type} error`, mcpError);
|
|
||||||
|
|
||||||
return {
|
|
||||||
contents: [
|
|
||||||
{
|
|
||||||
uri,
|
|
||||||
text: `Error: ${mcpError.message}`,
|
|
||||||
mimeType: 'text/plain',
|
|
||||||
description: `Error: ${mcpError.type}`,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle error in CLI context with improved user feedback
|
|
||||||
*/
|
|
||||||
export function handleCliError(error: unknown): never {
|
|
||||||
const methodLogger = Logger.forContext(
|
|
||||||
'utils/error.util.ts',
|
|
||||||
'handleCliError',
|
|
||||||
);
|
|
||||||
const mcpError = ensureMcpError(error);
|
|
||||||
methodLogger.error(`${mcpError.type} error`, mcpError);
|
|
||||||
|
|
||||||
// Print the error message
|
|
||||||
console.error(`Error: ${mcpError.message}`);
|
|
||||||
|
|
||||||
// Provide helpful context based on error type
|
|
||||||
if (mcpError.type === ErrorType.AUTH_MISSING) {
|
|
||||||
console.error(
|
|
||||||
'\nTip: Make sure to set up your API token in the configuration file or environment variables.',
|
|
||||||
);
|
|
||||||
} else if (mcpError.type === ErrorType.AUTH_INVALID) {
|
|
||||||
console.error(
|
|
||||||
'\nTip: Check that your API token is correct and has not expired.',
|
|
||||||
);
|
|
||||||
} else if (mcpError.type === ErrorType.API_ERROR) {
|
|
||||||
if (mcpError.statusCode === 429) {
|
|
||||||
console.error(
|
|
||||||
'\nTip: You may have exceeded your API rate limits. Try again later or upgrade your API plan.',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Display DEBUG tip
|
|
||||||
if (process.env.DEBUG !== 'mcp:*') {
|
|
||||||
console.error(
|
|
||||||
'\nFor more detailed error information, run with DEBUG=mcp:* environment variable.',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,131 +0,0 @@
|
|||||||
/**
|
|
||||||
* Standardized formatting utilities for consistent output across all CLI and Tool interfaces.
|
|
||||||
* These functions should be used by all formatters to ensure consistent formatting.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format a date in a standardized way: YYYY-MM-DD HH:MM:SS UTC
|
|
||||||
* @param dateString - ISO date string or Date object
|
|
||||||
* @returns Formatted date string
|
|
||||||
*/
|
|
||||||
export function formatDate(dateString?: string | Date): string {
|
|
||||||
if (!dateString) {
|
|
||||||
return 'Not available';
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const date =
|
|
||||||
typeof dateString === 'string' ? new Date(dateString) : dateString;
|
|
||||||
|
|
||||||
// Format: YYYY-MM-DD HH:MM:SS UTC
|
|
||||||
return date
|
|
||||||
.toISOString()
|
|
||||||
.replace('T', ' ')
|
|
||||||
.replace(/\.\d+Z$/, ' UTC');
|
|
||||||
} catch {
|
|
||||||
return 'Invalid date';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format a URL as a markdown link
|
|
||||||
* @param url - URL to format
|
|
||||||
* @param title - Link title
|
|
||||||
* @returns Formatted markdown link
|
|
||||||
*/
|
|
||||||
export function formatUrl(url?: string, title?: string): string {
|
|
||||||
if (!url) {
|
|
||||||
return 'Not available';
|
|
||||||
}
|
|
||||||
|
|
||||||
const linkTitle = title || url;
|
|
||||||
return `[${linkTitle}](${url})`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format a heading with consistent style
|
|
||||||
* @param text - Heading text
|
|
||||||
* @param level - Heading level (1-6)
|
|
||||||
* @returns Formatted heading
|
|
||||||
*/
|
|
||||||
export function formatHeading(text: string, level: number = 1): string {
|
|
||||||
const validLevel = Math.min(Math.max(level, 1), 6);
|
|
||||||
const prefix = '#'.repeat(validLevel);
|
|
||||||
return `${prefix} ${text}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format a list of key-value pairs as a bullet list
|
|
||||||
* @param items - Object with key-value pairs
|
|
||||||
* @param keyFormatter - Optional function to format keys
|
|
||||||
* @returns Formatted bullet list
|
|
||||||
*/
|
|
||||||
export function formatBulletList(
|
|
||||||
items: Record<string, unknown>,
|
|
||||||
keyFormatter?: (key: string) => string,
|
|
||||||
): string {
|
|
||||||
const lines: string[] = [];
|
|
||||||
|
|
||||||
for (const [key, value] of Object.entries(items)) {
|
|
||||||
if (value === undefined || value === null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const formattedKey = keyFormatter ? keyFormatter(key) : key;
|
|
||||||
const formattedValue = formatValue(value);
|
|
||||||
lines.push(`- **${formattedKey}**: ${formattedValue}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format a value based on its type
|
|
||||||
* @param value - Value to format
|
|
||||||
* @returns Formatted value
|
|
||||||
*/
|
|
||||||
function formatValue(value: unknown): string {
|
|
||||||
if (value === undefined || value === null) {
|
|
||||||
return 'Not available';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (value instanceof Date) {
|
|
||||||
return formatDate(value);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle URL objects with url and title properties
|
|
||||||
if (typeof value === 'object' && value !== null && 'url' in value) {
|
|
||||||
const urlObj = value as { url: string; title?: string };
|
|
||||||
if (typeof urlObj.url === 'string') {
|
|
||||||
return formatUrl(urlObj.url, urlObj.title);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof value === 'string') {
|
|
||||||
// Check if it's a URL
|
|
||||||
if (value.startsWith('http://') || value.startsWith('https://')) {
|
|
||||||
return formatUrl(value);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if it might be a date
|
|
||||||
if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/.test(value)) {
|
|
||||||
return formatDate(value);
|
|
||||||
}
|
|
||||||
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof value === 'boolean') {
|
|
||||||
return value ? 'Yes' : 'No';
|
|
||||||
}
|
|
||||||
|
|
||||||
return String(value);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format a separator line
|
|
||||||
* @returns Separator line
|
|
||||||
*/
|
|
||||||
export function formatSeparator(): string {
|
|
||||||
return '---';
|
|
||||||
}
|
|
||||||
@@ -194,7 +194,7 @@ let isLoggerInitialized = false;
|
|||||||
* 5. Set DEBUG environment variable to control which modules show debug logs:
|
* 5. Set DEBUG environment variable to control which modules show debug logs:
|
||||||
* - DEBUG=true (enable all debug logs)
|
* - DEBUG=true (enable all debug logs)
|
||||||
* - DEBUG=controllers/*,services/* (enable for specific module groups)
|
* - DEBUG=controllers/*,services/* (enable for specific module groups)
|
||||||
* - DEBUG=transport,utils/formatter* (enable specific modules, supports wildcards)
|
* - DEBUG=transport,utils/error* (enable specific modules, supports wildcards)
|
||||||
*/
|
*/
|
||||||
class Logger {
|
class Logger {
|
||||||
private context?: string;
|
private context?: string;
|
||||||
|
|||||||
@@ -1,143 +0,0 @@
|
|||||||
import { Logger } from './logger.util.js';
|
|
||||||
import {
|
|
||||||
createApiError,
|
|
||||||
createAuthInvalidError,
|
|
||||||
createUnexpectedError,
|
|
||||||
McpError,
|
|
||||||
} from './error.util.js';
|
|
||||||
|
|
||||||
// Create a contextualized logger for this file
|
|
||||||
const transportLogger = Logger.forContext('utils/transport.util.ts');
|
|
||||||
|
|
||||||
// Log transport utility initialization
|
|
||||||
transportLogger.debug('Transport utility initialized');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Interface for HTTP request options
|
|
||||||
*/
|
|
||||||
export interface RequestOptions {
|
|
||||||
method?: 'GET' | 'POST' | 'PUT' | 'DELETE';
|
|
||||||
headers?: Record<string, string>;
|
|
||||||
body?: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generic and reusable function to fetch data from any API endpoint.
|
|
||||||
* Handles standard HTTP request setup, response checking, basic error handling, and logging.
|
|
||||||
*
|
|
||||||
* @param url The full URL to fetch data from.
|
|
||||||
* @param options Request options including method, headers, and body.
|
|
||||||
* @returns The response data parsed as type T.
|
|
||||||
* @throws {McpError} If the request fails, including network errors, non-OK HTTP status, or JSON parsing issues.
|
|
||||||
*/
|
|
||||||
export async function fetchApi<T>(
|
|
||||||
url: string,
|
|
||||||
options: RequestOptions = {},
|
|
||||||
): Promise<T> {
|
|
||||||
const methodLogger = Logger.forContext(
|
|
||||||
'utils/transport.util.ts',
|
|
||||||
'fetchApi',
|
|
||||||
);
|
|
||||||
|
|
||||||
// Prepare standard request options
|
|
||||||
const requestOptions: RequestInit = {
|
|
||||||
method: options.method || 'GET',
|
|
||||||
headers: {
|
|
||||||
// Standard headers, allow overrides via options.headers
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
Accept: 'application/json',
|
|
||||||
...options.headers,
|
|
||||||
},
|
|
||||||
body: options.body ? JSON.stringify(options.body) : undefined,
|
|
||||||
};
|
|
||||||
|
|
||||||
methodLogger.debug(`Executing API call: ${requestOptions.method} ${url}`);
|
|
||||||
const startTime = performance.now(); // Track performance
|
|
||||||
|
|
||||||
try {
|
|
||||||
const response = await fetch(url, requestOptions);
|
|
||||||
const endTime = performance.now();
|
|
||||||
const duration = (endTime - startTime).toFixed(2);
|
|
||||||
|
|
||||||
methodLogger.debug(
|
|
||||||
`API call completed in ${duration}ms with status: ${response.status} ${response.statusText}`,
|
|
||||||
{ url, status: response.status },
|
|
||||||
);
|
|
||||||
|
|
||||||
// Check if the response status is OK (2xx)
|
|
||||||
if (!response.ok) {
|
|
||||||
const errorText = await response.text(); // Get error body for context
|
|
||||||
methodLogger.error(
|
|
||||||
`API error response (${response.status}):`,
|
|
||||||
errorText,
|
|
||||||
);
|
|
||||||
|
|
||||||
// Classify standard HTTP errors
|
|
||||||
if (response.status === 401) {
|
|
||||||
throw createAuthInvalidError(
|
|
||||||
'Authentication failed. Check API token if required.',
|
|
||||||
);
|
|
||||||
} else if (response.status === 403) {
|
|
||||||
throw createAuthInvalidError(
|
|
||||||
'Permission denied for the requested resource.',
|
|
||||||
);
|
|
||||||
} else if (response.status === 404) {
|
|
||||||
throw createApiError(
|
|
||||||
'Resource not found at the specified URL.',
|
|
||||||
response.status,
|
|
||||||
errorText,
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
// Generic API error for other non-2xx statuses
|
|
||||||
throw createApiError(
|
|
||||||
`API request failed with status ${response.status}: ${response.statusText}`,
|
|
||||||
response.status,
|
|
||||||
errorText,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Attempt to parse the response body as JSON
|
|
||||||
try {
|
|
||||||
const responseData = await response.json();
|
|
||||||
methodLogger.debug('Response body successfully parsed as JSON.');
|
|
||||||
// methodLogger.debug('Response Data:', responseData); // Uncomment for full response logging
|
|
||||||
return responseData as T;
|
|
||||||
} catch (parseError) {
|
|
||||||
methodLogger.error(
|
|
||||||
'Failed to parse API response JSON:',
|
|
||||||
parseError,
|
|
||||||
);
|
|
||||||
// Throw a specific error for JSON parsing failure
|
|
||||||
throw createApiError(
|
|
||||||
`Failed to parse API response JSON: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
|
|
||||||
response.status, // Include original status for context
|
|
||||||
parseError,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
const endTime = performance.now();
|
|
||||||
const duration = (endTime - startTime).toFixed(2);
|
|
||||||
methodLogger.error(
|
|
||||||
`API call failed after ${duration}ms for ${url}:`,
|
|
||||||
error,
|
|
||||||
);
|
|
||||||
|
|
||||||
// Rethrow if it's already an McpError (e.g., from status checks or parsing)
|
|
||||||
if (error instanceof McpError) {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle potential network errors (TypeError in fetch)
|
|
||||||
if (error instanceof TypeError) {
|
|
||||||
throw createApiError(
|
|
||||||
`Network error during API call: ${error.message}`,
|
|
||||||
undefined, // No specific HTTP status for network errors
|
|
||||||
error,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wrap any other unexpected errors
|
|
||||||
throw createUnexpectedError('Unexpected error during API call', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user