LLM API Integration Guide
This guide covers integrating various Large Language Model (LLM) APIs into your applications.
Available LLM APIs
Major Providers
| Provider | Models | Strengths | Best For |
|---|---|---|---|
| Anthropic | Claude | Long context, reasoning | Complex analysis, code |
| OpenAI | GPT-4, GPT-3.5 | General purpose, widely adopted | Versatile applications |
| Gemini, PaLM | Multimodal capabilities | Vision + language tasks | |
| Cohere | Command | Enterprise focus | Classification, search |
| Meta | Llama | Open source | Self-hosted solutions |
General Integration Pattern
Basic Structure
typescript
interface LLMRequest {
model: string;
messages: Message[];
temperature?: number;
max_tokens?: number;
stream?: boolean;
}
interface Message {
role: 'system' | 'user' | 'assistant';
content: string;
}
interface LLMResponse {
id: string;
content: string;
model: string;
usage: {
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
};
}API Client Wrapper
typescript
abstract class LLMClient {
abstract async complete(request: LLMRequest): Promise<LLMResponse>;
abstract async stream(request: LLMRequest): AsyncIterator<string>;
protected async retryWithBackoff<T>(
fn: () => Promise<T>,
maxRetries: number = 3
): Promise<T> {
for (let i = 0; i < maxRetries; i++) {
try {
return await fn();
} catch (error) {
if (i === maxRetries - 1) throw error;
await this.sleep(Math.pow(2, i) * 1000);
}
}
throw new Error('Max retries exceeded');
}
private sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
}Provider-Specific Implementations
Anthropic (Claude)
typescript
import Anthropic from '@anthropic-ai/sdk';
class AnthropicClient extends LLMClient {
private client: Anthropic;
constructor(apiKey: string) {
super();
this.client = new Anthropic({ apiKey });
}
async complete(request: LLMRequest): Promise<LLMResponse> {
const response = await this.retryWithBackoff(() =>
this.client.messages.create({
model: request.model,
messages: request.messages,
max_tokens: request.max_tokens || 4096,
temperature: request.temperature,
})
);
return {
id: response.id,
content: response.content[0].text,
model: response.model,
usage: {
prompt_tokens: response.usage.input_tokens,
completion_tokens: response.usage.output_tokens,
total_tokens: response.usage.input_tokens + response.usage.output_tokens,
},
};
}
async *stream(request: LLMRequest): AsyncIterator<string> {
const stream = await this.client.messages.create({
...request,
stream: true,
});
for await (const event of stream) {
if (event.type === 'content_block_delta') {
yield event.delta.text;
}
}
}
}OpenAI (GPT)
typescript
import OpenAI from 'openai';
class OpenAIClient extends LLMClient {
private client: OpenAI;
constructor(apiKey: string) {
super();
this.client = new OpenAI({ apiKey });
}
async complete(request: LLMRequest): Promise<LLMResponse> {
const response = await this.retryWithBackoff(() =>
this.client.chat.completions.create({
model: request.model,
messages: request.messages,
max_tokens: request.max_tokens,
temperature: request.temperature,
})
);
return {
id: response.id,
content: response.choices[0].message.content,
model: response.model,
usage: {
prompt_tokens: response.usage.prompt_tokens,
completion_tokens: response.usage.completion_tokens,
total_tokens: response.usage.total_tokens,
},
};
}
async *stream(request: LLMRequest): AsyncIterator<string> {
const stream = await this.client.chat.completions.create({
...request,
stream: true,
});
for await (const chunk of stream) {
const content = chunk.choices[0]?.delta?.content;
if (content) {
yield content;
}
}
}
}Unified Interface
Create a provider-agnostic interface:
typescript
class UnifiedLLMClient {
private clients: Map<string, LLMClient> = new Map();
registerProvider(name: string, client: LLMClient) {
this.clients.set(name, client);
}
async complete(provider: string, request: LLMRequest): Promise<LLMResponse> {
const client = this.clients.get(provider);
if (!client) throw new Error(`Provider ${provider} not found`);
return client.complete(request);
}
async *stream(provider: string, request: LLMRequest): AsyncIterator<string> {
const client = this.clients.get(provider);
if (!client) throw new Error(`Provider ${provider} not found`);
yield* client.stream(request);
}
}
// Usage
const llm = new UnifiedLLMClient();
llm.registerProvider('anthropic', new AnthropicClient(ANTHROPIC_KEY));
llm.registerProvider('openai', new OpenAIClient(OPENAI_KEY));
// Use any provider with the same interface
const response = await llm.complete('anthropic', {
model: 'claude-3-5-sonnet-20241022',
messages: [{ role: 'user', content: 'Hello!' }],
});Best Practices
1. Error Handling
typescript
class LLMError extends Error {
constructor(
message: string,
public provider: string,
public statusCode?: number,
public originalError?: Error
) {
super(message);
this.name = 'LLMError';
}
}
async function safeLLMCall<T>(
fn: () => Promise<T>,
provider: string
): Promise<T> {
try {
return await fn();
} catch (error) {
if (error.status === 429) {
throw new LLMError('Rate limit exceeded', provider, 429, error);
}
if (error.status === 401) {
throw new LLMError('Invalid API key', provider, 401, error);
}
throw new LLMError('API request failed', provider, error.status, error);
}
}2. Rate Limiting
typescript
import Bottleneck from 'bottleneck';
class RateLimitedLLMClient extends LLMClient {
private limiter: Bottleneck;
constructor(
private baseClient: LLMClient,
requestsPerMinute: number = 50
) {
super();
this.limiter = new Bottleneck({
minTime: 60000 / requestsPerMinute,
maxConcurrent: 5,
});
}
async complete(request: LLMRequest): Promise<LLMResponse> {
return this.limiter.schedule(() => this.baseClient.complete(request));
}
async *stream(request: LLMRequest): AsyncIterator<string> {
yield* await this.limiter.schedule(() => this.baseClient.stream(request));
}
}3. Caching
typescript
import { createHash } from 'crypto';
class CachedLLMClient extends LLMClient {
private cache: Map<string, { response: LLMResponse; timestamp: number }> = new Map();
private ttl: number = 3600000; // 1 hour
constructor(private baseClient: LLMClient) {
super();
}
private getCacheKey(request: LLMRequest): string {
return createHash('sha256')
.update(JSON.stringify(request))
.digest('hex');
}
async complete(request: LLMRequest): Promise<LLMResponse> {
const key = this.getCacheKey(request);
const cached = this.cache.get(key);
if (cached && Date.now() - cached.timestamp < this.ttl) {
return cached.response;
}
const response = await this.baseClient.complete(request);
this.cache.set(key, { response, timestamp: Date.now() });
return response;
}
}4. Token Management
typescript
interface TokenBudget {
maxPromptTokens: number;
maxCompletionTokens: number;
}
function estimateTokens(text: string): number {
// Rough estimation: 1 token ≈ 4 characters
return Math.ceil(text.length / 4);
}
function validateBudget(
request: LLMRequest,
budget: TokenBudget
): void {
const promptText = request.messages.map(m => m.content).join('');
const estimatedTokens = estimateTokens(promptText);
if (estimatedTokens > budget.maxPromptTokens) {
throw new Error(`Prompt exceeds token budget: ${estimatedTokens} > ${budget.maxPromptTokens}`);
}
if (request.max_tokens && request.max_tokens > budget.maxCompletionTokens) {
throw new Error(`Max tokens exceeds budget: ${request.max_tokens} > ${budget.maxCompletionTokens}`);
}
}5. Prompt Templates
typescript
class PromptTemplate {
constructor(private template: string) {}
render(variables: Record<string, string>): string {
return this.template.replace(
/\{\{(\w+)\}\}/g,
(_, key) => variables[key] || ''
);
}
}
// Usage
const codeReviewTemplate = new PromptTemplate(`
Review this {{language}} code for:
- Bugs and errors
- Security issues
- Best practices
- Performance
Code:
{{code}}
`);
const prompt = codeReviewTemplate.render({
language: 'TypeScript',
code: myCode,
});Security Considerations
API Key Management
typescript
// ❌ Never hardcode keys
const apiKey = 'sk-1234567890...';
// ✅ Use environment variables
const apiKey = process.env.ANTHROPIC_API_KEY;
// ✅ Use secret management
import { SecretsManager } from '@aws-sdk/client-secrets-manager';
const apiKey = await getSecret('anthropic-api-key');Request Sanitization
typescript
function sanitizeForLLM(data: any): string {
// Remove sensitive patterns
let text = JSON.stringify(data, null, 2);
// Remove API keys
text = text.replace(/["']?api[_-]?key["']?\s*[:=]\s*["'][^"']+["']/gi, '"api_key": "[REDACTED]"');
// Remove tokens
text = text.replace(/["']?token["']?\s*[:=]\s*["'][^"']+["']/gi, '"token": "[REDACTED]"');
// Remove emails
text = text.replace(/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g, '[email]');
return text;
}Response Validation
typescript
function validateLLMResponse(response: LLMResponse): void {
// Check for leaked secrets
const secretPatterns = [
/sk-[a-zA-Z0-9]{32,}/, // API keys
/-----BEGIN [A-Z ]+ KEY-----/, // Private keys
/[0-9]{16}/, // Credit card numbers
];
for (const pattern of secretPatterns) {
if (pattern.test(response.content)) {
throw new Error('Response contains potential secret');
}
}
}Monitoring and Logging
typescript
interface LLMMetrics {
requestCount: number;
errorCount: number;
totalTokens: number;
totalCost: number;
averageLatency: number;
}
class MonitoredLLMClient extends LLMClient {
private metrics: LLMMetrics = {
requestCount: 0,
errorCount: 0,
totalTokens: 0,
totalCost: 0,
averageLatency: 0,
};
constructor(
private baseClient: LLMClient,
private costPerToken: number
) {
super();
}
async complete(request: LLMRequest): Promise<LLMResponse> {
const startTime = Date.now();
try {
const response = await this.baseClient.complete(request);
this.recordSuccess(response, Date.now() - startTime);
return response;
} catch (error) {
this.recordError();
throw error;
}
}
private recordSuccess(response: LLMResponse, latency: number): void {
this.metrics.requestCount++;
this.metrics.totalTokens += response.usage.total_tokens;
this.metrics.totalCost += response.usage.total_tokens * this.costPerToken;
this.metrics.averageLatency =
(this.metrics.averageLatency * (this.metrics.requestCount - 1) + latency) /
this.metrics.requestCount;
}
private recordError(): void {
this.metrics.errorCount++;
}
getMetrics(): LLMMetrics {
return { ...this.metrics };
}
}Testing
Mock LLM Client for Testing
typescript
class MockLLMClient extends LLMClient {
constructor(private responses: Map<string, string>) {
super();
}
async complete(request: LLMRequest): Promise<LLMResponse> {
const key = request.messages[request.messages.length - 1].content;
const content = this.responses.get(key) || 'Mock response';
return {
id: 'mock-id',
content,
model: 'mock-model',
usage: { prompt_tokens: 10, completion_tokens: 10, total_tokens: 20 },
};
}
async *stream(request: LLMRequest): AsyncIterator<string> {
const response = await this.complete(request);
yield response.content;
}
}
// Usage in tests
describe('LLM Integration', () => {
it('should process requests', async () => {
const mockClient = new MockLLMClient(
new Map([['test prompt', 'test response']])
);
const response = await mockClient.complete({
model: 'test',
messages: [{ role: 'user', content: 'test prompt' }],
});
expect(response.content).toBe('test response');
});
});Resources
Remember: Always validate and sanitize data before sending to LLM APIs, and review responses before using them in production.