Skip to content

LLM API Integration Guide ​

This guide covers integrating various Large Language Model (LLM) APIs into your applications.

Available LLM APIs ​

Major Providers ​

ProviderModelsStrengthsBest For
AnthropicClaudeLong context, reasoningComplex analysis, code
OpenAIGPT-4, GPT-3.5General purpose, widely adoptedVersatile applications
GoogleGemini, PaLMMultimodal capabilitiesVision + language tasks
CohereCommandEnterprise focusClassification, search
MetaLlamaOpen sourceSelf-hosted solutions

General Integration Pattern ​

Basic Structure ​

typescript
interface LLMRequest {
  model: string;
  messages: Message[];
  temperature?: number;
  max_tokens?: number;
  stream?: boolean;
}

interface Message {
  role: 'system' | 'user' | 'assistant';
  content: string;
}

interface LLMResponse {
  id: string;
  content: string;
  model: string;
  usage: {
    prompt_tokens: number;
    completion_tokens: number;
    total_tokens: number;
  };
}

API Client Wrapper ​

typescript
abstract class LLMClient {
  abstract async complete(request: LLMRequest): Promise<LLMResponse>;
  abstract async stream(request: LLMRequest): AsyncIterator<string>;

  protected async retryWithBackoff<T>(
    fn: () => Promise<T>,
    maxRetries: number = 3
  ): Promise<T> {
    for (let i = 0; i < maxRetries; i++) {
      try {
        return await fn();
      } catch (error) {
        if (i === maxRetries - 1) throw error;
        await this.sleep(Math.pow(2, i) * 1000);
      }
    }
    throw new Error('Max retries exceeded');
  }

  private sleep(ms: number): Promise<void> {
    return new Promise(resolve => setTimeout(resolve, ms));
  }
}

Provider-Specific Implementations ​

Anthropic (Claude) ​

typescript
import Anthropic from '@anthropic-ai/sdk';

class AnthropicClient extends LLMClient {
  private client: Anthropic;

  constructor(apiKey: string) {
    super();
    this.client = new Anthropic({ apiKey });
  }

  async complete(request: LLMRequest): Promise<LLMResponse> {
    const response = await this.retryWithBackoff(() =>
      this.client.messages.create({
        model: request.model,
        messages: request.messages,
        max_tokens: request.max_tokens || 4096,
        temperature: request.temperature,
      })
    );

    return {
      id: response.id,
      content: response.content[0].text,
      model: response.model,
      usage: {
        prompt_tokens: response.usage.input_tokens,
        completion_tokens: response.usage.output_tokens,
        total_tokens: response.usage.input_tokens + response.usage.output_tokens,
      },
    };
  }

  async *stream(request: LLMRequest): AsyncIterator<string> {
    const stream = await this.client.messages.create({
      ...request,
      stream: true,
    });

    for await (const event of stream) {
      if (event.type === 'content_block_delta') {
        yield event.delta.text;
      }
    }
  }
}

OpenAI (GPT) ​

typescript
import OpenAI from 'openai';

class OpenAIClient extends LLMClient {
  private client: OpenAI;

  constructor(apiKey: string) {
    super();
    this.client = new OpenAI({ apiKey });
  }

  async complete(request: LLMRequest): Promise<LLMResponse> {
    const response = await this.retryWithBackoff(() =>
      this.client.chat.completions.create({
        model: request.model,
        messages: request.messages,
        max_tokens: request.max_tokens,
        temperature: request.temperature,
      })
    );

    return {
      id: response.id,
      content: response.choices[0].message.content,
      model: response.model,
      usage: {
        prompt_tokens: response.usage.prompt_tokens,
        completion_tokens: response.usage.completion_tokens,
        total_tokens: response.usage.total_tokens,
      },
    };
  }

  async *stream(request: LLMRequest): AsyncIterator<string> {
    const stream = await this.client.chat.completions.create({
      ...request,
      stream: true,
    });

    for await (const chunk of stream) {
      const content = chunk.choices[0]?.delta?.content;
      if (content) {
        yield content;
      }
    }
  }
}

Unified Interface ​

Create a provider-agnostic interface:

typescript
class UnifiedLLMClient {
  private clients: Map<string, LLMClient> = new Map();

  registerProvider(name: string, client: LLMClient) {
    this.clients.set(name, client);
  }

  async complete(provider: string, request: LLMRequest): Promise<LLMResponse> {
    const client = this.clients.get(provider);
    if (!client) throw new Error(`Provider ${provider} not found`);
    return client.complete(request);
  }

  async *stream(provider: string, request: LLMRequest): AsyncIterator<string> {
    const client = this.clients.get(provider);
    if (!client) throw new Error(`Provider ${provider} not found`);
    yield* client.stream(request);
  }
}

// Usage
const llm = new UnifiedLLMClient();
llm.registerProvider('anthropic', new AnthropicClient(ANTHROPIC_KEY));
llm.registerProvider('openai', new OpenAIClient(OPENAI_KEY));

// Use any provider with the same interface
const response = await llm.complete('anthropic', {
  model: 'claude-3-5-sonnet-20241022',
  messages: [{ role: 'user', content: 'Hello!' }],
});

Best Practices ​

1. Error Handling ​

typescript
class LLMError extends Error {
  constructor(
    message: string,
    public provider: string,
    public statusCode?: number,
    public originalError?: Error
  ) {
    super(message);
    this.name = 'LLMError';
  }
}

async function safeLLMCall<T>(
  fn: () => Promise<T>,
  provider: string
): Promise<T> {
  try {
    return await fn();
  } catch (error) {
    if (error.status === 429) {
      throw new LLMError('Rate limit exceeded', provider, 429, error);
    }
    if (error.status === 401) {
      throw new LLMError('Invalid API key', provider, 401, error);
    }
    throw new LLMError('API request failed', provider, error.status, error);
  }
}

2. Rate Limiting ​

typescript
import Bottleneck from 'bottleneck';

class RateLimitedLLMClient extends LLMClient {
  private limiter: Bottleneck;

  constructor(
    private baseClient: LLMClient,
    requestsPerMinute: number = 50
  ) {
    super();
    this.limiter = new Bottleneck({
      minTime: 60000 / requestsPerMinute,
      maxConcurrent: 5,
    });
  }

  async complete(request: LLMRequest): Promise<LLMResponse> {
    return this.limiter.schedule(() => this.baseClient.complete(request));
  }

  async *stream(request: LLMRequest): AsyncIterator<string> {
    yield* await this.limiter.schedule(() => this.baseClient.stream(request));
  }
}

3. Caching ​

typescript
import { createHash } from 'crypto';

class CachedLLMClient extends LLMClient {
  private cache: Map<string, { response: LLMResponse; timestamp: number }> = new Map();
  private ttl: number = 3600000; // 1 hour

  constructor(private baseClient: LLMClient) {
    super();
  }

  private getCacheKey(request: LLMRequest): string {
    return createHash('sha256')
      .update(JSON.stringify(request))
      .digest('hex');
  }

  async complete(request: LLMRequest): Promise<LLMResponse> {
    const key = this.getCacheKey(request);
    const cached = this.cache.get(key);

    if (cached && Date.now() - cached.timestamp < this.ttl) {
      return cached.response;
    }

    const response = await this.baseClient.complete(request);
    this.cache.set(key, { response, timestamp: Date.now() });
    return response;
  }
}

4. Token Management ​

typescript
interface TokenBudget {
  maxPromptTokens: number;
  maxCompletionTokens: number;
}

function estimateTokens(text: string): number {
  // Rough estimation: 1 token ≈ 4 characters
  return Math.ceil(text.length / 4);
}

function validateBudget(
  request: LLMRequest,
  budget: TokenBudget
): void {
  const promptText = request.messages.map(m => m.content).join('');
  const estimatedTokens = estimateTokens(promptText);

  if (estimatedTokens > budget.maxPromptTokens) {
    throw new Error(`Prompt exceeds token budget: ${estimatedTokens} > ${budget.maxPromptTokens}`);
  }

  if (request.max_tokens && request.max_tokens > budget.maxCompletionTokens) {
    throw new Error(`Max tokens exceeds budget: ${request.max_tokens} > ${budget.maxCompletionTokens}`);
  }
}

5. Prompt Templates ​

typescript
class PromptTemplate {
  constructor(private template: string) {}

  render(variables: Record<string, string>): string {
    return this.template.replace(
      /\{\{(\w+)\}\}/g,
      (_, key) => variables[key] || ''
    );
  }
}

// Usage
const codeReviewTemplate = new PromptTemplate(`
Review this {{language}} code for:
- Bugs and errors
- Security issues
- Best practices
- Performance

Code:
{{code}}
`);

const prompt = codeReviewTemplate.render({
  language: 'TypeScript',
  code: myCode,
});

Security Considerations ​

API Key Management ​

typescript
// ❌ Never hardcode keys
const apiKey = 'sk-1234567890...';

// ✅ Use environment variables
const apiKey = process.env.ANTHROPIC_API_KEY;

// ✅ Use secret management
import { SecretsManager } from '@aws-sdk/client-secrets-manager';
const apiKey = await getSecret('anthropic-api-key');

Request Sanitization ​

typescript
function sanitizeForLLM(data: any): string {
  // Remove sensitive patterns
  let text = JSON.stringify(data, null, 2);

  // Remove API keys
  text = text.replace(/["']?api[_-]?key["']?\s*[:=]\s*["'][^"']+["']/gi, '"api_key": "[REDACTED]"');

  // Remove tokens
  text = text.replace(/["']?token["']?\s*[:=]\s*["'][^"']+["']/gi, '"token": "[REDACTED]"');

  // Remove emails
  text = text.replace(/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g, '[email]');

  return text;
}

Response Validation ​

typescript
function validateLLMResponse(response: LLMResponse): void {
  // Check for leaked secrets
  const secretPatterns = [
    /sk-[a-zA-Z0-9]{32,}/,  // API keys
    /-----BEGIN [A-Z ]+ KEY-----/,  // Private keys
    /[0-9]{16}/,  // Credit card numbers
  ];

  for (const pattern of secretPatterns) {
    if (pattern.test(response.content)) {
      throw new Error('Response contains potential secret');
    }
  }
}

Monitoring and Logging ​

typescript
interface LLMMetrics {
  requestCount: number;
  errorCount: number;
  totalTokens: number;
  totalCost: number;
  averageLatency: number;
}

class MonitoredLLMClient extends LLMClient {
  private metrics: LLMMetrics = {
    requestCount: 0,
    errorCount: 0,
    totalTokens: 0,
    totalCost: 0,
    averageLatency: 0,
  };

  constructor(
    private baseClient: LLMClient,
    private costPerToken: number
  ) {
    super();
  }

  async complete(request: LLMRequest): Promise<LLMResponse> {
    const startTime = Date.now();

    try {
      const response = await this.baseClient.complete(request);
      this.recordSuccess(response, Date.now() - startTime);
      return response;
    } catch (error) {
      this.recordError();
      throw error;
    }
  }

  private recordSuccess(response: LLMResponse, latency: number): void {
    this.metrics.requestCount++;
    this.metrics.totalTokens += response.usage.total_tokens;
    this.metrics.totalCost += response.usage.total_tokens * this.costPerToken;
    this.metrics.averageLatency =
      (this.metrics.averageLatency * (this.metrics.requestCount - 1) + latency) /
      this.metrics.requestCount;
  }

  private recordError(): void {
    this.metrics.errorCount++;
  }

  getMetrics(): LLMMetrics {
    return { ...this.metrics };
  }
}

Testing ​

Mock LLM Client for Testing ​

typescript
class MockLLMClient extends LLMClient {
  constructor(private responses: Map<string, string>) {
    super();
  }

  async complete(request: LLMRequest): Promise<LLMResponse> {
    const key = request.messages[request.messages.length - 1].content;
    const content = this.responses.get(key) || 'Mock response';

    return {
      id: 'mock-id',
      content,
      model: 'mock-model',
      usage: { prompt_tokens: 10, completion_tokens: 10, total_tokens: 20 },
    };
  }

  async *stream(request: LLMRequest): AsyncIterator<string> {
    const response = await this.complete(request);
    yield response.content;
  }
}

// Usage in tests
describe('LLM Integration', () => {
  it('should process requests', async () => {
    const mockClient = new MockLLMClient(
      new Map([['test prompt', 'test response']])
    );

    const response = await mockClient.complete({
      model: 'test',
      messages: [{ role: 'user', content: 'test prompt' }],
    });

    expect(response.content).toBe('test response');
  });
});

Resources ​


Remember: Always validate and sanitize data before sending to LLM APIs, and review responses before using them in production.

Enterprise AI Documentation Hub