From 429efaf629e7c538c611c63ba45e8d21c0ad2a12 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Sat, 16 Aug 2025 16:25:45 -0700 Subject: [PATCH] cleaned up the LLM providers. --- src/agent/core/AgentCore.ts | 25 ---- src/agent/llm/ClaudeProvider.ts | 69 +--------- src/agent/llm/GeminiProvider.ts | 64 +-------- src/agent/llm/LLMProvider.ts | 14 +- src/agent/llm/OpenAIProvider.ts | 237 ++++++++++++-------------------- src/constants/llmConstants.ts | 10 ++ 6 files changed, 100 insertions(+), 319 deletions(-) create mode 100644 src/constants/llmConstants.ts diff --git a/src/agent/core/AgentCore.ts b/src/agent/core/AgentCore.ts index 6c4a213..66ea05a 100644 --- a/src/agent/core/AgentCore.ts +++ b/src/agent/core/AgentCore.ts @@ -93,31 +93,6 @@ export class AgentCore { } } - /** - * Process user input and get complete response (non-streaming) - */ - async processUserInputComplete(userInput: string): Promise { - if (!this.llmProvider) { - throw new Error('No LLM provider configured'); - } - - // Add user message to state - this.agentState.addMessage('user', userInput); - - // Get system prompt with current context - const systemPrompt = await SystemPrompts.getSystemPromptWithContext(); - - // Get full conversation history with preserved roles - const conversationHistory = this.agentState.getMessages(); - - // Generate complete response with full conversation context - const response = await this.llmProvider.generateCompletion(conversationHistory, systemPrompt); - - // Add assistant response to state - this.agentState.addMessage('assistant', response.content); - - return response.content; - } /** * Abort the current streaming request and clean up messages diff --git a/src/agent/llm/ClaudeProvider.ts b/src/agent/llm/ClaudeProvider.ts index 9b5faa6..e9ec58a 100644 --- a/src/agent/llm/ClaudeProvider.ts +++ b/src/agent/llm/ClaudeProvider.ts @@ -1,5 +1,5 @@ import { LLMProvider } from './LLMProvider'; -import type { StreamChunk, LLMResponse } from './StreamingTypes'; +import type { StreamChunk } from './StreamingTypes'; import type { Message } from '../core/AgentState'; import { ConfigManager } from '../../core/config/ConfigManager'; @@ -159,71 +159,4 @@ export class ClaudeProvider extends LLMProvider { } } - async generateCompletion( - messages: Message[], - systemPrompt?: string, - tools?: Record[] - ): Promise { - const { system, messages: claudeMessages } = this.convertMessages(messages, systemPrompt); - - const requestBody: { - model: string; - max_tokens: number; - messages: Array<{ role: 'user' | 'assistant'; content: string }>; - stream: boolean; - system?: string; - tools?: Record[]; - } = { - model: this.model, - max_tokens: 8192, - messages: claudeMessages, - stream: false - }; - - if (system) { - requestBody.system = system; - } - - if (tools && tools.length > 0) { - requestBody.tools = tools; - } - - const response = await fetch(this.apiEndpoint, { - method: 'POST', - headers: { - 'x-api-key': this.apiKey, - 'Content-Type': 'application/json', - 'anthropic-version': '2023-06-01' - }, - body: JSON.stringify(requestBody), - mode: 'cors' - }); - - if (!response.ok) { - throw new Error(`Claude API error: ${response.status} ${response.statusText}`); - } - - const data = await response.json(); - - // Extract content from Claude's response format - const content = data.content - ?.filter((block: { type: string }) => block.type === 'text') - ?.map((block: { text: string }) => block.text) - ?.join('') || ''; - - // Extract tool calls if present - const toolCalls = data.content - ?.filter((block: { type: string }) => block.type === 'tool_use') - ?.map((block: { id: string; name: string; input: Record }) => ({ - id: block.id, - name: block.name, - parameters: block.input - })) || []; - - return { - content, - toolCalls: toolCalls.length > 0 ? toolCalls : undefined, - finished: data.stop_reason === 'end_turn' || data.stop_reason === 'tool_use' - }; - } } \ No newline at end of file diff --git a/src/agent/llm/GeminiProvider.ts b/src/agent/llm/GeminiProvider.ts index 7a770b1..c62f0c5 100644 --- a/src/agent/llm/GeminiProvider.ts +++ b/src/agent/llm/GeminiProvider.ts @@ -1,5 +1,5 @@ import { LLMProvider } from './LLMProvider'; -import type { StreamChunk, LLMResponse } from './StreamingTypes'; +import type { StreamChunk } from './StreamingTypes'; import type { Message } from '../core/AgentState'; import { ConfigManager } from '../../core/config/ConfigManager'; @@ -180,66 +180,4 @@ export class GeminiProvider extends LLMProvider { reader.releaseLock(); } } - - async generateCompletion( - messages: Message[], - systemPrompt?: string, - tools?: Record[] - ): Promise { - const { systemInstruction, contents } = this.convertMessages(messages, systemPrompt); - - const requestBody: { - contents: Array<{ role: 'user' | 'model'; parts: Array<{ text: string }> }>; - generationConfig: { temperature: number; maxOutputTokens: number }; - systemInstruction?: { parts: Array<{ text: string }> }; - tools?: Record[]; - } = { - contents, - generationConfig: { - temperature: 0.7, - maxOutputTokens: 8192, - } - }; - - if (systemInstruction) { - requestBody.systemInstruction = systemInstruction; - } - - if (tools && tools.length > 0) { - requestBody.tools = tools; - } - - const response = await fetch(`${this.apiEndpoint}:generateContent?key=${this.apiKey}`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }); - - if (!response.ok) { - throw new Error(`Gemini API error: ${response.status} ${response.statusText}`); - } - - const data = await response.json(); - - // Extract content from Gemini's response format - const candidate = data.candidates?.[0]; - const content = candidate?.content?.parts?.[0]?.text || ''; - - // Extract tool calls if present (Gemini format) - const toolCalls = candidate?.content?.parts - ?.filter((part: { functionCall?: unknown }) => part.functionCall) - ?.map((part: { functionCall: { name: string; args: Record } }) => ({ - id: `tool_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`, // Generate ID - name: part.functionCall.name, - parameters: part.functionCall.args - })) || []; - - return { - content, - toolCalls: toolCalls.length > 0 ? toolCalls : undefined, - finished: candidate?.finishReason === 'STOP' || candidate?.finishReason === 'MAX_TOKENS' - }; - } } \ No newline at end of file diff --git a/src/agent/llm/LLMProvider.ts b/src/agent/llm/LLMProvider.ts index 67d6ee0..71d78f0 100644 --- a/src/agent/llm/LLMProvider.ts +++ b/src/agent/llm/LLMProvider.ts @@ -1,4 +1,4 @@ -import type { StreamChunk, LLMResponse } from './StreamingTypes'; +import type { StreamChunk } from './StreamingTypes'; import type { Message } from '../core/AgentState'; /** @@ -18,16 +18,4 @@ export abstract class LLMProvider { systemPrompt?: string, tools?: Record[] ): AsyncIterableIterator; - - /** - * Generate a complete response from the LLM (non-streaming) - * @param messages The full conversation history with preserved roles - * @param systemPrompt The system prompt (optional, can be included in messages) - * @param tools Available tools (optional for now) - */ - abstract generateCompletion( - messages: Message[], - systemPrompt?: string, - tools?: Record[] - ): Promise; } \ No newline at end of file diff --git a/src/agent/llm/OpenAIProvider.ts b/src/agent/llm/OpenAIProvider.ts index 3e10f0a..0a36ad7 100644 --- a/src/agent/llm/OpenAIProvider.ts +++ b/src/agent/llm/OpenAIProvider.ts @@ -1,8 +1,9 @@ import { LLMProvider } from './LLMProvider'; -import type { StreamChunk, LLMResponse } from './StreamingTypes'; +import type { StreamChunk } from './StreamingTypes'; import type { Message } from '../core/AgentState'; import { ConfigManager } from '../../core/config/ConfigManager'; import { URL_CONSTANTS } from '../../constants/coreConstants'; +import { LLM_PROTOCOL } from '../../constants/llmConstants'; /** * OpenAI API provider implementation @@ -16,6 +17,78 @@ export class OpenAIProvider extends LLMProvider { super(); } + /** + * Build OpenAI-compatible messages array from input messages and system prompt + */ + private buildRequestMessages(messages: Message[], systemPrompt?: string): Array<{ role: string; content: string }> { + const openAIMessages: Array<{ role: string; content: string }> = []; + + // Add system prompt if provided + if (systemPrompt) { + openAIMessages.push({ role: 'system', content: systemPrompt }); + } + + // Add conversation history with preserved roles + openAIMessages.push(...messages.map(msg => ({ + role: msg.role, + content: msg.content + }))); + + return openAIMessages; + } + + /** + * Create API request with proper headers and body + */ + private async createApiRequest(messages: Array<{ role: string; content: string }>, config: ReturnType, streaming: boolean): Promise { + const response = await fetch(config.apiEndpoint, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${config.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: config.model, + ...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}), + messages, + stream: streaming, + }), + }); + + if (!response.ok) { + const errorText = await response.text().catch(() => 'Unknown error'); + throw new Error(`${this.name} API request failed (${response.status}): ${response.statusText}. ${errorText}`); + } + + return response; + } + + /** + * Process thinking and content chunks, yielding appropriate StreamChunks + */ + private async *processContentChunk( + thinking: string | undefined, + content: string | undefined, + lastSegmentType: { current: 'thinking' | 'content' | null } + ): AsyncIterableIterator { + if (typeof thinking === 'string' && thinking.length > 0) { + if (lastSegmentType.current && lastSegmentType.current !== 'thinking') { + yield { type: 'text', content: LLM_PROTOCOL.SEGMENT_SEPARATOR }; + } + yield { type: 'text', content: thinking }; + lastSegmentType.current = 'thinking'; + } + + if (typeof content === 'string' && content.length > 0) { + if (lastSegmentType.current && lastSegmentType.current !== 'content') { + yield { type: 'text', content: LLM_PROTOCOL.SEGMENT_SEPARATOR }; + } + yield { type: 'text', content: content }; + lastSegmentType.current = 'content'; + } + } + + /** * Get current configuration values from ConfigManager */ @@ -50,7 +123,7 @@ export class OpenAIProvider extends LLMProvider { */ private detectStreamFormat(firstChunk: string): boolean { // If it starts with "data: ", it's OpenAI SSE format - if (firstChunk.trim().startsWith('data: ')) { + if (firstChunk.trim().startsWith(LLM_PROTOCOL.SSE_DATA_PREFIX)) { return false; // Not Ollama format } @@ -85,12 +158,12 @@ export class OpenAIProvider extends LLMProvider { * Parse OpenAI's SSE format chunk */ private parseOpenAIChunk(line: string): { thinking?: string; content?: string; isDone?: boolean } { - if (!line.startsWith('data: ')) { + if (!line.startsWith(LLM_PROTOCOL.SSE_DATA_PREFIX)) { return {}; } - const data = line.slice(6); - if (data === '[DONE]') { + const data = line.slice(LLM_PROTOCOL.SSE_DATA_PREFIX.length); + if (data === LLM_PROTOCOL.SSE_DONE_MARKER) { return { isDone: true }; } @@ -109,52 +182,20 @@ export class OpenAIProvider extends LLMProvider { messages: Message[], systemPrompt?: string ): AsyncIterableIterator { - // Get fresh config values const config = this.getCurrentConfig(); - - // Build OpenAI messages array with role preservation - const openAIMessages: Array<{ role: string; content: string }> = []; - - // Add system prompt if provided - if (systemPrompt) { - openAIMessages.push({ role: 'system', content: systemPrompt }); - } - - // Add conversation history with preserved roles - openAIMessages.push(...messages.map(msg => ({ - role: msg.role, - content: msg.content - }))); - - const response = await fetch(config.apiEndpoint, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${config.apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - model: config.model, - ...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}), - messages: openAIMessages, - stream: true, - }), - }); - - if (!response.ok) { - throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`); - } + const requestMessages = this.buildRequestMessages(messages, systemPrompt); + const response = await this.createApiRequest(requestMessages, config, true); const reader = response.body?.getReader(); if (!reader) { - throw new Error('Failed to get response reader'); + throw new Error(`${this.name} streaming: Failed to get response reader from API response`); } const decoder = new TextDecoder(); let buffer = ''; let firstChunkProcessed = false; - let lastSegmentType: 'thinking' | 'content' | null = null; + const lastSegmentType = { current: null as 'thinking' | 'content' | null }; - try { while (true) { const { done, value } = await reader.read(); @@ -177,124 +218,20 @@ export class OpenAIProvider extends LLMProvider { firstChunkProcessed = true; } - if (this.isOllamaFormat) { - const { thinking, content, isDone } = this.parseOllamaChunk(trimmedLine); - if (isDone) { - yield { type: 'done', content: '' }; - return; - } - - if (typeof thinking === 'string' && thinking.length > 0) { - if (lastSegmentType && lastSegmentType !== 'thinking') { - yield { type: 'text', content: '\n\n' }; - } - yield { type: 'text', content: thinking }; - lastSegmentType = 'thinking'; - } - - if (typeof content === 'string' && content.length > 0) { - if (lastSegmentType && lastSegmentType !== 'content') { - yield { type: 'text', content: '\n\n' }; - } - yield { type: 'text', content: content }; - lastSegmentType = 'content'; - } - - continue; - } - - const { thinking, content, isDone } = this.parseOpenAIChunk(trimmedLine); + const { thinking, content, isDone } = this.isOllamaFormat + ? this.parseOllamaChunk(trimmedLine) + : this.parseOpenAIChunk(trimmedLine); + if (isDone) { yield { type: 'done', content: '' }; return; } - - if (typeof thinking === 'string' && thinking.length > 0) { - if (lastSegmentType && lastSegmentType !== 'thinking') { - yield { type: 'text', content: '\n\n' }; - } - yield { type: 'text', content: thinking }; - lastSegmentType = 'thinking'; - } - if (typeof content === 'string' && content.length > 0) { - if (lastSegmentType && lastSegmentType !== 'content') { - yield { type: 'text', content: '\n\n' }; - } - yield { type: 'text', content }; - lastSegmentType = 'content'; - } + yield* this.processContentChunk(thinking, content, lastSegmentType); } } } finally { reader.releaseLock(); } } - - async generateCompletion( - messages: Message[], - systemPrompt?: string - ): Promise { - // Get fresh config values - const config = this.getCurrentConfig(); - - // Build OpenAI messages array with role preservation - const openAIMessages: Array<{ role: string; content: string }> = []; - - // Add system prompt if provided - if (systemPrompt) { - openAIMessages.push({ role: 'system', content: systemPrompt }); - } - - // Add conversation history with preserved roles - openAIMessages.push(...messages.map(msg => ({ - role: msg.role, - content: msg.content - }))); - - const response = await fetch(config.apiEndpoint, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${config.apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - model: config.model, - ...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}), - messages: openAIMessages, - stream: false, - }), - }); - - if (!response.ok) { - throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`); - } - - const data = await response.json(); - - // Handle different response formats - - if (data.message) { - // Ollama/compatible format - const thinking: string = data.message.thinking || ''; - const contentText: string = data.message.content || data.response || ''; - const textCombined = thinking && contentText ? `${thinking}\n\n${contentText}` : (thinking || contentText); - - return { - content: textCombined, - finished: data.done === true || data.done_reason === 'stop' - }; - } else { - // OpenAI format - const choice = data.choices?.[0]; - const thinking: string = choice?.message?.thinking || ''; - const contentText: string = choice?.message?.content || ''; - const textCombined = thinking && contentText ? `${thinking}\n\n${contentText}` : (thinking || contentText); - - return { - content: textCombined, - finished: choice?.finish_reason === 'stop' - }; - } - } } \ No newline at end of file diff --git a/src/constants/llmConstants.ts b/src/constants/llmConstants.ts new file mode 100644 index 0000000..1674081 --- /dev/null +++ b/src/constants/llmConstants.ts @@ -0,0 +1,10 @@ +/** + * Constants related to LLM providers and API interactions + */ + +// Protocol markers and special character combinations used in LLM communications +export const LLM_PROTOCOL = { + SSE_DATA_PREFIX: 'data: ', + SSE_DONE_MARKER: '[DONE]', + SEGMENT_SEPARATOR: '\n\n' +} as const; \ No newline at end of file