3dtours/KGStudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implemented the v1 browser embedded LLM support (Gemma 4 E4B based)

Browse Source
This commit is contained in:
Xiaohan-Tian
2026-05-14 18:55:23 -07:00
parent 0e6e2736d0
commit 98c2b97413
30 changed files with 19462 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agent/core/AgentCore.ts
+4 -2
View File
@@ -1,4 +1,4 @@
import { LLMProvider } from '../llm/LLMProvider';
import type { LLMProvider } from '../llm/LLMProvider';
import { AgentState } from './AgentState';
import { SystemPrompts } from './SystemPrompts';
import { AVAILABLE_TOOLS } from '../tools';
@@ -92,7 +92,9 @@ export class AgentCore {
// Add user message to state
this.currentUserMessageId = this.agentState.addMessage('user', userInput);
const systemPrompt = await SystemPrompts.getSystemPromptWithContext();
const systemPrompt = await SystemPrompts.getSystemPromptWithContext(
this.llmProvider.getPreferredSystemPromptPath?.(),
);
const tools = this.getToolDefinitions();
try {
src/agent/core/SystemPrompts.ts
+11 -10
View File
@@ -22,25 +22,26 @@ interface SystemPromptContext {
* System prompts for the AI agent with dynamic context loading
*/
export class SystemPrompts {
private static cachedTemplate: string | null = null;
private static cachedTemplates: Map<string, string> = new Map();
private static readonly FALLBACK_PROMPT = `You are K.G.Studio Musician Assistant Agent, a highly skilled music musician with extensive knowledge in music theory, composition, and production.`;
/**
* Load the system prompt template from the public folder
*/
private static async loadTemplate(): Promise<string> {
if (this.cachedTemplate) {
return this.cachedTemplate;
private static async loadTemplate(templatePath: string = 'prompts/system.md'): Promise<string> {
if (this.cachedTemplates.has(templatePath)) {
return this.cachedTemplates.get(templatePath)!;
}
try {
const response = await fetch(`${import.meta.env.BASE_URL}prompts/system.md`);
const response = await fetch(`${import.meta.env.BASE_URL}${templatePath}`);
if (!response.ok) {
throw new Error(`Failed to load system prompt: ${response.status}`);
}
this.cachedTemplate = await response.text();
return this.cachedTemplate;
const template = await response.text();
this.cachedTemplates.set(templatePath, template);
return template;
} catch (error) {
console.error('Failed to load system prompt template:', error);
return this.FALLBACK_PROMPT;
@@ -231,9 +232,9 @@ export class SystemPrompts {
/**
* Get the system prompt with current context applied (backward compatible)
*/
static async getSystemPromptWithContext(): Promise<string> {
static async getSystemPromptWithContext(templatePath?: string): Promise<string> {
try {
const template = await this.loadTemplate();
const template = await this.loadTemplate(templatePath);
let promptWithContext = await this.getPromptWithContext(template);
// Append custom instructions from config if provided
@@ -263,4 +264,4 @@ export class SystemPrompts {
static clearCache(): void {
this.cachedTemplate = null;
}
}
}
src/agent/llm/LLMProvider.ts
+13 -22
View File
@@ -3,16 +3,24 @@ import type { StreamChunk } from './StreamingTypes';
import type { Message, ToolCall } from '../core/AgentState';
import type { OpenAIToolDefinition } from '../tools/BaseTool';
export interface LLMProvider {
getPreferredSystemPromptPath?(): string | undefined;
generateStream(
messages: Message[],
systemPrompt?: string,
tools?: OpenAIToolDefinition[],
): AsyncIterableIterator<StreamChunk>;
}
/**
* LLM provider using the OpenAI SDK.
* Works with any OpenAI-compatible API (OpenAI, OpenRouter, Ollama, vLLM, etc.)
* OpenAI-compatible provider implementation.
* Works with OpenAI and OpenAI-compatible APIs (OpenRouter, Ollama, vLLM, etc.)
*/
export class LLMProvider {
export class OpenAICompatibleLLMProvider implements LLMProvider {
private client: OpenAI;
private model: string;
constructor(apiKey: string, model: string, baseURL?: string) {
// The OpenAI SDK appends /chat/completions itself, so strip it if the user included it
const normalizedBaseURL = baseURL?.replace(/\/chat\/completions\/?$/, '') || undefined;
this.client = new OpenAI({
@@ -23,12 +31,9 @@ export class LLMProvider {
this.model = model;
}
/**
* Convert internal Message[] to OpenAI ChatCompletionMessageParam[]
*/
private convertMessages(
messages: Message[],
systemPrompt?: string
systemPrompt?: string,
): OpenAI.ChatCompletionMessageParam[] {
const result: OpenAI.ChatCompletionMessageParam[] = [];
@@ -64,10 +69,6 @@ export class LLMProvider {
return result;
}
/**
* Generate a streaming response from the LLM.
* Yields StreamChunks for text content and tool calls.
*/
async *generateStream(
messages: Message[],
systemPrompt?: string,
@@ -87,8 +88,6 @@ export class LLMProvider {
}
const stream = this.client.chat.completions.stream(requestParams);
// Accumulate tool calls across chunks (they arrive incrementally)
const toolCallAccumulator = new Map<number, { id: string; name: string; arguments: string }>();
for await (const chunk of stream) {
@@ -97,23 +96,18 @@ export class LLMProvider {
if (!choice) continue;
const delta = choice.delta;
// Yield text content
if (delta.content) {
yield { type: 'text', content: delta.content };
}
// Accumulate tool calls from deltas
if (delta.tool_calls) {
for (const tc of delta.tool_calls) {
const existing = toolCallAccumulator.get(tc.index);
if (existing) {
// Append to existing tool call
if (tc.function?.arguments) {
existing.arguments += tc.function.arguments;
}
} else {
// New tool call
toolCallAccumulator.set(tc.index, {
id: tc.id ?? '',
name: tc.function?.name ?? '',
@@ -124,11 +118,9 @@ export class LLMProvider {
}
}
// After stream ends, get the final completion for finish_reason
const finalCompletion = await stream.finalChatCompletion();
const finishReason = finalCompletion.choices[0]?.finish_reason ?? 'stop';
// Emit accumulated tool calls
if (toolCallAccumulator.size > 0) {
for (const [, tc] of toolCallAccumulator) {
const toolCall: ToolCall = {
@@ -140,7 +132,6 @@ export class LLMProvider {
}
}
// Signal completion
yield { type: 'done', content: '', finishReason };
}
}
src/agent/llm/LocalBrowserLLMProvider.ts
+301
View File
@@ -0,0 +1,301 @@
import type { StreamChunk } from './StreamingTypes';
import type { Message, ToolCall } from '../core/AgentState';
import type { OpenAIToolDefinition } from '../tools/BaseTool';
import { LocalLLMModelManager } from '../../util/localLLMModelManager';
import {
formatToolCall,
formatToolDeclaration,
formatToolResponse,
parseToolCalls,
stripToolProtocol,
} from './gemmaToolProtocol';
import { LOCAL_LLM_MODEL_FILENAME, LOCAL_LLM_MODEL_URL } from '../../util/localLLMConfig';
import { LocalLLMModelCache } from '../../util/localLLMModelCache';
import type { LLMProvider } from './LLMProvider';
type MediaPipeGenAI = {
FilesetResolver: {
forGenAiTasks(basePath: string): Promise<unknown>;
};
LlmInference: {
createFromOptions(fileset: unknown, options: Record<string, unknown>): Promise<GemmaInference>;
};
};
type GemmaInference = {
generateResponse(
prompt: string,
callback: (partial: string, done: boolean) => void,
): Promise<void> | void;
sizeInTokens(text: string): number;
close?: () => void;
};
interface PromptTemplatePart {
pre: string;
post: string;
}
const PROMPT_TEMPLATE: Record<'user' | 'model' | 'system', PromptTemplatePart> = {
user: { pre: '<|turn>user\n', post: '<turn|>\n' },
model: { pre: '<|turn>model\n', post: '<turn|>\n' },
system: { pre: '<|turn>system\n', post: '<turn|>\n' },
};
async function importMediaPipe(): Promise<MediaPipeGenAI> {
const bundleUrl = new URL(`${import.meta.env.BASE_URL}mediapipe/genai_bundle.mjs`, window.location.origin).href;
return import(/* @vite-ignore */ bundleUrl) as Promise<MediaPipeGenAI>;
}
export class LocalBrowserLLMProvider implements LLMProvider {
private inference: GemmaInference | null = null;
getPreferredSystemPromptPath(): string | undefined {
return 'prompts/system_compact.md';
}
private async ensureInference(): Promise<GemmaInference> {
await LocalLLMModelManager.ensureRuntimeSupported();
if (this.inference) {
return this.inference;
}
const [{ FilesetResolver, LlmInference }, modelLoad] = await Promise.all([
this.getMediaPipeModule(),
LocalLLMModelCache.loadModelReaderWithCache(
LOCAL_LLM_MODEL_URL,
LOCAL_LLM_MODEL_FILENAME,
progress => {
if (progress.fromCache) {
LocalLLMModelManager.notifyLoadStart(true);
} else {
LocalLLMModelManager.notifyLoadProgress(progress.receivedBytes, progress.totalBytes, false);
}
},
),
]);
LocalLLMModelManager.notifyLoadStart(modelLoad.fromCache);
console.log('[localLLM] Model stream prepared for MediaPipe.', {
filename: LOCAL_LLM_MODEL_FILENAME,
totalBytes: modelLoad.totalBytes,
fromCache: modelLoad.fromCache,
});
const fileset = await FilesetResolver.forGenAiTasks(`${import.meta.env.BASE_URL}mediapipe/wasm`);
console.log('[localLLM] MediaPipe fileset resolved. Creating inference engine...');
try {
this.inference = await LlmInference.createFromOptions(fileset, {
baseOptions: {
modelAssetBuffer: modelLoad.reader,
},
numResponses: 1,
maxTokens: 32768,
topK: 64,
temperature: 1.0,
});
console.log('[localLLM] MediaPipe inference engine created successfully.');
if (modelLoad.cacheWritePromise) {
void modelLoad.cacheWritePromise.then(() => {
LocalLLMModelManager.notifyCacheReady();
}).catch(error => {
console.error('[localLLM] Background cache write failed after inference creation.', error);
LocalLLMModelManager.notifyLoadError(error);
});
} else {
LocalLLMModelManager.notifyCacheReady();
}
return this.inference;
} catch (error) {
LocalLLMModelManager.notifyLoadError(error);
throw error;
}
}
private async getMediaPipeModule(): Promise<MediaPipeGenAI> {
return importMediaPipe();
}
private applyTemplate(message: { role: 'user' | 'model'; text: string }): string {
const template = PROMPT_TEMPLATE[message.role];
return `${template.pre}${message.text}${template.post}`;
}
private renderPrompt(
messages: Message[],
systemPrompt: string | undefined,
tools: OpenAIToolDefinition[] | undefined,
): string {
const thinkPrefix = '<|think|>';
const toolDeclarations = (tools ?? []).map(formatToolDeclaration).join('');
const systemContent = `${thinkPrefix}${systemPrompt ?? ''}${toolDeclarations}`;
const systemSection = systemContent
? `${PROMPT_TEMPLATE.system.pre}${systemContent}${PROMPT_TEMPLATE.system.post}`
: '';
const conversationParts: string[] = [];
for (let i = 0; i < messages.length; i += 1) {
const message = messages[i];
if (message.role === 'user') {
conversationParts.push(this.applyTemplate({ role: 'user', text: message.content ?? '' }));
continue;
}
if (message.role === 'assistant') {
let modelText = message.content ?? '';
if (message.tool_calls?.length) {
for (const toolCall of message.tool_calls) {
modelText += formatToolCall(toolCall.function.name, toolCall.function.arguments);
}
let scanIndex = i + 1;
while (scanIndex < messages.length && messages[scanIndex].role === 'tool') {
const toolMessage = messages[scanIndex];
const matchedCall = message.tool_calls.find(call => call.id === toolMessage.tool_call_id);
if (matchedCall) {
let parsedResult: unknown = toolMessage.content ?? '';
try {
parsedResult = JSON.parse(toolMessage.content ?? '{}');
} catch {
parsedResult = toolMessage.content ?? '';
}
modelText += formatToolResponse(matchedCall.function.name, parsedResult);
}
scanIndex += 1;
}
}
conversationParts.push(this.applyTemplate({ role: 'model', text: modelText }));
}
}
return `${systemSection}${conversationParts.join('')}${PROMPT_TEMPLATE.model.pre}`;
}
async *generateStream(
messages: Message[],
systemPrompt?: string,
tools?: OpenAIToolDefinition[],
): AsyncIterableIterator<StreamChunk> {
const inference = await this.ensureInference();
const prompt = this.renderPrompt(messages, systemPrompt, tools);
console.log('------------ LOCAL RAW PROMPT ------------');
console.log(prompt);
console.log('------------------------------------------');
const start = performance.now();
let firstTokenTime: number | null = null;
let rawResponse = '';
let streamedVisibleText = '';
const pendingTextDeltas: string[] = [];
let generationError: unknown = null;
let generationDone = false;
let notifyWaiting: (() => void) | null = null;
const wake = () => {
if (notifyWaiting) {
const resolve = notifyWaiting;
notifyWaiting = null;
resolve();
}
};
const generationPromise = new Promise<void>((resolve, reject) => {
try {
const result = inference.generateResponse(prompt, (partial, done) => {
if (firstTokenTime === null) {
firstTokenTime = performance.now();
}
rawResponse += partial;
const visibleText = stripToolProtocol(rawResponse);
if (visibleText.startsWith(streamedVisibleText)) {
const delta = visibleText.slice(streamedVisibleText.length);
if (delta) {
streamedVisibleText = visibleText;
pendingTextDeltas.push(delta);
wake();
}
} else if (visibleText && visibleText !== streamedVisibleText) {
const delta = visibleText.slice(streamedVisibleText.length) || visibleText;
streamedVisibleText = visibleText;
pendingTextDeltas.push(delta);
wake();
}
if (done) {
generationDone = true;
wake();
setTimeout(resolve, 0);
}
});
Promise.resolve(result).catch(error => {
generationError = error;
generationDone = true;
wake();
reject(error);
});
} catch (error) {
generationError = error;
generationDone = true;
wake();
reject(error);
}
});
while (!generationDone || pendingTextDeltas.length > 0) {
while (pendingTextDeltas.length > 0) {
const delta = pendingTextDeltas.shift();
if (delta) {
yield { type: 'text', content: delta };
}
}
if (generationDone) {
break;
}
await new Promise<void>(resolve => {
notifyWaiting = resolve;
});
}
await generationPromise;
if (generationError) {
throw generationError;
}
console.log('------------ LOCAL RAW RESPONSE ------------');
console.log(rawResponse);
console.log('--------------------------------------------');
const toolCalls = parseToolCalls(rawResponse);
const finishReason = toolCalls.length > 0 ? 'tool_calls' : 'stop';
const promptTokenCount = inference.sizeInTokens(prompt);
const generatedTokenCount = inference.sizeInTokens(rawResponse);
const totalEnd = performance.now();
const prefillMs = firstTokenTime !== null ? firstTokenTime - start : 0;
const decodeMs = Math.max(0, totalEnd - start - prefillMs);
const prefillTps = prefillMs > 0 ? promptTokenCount / (prefillMs / 1000) : 0;
const generationTps = decodeMs > 0 ? generatedTokenCount / (decodeMs / 1000) : 0;
console.log(`[localLLM] prefill t/s: ${prefillTps.toFixed(1)}`);
console.log(`[localLLM] generation t/s: ${generationTps.toFixed(1)}`);
for (const parsed of toolCalls) {
const toolCall: ToolCall = {
id: `gemma_tool_${Date.now()}_${Math.random().toString(36).slice(2)}`,
type: 'function',
function: {
name: parsed.name,
arguments: JSON.stringify(parsed.args),
},
};
yield { type: 'tool_call', content: '', toolCall };
}
yield { type: 'done', content: '', finishReason };
}
}
src/agent/llm/gemmaToolProtocol.streaming.test.ts
+20
View File
@@ -0,0 +1,20 @@
import { describe, expect, it } from 'vitest';
import { stripToolProtocol } from './gemmaToolProtocol';
describe('stripToolProtocol streaming safety', () => {
it('hides incomplete thought blocks entirely until they are closed', () => {
const partial = `<|channel>thought
I have successfully read the music.
I will summarize this for the user.`;
expect(stripToolProtocol(partial)).toBe('');
});
it('reveals visible answer cleanly after a thought block closes', () => {
const completed = `<|channel>thought
Internal reasoning here.
<channel|>Here is the sheet music I read.`;
expect(stripToolProtocol(completed)).toBe('Here is the sheet music I read.');
});
});
src/agent/llm/gemmaToolProtocol.test.ts
+25
View File
@@ -0,0 +1,25 @@
import { describe, expect, it } from 'vitest';
import { parseToolCalls } from './gemmaToolProtocol';
describe('parseToolCalls', () => {
it('parses nested object arguments emitted by Gemma tool calling', () => {
const text = '<|tool_call>call:add_notes{notes:[{pitch:<|"|>C4<|"|>,meta:{velocity:90,length:1.5}}],replaceExisting:false}<tool_call|>';
const parsed = parseToolCalls(text);
expect(parsed).toHaveLength(1);
expect(parsed[0].name).toBe('add_notes');
expect(parsed[0].args).toEqual({
notes: [
{
pitch: 'C4',
meta: {
velocity: 90,
length: 1.5,
},
},
],
replaceExisting: false,
});
});
});
src/agent/llm/gemmaToolProtocol.ts
+154
View File
@@ -0,0 +1,154 @@
import type { OpenAIToolDefinition } from '../tools/BaseTool';
function gemmaValue(value: unknown): string {
if (typeof value === 'string') return `<|"|>${value}<|"|>`;
if (Array.isArray(value)) return `[${value.map(gemmaValue).join(',')}]`;
if (typeof value === 'object' && value !== null) {
return `{${Object.entries(value as Record<string, unknown>).map(([key, nested]) => `${key}:${gemmaValue(nested)}`).join(',')}}`;
}
return String(value);
}
export function formatToolDeclaration(tool: OpenAIToolDefinition): string {
const body = {
description: tool.function.description,
parameters: tool.function.parameters,
};
return `<|tool>declaration:${tool.function.name}${gemmaValue(body)}<tool|>`;
}
export function formatToolCall(name: string, args: string): string {
const parsed = JSON.parse(args) as Record<string, unknown>;
return `<|tool_call>call:${name}${gemmaValue(parsed)}<tool_call|>`;
}
export function formatToolResponse(name: string, result: unknown): string {
return `<|tool_response>response:${name}${gemmaValue(result)}<tool_response|>`;
}
function parseGemmaValue(str: string, pos: number): { value: unknown; next: number } {
while (pos < str.length && str[pos] === ' ') pos += 1;
const stringDelimiter = '<|"|>';
if (str.startsWith(stringDelimiter, pos)) {
const start = pos + stringDelimiter.length;
const end = str.indexOf(stringDelimiter, start);
if (end === -1) {
return { value: '', next: str.length };
}
return { value: str.slice(start, end), next: end + stringDelimiter.length };
}
if (str[pos] === '{') {
const result: Record<string, unknown> = {};
pos += 1;
while (pos < str.length && str[pos] !== '}') {
while (pos < str.length && (str[pos] === ',' || str[pos] === ' ')) pos += 1;
if (str[pos] === '}') break;
const colonIndex = str.indexOf(':', pos);
if (colonIndex === -1) break;
const key = str.slice(pos, colonIndex).trim();
pos = colonIndex + 1;
const nested = parseGemmaValue(str, pos);
result[key] = nested.value;
pos = nested.next;
}
return { value: result, next: pos + 1 };
}
if (str[pos] === '[') {
const result: unknown[] = [];
pos += 1;
while (pos < str.length && str[pos] !== ']') {
while (pos < str.length && (str[pos] === ',' || str[pos] === ' ')) pos += 1;
if (str[pos] === ']') break;
const nested = parseGemmaValue(str, pos);
result.push(nested.value);
pos = nested.next;
}
return { value: result, next: pos + 1 };
}
let end = pos;
while (end < str.length && str[end] !== ',' && str[end] !== '}' && str[end] !== ']') end += 1;
const raw = str.slice(pos, end).trim();
if (raw === 'true') return { value: true, next: end };
if (raw === 'false') return { value: false, next: end };
if (raw !== '' && !Number.isNaN(Number(raw))) return { value: Number(raw), next: end };
return { value: raw, next: end };
}
function parseArgs(argsStr: string): Record<string, unknown> {
const parsed = parseGemmaValue(`{${argsStr}}`, 0).value;
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
return {};
}
return parsed as Record<string, unknown>;
}
export interface ParsedGemmaToolCall {
name: string;
args: Record<string, unknown>;
fullMatch: string;
index: number;
endIndex: number;
}
export function parseToolCalls(text: string): ParsedGemmaToolCall[] {
const prefix = '<|tool_call>call:';
const suffix = '<tool_call|>';
const stringDelimiter = '<|"|>';
const calls: ParsedGemmaToolCall[] = [];
let searchFrom = 0;
while (true) {
const prefixIndex = text.indexOf(prefix, searchFrom);
if (prefixIndex === -1) break;
const braceStart = text.indexOf('{', prefixIndex + prefix.length);
if (braceStart === -1) break;
const name = text.slice(prefixIndex + prefix.length, braceStart).trim();
let depth = 1;
let index = braceStart + 1;
while (index < text.length && depth > 0) {
if (text.startsWith(stringDelimiter, index)) {
const stringEnd = text.indexOf(stringDelimiter, index + stringDelimiter.length);
index = stringEnd === -1 ? text.length : stringEnd + stringDelimiter.length;
continue;
}
if (text[index] === '{') depth += 1;
else if (text[index] === '}') depth -= 1;
index += 1;
}
const braceEnd = index;
const suffixIndex = text.indexOf(suffix, braceEnd);
if (suffixIndex === -1) break;
const argsStr = text.slice(braceStart + 1, braceEnd - 1);
const endIndex = suffixIndex + suffix.length;
calls.push({
name,
args: parseArgs(argsStr),
fullMatch: text.slice(prefixIndex, endIndex),
index: prefixIndex,
endIndex,
});
searchFrom = endIndex;
}
return calls;
}
export function stripToolProtocol(text: string): string {
let result = text.replace(
/<\|tool_call>call:(\w+)\{[\s\S]*?\}<tool_call\|><\|tool_response>[\s\S]*?<tool_response\|>/g,
'[Tool call completed]',
);
result = result.replace(/<\|channel>thought[\s\S]*?<channel\|>/g, '');
result = result.replace(/<\|channel>thought[\s\S]*/g, '');
result = result.replace(/<\|tool_call>[\s\S]*/g, '');
result = result.replace(/<\|tool_response>[\s\S]*/g, '');
result = result.replace(/<\|"\|>/g, '');
return result.trimStart();
}
src/components/ChatBox.tsx
+56 -3
View File
@@ -3,7 +3,8 @@ import './ChatBox.css';
import { FaPlus, FaBan, FaDownload } from 'react-icons/fa';
import { UserMessage, AssistantMessage } from './chat';
import { AgentCore } from '../agent/core/AgentCore';
import { LLMProvider } from '../agent/llm/LLMProvider';
import { OpenAICompatibleLLMProvider, type LLMProvider } from '../agent/llm/LLMProvider';
import { LocalBrowserLLMProvider } from '../agent/llm/LocalBrowserLLMProvider';
import { ConfigManager } from '../core/config/ConfigManager';
import { useProjectStore } from '../stores/projectStore';
import { SystemPrompts } from '../agent/core/SystemPrompts';
@@ -13,6 +14,8 @@ import { useStreamProcessor } from '../hooks/useStreamProcessor';
import { createMessage, addWelcomeMessage } from '../utils/chatMessageUtils';
import { formatLocalDateTime } from '../util/timeUtil';
import { downloadBlob, buildTimestampSuffix } from '../util/miscUtil';
import { LocalLLMModelManager, type LocalLLMModelState } from '../util/localLLMModelManager';
import { LOCAL_LLM_DISPLAY_NAME, LOCAL_LLM_PROVIDER_KEY } from '../util/localLLMConfig';
import KGDropdown from './common/KGDropdown';
import type { ChatMessage } from '../types/projectTypes';
@@ -32,6 +35,8 @@ const createLLMProviderFromConfig = (): LLMProvider => {
let baseURL: string | undefined;
switch (providerType) {
case LOCAL_LLM_PROVIDER_KEY:
return new LocalBrowserLLMProvider();
case 'openai':
apiKey = configManager.get('general.openai.api_key') as string;
model = configManager.get('general.openai.model') as string;
@@ -50,7 +55,7 @@ const createLLMProviderFromConfig = (): LLMProvider => {
break;
}
return new LLMProvider(apiKey, model, baseURL);
return new OpenAICompatibleLLMProvider(apiKey, model, baseURL);
};
interface ChatBoxProps {
@@ -64,6 +69,8 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
const [messages, setMessages] = useState<ChatMessage[]>([]);
const [isProcessing, setIsProcessing] = useState(false);
const [lastUserMessage, setLastUserMessage] = useState<string>('');
const [localModelState, setLocalModelState] = useState<LocalLLMModelState>(LocalLLMModelManager.getState());
const [activeProvider, setActiveProvider] = useState<string>('openai');
// Track if this is the first message (for system prompt logging)
const [isFirstMessage, setIsFirstMessage] = useState(true);
@@ -180,9 +187,11 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
}
const applyProviderFromConfig = () => {
const providerType = (configManager.get('general.llm_provider') as string) || 'openai';
const provider = createLLMProviderFromConfig();
const agentCore = AgentCore.instance();
agentCore.setLLMProvider(provider);
setActiveProvider(providerType);
console.log('LLM provider configured');
};
@@ -204,6 +213,7 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
registerClearChatUICallback(clearChatUI);
const maybeUnsubscribePromise = initializeProvider();
const unsubscribeLocalModel = LocalLLMModelManager.subscribe(setLocalModelState);
(async () => {
if (hasShownWelcomeOnceInRuntime) return;
@@ -216,6 +226,7 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
})();
return () => {
unsubscribeLocalModel();
Promise.resolve(maybeUnsubscribePromise).then((cleanup) => {
if (typeof cleanup === 'function') cleanup();
}).catch(() => {});
@@ -266,7 +277,10 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
// Log system prompt only for first message
if (isFirstMessage) {
try {
const systemPrompt = await SystemPrompts.getSystemPromptWithContext();
const provider = AgentCore.instance().getLLMProvider();
const systemPrompt = await SystemPrompts.getSystemPromptWithContext(
provider?.getPreferredSystemPromptPath?.(),
);
console.log('------------ SYSTEM ------------');
console.log(systemPrompt);
console.log('--------------------------------');
@@ -372,6 +386,45 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
</div>
</div>
{activeProvider === LOCAL_LLM_PROVIDER_KEY && (
<div style={{ padding: '10px 14px 0 14px' }}>
<div className="settings-group" style={{ marginBottom: '12px', padding: '14px' }}>
<h4 style={{ marginBottom: '10px' }}>{LOCAL_LLM_DISPLAY_NAME} Local Runtime</h4>
{!localModelState.runtimeSupport.supported && (
<div className="settings-help" style={{ fontSize: '12px', color: '#d0a56b', marginBottom: '8px' }}>
{localModelState.runtimeSupport.reason}
</div>
)}
{!localModelState.isCached && !localModelState.isDownloading && localModelState.runtimeSupport.supported && (
<div className="settings-help" style={{ fontSize: '12px', color: '#b0b0b0', marginBottom: '8px' }}>
The local language model has not been downloaded yet. It will be downloaded automatically the next time you send a chat request with this provider.
</div>
)}
{(localModelState.isChecking || localModelState.isDownloading || localModelState.progressText) && (
<div className="settings-progress-block">
<div
className="settings-progress-track"
role="progressbar"
aria-valuemin={0}
aria-valuemax={100}
aria-valuenow={Math.max(0, Math.min(100, localModelState.progressPercent))}
>
<div className="settings-progress-fill" style={{ width: `${Math.max(0, Math.min(100, localModelState.progressPercent))}%` }} />
</div>
<div className="settings-help" style={{ fontSize: '12px', color: '#b0b0b0', marginTop: '6px' }}>
{localModelState.isChecking ? 'Checking local model cache...' : localModelState.progressText}
</div>
</div>
)}
{localModelState.error && (
<div className="settings-help" style={{ fontSize: '12px', color: '#d45a5a' }}>
{localModelState.error}
</div>
)}
</div>
</div>
)}
<div className="chatbox-messages">
{messages.map((message) => (
message.role === 'user' ? (
src/components/settings/Settings.css
+19
View File
@@ -293,6 +293,25 @@
font-size: 12px;
}
.settings-progress-block {
margin-top: 10px;
}
.settings-progress-track {
width: 100%;
height: 10px;
border-radius: 999px;
overflow: hidden;
background-color: #3a3a3a;
border: 1px solid #4a4a4a;
}
.settings-progress-fill {
height: 100%;
background: linear-gradient(90deg, #5a9fd4 0%, #76c28f 100%);
transition: width 0.2s ease;
}
/* Settings Help Links */
.settings-help-links {
display: flex;
src/components/settings/sections/GeneralSettings.tsx
+80 -3
View File
@@ -1,8 +1,10 @@
import React, { useState, useEffect, useCallback, useMemo } from 'react';
import { ConfigManager } from '../../../core/config/ConfigManager';
import { LocalLLMModelManager, type LocalLLMModelState } from '../../../util/localLLMModelManager';
import { LOCAL_LLM_DISPLAY_NAME, LOCAL_LLM_PROVIDER_KEY } from '../../../util/localLLMConfig';
const GeneralSettings: React.FC = () => {
const [llmProvider, setLlmProvider] = useState<string>('openai');
const [llmProvider, setLlmProvider] = useState<string>(LOCAL_LLM_PROVIDER_KEY);
const [openaiKey, setOpenaiKey] = useState<string>('');
const [openaiModel, setOpenaiModel] = useState<string>('');
const [geminiKey, setGeminiKey] = useState<string>('');
@@ -22,6 +24,7 @@ const GeneralSettings: React.FC = () => {
const [kgoneBaseUrl, setKgoneBaseUrl] = useState<string>('');
const [kgoneServerManaged, setKgoneServerManaged] = useState<boolean>(false);
const [soundfontServerManaged, setSoundfontServerManaged] = useState<boolean>(false);
const [localModelState, setLocalModelState] = useState<LocalLLMModelState>(LocalLLMModelManager.getState());
const configManager = ConfigManager.instance();
@@ -46,7 +49,7 @@ const GeneralSettings: React.FC = () => {
await configManager.initialize();
}
setLlmProvider((configManager.get('general.llm_provider') as string) || 'openai');
setLlmProvider((configManager.get('general.llm_provider') as string) || LOCAL_LLM_PROVIDER_KEY);
setOpenaiKey((configManager.get('general.openai.api_key') as string) || '');
setOpenaiModel((configManager.get('general.openai.model') as string) || '');
setOpenaiFlex((configManager.get('general.openai.flex') as boolean) ?? false);
@@ -69,6 +72,8 @@ const GeneralSettings: React.FC = () => {
};
loadConfig();
const unsubscribe = LocalLLMModelManager.subscribe(setLocalModelState);
return unsubscribe;
}, [configManager]);
// Debounced save function for text inputs
@@ -197,6 +202,14 @@ const GeneralSettings: React.FC = () => {
debouncedSave('general.kgone.base_url', value);
};
const handleDeleteLocalModel = async () => {
try {
await LocalLLMModelManager.deleteCachedModel();
} catch (error) {
console.error('Failed to delete local language model cache:', error);
}
};
// NOTE: Gemini and Claude are not supported yet due to CORS issues.
return (
<div className="settings-section">
@@ -217,6 +230,7 @@ const GeneralSettings: React.FC = () => {
value={llmProvider}
onChange={(e) => handleLlmProviderChange(e.target.value)}
>
<option value={LOCAL_LLM_PROVIDER_KEY}>Local LLM (Browser)</option>
<option value="openai">OpenAI</option>
{/* <option value="gemini">Gemini</option>
<option value="claude">Claude</option> */}
@@ -243,6 +257,69 @@ const GeneralSettings: React.FC = () => {
</div>
</div>
<div className="settings-group">
<h4>{LOCAL_LLM_DISPLAY_NAME} Local Runtime</h4>
{!localModelState.runtimeSupport.supported && (
<div className="settings-help" style={{ fontSize: '12px', color: '#d0a56b', marginTop: '4px', marginBottom: '8px' }}>
{localModelState.runtimeSupport.reason}
</div>
)}
<div className="settings-item">
<label className="settings-label">
Cached Model Status
</label>
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
{localModelState.isChecking
? 'Checking local model cache...'
: localModelState.isCached
? 'Downloaded in browser cache.'
: 'Not downloaded yet.'}
</div>
</div>
{!localModelState.isCached && !localModelState.isDownloading && localModelState.runtimeSupport.supported && (
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px', marginBottom: '8px' }}>
The local model downloads automatically the next time you chat with `Local LLM (Browser)`.
</div>
)}
{(localModelState.isDownloading || localModelState.progressText) && (
<div className="settings-progress-block">
<div
className="settings-progress-track"
role="progressbar"
aria-valuemin={0}
aria-valuemax={100}
aria-valuenow={Math.max(0, Math.min(100, localModelState.progressPercent))}
>
<div className="settings-progress-fill" style={{ width: `${Math.max(0, Math.min(100, localModelState.progressPercent))}%` }} />
</div>
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '6px' }}>
{localModelState.progressText}
</div>
</div>
)}
{localModelState.error && (
<div className="settings-help" style={{ fontSize: '12px', color: '#d45a5a', marginTop: '8px' }}>
{localModelState.error}
</div>
)}
<div className="settings-item" style={{ marginTop: '12px' }}>
<button
type="button"
className="settings-btn settings-btn-danger"
onClick={() => void handleDeleteLocalModel()}
disabled={localModelState.isDeleting || localModelState.isDownloading || !localModelState.isCached}
>
{localModelState.isDeleting ? 'Deleting...' : 'Delete Cached Model'}
</button>
</div>
</div>
<div className="settings-group">
<h4>OpenAI</h4>
@@ -600,4 +677,4 @@ const GeneralSettings: React.FC = () => {
);
};
export default GeneralSettings;
export default GeneralSettings;
src/constants/coreConstants.ts
+1 -1
View File
@@ -104,7 +104,7 @@ export const OPFS_CONSTANTS = {
export const CONFIG_UPGRADER_CONSTANTS = {
VERSION_KEY: '__config_version',
CURRENT_VERSION: 1,
CURRENT_VERSION: 2,
};
export const URL_CONSTANTS = {
src/core/config-upgrader/KGConfigUpgrader.ts
+5
View File
@@ -1,6 +1,7 @@
import { KGConfigStorage } from '../io/KGConfigStorage';
import { CONFIG_UPGRADER_CONSTANTS } from '../../constants/coreConstants';
import { upgradeConfigToV1 } from './upgradeConfigToV1';
import { upgradeConfigToV2 } from './upgradeConfigToV2';
/**
* KGConfigUpgrader — Orchestrates app-level migrations (e.g., storage backend changes).
@@ -33,6 +34,10 @@ export class KGConfigUpgrader {
await upgradeConfigToV1();
break;
}
case 2: {
await upgradeConfigToV2();
break;
}
default: {
throw new Error(`No config upgrader found for version ${nextVersion}`);
}
src/core/config-upgrader/upgradeConfigToV2.test.ts
+54
View File
@@ -0,0 +1,54 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
const configStore = new Map<string, { name: string; data: Record<string, unknown>; lastModified: number }>();
vi.mock('../io/KGConfigStorage', () => ({
KGConfigStorage: {
getInstance: () => ({
getRaw: vi.fn(async (name: string) => configStore.get(name)?.data ?? null),
saveRaw: vi.fn(async (name: string, data: Record<string, unknown>) => {
configStore.set(name, { name, data, lastModified: Date.now() });
}),
}),
},
}));
import { upgradeConfigToV2 } from './upgradeConfigToV2';
describe('upgradeConfigToV2', () => {
beforeEach(() => {
configStore.clear();
});
it('pins legacy installs without an explicit provider to the old default provider', async () => {
configStore.set('userConfig', {
name: 'userConfig',
data: {
general: {
openai: { api_key: '', model: 'gpt-5.4-mini', flex: false },
},
},
lastModified: Date.now(),
});
await upgradeConfigToV2();
expect((configStore.get('userConfig')?.data.general as Record<string, unknown>).llm_provider).toBe('openai');
});
it('leaves explicit providers unchanged', async () => {
configStore.set('userConfig', {
name: 'userConfig',
data: {
general: {
llm_provider: 'openai_compatible',
},
},
lastModified: Date.now(),
});
await upgradeConfigToV2();
expect((configStore.get('userConfig')?.data.general as Record<string, unknown>).llm_provider).toBe('openai_compatible');
});
});
src/core/config-upgrader/upgradeConfigToV2.ts
+25
View File
@@ -0,0 +1,25 @@
import { KGConfigStorage } from '../io/KGConfigStorage';
const CONFIG_KEY = 'userConfig';
const LEGACY_DEFAULT_PROVIDER = 'openai';
export async function upgradeConfigToV2(): Promise<void> {
const storage = KGConfigStorage.getInstance();
const rawConfig = await storage.getRaw(CONFIG_KEY);
if (!rawConfig || typeof rawConfig !== 'object') {
return;
}
const config = rawConfig as Record<string, unknown>;
const general = config.general;
if (!general || typeof general !== 'object') {
return;
}
if ('llm_provider' in (general as Record<string, unknown>)) {
return;
}
(general as Record<string, unknown>).llm_provider = LEGACY_DEFAULT_PROVIDER;
await storage.saveRaw(CONFIG_KEY, config);
}
src/core/config/ConfigManager.ts
+2 -2
View File
@@ -6,7 +6,7 @@ import { KGConfigStorage } from '../io/KGConfigStorage';
interface AppConfig {
general: {
language: string;
llm_provider: 'openai' | 'gemini' | 'claude' | 'claude_openrouter' | 'openai_compatible';
llm_provider: 'local_browser' | 'openai' | 'gemini' | 'claude' | 'claude_openrouter' | 'openai_compatible';
persist_api_keys_non_localhost: boolean;
openai: {
api_key: string;
@@ -183,7 +183,7 @@ export class ConfigManager {
this.defaultConfig = {
general: {
language: 'en_us',
llm_provider: 'openai',
llm_provider: 'local_browser',
persist_api_keys_non_localhost: false,
openai: {
api_key: '',
src/core/io/LocalSeparatorModelCache.test.ts
+8 -4
View File
@@ -9,10 +9,12 @@ class MockWritableFileStream {
this.handle = handle;
}
async write(content: ArrayBuffer | ArrayBufferView): Promise<void> {
const bytes = content instanceof ArrayBuffer
? new Uint8Array(content)
: new Uint8Array(content.buffer, content.byteOffset, content.byteLength);
async write(content: ArrayBuffer | ArrayBufferView | string): Promise<void> {
const bytes = typeof content === 'string'
? new TextEncoder().encode(content)
: content instanceof ArrayBuffer
? new Uint8Array(content)
: new Uint8Array(content.buffer, content.byteOffset, content.byteLength);
this.chunks.push(new Uint8Array(bytes));
}
@@ -44,6 +46,8 @@ class MockFileSystemFileHandle {
async getFile(): Promise<File> {
return {
size: this.content.byteLength,
text: async () => new TextDecoder().decode(this.content),
arrayBuffer: async () => this.content.buffer.slice(0),
} as unknown as File;
}
src/util/localLLMConfig.ts
+42
View File
@@ -0,0 +1,42 @@
export const LOCAL_LLM_PROVIDER_KEY = 'local_browser';
export const LOCAL_LLM_MODEL_URL =
'http://localhost:3000/models/gemma-4-E4B-it-web.task';
export const LOCAL_LLM_MODEL_FILENAME = 'gemma-4-E4B-it-web.task';
export const LOCAL_LLM_DISPLAY_NAME = 'Gemma 4 E4B';
export const LOCAL_LLM_LEGACY_FILENAMES = [
'gemma-3n-E4B-it-int4-Web.litertlm',
];
export interface LocalLLMRuntimeSupport {
supported: boolean;
webgpuExposed: boolean;
crossOriginIsolated: boolean;
sharedArrayBufferAvailable: boolean;
secureContext: boolean;
reason: string | null;
}
export function detectLocalLLMRuntimeSupport(): LocalLLMRuntimeSupport {
const secureContext = typeof window !== 'undefined' ? window.isSecureContext : false;
const crossOriginIsolated = typeof window !== 'undefined' ? window.crossOriginIsolated : false;
const sharedArrayBufferAvailable = typeof SharedArrayBuffer !== 'undefined';
const webgpuExposed = typeof navigator !== 'undefined' && 'gpu' in navigator;
let reason: string | null = null;
if (!secureContext) {
reason = 'Local browser LLM requires a secure context (HTTPS or localhost).';
} else if (!crossOriginIsolated || !sharedArrayBufferAvailable) {
reason = 'Local browser LLM requires SharedArrayBuffer support. Ensure COOP/COEP headers are enabled.';
} else if (!webgpuExposed) {
reason = 'Local browser LLM currently requires a browser with WebGPU support.';
}
return {
supported: reason === null,
webgpuExposed,
crossOriginIsolated,
sharedArrayBufferAvailable,
secureContext,
reason,
};
}
src/util/localLLMModelCache.ts
+92
View File
@@ -0,0 +1,92 @@
import { OpfsModelCache, type ModelDownloadProgress } from './opfsModelCache';
import { LOCAL_LLM_MODEL_FILENAME } from './localLLMConfig';
const cache = new OpfsModelCache({ directoryName: 'models' });
let writingToCachePromise: Promise<void> | null = null;
export { type ModelDownloadProgress };
export interface CachedModelStreamResult {
reader: ReadableStreamDefaultReader<Uint8Array>;
totalBytes: number;
fromCache: boolean;
cacheWritePromise: Promise<void> | null;
}
export class LocalLLMModelCache {
public static async exists(filename: string = LOCAL_LLM_MODEL_FILENAME): Promise<boolean> {
return cache.exists(filename);
}
public static async getFile(filename: string = LOCAL_LLM_MODEL_FILENAME): Promise<File> {
return cache.getFile(filename);
}
public static async getArrayBuffer(filename: string = LOCAL_LLM_MODEL_FILENAME): Promise<ArrayBuffer> {
return cache.getArrayBuffer(filename);
}
public static async delete(filename: string = LOCAL_LLM_MODEL_FILENAME): Promise<void> {
await cache.delete(filename);
}
public static async loadModelReaderWithCache(
sourceUrl: string,
filename: string = LOCAL_LLM_MODEL_FILENAME,
onProgress?: (progress: ModelDownloadProgress & { fromCache: boolean }) => void,
): Promise<CachedModelStreamResult> {
if (writingToCachePromise) {
await writingToCachePromise.catch(() => {});
}
if (await this.exists(filename)) {
const file = await this.getFile(filename);
onProgress?.({
receivedBytes: file.size,
totalBytes: file.size,
percent: 100,
fromCache: true,
});
return {
reader: file.stream().getReader(),
totalBytes: file.size,
fromCache: true,
cacheWritePromise: null,
};
}
const response = await fetch(sourceUrl);
if (!response.ok || !response.body) {
throw new Error(`Model download failed (${response.status})`);
}
const totalBytesHeader = response.headers.get('Content-Length');
const totalBytes = totalBytesHeader ? Number(totalBytesHeader) : 0;
const [streamForConsumer, streamForCache] = response.body.tee();
writingToCachePromise = cache.downloadStream(
streamForCache,
filename,
totalBytes > 0 ? totalBytes : null,
progress => onProgress?.({ ...progress, fromCache: false }),
);
writingToCachePromise = writingToCachePromise.finally(() => {
writingToCachePromise = null;
});
return {
reader: streamForConsumer.getReader(),
totalBytes,
fromCache: false,
cacheWritePromise: writingToCachePromise,
};
}
public static async download(
sourceUrl: string,
filename: string = LOCAL_LLM_MODEL_FILENAME,
onProgress?: (progress: ModelDownloadProgress) => void,
): Promise<void> {
await cache.download(sourceUrl, filename, onProgress);
}
}
src/util/localLLMModelManager.ts
+159
View File
@@ -0,0 +1,159 @@
import {
detectLocalLLMRuntimeSupport,
LOCAL_LLM_LEGACY_FILENAMES,
LOCAL_LLM_MODEL_FILENAME,
LOCAL_LLM_MODEL_URL,
type LocalLLMRuntimeSupport,
} from './localLLMConfig';
import { LocalLLMModelCache } from './localLLMModelCache';
export interface LocalLLMModelState {
isCached: boolean;
isChecking: boolean;
isDownloading: boolean;
isDeleting: boolean;
progressPercent: number;
progressText: string;
error: string;
runtimeSupport: LocalLLMRuntimeSupport;
}
type Listener = (state: LocalLLMModelState) => void;
export class LocalLLMModelManager {
private static listeners = new Set<Listener>();
private static initialized = false;
private static state: LocalLLMModelState = {
isCached: false,
isChecking: false,
isDownloading: false,
isDeleting: false,
progressPercent: 0,
progressText: '',
error: '',
runtimeSupport: detectLocalLLMRuntimeSupport(),
};
public static subscribe(listener: Listener): () => void {
this.listeners.add(listener);
listener(this.getState());
if (!this.initialized) {
this.initialized = true;
void this.refresh();
}
return () => this.listeners.delete(listener);
}
public static getState(): LocalLLMModelState {
return { ...this.state, runtimeSupport: { ...this.state.runtimeSupport } };
}
public static async refresh(): Promise<void> {
this.setState({
isChecking: true,
runtimeSupport: detectLocalLLMRuntimeSupport(),
});
try {
await this.cleanupLegacyEntries();
const isCached = await LocalLLMModelCache.exists();
this.setState({ isCached, error: '' });
} catch (error) {
this.setState({ error: error instanceof Error ? error.message : String(error) });
} finally {
this.setState({ isChecking: false });
}
}
public static async ensureRuntimeSupported(): Promise<void> {
const runtimeSupport = detectLocalLLMRuntimeSupport();
this.setState({ runtimeSupport });
if (!runtimeSupport.supported) {
throw new Error(runtimeSupport.reason ?? 'Local browser LLM is not supported in this browser.');
}
await this.cleanupLegacyEntries();
}
public static async deleteCachedModel(): Promise<void> {
this.setState({ isDeleting: true, error: '' });
try {
await LocalLLMModelCache.delete();
await this.cleanupLegacyEntries();
this.setState({
isCached: false,
progressPercent: 0,
progressText: '',
});
} catch (error) {
this.setState({ error: error instanceof Error ? error.message : String(error) });
throw error;
} finally {
this.setState({ isDeleting: false });
}
}
private static setState(partial: Partial<LocalLLMModelState>): void {
this.state = {
...this.state,
...partial,
};
for (const listener of this.listeners) {
listener(this.getState());
}
}
private static async cleanupLegacyEntries(): Promise<void> {
await Promise.all(
LOCAL_LLM_LEGACY_FILENAMES.map(async legacyFilename => {
try {
await LocalLLMModelCache.delete(legacyFilename);
} catch {
// Ignore best-effort legacy cleanup failures.
}
}),
);
}
public static notifyLoadStart(fromCache: boolean): void {
this.setState({
isDownloading: !fromCache,
progressPercent: fromCache ? 100 : 0,
progressText: fromCache ? 'Loading local language model from browser cache...' : 'Downloading local language model...',
error: '',
});
}
public static notifyLoadProgress(receivedBytes: number, totalBytes: number | null, fromCache: boolean): void {
const receivedMb = (receivedBytes / (1024 * 1024)).toFixed(1);
const totalMb = totalBytes ? (totalBytes / (1024 * 1024)).toFixed(1) : null;
this.setState({
isDownloading: !fromCache,
progressPercent: totalBytes ? (receivedBytes / totalBytes) * 100 : 0,
progressText: fromCache
? 'Loading local language model from browser cache...'
: totalMb
? `Downloading local language model... ${receivedMb} / ${totalMb} MB`
: `Downloading local language model... ${receivedMb} MB`,
error: '',
});
}
public static notifyCacheReady(): void {
this.setState({
isCached: true,
isDownloading: false,
progressPercent: 100,
progressText: 'Local language model is ready.',
error: '',
});
}
public static notifyLoadError(error: unknown): void {
this.setState({
isDownloading: false,
progressPercent: 0,
progressText: '',
error: error instanceof Error ? error.message : String(error),
});
}
}
src/util/localSeparatorModelCache.ts
+9 -103
View File
@@ -1,50 +1,25 @@
import { LOCAL_SEPARATOR_MODEL_FILENAME } from './localSeparatorConfig';
import { OpfsModelCache, type ModelDownloadProgress } from './opfsModelCache';
export interface ModelDownloadProgress {
receivedBytes: number;
totalBytes: number | null;
percent: number;
}
const cache = new OpfsModelCache({ directoryName: 'models' });
export { type ModelDownloadProgress };
export class LocalSeparatorModelCache {
private static readonly MODELS_DIR = 'models';
private static readonly TEMP_SUFFIX = '.download';
public static async exists(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<boolean> {
try {
const modelsDir = await this.getModelsDir();
await modelsDir.getFileHandle(filename);
return true;
} catch {
return false;
}
return cache.exists(filename);
}
public static async getFile(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<File> {
const modelsDir = await this.getModelsDir();
const fileHandle = await modelsDir.getFileHandle(filename);
return fileHandle.getFile();
return cache.getFile(filename);
}
public static async getArrayBuffer(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<ArrayBuffer> {
const file = await this.getFile(filename);
return file.arrayBuffer();
return cache.getArrayBuffer(filename);
}
public static async delete(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<void> {
try {
const modelsDir = await this.getModelsDir();
await modelsDir.removeEntry(filename);
} catch {
// Ignore missing file cleanup.
}
try {
const modelsDir = await this.getModelsDir();
await modelsDir.removeEntry(`${filename}${this.TEMP_SUFFIX}`);
} catch {
// Ignore missing temp file cleanup.
}
await cache.delete(filename);
}
public static async download(
@@ -52,75 +27,6 @@ export class LocalSeparatorModelCache {
filename: string = LOCAL_SEPARATOR_MODEL_FILENAME,
onProgress?: (progress: ModelDownloadProgress) => void,
): Promise<void> {
const response = await fetch(sourceUrl);
if (!response.ok) {
throw new Error(`Model download failed (${response.status})`);
}
const modelsDir = await this.getModelsDir();
const tempName = `${filename}${this.TEMP_SUFFIX}`;
await this.delete(filename);
const tempHandle = await modelsDir.getFileHandle(tempName, { create: true });
const writable = await tempHandle.createWritable();
try {
const totalBytesHeader = response.headers.get('Content-Length');
const totalBytes = totalBytesHeader ? Number(totalBytesHeader) : null;
if (!response.body) {
const buffer = await response.arrayBuffer();
await writable.write(buffer);
onProgress?.({
receivedBytes: buffer.byteLength,
totalBytes,
percent: 100,
});
} else {
const reader = response.body.getReader();
let receivedBytes = 0;
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (!value) continue;
await writable.write(value);
receivedBytes += value.byteLength;
onProgress?.({
receivedBytes,
totalBytes,
percent: totalBytes ? (receivedBytes / totalBytes) * 100 : 0,
});
}
}
} catch (error) {
await writable.abort();
await this.delete(filename);
throw error;
}
await writable.close();
const finalHandle = await modelsDir.getFileHandle(filename, { create: true });
const finalWritable = await finalHandle.createWritable();
try {
const tempFile = await tempHandle.getFile();
await finalWritable.write(await tempFile.arrayBuffer());
await finalWritable.close();
} catch (error) {
await finalWritable.abort();
throw error;
} finally {
try {
await modelsDir.removeEntry(tempName);
} catch {
// Ignore temp cleanup errors.
}
}
}
private static async getModelsDir(): Promise<FileSystemDirectoryHandle> {
const root = await navigator.storage.getDirectory();
return root.getDirectoryHandle(this.MODELS_DIR, { create: true });
await cache.download(sourceUrl, filename, onProgress);
}
}
src/util/messageFilter/UserMessageFilter.ts
+13 -2
View File
@@ -2,6 +2,7 @@ import { clearChatHistoryAndUI } from '../chatUtil';
import { useProjectStore } from '../../stores/projectStore';
import { ConfigManager } from '../../core/config/ConfigManager';
import { SystemPrompts } from '../../agent/core/SystemPrompts';
import { detectLocalLLMRuntimeSupport, LOCAL_LLM_PROVIDER_KEY } from '../localLLMConfig';
export interface UserMessageFilterResult {
// Whether to render the user message bubble (div.message-user)
@@ -134,7 +135,18 @@ export async function processUserMessage(originalMessage: string): Promise<UserM
}
const provider = (configManager.get('general.llm_provider') as string) || 'openai';
if (provider === 'openai') {
if (provider === LOCAL_LLM_PROVIDER_KEY) {
const runtimeSupport = detectLocalLLMRuntimeSupport();
if (!runtimeSupport.supported) {
return {
displayUserMessage: true,
sendToLLM: false,
finalMessageForLLM: null,
pseudoAssistantResponse: runtimeSupport.reason ?? 'Local browser LLM is not supported in this environment.',
metadata: { error: 'local_browser_unsupported' }
};
}
} else if (provider === 'openai') {
const openaiKey = (configManager.get('general.openai.api_key') as string) || '';
if (openaiKey.trim() === '') {
const url = `${import.meta.env.BASE_URL}chat/error_no_openai_key.md`;
@@ -256,4 +268,3 @@ export async function processUserMessage(originalMessage: string): Promise<UserM
}
}
src/util/opfsModelCache.ts
+188
View File
@@ -0,0 +1,188 @@
export interface ModelDownloadProgress {
receivedBytes: number;
totalBytes: number | null;
percent: number;
}
interface OpfsModelCacheOptions {
directoryName?: string;
sizeSuffix?: string;
tempSuffix?: string;
}
export class OpfsModelCache {
private readonly directoryName: string;
private readonly sizeSuffix: string;
private readonly tempSuffix: string;
constructor(options: OpfsModelCacheOptions = {}) {
this.directoryName = options.directoryName ?? 'models';
this.sizeSuffix = options.sizeSuffix ?? '.size';
this.tempSuffix = options.tempSuffix ?? '.download';
}
public async exists(filename: string): Promise<boolean> {
try {
const dir = await this.getDir();
const fileHandle = await dir.getFileHandle(filename);
const sizeHandle = await dir.getFileHandle(this.getSizeFilename(filename));
const [file, sizeFile] = await Promise.all([fileHandle.getFile(), sizeHandle.getFile()]);
const expectedSize = Number(await sizeFile.text());
if (!Number.isFinite(expectedSize) || expectedSize <= 0) {
await this.delete(filename);
return false;
}
if (file.size !== expectedSize) {
await this.delete(filename);
return false;
}
return true;
} catch {
return false;
}
}
public async getFile(filename: string): Promise<File> {
const dir = await this.getDir();
const handle = await dir.getFileHandle(filename);
const file = await handle.getFile();
console.log('[opfsModelCache] Opened cached file.', {
filename,
size: file.size,
});
return file;
}
public async getArrayBuffer(filename: string): Promise<ArrayBuffer> {
const file = await this.getFile(filename);
return file.arrayBuffer();
}
public async delete(filename: string): Promise<void> {
const dir = await this.getDir();
await this.removeIfExists(dir, filename);
await this.removeIfExists(dir, this.getSizeFilename(filename));
await this.removeIfExists(dir, `${filename}${this.tempSuffix}`);
await this.removeIfExists(dir, `${this.getSizeFilename(filename)}${this.tempSuffix}`);
}
public async download(
sourceUrl: string,
filename: string,
onProgress?: (progress: ModelDownloadProgress) => void,
): Promise<void> {
const response = await fetch(sourceUrl);
if (!response.ok) {
throw new Error(`Model download failed (${response.status})`);
}
const totalBytesHeader = response.headers.get('Content-Length');
const totalBytes = totalBytesHeader ? Number(totalBytesHeader) : null;
if (!response.body) {
throw new Error('Model download response did not include a readable body.');
}
await this.downloadStream(response.body, filename, totalBytes, onProgress);
}
public async downloadStream(
stream: ReadableStream<Uint8Array>,
filename: string,
totalBytes: number | null,
onProgress?: (progress: ModelDownloadProgress) => void,
): Promise<void> {
const dir = await this.getDir();
await this.delete(filename);
const tempFilename = `${filename}${this.tempSuffix}`;
const tempHandle = await dir.getFileHandle(tempFilename, { create: true });
const tempWritable = await tempHandle.createWritable();
const reader = stream.getReader();
let receivedBytes = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (!value) continue;
await tempWritable.write(value);
receivedBytes += value.byteLength;
onProgress?.({
receivedBytes,
totalBytes,
percent: totalBytes ? (receivedBytes / totalBytes) * 100 : 0,
});
}
await tempWritable.close();
const sizeValue = totalBytes ?? receivedBytes;
if (!Number.isFinite(sizeValue) || sizeValue <= 0) {
throw new Error('Model download did not provide a valid size.');
}
console.log(`[opfsModelCache] Finalizing cached model ${filename} from temp file ${tempFilename}.`);
const finalHandle = await dir.getFileHandle(filename, { create: true });
const finalWritable = await finalHandle.createWritable();
try {
const tempFile = await tempHandle.getFile();
console.log('[opfsModelCache] Temp file ready for finalize copy.', {
filename,
tempFilename,
tempSize: tempFile.size,
expectedSize: sizeValue,
});
await finalWritable.write(tempFile);
await finalWritable.close();
} catch (error) {
await finalWritable.abort();
throw error;
}
const sizeHandle = await dir.getFileHandle(this.getSizeFilename(filename), { create: true });
const sizeWritable = await sizeHandle.createWritable();
try {
await sizeWritable.write(String(sizeValue));
await sizeWritable.close();
} catch (error) {
await sizeWritable.abort();
throw error;
}
onProgress?.({
receivedBytes: sizeValue,
totalBytes: sizeValue,
percent: 100,
});
console.log('[opfsModelCache] Cached model finalize completed.', {
filename,
size: sizeValue,
});
} catch (error) {
try {
await tempWritable.abort();
} catch {
// Ignore abort cleanup errors.
}
await this.delete(filename);
throw error;
} finally {
await this.removeIfExists(dir, tempFilename);
reader.releaseLock();
}
}
private getSizeFilename(filename: string): string {
return `${filename}${this.sizeSuffix}`;
}
private async getDir(): Promise<FileSystemDirectoryHandle> {
const root = await navigator.storage.getDirectory();
return root.getDirectoryHandle(this.directoryName, { create: true });
}
private async removeIfExists(dir: FileSystemDirectoryHandle, name: string): Promise<void> {
try {
await dir.removeEntry(name);
} catch {
// Ignore missing entry cleanup.
}
}
}