feat: implemented the v1 browser embedded LLM support (Gemma 4 E4B based)
This commit is contained in:
Xiaohan-Tian
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"general": {
|
||||
"language": "en_us",
|
||||
"llm_provider": "openai",
|
||||
"llm_provider": "local_browser",
|
||||
"persist_api_keys_non_localhost": false,
|
||||
"openai": {
|
||||
"api_key": "",
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
@@ -0,0 +1,102 @@
|
||||
You are K.G.Studio Music Assistant Agent.
|
||||
|
||||
You are a skilled AI music assistant focused on composition, harmony, arrangement, MIDI editing, and music production.
|
||||
|
||||
Your job:
|
||||
- Understand the user request
|
||||
- Read existing music if needed
|
||||
- Edit music step-by-step using tools
|
||||
- Make musically coherent decisions
|
||||
- Return a short summary after finishing
|
||||
|
||||
====
|
||||
|
||||
TOOLS
|
||||
|
||||
## read_music
|
||||
Reads existing music in ABC notation.
|
||||
|
||||
## remove_notes
|
||||
Removes notes from a beat range.
|
||||
|
||||
## add_notes
|
||||
Adds notes to the current region.
|
||||
|
||||
Pitch format:
|
||||
- Scientific notation
|
||||
- Examples: C4, F#3, Bb2
|
||||
|
||||
Important:
|
||||
- `start` is ALWAYS the absolute beat position in the project timeline.
|
||||
|
||||
====
|
||||
|
||||
TOOL RULES
|
||||
|
||||
- Use tools step-by-step.
|
||||
- Check tool results before continuing.
|
||||
- Do not assume success.
|
||||
- If information is missing, ask the user.
|
||||
- Use `read_music` before editing when musical context is needed.
|
||||
- Do not ask the user to manually provide existing music before using `read_music`.
|
||||
|
||||
====
|
||||
|
||||
EDITING RULES
|
||||
|
||||
When adding notes:
|
||||
- Explicitly include ALL notes.
|
||||
- Never omit repeated notes with "..." or summaries.
|
||||
- Chords = multiple notes sharing the same `start`.
|
||||
- Melodies = sequential `start` values.
|
||||
|
||||
Before adding notes:
|
||||
- Remove conflicting notes if necessary.
|
||||
- Avoid stacking unintended duplicate notes.
|
||||
|
||||
Music quality guidelines:
|
||||
- Stay in key unless stylistically appropriate.
|
||||
- Use reasonable instrument ranges.
|
||||
- Respect the time signature grid.
|
||||
- Keep harmony and rhythm musically coherent.
|
||||
- Prefer smooth melodic and harmonic movement.
|
||||
|
||||
====
|
||||
|
||||
DRUM NOTE MAP
|
||||
|
||||
Common drum pitches:
|
||||
- C2 = Kick
|
||||
- D2 = Snare
|
||||
- F#2 = Closed Hi-Hat
|
||||
- Bb2 = Open Hi-Hat
|
||||
- Db3 = Crash
|
||||
- Eb3 = Ride
|
||||
|
||||
====
|
||||
|
||||
CAPABILITIES
|
||||
|
||||
You can:
|
||||
- Analyze existing music
|
||||
- Continue musical ideas
|
||||
- Create melodies, chords, basslines, and drums
|
||||
- Adapt to musical style and genre
|
||||
- Make arrangement decisions using music theory knowledge
|
||||
|
||||
Project information such as BPM, key, time signature, track instrument, and region boundaries will be provided dynamically.
|
||||
|
||||
Focus mainly on the current region.
|
||||
|
||||
====
|
||||
|
||||
WORKFLOW
|
||||
|
||||
1. Understand the task
|
||||
2. Read music if needed
|
||||
3. Plan musical changes
|
||||
4. Edit step-by-step with tools
|
||||
5. Verify results
|
||||
6. Return a concise summary
|
||||
|
||||
Do not endlessly continue conversations after finishing the task.
|
||||
@@ -1,4 +1,4 @@
|
||||
import { LLMProvider } from '../llm/LLMProvider';
|
||||
import type { LLMProvider } from '../llm/LLMProvider';
|
||||
import { AgentState } from './AgentState';
|
||||
import { SystemPrompts } from './SystemPrompts';
|
||||
import { AVAILABLE_TOOLS } from '../tools';
|
||||
@@ -92,7 +92,9 @@ export class AgentCore {
|
||||
// Add user message to state
|
||||
this.currentUserMessageId = this.agentState.addMessage('user', userInput);
|
||||
|
||||
const systemPrompt = await SystemPrompts.getSystemPromptWithContext();
|
||||
const systemPrompt = await SystemPrompts.getSystemPromptWithContext(
|
||||
this.llmProvider.getPreferredSystemPromptPath?.(),
|
||||
);
|
||||
const tools = this.getToolDefinitions();
|
||||
|
||||
try {
|
||||
|
||||
@@ -22,25 +22,26 @@ interface SystemPromptContext {
|
||||
* System prompts for the AI agent with dynamic context loading
|
||||
*/
|
||||
export class SystemPrompts {
|
||||
private static cachedTemplate: string | null = null;
|
||||
private static cachedTemplates: Map<string, string> = new Map();
|
||||
private static readonly FALLBACK_PROMPT = `You are K.G.Studio Musician Assistant Agent, a highly skilled music musician with extensive knowledge in music theory, composition, and production.`;
|
||||
|
||||
/**
|
||||
* Load the system prompt template from the public folder
|
||||
*/
|
||||
private static async loadTemplate(): Promise<string> {
|
||||
if (this.cachedTemplate) {
|
||||
return this.cachedTemplate;
|
||||
private static async loadTemplate(templatePath: string = 'prompts/system.md'): Promise<string> {
|
||||
if (this.cachedTemplates.has(templatePath)) {
|
||||
return this.cachedTemplates.get(templatePath)!;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`${import.meta.env.BASE_URL}prompts/system.md`);
|
||||
const response = await fetch(`${import.meta.env.BASE_URL}${templatePath}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to load system prompt: ${response.status}`);
|
||||
}
|
||||
|
||||
this.cachedTemplate = await response.text();
|
||||
return this.cachedTemplate;
|
||||
const template = await response.text();
|
||||
this.cachedTemplates.set(templatePath, template);
|
||||
return template;
|
||||
} catch (error) {
|
||||
console.error('Failed to load system prompt template:', error);
|
||||
return this.FALLBACK_PROMPT;
|
||||
@@ -231,9 +232,9 @@ export class SystemPrompts {
|
||||
/**
|
||||
* Get the system prompt with current context applied (backward compatible)
|
||||
*/
|
||||
static async getSystemPromptWithContext(): Promise<string> {
|
||||
static async getSystemPromptWithContext(templatePath?: string): Promise<string> {
|
||||
try {
|
||||
const template = await this.loadTemplate();
|
||||
const template = await this.loadTemplate(templatePath);
|
||||
let promptWithContext = await this.getPromptWithContext(template);
|
||||
|
||||
// Append custom instructions from config if provided
|
||||
|
||||
@@ -3,16 +3,24 @@ import type { StreamChunk } from './StreamingTypes';
|
||||
import type { Message, ToolCall } from '../core/AgentState';
|
||||
import type { OpenAIToolDefinition } from '../tools/BaseTool';
|
||||
|
||||
export interface LLMProvider {
|
||||
getPreferredSystemPromptPath?(): string | undefined;
|
||||
generateStream(
|
||||
messages: Message[],
|
||||
systemPrompt?: string,
|
||||
tools?: OpenAIToolDefinition[],
|
||||
): AsyncIterableIterator<StreamChunk>;
|
||||
}
|
||||
|
||||
/**
|
||||
* LLM provider using the OpenAI SDK.
|
||||
* Works with any OpenAI-compatible API (OpenAI, OpenRouter, Ollama, vLLM, etc.)
|
||||
* OpenAI-compatible provider implementation.
|
||||
* Works with OpenAI and OpenAI-compatible APIs (OpenRouter, Ollama, vLLM, etc.)
|
||||
*/
|
||||
export class LLMProvider {
|
||||
export class OpenAICompatibleLLMProvider implements LLMProvider {
|
||||
private client: OpenAI;
|
||||
private model: string;
|
||||
|
||||
constructor(apiKey: string, model: string, baseURL?: string) {
|
||||
// The OpenAI SDK appends /chat/completions itself, so strip it if the user included it
|
||||
const normalizedBaseURL = baseURL?.replace(/\/chat\/completions\/?$/, '') || undefined;
|
||||
|
||||
this.client = new OpenAI({
|
||||
@@ -23,12 +31,9 @@ export class LLMProvider {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert internal Message[] to OpenAI ChatCompletionMessageParam[]
|
||||
*/
|
||||
private convertMessages(
|
||||
messages: Message[],
|
||||
systemPrompt?: string
|
||||
systemPrompt?: string,
|
||||
): OpenAI.ChatCompletionMessageParam[] {
|
||||
const result: OpenAI.ChatCompletionMessageParam[] = [];
|
||||
|
||||
@@ -64,10 +69,6 @@ export class LLMProvider {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a streaming response from the LLM.
|
||||
* Yields StreamChunks for text content and tool calls.
|
||||
*/
|
||||
async *generateStream(
|
||||
messages: Message[],
|
||||
systemPrompt?: string,
|
||||
@@ -87,8 +88,6 @@ export class LLMProvider {
|
||||
}
|
||||
|
||||
const stream = this.client.chat.completions.stream(requestParams);
|
||||
|
||||
// Accumulate tool calls across chunks (they arrive incrementally)
|
||||
const toolCallAccumulator = new Map<number, { id: string; name: string; arguments: string }>();
|
||||
|
||||
for await (const chunk of stream) {
|
||||
@@ -97,23 +96,18 @@ export class LLMProvider {
|
||||
if (!choice) continue;
|
||||
|
||||
const delta = choice.delta;
|
||||
|
||||
// Yield text content
|
||||
if (delta.content) {
|
||||
yield { type: 'text', content: delta.content };
|
||||
}
|
||||
|
||||
// Accumulate tool calls from deltas
|
||||
if (delta.tool_calls) {
|
||||
for (const tc of delta.tool_calls) {
|
||||
const existing = toolCallAccumulator.get(tc.index);
|
||||
if (existing) {
|
||||
// Append to existing tool call
|
||||
if (tc.function?.arguments) {
|
||||
existing.arguments += tc.function.arguments;
|
||||
}
|
||||
} else {
|
||||
// New tool call
|
||||
toolCallAccumulator.set(tc.index, {
|
||||
id: tc.id ?? '',
|
||||
name: tc.function?.name ?? '',
|
||||
@@ -124,11 +118,9 @@ export class LLMProvider {
|
||||
}
|
||||
}
|
||||
|
||||
// After stream ends, get the final completion for finish_reason
|
||||
const finalCompletion = await stream.finalChatCompletion();
|
||||
const finishReason = finalCompletion.choices[0]?.finish_reason ?? 'stop';
|
||||
|
||||
// Emit accumulated tool calls
|
||||
if (toolCallAccumulator.size > 0) {
|
||||
for (const [, tc] of toolCallAccumulator) {
|
||||
const toolCall: ToolCall = {
|
||||
@@ -140,7 +132,6 @@ export class LLMProvider {
|
||||
}
|
||||
}
|
||||
|
||||
// Signal completion
|
||||
yield { type: 'done', content: '', finishReason };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
import type { StreamChunk } from './StreamingTypes';
|
||||
import type { Message, ToolCall } from '../core/AgentState';
|
||||
import type { OpenAIToolDefinition } from '../tools/BaseTool';
|
||||
import { LocalLLMModelManager } from '../../util/localLLMModelManager';
|
||||
import {
|
||||
formatToolCall,
|
||||
formatToolDeclaration,
|
||||
formatToolResponse,
|
||||
parseToolCalls,
|
||||
stripToolProtocol,
|
||||
} from './gemmaToolProtocol';
|
||||
import { LOCAL_LLM_MODEL_FILENAME, LOCAL_LLM_MODEL_URL } from '../../util/localLLMConfig';
|
||||
import { LocalLLMModelCache } from '../../util/localLLMModelCache';
|
||||
import type { LLMProvider } from './LLMProvider';
|
||||
|
||||
type MediaPipeGenAI = {
|
||||
FilesetResolver: {
|
||||
forGenAiTasks(basePath: string): Promise<unknown>;
|
||||
};
|
||||
LlmInference: {
|
||||
createFromOptions(fileset: unknown, options: Record<string, unknown>): Promise<GemmaInference>;
|
||||
};
|
||||
};
|
||||
|
||||
type GemmaInference = {
|
||||
generateResponse(
|
||||
prompt: string,
|
||||
callback: (partial: string, done: boolean) => void,
|
||||
): Promise<void> | void;
|
||||
sizeInTokens(text: string): number;
|
||||
close?: () => void;
|
||||
};
|
||||
|
||||
interface PromptTemplatePart {
|
||||
pre: string;
|
||||
post: string;
|
||||
}
|
||||
|
||||
const PROMPT_TEMPLATE: Record<'user' | 'model' | 'system', PromptTemplatePart> = {
|
||||
user: { pre: '<|turn>user\n', post: '<turn|>\n' },
|
||||
model: { pre: '<|turn>model\n', post: '<turn|>\n' },
|
||||
system: { pre: '<|turn>system\n', post: '<turn|>\n' },
|
||||
};
|
||||
|
||||
async function importMediaPipe(): Promise<MediaPipeGenAI> {
|
||||
const bundleUrl = new URL(`${import.meta.env.BASE_URL}mediapipe/genai_bundle.mjs`, window.location.origin).href;
|
||||
return import(/* @vite-ignore */ bundleUrl) as Promise<MediaPipeGenAI>;
|
||||
}
|
||||
|
||||
export class LocalBrowserLLMProvider implements LLMProvider {
|
||||
private inference: GemmaInference | null = null;
|
||||
|
||||
getPreferredSystemPromptPath(): string | undefined {
|
||||
return 'prompts/system_compact.md';
|
||||
}
|
||||
|
||||
private async ensureInference(): Promise<GemmaInference> {
|
||||
await LocalLLMModelManager.ensureRuntimeSupported();
|
||||
if (this.inference) {
|
||||
return this.inference;
|
||||
}
|
||||
|
||||
const [{ FilesetResolver, LlmInference }, modelLoad] = await Promise.all([
|
||||
this.getMediaPipeModule(),
|
||||
LocalLLMModelCache.loadModelReaderWithCache(
|
||||
LOCAL_LLM_MODEL_URL,
|
||||
LOCAL_LLM_MODEL_FILENAME,
|
||||
progress => {
|
||||
if (progress.fromCache) {
|
||||
LocalLLMModelManager.notifyLoadStart(true);
|
||||
} else {
|
||||
LocalLLMModelManager.notifyLoadProgress(progress.receivedBytes, progress.totalBytes, false);
|
||||
}
|
||||
},
|
||||
),
|
||||
]);
|
||||
|
||||
LocalLLMModelManager.notifyLoadStart(modelLoad.fromCache);
|
||||
console.log('[localLLM] Model stream prepared for MediaPipe.', {
|
||||
filename: LOCAL_LLM_MODEL_FILENAME,
|
||||
totalBytes: modelLoad.totalBytes,
|
||||
fromCache: modelLoad.fromCache,
|
||||
});
|
||||
|
||||
const fileset = await FilesetResolver.forGenAiTasks(`${import.meta.env.BASE_URL}mediapipe/wasm`);
|
||||
console.log('[localLLM] MediaPipe fileset resolved. Creating inference engine...');
|
||||
try {
|
||||
this.inference = await LlmInference.createFromOptions(fileset, {
|
||||
baseOptions: {
|
||||
modelAssetBuffer: modelLoad.reader,
|
||||
},
|
||||
numResponses: 1,
|
||||
maxTokens: 32768,
|
||||
topK: 64,
|
||||
temperature: 1.0,
|
||||
});
|
||||
console.log('[localLLM] MediaPipe inference engine created successfully.');
|
||||
if (modelLoad.cacheWritePromise) {
|
||||
void modelLoad.cacheWritePromise.then(() => {
|
||||
LocalLLMModelManager.notifyCacheReady();
|
||||
}).catch(error => {
|
||||
console.error('[localLLM] Background cache write failed after inference creation.', error);
|
||||
LocalLLMModelManager.notifyLoadError(error);
|
||||
});
|
||||
} else {
|
||||
LocalLLMModelManager.notifyCacheReady();
|
||||
}
|
||||
return this.inference;
|
||||
} catch (error) {
|
||||
LocalLLMModelManager.notifyLoadError(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async getMediaPipeModule(): Promise<MediaPipeGenAI> {
|
||||
return importMediaPipe();
|
||||
}
|
||||
|
||||
private applyTemplate(message: { role: 'user' | 'model'; text: string }): string {
|
||||
const template = PROMPT_TEMPLATE[message.role];
|
||||
return `${template.pre}${message.text}${template.post}`;
|
||||
}
|
||||
|
||||
private renderPrompt(
|
||||
messages: Message[],
|
||||
systemPrompt: string | undefined,
|
||||
tools: OpenAIToolDefinition[] | undefined,
|
||||
): string {
|
||||
const thinkPrefix = '<|think|>';
|
||||
const toolDeclarations = (tools ?? []).map(formatToolDeclaration).join('');
|
||||
const systemContent = `${thinkPrefix}${systemPrompt ?? ''}${toolDeclarations}`;
|
||||
const systemSection = systemContent
|
||||
? `${PROMPT_TEMPLATE.system.pre}${systemContent}${PROMPT_TEMPLATE.system.post}`
|
||||
: '';
|
||||
|
||||
const conversationParts: string[] = [];
|
||||
|
||||
for (let i = 0; i < messages.length; i += 1) {
|
||||
const message = messages[i];
|
||||
if (message.role === 'user') {
|
||||
conversationParts.push(this.applyTemplate({ role: 'user', text: message.content ?? '' }));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (message.role === 'assistant') {
|
||||
let modelText = message.content ?? '';
|
||||
if (message.tool_calls?.length) {
|
||||
for (const toolCall of message.tool_calls) {
|
||||
modelText += formatToolCall(toolCall.function.name, toolCall.function.arguments);
|
||||
}
|
||||
|
||||
let scanIndex = i + 1;
|
||||
while (scanIndex < messages.length && messages[scanIndex].role === 'tool') {
|
||||
const toolMessage = messages[scanIndex];
|
||||
const matchedCall = message.tool_calls.find(call => call.id === toolMessage.tool_call_id);
|
||||
if (matchedCall) {
|
||||
let parsedResult: unknown = toolMessage.content ?? '';
|
||||
try {
|
||||
parsedResult = JSON.parse(toolMessage.content ?? '{}');
|
||||
} catch {
|
||||
parsedResult = toolMessage.content ?? '';
|
||||
}
|
||||
modelText += formatToolResponse(matchedCall.function.name, parsedResult);
|
||||
}
|
||||
scanIndex += 1;
|
||||
}
|
||||
}
|
||||
|
||||
conversationParts.push(this.applyTemplate({ role: 'model', text: modelText }));
|
||||
}
|
||||
}
|
||||
|
||||
return `${systemSection}${conversationParts.join('')}${PROMPT_TEMPLATE.model.pre}`;
|
||||
}
|
||||
|
||||
async *generateStream(
|
||||
messages: Message[],
|
||||
systemPrompt?: string,
|
||||
tools?: OpenAIToolDefinition[],
|
||||
): AsyncIterableIterator<StreamChunk> {
|
||||
const inference = await this.ensureInference();
|
||||
const prompt = this.renderPrompt(messages, systemPrompt, tools);
|
||||
console.log('------------ LOCAL RAW PROMPT ------------');
|
||||
console.log(prompt);
|
||||
console.log('------------------------------------------');
|
||||
|
||||
const start = performance.now();
|
||||
let firstTokenTime: number | null = null;
|
||||
let rawResponse = '';
|
||||
let streamedVisibleText = '';
|
||||
const pendingTextDeltas: string[] = [];
|
||||
let generationError: unknown = null;
|
||||
let generationDone = false;
|
||||
let notifyWaiting: (() => void) | null = null;
|
||||
|
||||
const wake = () => {
|
||||
if (notifyWaiting) {
|
||||
const resolve = notifyWaiting;
|
||||
notifyWaiting = null;
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
const generationPromise = new Promise<void>((resolve, reject) => {
|
||||
try {
|
||||
const result = inference.generateResponse(prompt, (partial, done) => {
|
||||
if (firstTokenTime === null) {
|
||||
firstTokenTime = performance.now();
|
||||
}
|
||||
rawResponse += partial;
|
||||
|
||||
const visibleText = stripToolProtocol(rawResponse);
|
||||
if (visibleText.startsWith(streamedVisibleText)) {
|
||||
const delta = visibleText.slice(streamedVisibleText.length);
|
||||
if (delta) {
|
||||
streamedVisibleText = visibleText;
|
||||
pendingTextDeltas.push(delta);
|
||||
wake();
|
||||
}
|
||||
} else if (visibleText && visibleText !== streamedVisibleText) {
|
||||
const delta = visibleText.slice(streamedVisibleText.length) || visibleText;
|
||||
streamedVisibleText = visibleText;
|
||||
pendingTextDeltas.push(delta);
|
||||
wake();
|
||||
}
|
||||
|
||||
if (done) {
|
||||
generationDone = true;
|
||||
wake();
|
||||
setTimeout(resolve, 0);
|
||||
}
|
||||
});
|
||||
|
||||
Promise.resolve(result).catch(error => {
|
||||
generationError = error;
|
||||
generationDone = true;
|
||||
wake();
|
||||
reject(error);
|
||||
});
|
||||
} catch (error) {
|
||||
generationError = error;
|
||||
generationDone = true;
|
||||
wake();
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
|
||||
while (!generationDone || pendingTextDeltas.length > 0) {
|
||||
while (pendingTextDeltas.length > 0) {
|
||||
const delta = pendingTextDeltas.shift();
|
||||
if (delta) {
|
||||
yield { type: 'text', content: delta };
|
||||
}
|
||||
}
|
||||
|
||||
if (generationDone) {
|
||||
break;
|
||||
}
|
||||
|
||||
await new Promise<void>(resolve => {
|
||||
notifyWaiting = resolve;
|
||||
});
|
||||
}
|
||||
|
||||
await generationPromise;
|
||||
if (generationError) {
|
||||
throw generationError;
|
||||
}
|
||||
|
||||
console.log('------------ LOCAL RAW RESPONSE ------------');
|
||||
console.log(rawResponse);
|
||||
console.log('--------------------------------------------');
|
||||
|
||||
const toolCalls = parseToolCalls(rawResponse);
|
||||
const finishReason = toolCalls.length > 0 ? 'tool_calls' : 'stop';
|
||||
|
||||
const promptTokenCount = inference.sizeInTokens(prompt);
|
||||
const generatedTokenCount = inference.sizeInTokens(rawResponse);
|
||||
const totalEnd = performance.now();
|
||||
const prefillMs = firstTokenTime !== null ? firstTokenTime - start : 0;
|
||||
const decodeMs = Math.max(0, totalEnd - start - prefillMs);
|
||||
const prefillTps = prefillMs > 0 ? promptTokenCount / (prefillMs / 1000) : 0;
|
||||
const generationTps = decodeMs > 0 ? generatedTokenCount / (decodeMs / 1000) : 0;
|
||||
console.log(`[localLLM] prefill t/s: ${prefillTps.toFixed(1)}`);
|
||||
console.log(`[localLLM] generation t/s: ${generationTps.toFixed(1)}`);
|
||||
|
||||
for (const parsed of toolCalls) {
|
||||
const toolCall: ToolCall = {
|
||||
id: `gemma_tool_${Date.now()}_${Math.random().toString(36).slice(2)}`,
|
||||
type: 'function',
|
||||
function: {
|
||||
name: parsed.name,
|
||||
arguments: JSON.stringify(parsed.args),
|
||||
},
|
||||
};
|
||||
yield { type: 'tool_call', content: '', toolCall };
|
||||
}
|
||||
|
||||
yield { type: 'done', content: '', finishReason };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { stripToolProtocol } from './gemmaToolProtocol';
|
||||
|
||||
describe('stripToolProtocol streaming safety', () => {
|
||||
it('hides incomplete thought blocks entirely until they are closed', () => {
|
||||
const partial = `<|channel>thought
|
||||
I have successfully read the music.
|
||||
I will summarize this for the user.`;
|
||||
|
||||
expect(stripToolProtocol(partial)).toBe('');
|
||||
});
|
||||
|
||||
it('reveals visible answer cleanly after a thought block closes', () => {
|
||||
const completed = `<|channel>thought
|
||||
Internal reasoning here.
|
||||
<channel|>Here is the sheet music I read.`;
|
||||
|
||||
expect(stripToolProtocol(completed)).toBe('Here is the sheet music I read.');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { parseToolCalls } from './gemmaToolProtocol';
|
||||
|
||||
describe('parseToolCalls', () => {
|
||||
it('parses nested object arguments emitted by Gemma tool calling', () => {
|
||||
const text = '<|tool_call>call:add_notes{notes:[{pitch:<|"|>C4<|"|>,meta:{velocity:90,length:1.5}}],replaceExisting:false}<tool_call|>';
|
||||
|
||||
const parsed = parseToolCalls(text);
|
||||
|
||||
expect(parsed).toHaveLength(1);
|
||||
expect(parsed[0].name).toBe('add_notes');
|
||||
expect(parsed[0].args).toEqual({
|
||||
notes: [
|
||||
{
|
||||
pitch: 'C4',
|
||||
meta: {
|
||||
velocity: 90,
|
||||
length: 1.5,
|
||||
},
|
||||
},
|
||||
],
|
||||
replaceExisting: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,154 @@
|
||||
import type { OpenAIToolDefinition } from '../tools/BaseTool';
|
||||
|
||||
function gemmaValue(value: unknown): string {
|
||||
if (typeof value === 'string') return `<|"|>${value}<|"|>`;
|
||||
if (Array.isArray(value)) return `[${value.map(gemmaValue).join(',')}]`;
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
return `{${Object.entries(value as Record<string, unknown>).map(([key, nested]) => `${key}:${gemmaValue(nested)}`).join(',')}}`;
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
export function formatToolDeclaration(tool: OpenAIToolDefinition): string {
|
||||
const body = {
|
||||
description: tool.function.description,
|
||||
parameters: tool.function.parameters,
|
||||
};
|
||||
return `<|tool>declaration:${tool.function.name}${gemmaValue(body)}<tool|>`;
|
||||
}
|
||||
|
||||
export function formatToolCall(name: string, args: string): string {
|
||||
const parsed = JSON.parse(args) as Record<string, unknown>;
|
||||
return `<|tool_call>call:${name}${gemmaValue(parsed)}<tool_call|>`;
|
||||
}
|
||||
|
||||
export function formatToolResponse(name: string, result: unknown): string {
|
||||
return `<|tool_response>response:${name}${gemmaValue(result)}<tool_response|>`;
|
||||
}
|
||||
|
||||
function parseGemmaValue(str: string, pos: number): { value: unknown; next: number } {
|
||||
while (pos < str.length && str[pos] === ' ') pos += 1;
|
||||
|
||||
const stringDelimiter = '<|"|>';
|
||||
if (str.startsWith(stringDelimiter, pos)) {
|
||||
const start = pos + stringDelimiter.length;
|
||||
const end = str.indexOf(stringDelimiter, start);
|
||||
if (end === -1) {
|
||||
return { value: '', next: str.length };
|
||||
}
|
||||
return { value: str.slice(start, end), next: end + stringDelimiter.length };
|
||||
}
|
||||
|
||||
if (str[pos] === '{') {
|
||||
const result: Record<string, unknown> = {};
|
||||
pos += 1;
|
||||
while (pos < str.length && str[pos] !== '}') {
|
||||
while (pos < str.length && (str[pos] === ',' || str[pos] === ' ')) pos += 1;
|
||||
if (str[pos] === '}') break;
|
||||
const colonIndex = str.indexOf(':', pos);
|
||||
if (colonIndex === -1) break;
|
||||
const key = str.slice(pos, colonIndex).trim();
|
||||
pos = colonIndex + 1;
|
||||
const nested = parseGemmaValue(str, pos);
|
||||
result[key] = nested.value;
|
||||
pos = nested.next;
|
||||
}
|
||||
return { value: result, next: pos + 1 };
|
||||
}
|
||||
|
||||
if (str[pos] === '[') {
|
||||
const result: unknown[] = [];
|
||||
pos += 1;
|
||||
while (pos < str.length && str[pos] !== ']') {
|
||||
while (pos < str.length && (str[pos] === ',' || str[pos] === ' ')) pos += 1;
|
||||
if (str[pos] === ']') break;
|
||||
const nested = parseGemmaValue(str, pos);
|
||||
result.push(nested.value);
|
||||
pos = nested.next;
|
||||
}
|
||||
return { value: result, next: pos + 1 };
|
||||
}
|
||||
|
||||
let end = pos;
|
||||
while (end < str.length && str[end] !== ',' && str[end] !== '}' && str[end] !== ']') end += 1;
|
||||
const raw = str.slice(pos, end).trim();
|
||||
if (raw === 'true') return { value: true, next: end };
|
||||
if (raw === 'false') return { value: false, next: end };
|
||||
if (raw !== '' && !Number.isNaN(Number(raw))) return { value: Number(raw), next: end };
|
||||
return { value: raw, next: end };
|
||||
}
|
||||
|
||||
function parseArgs(argsStr: string): Record<string, unknown> {
|
||||
const parsed = parseGemmaValue(`{${argsStr}}`, 0).value;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
return {};
|
||||
}
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ParsedGemmaToolCall {
|
||||
name: string;
|
||||
args: Record<string, unknown>;
|
||||
fullMatch: string;
|
||||
index: number;
|
||||
endIndex: number;
|
||||
}
|
||||
|
||||
export function parseToolCalls(text: string): ParsedGemmaToolCall[] {
|
||||
const prefix = '<|tool_call>call:';
|
||||
const suffix = '<tool_call|>';
|
||||
const stringDelimiter = '<|"|>';
|
||||
const calls: ParsedGemmaToolCall[] = [];
|
||||
let searchFrom = 0;
|
||||
|
||||
while (true) {
|
||||
const prefixIndex = text.indexOf(prefix, searchFrom);
|
||||
if (prefixIndex === -1) break;
|
||||
|
||||
const braceStart = text.indexOf('{', prefixIndex + prefix.length);
|
||||
if (braceStart === -1) break;
|
||||
const name = text.slice(prefixIndex + prefix.length, braceStart).trim();
|
||||
|
||||
let depth = 1;
|
||||
let index = braceStart + 1;
|
||||
while (index < text.length && depth > 0) {
|
||||
if (text.startsWith(stringDelimiter, index)) {
|
||||
const stringEnd = text.indexOf(stringDelimiter, index + stringDelimiter.length);
|
||||
index = stringEnd === -1 ? text.length : stringEnd + stringDelimiter.length;
|
||||
continue;
|
||||
}
|
||||
if (text[index] === '{') depth += 1;
|
||||
else if (text[index] === '}') depth -= 1;
|
||||
index += 1;
|
||||
}
|
||||
const braceEnd = index;
|
||||
const suffixIndex = text.indexOf(suffix, braceEnd);
|
||||
if (suffixIndex === -1) break;
|
||||
|
||||
const argsStr = text.slice(braceStart + 1, braceEnd - 1);
|
||||
const endIndex = suffixIndex + suffix.length;
|
||||
calls.push({
|
||||
name,
|
||||
args: parseArgs(argsStr),
|
||||
fullMatch: text.slice(prefixIndex, endIndex),
|
||||
index: prefixIndex,
|
||||
endIndex,
|
||||
});
|
||||
searchFrom = endIndex;
|
||||
}
|
||||
|
||||
return calls;
|
||||
}
|
||||
|
||||
export function stripToolProtocol(text: string): string {
|
||||
let result = text.replace(
|
||||
/<\|tool_call>call:(\w+)\{[\s\S]*?\}<tool_call\|><\|tool_response>[\s\S]*?<tool_response\|>/g,
|
||||
'[Tool call completed]',
|
||||
);
|
||||
result = result.replace(/<\|channel>thought[\s\S]*?<channel\|>/g, '');
|
||||
result = result.replace(/<\|channel>thought[\s\S]*/g, '');
|
||||
result = result.replace(/<\|tool_call>[\s\S]*/g, '');
|
||||
result = result.replace(/<\|tool_response>[\s\S]*/g, '');
|
||||
result = result.replace(/<\|"\|>/g, '');
|
||||
return result.trimStart();
|
||||
}
|
||||
@@ -3,7 +3,8 @@ import './ChatBox.css';
|
||||
import { FaPlus, FaBan, FaDownload } from 'react-icons/fa';
|
||||
import { UserMessage, AssistantMessage } from './chat';
|
||||
import { AgentCore } from '../agent/core/AgentCore';
|
||||
import { LLMProvider } from '../agent/llm/LLMProvider';
|
||||
import { OpenAICompatibleLLMProvider, type LLMProvider } from '../agent/llm/LLMProvider';
|
||||
import { LocalBrowserLLMProvider } from '../agent/llm/LocalBrowserLLMProvider';
|
||||
import { ConfigManager } from '../core/config/ConfigManager';
|
||||
import { useProjectStore } from '../stores/projectStore';
|
||||
import { SystemPrompts } from '../agent/core/SystemPrompts';
|
||||
@@ -13,6 +14,8 @@ import { useStreamProcessor } from '../hooks/useStreamProcessor';
|
||||
import { createMessage, addWelcomeMessage } from '../utils/chatMessageUtils';
|
||||
import { formatLocalDateTime } from '../util/timeUtil';
|
||||
import { downloadBlob, buildTimestampSuffix } from '../util/miscUtil';
|
||||
import { LocalLLMModelManager, type LocalLLMModelState } from '../util/localLLMModelManager';
|
||||
import { LOCAL_LLM_DISPLAY_NAME, LOCAL_LLM_PROVIDER_KEY } from '../util/localLLMConfig';
|
||||
import KGDropdown from './common/KGDropdown';
|
||||
|
||||
import type { ChatMessage } from '../types/projectTypes';
|
||||
@@ -32,6 +35,8 @@ const createLLMProviderFromConfig = (): LLMProvider => {
|
||||
let baseURL: string | undefined;
|
||||
|
||||
switch (providerType) {
|
||||
case LOCAL_LLM_PROVIDER_KEY:
|
||||
return new LocalBrowserLLMProvider();
|
||||
case 'openai':
|
||||
apiKey = configManager.get('general.openai.api_key') as string;
|
||||
model = configManager.get('general.openai.model') as string;
|
||||
@@ -50,7 +55,7 @@ const createLLMProviderFromConfig = (): LLMProvider => {
|
||||
break;
|
||||
}
|
||||
|
||||
return new LLMProvider(apiKey, model, baseURL);
|
||||
return new OpenAICompatibleLLMProvider(apiKey, model, baseURL);
|
||||
};
|
||||
|
||||
interface ChatBoxProps {
|
||||
@@ -64,6 +69,8 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||
const [isProcessing, setIsProcessing] = useState(false);
|
||||
const [lastUserMessage, setLastUserMessage] = useState<string>('');
|
||||
const [localModelState, setLocalModelState] = useState<LocalLLMModelState>(LocalLLMModelManager.getState());
|
||||
const [activeProvider, setActiveProvider] = useState<string>('openai');
|
||||
|
||||
// Track if this is the first message (for system prompt logging)
|
||||
const [isFirstMessage, setIsFirstMessage] = useState(true);
|
||||
@@ -180,9 +187,11 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
||||
}
|
||||
|
||||
const applyProviderFromConfig = () => {
|
||||
const providerType = (configManager.get('general.llm_provider') as string) || 'openai';
|
||||
const provider = createLLMProviderFromConfig();
|
||||
const agentCore = AgentCore.instance();
|
||||
agentCore.setLLMProvider(provider);
|
||||
setActiveProvider(providerType);
|
||||
console.log('LLM provider configured');
|
||||
};
|
||||
|
||||
@@ -204,6 +213,7 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
||||
registerClearChatUICallback(clearChatUI);
|
||||
|
||||
const maybeUnsubscribePromise = initializeProvider();
|
||||
const unsubscribeLocalModel = LocalLLMModelManager.subscribe(setLocalModelState);
|
||||
|
||||
(async () => {
|
||||
if (hasShownWelcomeOnceInRuntime) return;
|
||||
@@ -216,6 +226,7 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
||||
})();
|
||||
|
||||
return () => {
|
||||
unsubscribeLocalModel();
|
||||
Promise.resolve(maybeUnsubscribePromise).then((cleanup) => {
|
||||
if (typeof cleanup === 'function') cleanup();
|
||||
}).catch(() => {});
|
||||
@@ -266,7 +277,10 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
||||
// Log system prompt only for first message
|
||||
if (isFirstMessage) {
|
||||
try {
|
||||
const systemPrompt = await SystemPrompts.getSystemPromptWithContext();
|
||||
const provider = AgentCore.instance().getLLMProvider();
|
||||
const systemPrompt = await SystemPrompts.getSystemPromptWithContext(
|
||||
provider?.getPreferredSystemPromptPath?.(),
|
||||
);
|
||||
console.log('------------ SYSTEM ------------');
|
||||
console.log(systemPrompt);
|
||||
console.log('--------------------------------');
|
||||
@@ -372,6 +386,45 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{activeProvider === LOCAL_LLM_PROVIDER_KEY && (
|
||||
<div style={{ padding: '10px 14px 0 14px' }}>
|
||||
<div className="settings-group" style={{ marginBottom: '12px', padding: '14px' }}>
|
||||
<h4 style={{ marginBottom: '10px' }}>{LOCAL_LLM_DISPLAY_NAME} Local Runtime</h4>
|
||||
{!localModelState.runtimeSupport.supported && (
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#d0a56b', marginBottom: '8px' }}>
|
||||
{localModelState.runtimeSupport.reason}
|
||||
</div>
|
||||
)}
|
||||
{!localModelState.isCached && !localModelState.isDownloading && localModelState.runtimeSupport.supported && (
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#b0b0b0', marginBottom: '8px' }}>
|
||||
The local language model has not been downloaded yet. It will be downloaded automatically the next time you send a chat request with this provider.
|
||||
</div>
|
||||
)}
|
||||
{(localModelState.isChecking || localModelState.isDownloading || localModelState.progressText) && (
|
||||
<div className="settings-progress-block">
|
||||
<div
|
||||
className="settings-progress-track"
|
||||
role="progressbar"
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={100}
|
||||
aria-valuenow={Math.max(0, Math.min(100, localModelState.progressPercent))}
|
||||
>
|
||||
<div className="settings-progress-fill" style={{ width: `${Math.max(0, Math.min(100, localModelState.progressPercent))}%` }} />
|
||||
</div>
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#b0b0b0', marginTop: '6px' }}>
|
||||
{localModelState.isChecking ? 'Checking local model cache...' : localModelState.progressText}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{localModelState.error && (
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#d45a5a' }}>
|
||||
{localModelState.error}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="chatbox-messages">
|
||||
{messages.map((message) => (
|
||||
message.role === 'user' ? (
|
||||
|
||||
@@ -293,6 +293,25 @@
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.settings-progress-block {
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.settings-progress-track {
|
||||
width: 100%;
|
||||
height: 10px;
|
||||
border-radius: 999px;
|
||||
overflow: hidden;
|
||||
background-color: #3a3a3a;
|
||||
border: 1px solid #4a4a4a;
|
||||
}
|
||||
|
||||
.settings-progress-fill {
|
||||
height: 100%;
|
||||
background: linear-gradient(90deg, #5a9fd4 0%, #76c28f 100%);
|
||||
transition: width 0.2s ease;
|
||||
}
|
||||
|
||||
/* Settings Help Links */
|
||||
.settings-help-links {
|
||||
display: flex;
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import React, { useState, useEffect, useCallback, useMemo } from 'react';
|
||||
import { ConfigManager } from '../../../core/config/ConfigManager';
|
||||
import { LocalLLMModelManager, type LocalLLMModelState } from '../../../util/localLLMModelManager';
|
||||
import { LOCAL_LLM_DISPLAY_NAME, LOCAL_LLM_PROVIDER_KEY } from '../../../util/localLLMConfig';
|
||||
|
||||
const GeneralSettings: React.FC = () => {
|
||||
const [llmProvider, setLlmProvider] = useState<string>('openai');
|
||||
const [llmProvider, setLlmProvider] = useState<string>(LOCAL_LLM_PROVIDER_KEY);
|
||||
const [openaiKey, setOpenaiKey] = useState<string>('');
|
||||
const [openaiModel, setOpenaiModel] = useState<string>('');
|
||||
const [geminiKey, setGeminiKey] = useState<string>('');
|
||||
@@ -22,6 +24,7 @@ const GeneralSettings: React.FC = () => {
|
||||
const [kgoneBaseUrl, setKgoneBaseUrl] = useState<string>('');
|
||||
const [kgoneServerManaged, setKgoneServerManaged] = useState<boolean>(false);
|
||||
const [soundfontServerManaged, setSoundfontServerManaged] = useState<boolean>(false);
|
||||
const [localModelState, setLocalModelState] = useState<LocalLLMModelState>(LocalLLMModelManager.getState());
|
||||
|
||||
const configManager = ConfigManager.instance();
|
||||
|
||||
@@ -46,7 +49,7 @@ const GeneralSettings: React.FC = () => {
|
||||
await configManager.initialize();
|
||||
}
|
||||
|
||||
setLlmProvider((configManager.get('general.llm_provider') as string) || 'openai');
|
||||
setLlmProvider((configManager.get('general.llm_provider') as string) || LOCAL_LLM_PROVIDER_KEY);
|
||||
setOpenaiKey((configManager.get('general.openai.api_key') as string) || '');
|
||||
setOpenaiModel((configManager.get('general.openai.model') as string) || '');
|
||||
setOpenaiFlex((configManager.get('general.openai.flex') as boolean) ?? false);
|
||||
@@ -69,6 +72,8 @@ const GeneralSettings: React.FC = () => {
|
||||
};
|
||||
|
||||
loadConfig();
|
||||
const unsubscribe = LocalLLMModelManager.subscribe(setLocalModelState);
|
||||
return unsubscribe;
|
||||
}, [configManager]);
|
||||
|
||||
// Debounced save function for text inputs
|
||||
@@ -197,6 +202,14 @@ const GeneralSettings: React.FC = () => {
|
||||
debouncedSave('general.kgone.base_url', value);
|
||||
};
|
||||
|
||||
const handleDeleteLocalModel = async () => {
|
||||
try {
|
||||
await LocalLLMModelManager.deleteCachedModel();
|
||||
} catch (error) {
|
||||
console.error('Failed to delete local language model cache:', error);
|
||||
}
|
||||
};
|
||||
|
||||
// NOTE: Gemini and Claude are not supported yet due to CORS issues.
|
||||
return (
|
||||
<div className="settings-section">
|
||||
@@ -217,6 +230,7 @@ const GeneralSettings: React.FC = () => {
|
||||
value={llmProvider}
|
||||
onChange={(e) => handleLlmProviderChange(e.target.value)}
|
||||
>
|
||||
<option value={LOCAL_LLM_PROVIDER_KEY}>Local LLM (Browser)</option>
|
||||
<option value="openai">OpenAI</option>
|
||||
{/* <option value="gemini">Gemini</option>
|
||||
<option value="claude">Claude</option> */}
|
||||
@@ -243,6 +257,69 @@ const GeneralSettings: React.FC = () => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="settings-group">
|
||||
<h4>{LOCAL_LLM_DISPLAY_NAME} Local Runtime</h4>
|
||||
|
||||
{!localModelState.runtimeSupport.supported && (
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#d0a56b', marginTop: '4px', marginBottom: '8px' }}>
|
||||
{localModelState.runtimeSupport.reason}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="settings-item">
|
||||
<label className="settings-label">
|
||||
Cached Model Status
|
||||
</label>
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
|
||||
{localModelState.isChecking
|
||||
? 'Checking local model cache...'
|
||||
: localModelState.isCached
|
||||
? 'Downloaded in browser cache.'
|
||||
: 'Not downloaded yet.'}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!localModelState.isCached && !localModelState.isDownloading && localModelState.runtimeSupport.supported && (
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px', marginBottom: '8px' }}>
|
||||
The local model downloads automatically the next time you chat with `Local LLM (Browser)`.
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(localModelState.isDownloading || localModelState.progressText) && (
|
||||
<div className="settings-progress-block">
|
||||
<div
|
||||
className="settings-progress-track"
|
||||
role="progressbar"
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={100}
|
||||
aria-valuenow={Math.max(0, Math.min(100, localModelState.progressPercent))}
|
||||
>
|
||||
<div className="settings-progress-fill" style={{ width: `${Math.max(0, Math.min(100, localModelState.progressPercent))}%` }} />
|
||||
</div>
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '6px' }}>
|
||||
{localModelState.progressText}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{localModelState.error && (
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#d45a5a', marginTop: '8px' }}>
|
||||
{localModelState.error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="settings-item" style={{ marginTop: '12px' }}>
|
||||
<button
|
||||
type="button"
|
||||
className="settings-btn settings-btn-danger"
|
||||
onClick={() => void handleDeleteLocalModel()}
|
||||
disabled={localModelState.isDeleting || localModelState.isDownloading || !localModelState.isCached}
|
||||
>
|
||||
{localModelState.isDeleting ? 'Deleting...' : 'Delete Cached Model'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="settings-group">
|
||||
<h4>OpenAI</h4>
|
||||
|
||||
|
||||
@@ -104,7 +104,7 @@ export const OPFS_CONSTANTS = {
|
||||
|
||||
export const CONFIG_UPGRADER_CONSTANTS = {
|
||||
VERSION_KEY: '__config_version',
|
||||
CURRENT_VERSION: 1,
|
||||
CURRENT_VERSION: 2,
|
||||
};
|
||||
|
||||
export const URL_CONSTANTS = {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { KGConfigStorage } from '../io/KGConfigStorage';
|
||||
import { CONFIG_UPGRADER_CONSTANTS } from '../../constants/coreConstants';
|
||||
import { upgradeConfigToV1 } from './upgradeConfigToV1';
|
||||
import { upgradeConfigToV2 } from './upgradeConfigToV2';
|
||||
|
||||
/**
|
||||
* KGConfigUpgrader — Orchestrates app-level migrations (e.g., storage backend changes).
|
||||
@@ -33,6 +34,10 @@ export class KGConfigUpgrader {
|
||||
await upgradeConfigToV1();
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
await upgradeConfigToV2();
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
throw new Error(`No config upgrader found for version ${nextVersion}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const configStore = new Map<string, { name: string; data: Record<string, unknown>; lastModified: number }>();
|
||||
|
||||
vi.mock('../io/KGConfigStorage', () => ({
|
||||
KGConfigStorage: {
|
||||
getInstance: () => ({
|
||||
getRaw: vi.fn(async (name: string) => configStore.get(name)?.data ?? null),
|
||||
saveRaw: vi.fn(async (name: string, data: Record<string, unknown>) => {
|
||||
configStore.set(name, { name, data, lastModified: Date.now() });
|
||||
}),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
import { upgradeConfigToV2 } from './upgradeConfigToV2';
|
||||
|
||||
describe('upgradeConfigToV2', () => {
|
||||
beforeEach(() => {
|
||||
configStore.clear();
|
||||
});
|
||||
|
||||
it('pins legacy installs without an explicit provider to the old default provider', async () => {
|
||||
configStore.set('userConfig', {
|
||||
name: 'userConfig',
|
||||
data: {
|
||||
general: {
|
||||
openai: { api_key: '', model: 'gpt-5.4-mini', flex: false },
|
||||
},
|
||||
},
|
||||
lastModified: Date.now(),
|
||||
});
|
||||
|
||||
await upgradeConfigToV2();
|
||||
|
||||
expect((configStore.get('userConfig')?.data.general as Record<string, unknown>).llm_provider).toBe('openai');
|
||||
});
|
||||
|
||||
it('leaves explicit providers unchanged', async () => {
|
||||
configStore.set('userConfig', {
|
||||
name: 'userConfig',
|
||||
data: {
|
||||
general: {
|
||||
llm_provider: 'openai_compatible',
|
||||
},
|
||||
},
|
||||
lastModified: Date.now(),
|
||||
});
|
||||
|
||||
await upgradeConfigToV2();
|
||||
|
||||
expect((configStore.get('userConfig')?.data.general as Record<string, unknown>).llm_provider).toBe('openai_compatible');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
import { KGConfigStorage } from '../io/KGConfigStorage';
|
||||
|
||||
const CONFIG_KEY = 'userConfig';
|
||||
const LEGACY_DEFAULT_PROVIDER = 'openai';
|
||||
|
||||
export async function upgradeConfigToV2(): Promise<void> {
|
||||
const storage = KGConfigStorage.getInstance();
|
||||
const rawConfig = await storage.getRaw(CONFIG_KEY);
|
||||
if (!rawConfig || typeof rawConfig !== 'object') {
|
||||
return;
|
||||
}
|
||||
|
||||
const config = rawConfig as Record<string, unknown>;
|
||||
const general = config.general;
|
||||
if (!general || typeof general !== 'object') {
|
||||
return;
|
||||
}
|
||||
|
||||
if ('llm_provider' in (general as Record<string, unknown>)) {
|
||||
return;
|
||||
}
|
||||
|
||||
(general as Record<string, unknown>).llm_provider = LEGACY_DEFAULT_PROVIDER;
|
||||
await storage.saveRaw(CONFIG_KEY, config);
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import { KGConfigStorage } from '../io/KGConfigStorage';
|
||||
interface AppConfig {
|
||||
general: {
|
||||
language: string;
|
||||
llm_provider: 'openai' | 'gemini' | 'claude' | 'claude_openrouter' | 'openai_compatible';
|
||||
llm_provider: 'local_browser' | 'openai' | 'gemini' | 'claude' | 'claude_openrouter' | 'openai_compatible';
|
||||
persist_api_keys_non_localhost: boolean;
|
||||
openai: {
|
||||
api_key: string;
|
||||
@@ -183,7 +183,7 @@ export class ConfigManager {
|
||||
this.defaultConfig = {
|
||||
general: {
|
||||
language: 'en_us',
|
||||
llm_provider: 'openai',
|
||||
llm_provider: 'local_browser',
|
||||
persist_api_keys_non_localhost: false,
|
||||
openai: {
|
||||
api_key: '',
|
||||
|
||||
@@ -9,10 +9,12 @@ class MockWritableFileStream {
|
||||
this.handle = handle;
|
||||
}
|
||||
|
||||
async write(content: ArrayBuffer | ArrayBufferView): Promise<void> {
|
||||
const bytes = content instanceof ArrayBuffer
|
||||
? new Uint8Array(content)
|
||||
: new Uint8Array(content.buffer, content.byteOffset, content.byteLength);
|
||||
async write(content: ArrayBuffer | ArrayBufferView | string): Promise<void> {
|
||||
const bytes = typeof content === 'string'
|
||||
? new TextEncoder().encode(content)
|
||||
: content instanceof ArrayBuffer
|
||||
? new Uint8Array(content)
|
||||
: new Uint8Array(content.buffer, content.byteOffset, content.byteLength);
|
||||
this.chunks.push(new Uint8Array(bytes));
|
||||
}
|
||||
|
||||
@@ -44,6 +46,8 @@ class MockFileSystemFileHandle {
|
||||
|
||||
async getFile(): Promise<File> {
|
||||
return {
|
||||
size: this.content.byteLength,
|
||||
text: async () => new TextDecoder().decode(this.content),
|
||||
arrayBuffer: async () => this.content.buffer.slice(0),
|
||||
} as unknown as File;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
export const LOCAL_LLM_PROVIDER_KEY = 'local_browser';
|
||||
export const LOCAL_LLM_MODEL_URL =
|
||||
'http://localhost:3000/models/gemma-4-E4B-it-web.task';
|
||||
export const LOCAL_LLM_MODEL_FILENAME = 'gemma-4-E4B-it-web.task';
|
||||
export const LOCAL_LLM_DISPLAY_NAME = 'Gemma 4 E4B';
|
||||
export const LOCAL_LLM_LEGACY_FILENAMES = [
|
||||
'gemma-3n-E4B-it-int4-Web.litertlm',
|
||||
];
|
||||
|
||||
export interface LocalLLMRuntimeSupport {
|
||||
supported: boolean;
|
||||
webgpuExposed: boolean;
|
||||
crossOriginIsolated: boolean;
|
||||
sharedArrayBufferAvailable: boolean;
|
||||
secureContext: boolean;
|
||||
reason: string | null;
|
||||
}
|
||||
|
||||
export function detectLocalLLMRuntimeSupport(): LocalLLMRuntimeSupport {
|
||||
const secureContext = typeof window !== 'undefined' ? window.isSecureContext : false;
|
||||
const crossOriginIsolated = typeof window !== 'undefined' ? window.crossOriginIsolated : false;
|
||||
const sharedArrayBufferAvailable = typeof SharedArrayBuffer !== 'undefined';
|
||||
const webgpuExposed = typeof navigator !== 'undefined' && 'gpu' in navigator;
|
||||
|
||||
let reason: string | null = null;
|
||||
if (!secureContext) {
|
||||
reason = 'Local browser LLM requires a secure context (HTTPS or localhost).';
|
||||
} else if (!crossOriginIsolated || !sharedArrayBufferAvailable) {
|
||||
reason = 'Local browser LLM requires SharedArrayBuffer support. Ensure COOP/COEP headers are enabled.';
|
||||
} else if (!webgpuExposed) {
|
||||
reason = 'Local browser LLM currently requires a browser with WebGPU support.';
|
||||
}
|
||||
|
||||
return {
|
||||
supported: reason === null,
|
||||
webgpuExposed,
|
||||
crossOriginIsolated,
|
||||
sharedArrayBufferAvailable,
|
||||
secureContext,
|
||||
reason,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { OpfsModelCache, type ModelDownloadProgress } from './opfsModelCache';
|
||||
import { LOCAL_LLM_MODEL_FILENAME } from './localLLMConfig';
|
||||
|
||||
const cache = new OpfsModelCache({ directoryName: 'models' });
|
||||
let writingToCachePromise: Promise<void> | null = null;
|
||||
|
||||
export { type ModelDownloadProgress };
|
||||
|
||||
export interface CachedModelStreamResult {
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>;
|
||||
totalBytes: number;
|
||||
fromCache: boolean;
|
||||
cacheWritePromise: Promise<void> | null;
|
||||
}
|
||||
|
||||
export class LocalLLMModelCache {
|
||||
public static async exists(filename: string = LOCAL_LLM_MODEL_FILENAME): Promise<boolean> {
|
||||
return cache.exists(filename);
|
||||
}
|
||||
|
||||
public static async getFile(filename: string = LOCAL_LLM_MODEL_FILENAME): Promise<File> {
|
||||
return cache.getFile(filename);
|
||||
}
|
||||
|
||||
public static async getArrayBuffer(filename: string = LOCAL_LLM_MODEL_FILENAME): Promise<ArrayBuffer> {
|
||||
return cache.getArrayBuffer(filename);
|
||||
}
|
||||
|
||||
public static async delete(filename: string = LOCAL_LLM_MODEL_FILENAME): Promise<void> {
|
||||
await cache.delete(filename);
|
||||
}
|
||||
|
||||
public static async loadModelReaderWithCache(
|
||||
sourceUrl: string,
|
||||
filename: string = LOCAL_LLM_MODEL_FILENAME,
|
||||
onProgress?: (progress: ModelDownloadProgress & { fromCache: boolean }) => void,
|
||||
): Promise<CachedModelStreamResult> {
|
||||
if (writingToCachePromise) {
|
||||
await writingToCachePromise.catch(() => {});
|
||||
}
|
||||
|
||||
if (await this.exists(filename)) {
|
||||
const file = await this.getFile(filename);
|
||||
onProgress?.({
|
||||
receivedBytes: file.size,
|
||||
totalBytes: file.size,
|
||||
percent: 100,
|
||||
fromCache: true,
|
||||
});
|
||||
return {
|
||||
reader: file.stream().getReader(),
|
||||
totalBytes: file.size,
|
||||
fromCache: true,
|
||||
cacheWritePromise: null,
|
||||
};
|
||||
}
|
||||
|
||||
const response = await fetch(sourceUrl);
|
||||
if (!response.ok || !response.body) {
|
||||
throw new Error(`Model download failed (${response.status})`);
|
||||
}
|
||||
|
||||
const totalBytesHeader = response.headers.get('Content-Length');
|
||||
const totalBytes = totalBytesHeader ? Number(totalBytesHeader) : 0;
|
||||
const [streamForConsumer, streamForCache] = response.body.tee();
|
||||
|
||||
writingToCachePromise = cache.downloadStream(
|
||||
streamForCache,
|
||||
filename,
|
||||
totalBytes > 0 ? totalBytes : null,
|
||||
progress => onProgress?.({ ...progress, fromCache: false }),
|
||||
);
|
||||
writingToCachePromise = writingToCachePromise.finally(() => {
|
||||
writingToCachePromise = null;
|
||||
});
|
||||
|
||||
return {
|
||||
reader: streamForConsumer.getReader(),
|
||||
totalBytes,
|
||||
fromCache: false,
|
||||
cacheWritePromise: writingToCachePromise,
|
||||
};
|
||||
}
|
||||
|
||||
public static async download(
|
||||
sourceUrl: string,
|
||||
filename: string = LOCAL_LLM_MODEL_FILENAME,
|
||||
onProgress?: (progress: ModelDownloadProgress) => void,
|
||||
): Promise<void> {
|
||||
await cache.download(sourceUrl, filename, onProgress);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import {
|
||||
detectLocalLLMRuntimeSupport,
|
||||
LOCAL_LLM_LEGACY_FILENAMES,
|
||||
LOCAL_LLM_MODEL_FILENAME,
|
||||
LOCAL_LLM_MODEL_URL,
|
||||
type LocalLLMRuntimeSupport,
|
||||
} from './localLLMConfig';
|
||||
import { LocalLLMModelCache } from './localLLMModelCache';
|
||||
|
||||
export interface LocalLLMModelState {
|
||||
isCached: boolean;
|
||||
isChecking: boolean;
|
||||
isDownloading: boolean;
|
||||
isDeleting: boolean;
|
||||
progressPercent: number;
|
||||
progressText: string;
|
||||
error: string;
|
||||
runtimeSupport: LocalLLMRuntimeSupport;
|
||||
}
|
||||
|
||||
type Listener = (state: LocalLLMModelState) => void;
|
||||
|
||||
export class LocalLLMModelManager {
|
||||
private static listeners = new Set<Listener>();
|
||||
private static initialized = false;
|
||||
private static state: LocalLLMModelState = {
|
||||
isCached: false,
|
||||
isChecking: false,
|
||||
isDownloading: false,
|
||||
isDeleting: false,
|
||||
progressPercent: 0,
|
||||
progressText: '',
|
||||
error: '',
|
||||
runtimeSupport: detectLocalLLMRuntimeSupport(),
|
||||
};
|
||||
|
||||
public static subscribe(listener: Listener): () => void {
|
||||
this.listeners.add(listener);
|
||||
listener(this.getState());
|
||||
if (!this.initialized) {
|
||||
this.initialized = true;
|
||||
void this.refresh();
|
||||
}
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
public static getState(): LocalLLMModelState {
|
||||
return { ...this.state, runtimeSupport: { ...this.state.runtimeSupport } };
|
||||
}
|
||||
|
||||
public static async refresh(): Promise<void> {
|
||||
this.setState({
|
||||
isChecking: true,
|
||||
runtimeSupport: detectLocalLLMRuntimeSupport(),
|
||||
});
|
||||
try {
|
||||
await this.cleanupLegacyEntries();
|
||||
const isCached = await LocalLLMModelCache.exists();
|
||||
this.setState({ isCached, error: '' });
|
||||
} catch (error) {
|
||||
this.setState({ error: error instanceof Error ? error.message : String(error) });
|
||||
} finally {
|
||||
this.setState({ isChecking: false });
|
||||
}
|
||||
}
|
||||
|
||||
public static async ensureRuntimeSupported(): Promise<void> {
|
||||
const runtimeSupport = detectLocalLLMRuntimeSupport();
|
||||
this.setState({ runtimeSupport });
|
||||
if (!runtimeSupport.supported) {
|
||||
throw new Error(runtimeSupport.reason ?? 'Local browser LLM is not supported in this browser.');
|
||||
}
|
||||
|
||||
await this.cleanupLegacyEntries();
|
||||
}
|
||||
|
||||
public static async deleteCachedModel(): Promise<void> {
|
||||
this.setState({ isDeleting: true, error: '' });
|
||||
try {
|
||||
await LocalLLMModelCache.delete();
|
||||
await this.cleanupLegacyEntries();
|
||||
this.setState({
|
||||
isCached: false,
|
||||
progressPercent: 0,
|
||||
progressText: '',
|
||||
});
|
||||
} catch (error) {
|
||||
this.setState({ error: error instanceof Error ? error.message : String(error) });
|
||||
throw error;
|
||||
} finally {
|
||||
this.setState({ isDeleting: false });
|
||||
}
|
||||
}
|
||||
|
||||
private static setState(partial: Partial<LocalLLMModelState>): void {
|
||||
this.state = {
|
||||
...this.state,
|
||||
...partial,
|
||||
};
|
||||
for (const listener of this.listeners) {
|
||||
listener(this.getState());
|
||||
}
|
||||
}
|
||||
|
||||
private static async cleanupLegacyEntries(): Promise<void> {
|
||||
await Promise.all(
|
||||
LOCAL_LLM_LEGACY_FILENAMES.map(async legacyFilename => {
|
||||
try {
|
||||
await LocalLLMModelCache.delete(legacyFilename);
|
||||
} catch {
|
||||
// Ignore best-effort legacy cleanup failures.
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
public static notifyLoadStart(fromCache: boolean): void {
|
||||
this.setState({
|
||||
isDownloading: !fromCache,
|
||||
progressPercent: fromCache ? 100 : 0,
|
||||
progressText: fromCache ? 'Loading local language model from browser cache...' : 'Downloading local language model...',
|
||||
error: '',
|
||||
});
|
||||
}
|
||||
|
||||
public static notifyLoadProgress(receivedBytes: number, totalBytes: number | null, fromCache: boolean): void {
|
||||
const receivedMb = (receivedBytes / (1024 * 1024)).toFixed(1);
|
||||
const totalMb = totalBytes ? (totalBytes / (1024 * 1024)).toFixed(1) : null;
|
||||
this.setState({
|
||||
isDownloading: !fromCache,
|
||||
progressPercent: totalBytes ? (receivedBytes / totalBytes) * 100 : 0,
|
||||
progressText: fromCache
|
||||
? 'Loading local language model from browser cache...'
|
||||
: totalMb
|
||||
? `Downloading local language model... ${receivedMb} / ${totalMb} MB`
|
||||
: `Downloading local language model... ${receivedMb} MB`,
|
||||
error: '',
|
||||
});
|
||||
}
|
||||
|
||||
public static notifyCacheReady(): void {
|
||||
this.setState({
|
||||
isCached: true,
|
||||
isDownloading: false,
|
||||
progressPercent: 100,
|
||||
progressText: 'Local language model is ready.',
|
||||
error: '',
|
||||
});
|
||||
}
|
||||
|
||||
public static notifyLoadError(error: unknown): void {
|
||||
this.setState({
|
||||
isDownloading: false,
|
||||
progressPercent: 0,
|
||||
progressText: '',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,50 +1,25 @@
|
||||
import { LOCAL_SEPARATOR_MODEL_FILENAME } from './localSeparatorConfig';
|
||||
import { OpfsModelCache, type ModelDownloadProgress } from './opfsModelCache';
|
||||
|
||||
export interface ModelDownloadProgress {
|
||||
receivedBytes: number;
|
||||
totalBytes: number | null;
|
||||
percent: number;
|
||||
}
|
||||
const cache = new OpfsModelCache({ directoryName: 'models' });
|
||||
|
||||
export { type ModelDownloadProgress };
|
||||
|
||||
export class LocalSeparatorModelCache {
|
||||
private static readonly MODELS_DIR = 'models';
|
||||
private static readonly TEMP_SUFFIX = '.download';
|
||||
|
||||
public static async exists(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<boolean> {
|
||||
try {
|
||||
const modelsDir = await this.getModelsDir();
|
||||
await modelsDir.getFileHandle(filename);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return cache.exists(filename);
|
||||
}
|
||||
|
||||
public static async getFile(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<File> {
|
||||
const modelsDir = await this.getModelsDir();
|
||||
const fileHandle = await modelsDir.getFileHandle(filename);
|
||||
return fileHandle.getFile();
|
||||
return cache.getFile(filename);
|
||||
}
|
||||
|
||||
public static async getArrayBuffer(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<ArrayBuffer> {
|
||||
const file = await this.getFile(filename);
|
||||
return file.arrayBuffer();
|
||||
return cache.getArrayBuffer(filename);
|
||||
}
|
||||
|
||||
public static async delete(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<void> {
|
||||
try {
|
||||
const modelsDir = await this.getModelsDir();
|
||||
await modelsDir.removeEntry(filename);
|
||||
} catch {
|
||||
// Ignore missing file cleanup.
|
||||
}
|
||||
|
||||
try {
|
||||
const modelsDir = await this.getModelsDir();
|
||||
await modelsDir.removeEntry(`${filename}${this.TEMP_SUFFIX}`);
|
||||
} catch {
|
||||
// Ignore missing temp file cleanup.
|
||||
}
|
||||
await cache.delete(filename);
|
||||
}
|
||||
|
||||
public static async download(
|
||||
@@ -52,75 +27,6 @@ export class LocalSeparatorModelCache {
|
||||
filename: string = LOCAL_SEPARATOR_MODEL_FILENAME,
|
||||
onProgress?: (progress: ModelDownloadProgress) => void,
|
||||
): Promise<void> {
|
||||
const response = await fetch(sourceUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Model download failed (${response.status})`);
|
||||
}
|
||||
|
||||
const modelsDir = await this.getModelsDir();
|
||||
const tempName = `${filename}${this.TEMP_SUFFIX}`;
|
||||
await this.delete(filename);
|
||||
|
||||
const tempHandle = await modelsDir.getFileHandle(tempName, { create: true });
|
||||
const writable = await tempHandle.createWritable();
|
||||
|
||||
try {
|
||||
const totalBytesHeader = response.headers.get('Content-Length');
|
||||
const totalBytes = totalBytesHeader ? Number(totalBytesHeader) : null;
|
||||
|
||||
if (!response.body) {
|
||||
const buffer = await response.arrayBuffer();
|
||||
await writable.write(buffer);
|
||||
onProgress?.({
|
||||
receivedBytes: buffer.byteLength,
|
||||
totalBytes,
|
||||
percent: 100,
|
||||
});
|
||||
} else {
|
||||
const reader = response.body.getReader();
|
||||
let receivedBytes = 0;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value) continue;
|
||||
await writable.write(value);
|
||||
receivedBytes += value.byteLength;
|
||||
onProgress?.({
|
||||
receivedBytes,
|
||||
totalBytes,
|
||||
percent: totalBytes ? (receivedBytes / totalBytes) * 100 : 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
await writable.abort();
|
||||
await this.delete(filename);
|
||||
throw error;
|
||||
}
|
||||
|
||||
await writable.close();
|
||||
|
||||
const finalHandle = await modelsDir.getFileHandle(filename, { create: true });
|
||||
const finalWritable = await finalHandle.createWritable();
|
||||
try {
|
||||
const tempFile = await tempHandle.getFile();
|
||||
await finalWritable.write(await tempFile.arrayBuffer());
|
||||
await finalWritable.close();
|
||||
} catch (error) {
|
||||
await finalWritable.abort();
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
await modelsDir.removeEntry(tempName);
|
||||
} catch {
|
||||
// Ignore temp cleanup errors.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async getModelsDir(): Promise<FileSystemDirectoryHandle> {
|
||||
const root = await navigator.storage.getDirectory();
|
||||
return root.getDirectoryHandle(this.MODELS_DIR, { create: true });
|
||||
await cache.download(sourceUrl, filename, onProgress);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { clearChatHistoryAndUI } from '../chatUtil';
|
||||
import { useProjectStore } from '../../stores/projectStore';
|
||||
import { ConfigManager } from '../../core/config/ConfigManager';
|
||||
import { SystemPrompts } from '../../agent/core/SystemPrompts';
|
||||
import { detectLocalLLMRuntimeSupport, LOCAL_LLM_PROVIDER_KEY } from '../localLLMConfig';
|
||||
|
||||
export interface UserMessageFilterResult {
|
||||
// Whether to render the user message bubble (div.message-user)
|
||||
@@ -134,7 +135,18 @@ export async function processUserMessage(originalMessage: string): Promise<UserM
|
||||
}
|
||||
const provider = (configManager.get('general.llm_provider') as string) || 'openai';
|
||||
|
||||
if (provider === 'openai') {
|
||||
if (provider === LOCAL_LLM_PROVIDER_KEY) {
|
||||
const runtimeSupport = detectLocalLLMRuntimeSupport();
|
||||
if (!runtimeSupport.supported) {
|
||||
return {
|
||||
displayUserMessage: true,
|
||||
sendToLLM: false,
|
||||
finalMessageForLLM: null,
|
||||
pseudoAssistantResponse: runtimeSupport.reason ?? 'Local browser LLM is not supported in this environment.',
|
||||
metadata: { error: 'local_browser_unsupported' }
|
||||
};
|
||||
}
|
||||
} else if (provider === 'openai') {
|
||||
const openaiKey = (configManager.get('general.openai.api_key') as string) || '';
|
||||
if (openaiKey.trim() === '') {
|
||||
const url = `${import.meta.env.BASE_URL}chat/error_no_openai_key.md`;
|
||||
@@ -256,4 +268,3 @@ export async function processUserMessage(originalMessage: string): Promise<UserM
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
export interface ModelDownloadProgress {
|
||||
receivedBytes: number;
|
||||
totalBytes: number | null;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
interface OpfsModelCacheOptions {
|
||||
directoryName?: string;
|
||||
sizeSuffix?: string;
|
||||
tempSuffix?: string;
|
||||
}
|
||||
|
||||
export class OpfsModelCache {
|
||||
private readonly directoryName: string;
|
||||
private readonly sizeSuffix: string;
|
||||
private readonly tempSuffix: string;
|
||||
|
||||
constructor(options: OpfsModelCacheOptions = {}) {
|
||||
this.directoryName = options.directoryName ?? 'models';
|
||||
this.sizeSuffix = options.sizeSuffix ?? '.size';
|
||||
this.tempSuffix = options.tempSuffix ?? '.download';
|
||||
}
|
||||
|
||||
public async exists(filename: string): Promise<boolean> {
|
||||
try {
|
||||
const dir = await this.getDir();
|
||||
const fileHandle = await dir.getFileHandle(filename);
|
||||
const sizeHandle = await dir.getFileHandle(this.getSizeFilename(filename));
|
||||
const [file, sizeFile] = await Promise.all([fileHandle.getFile(), sizeHandle.getFile()]);
|
||||
const expectedSize = Number(await sizeFile.text());
|
||||
if (!Number.isFinite(expectedSize) || expectedSize <= 0) {
|
||||
await this.delete(filename);
|
||||
return false;
|
||||
}
|
||||
if (file.size !== expectedSize) {
|
||||
await this.delete(filename);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async getFile(filename: string): Promise<File> {
|
||||
const dir = await this.getDir();
|
||||
const handle = await dir.getFileHandle(filename);
|
||||
const file = await handle.getFile();
|
||||
console.log('[opfsModelCache] Opened cached file.', {
|
||||
filename,
|
||||
size: file.size,
|
||||
});
|
||||
return file;
|
||||
}
|
||||
|
||||
public async getArrayBuffer(filename: string): Promise<ArrayBuffer> {
|
||||
const file = await this.getFile(filename);
|
||||
return file.arrayBuffer();
|
||||
}
|
||||
|
||||
public async delete(filename: string): Promise<void> {
|
||||
const dir = await this.getDir();
|
||||
await this.removeIfExists(dir, filename);
|
||||
await this.removeIfExists(dir, this.getSizeFilename(filename));
|
||||
await this.removeIfExists(dir, `${filename}${this.tempSuffix}`);
|
||||
await this.removeIfExists(dir, `${this.getSizeFilename(filename)}${this.tempSuffix}`);
|
||||
}
|
||||
|
||||
public async download(
|
||||
sourceUrl: string,
|
||||
filename: string,
|
||||
onProgress?: (progress: ModelDownloadProgress) => void,
|
||||
): Promise<void> {
|
||||
const response = await fetch(sourceUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Model download failed (${response.status})`);
|
||||
}
|
||||
const totalBytesHeader = response.headers.get('Content-Length');
|
||||
const totalBytes = totalBytesHeader ? Number(totalBytesHeader) : null;
|
||||
if (!response.body) {
|
||||
throw new Error('Model download response did not include a readable body.');
|
||||
}
|
||||
await this.downloadStream(response.body, filename, totalBytes, onProgress);
|
||||
}
|
||||
|
||||
public async downloadStream(
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
filename: string,
|
||||
totalBytes: number | null,
|
||||
onProgress?: (progress: ModelDownloadProgress) => void,
|
||||
): Promise<void> {
|
||||
const dir = await this.getDir();
|
||||
await this.delete(filename);
|
||||
|
||||
const tempFilename = `${filename}${this.tempSuffix}`;
|
||||
const tempHandle = await dir.getFileHandle(tempFilename, { create: true });
|
||||
const tempWritable = await tempHandle.createWritable();
|
||||
const reader = stream.getReader();
|
||||
let receivedBytes = 0;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value) continue;
|
||||
await tempWritable.write(value);
|
||||
receivedBytes += value.byteLength;
|
||||
onProgress?.({
|
||||
receivedBytes,
|
||||
totalBytes,
|
||||
percent: totalBytes ? (receivedBytes / totalBytes) * 100 : 0,
|
||||
});
|
||||
}
|
||||
await tempWritable.close();
|
||||
|
||||
const sizeValue = totalBytes ?? receivedBytes;
|
||||
if (!Number.isFinite(sizeValue) || sizeValue <= 0) {
|
||||
throw new Error('Model download did not provide a valid size.');
|
||||
}
|
||||
|
||||
console.log(`[opfsModelCache] Finalizing cached model ${filename} from temp file ${tempFilename}.`);
|
||||
const finalHandle = await dir.getFileHandle(filename, { create: true });
|
||||
const finalWritable = await finalHandle.createWritable();
|
||||
try {
|
||||
const tempFile = await tempHandle.getFile();
|
||||
console.log('[opfsModelCache] Temp file ready for finalize copy.', {
|
||||
filename,
|
||||
tempFilename,
|
||||
tempSize: tempFile.size,
|
||||
expectedSize: sizeValue,
|
||||
});
|
||||
await finalWritable.write(tempFile);
|
||||
await finalWritable.close();
|
||||
} catch (error) {
|
||||
await finalWritable.abort();
|
||||
throw error;
|
||||
}
|
||||
|
||||
const sizeHandle = await dir.getFileHandle(this.getSizeFilename(filename), { create: true });
|
||||
const sizeWritable = await sizeHandle.createWritable();
|
||||
try {
|
||||
await sizeWritable.write(String(sizeValue));
|
||||
await sizeWritable.close();
|
||||
} catch (error) {
|
||||
await sizeWritable.abort();
|
||||
throw error;
|
||||
}
|
||||
|
||||
onProgress?.({
|
||||
receivedBytes: sizeValue,
|
||||
totalBytes: sizeValue,
|
||||
percent: 100,
|
||||
});
|
||||
console.log('[opfsModelCache] Cached model finalize completed.', {
|
||||
filename,
|
||||
size: sizeValue,
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await tempWritable.abort();
|
||||
} catch {
|
||||
// Ignore abort cleanup errors.
|
||||
}
|
||||
await this.delete(filename);
|
||||
throw error;
|
||||
} finally {
|
||||
await this.removeIfExists(dir, tempFilename);
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
private getSizeFilename(filename: string): string {
|
||||
return `${filename}${this.sizeSuffix}`;
|
||||
}
|
||||
|
||||
private async getDir(): Promise<FileSystemDirectoryHandle> {
|
||||
const root = await navigator.storage.getDirectory();
|
||||
return root.getDirectoryHandle(this.directoryName, { create: true });
|
||||
}
|
||||
|
||||
private async removeIfExists(dir: FileSystemDirectoryHandle, name: string): Promise<void> {
|
||||
try {
|
||||
await dir.removeEntry(name);
|
||||
} catch {
|
||||
// Ignore missing entry cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,17 @@ export default defineConfig({
|
||||
},
|
||||
server: {
|
||||
host: true,
|
||||
headers: {
|
||||
'Cross-Origin-Opener-Policy': 'same-origin',
|
||||
'Cross-Origin-Embedder-Policy': 'require-corp',
|
||||
},
|
||||
// IMPORTANT: MAKE SURE TO UPDATE YOUR OS HOSTS FILE TO POINT `testlocal.com` TO YOUR LOCAL IP (e.g. 127.0.0.1).
|
||||
allowedHosts: ['testlocal.com', '.testlocal.com', 'localhost', '127.0.0.1'],
|
||||
},
|
||||
preview: {
|
||||
headers: {
|
||||
'Cross-Origin-Opener-Policy': 'same-origin',
|
||||
'Cross-Origin-Embedder-Policy': 'require-corp',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user