diff --git a/public/config.json b/public/config.json index 4d2f9ea..015918a 100644 --- a/public/config.json +++ b/public/config.json @@ -25,6 +25,9 @@ "api_key": "", "base_url": "", "model": "" + }, + "local_browser": { + "context_length": 32768 }, "soundfont": { "base_url": "https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/" diff --git a/src/agent/llm/LocalBrowserLLMProvider.test.ts b/src/agent/llm/LocalBrowserLLMProvider.test.ts new file mode 100644 index 0000000..1284c9d --- /dev/null +++ b/src/agent/llm/LocalBrowserLLMProvider.test.ts @@ -0,0 +1,106 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { + configGetMock, + ensureRuntimeSupportedMock, + notifyLoadProgressMock, + notifyLoadStartMock, + notifyCacheReadyMock, + notifyLoadErrorMock, + loadModelReaderWithCacheMock, +} = vi.hoisted(() => ({ + configGetMock: vi.fn(), + ensureRuntimeSupportedMock: vi.fn(async () => undefined), + notifyLoadProgressMock: vi.fn(), + notifyLoadStartMock: vi.fn(), + notifyCacheReadyMock: vi.fn(), + notifyLoadErrorMock: vi.fn(), + loadModelReaderWithCacheMock: vi.fn(), +})); + +vi.mock('../../core/config/ConfigManager', () => ({ + ConfigManager: { + instance: () => ({ + get: configGetMock, + }), + }, +})); + +vi.mock('../../util/localLLMModelManager', () => ({ + LocalLLMModelManager: { + ensureRuntimeSupported: ensureRuntimeSupportedMock, + notifyLoadProgress: notifyLoadProgressMock, + notifyLoadStart: notifyLoadStartMock, + notifyCacheReady: notifyCacheReadyMock, + notifyLoadError: notifyLoadErrorMock, + }, +})); + +vi.mock('../../util/localLLMModelCache', () => ({ + LocalLLMModelCache: { + loadModelReaderWithCache: loadModelReaderWithCacheMock, + }, +})); + +import { LocalBrowserLLMProvider } from './LocalBrowserLLMProvider'; + +describe('LocalBrowserLLMProvider', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + beforeEach(() => { + configGetMock.mockReset(); + ensureRuntimeSupportedMock.mockClear(); + notifyLoadProgressMock.mockClear(); + notifyLoadStartMock.mockClear(); + notifyCacheReadyMock.mockClear(); + notifyLoadErrorMock.mockClear(); + loadModelReaderWithCacheMock.mockReset(); + }); + + async function runProviderAndCaptureOptions(configValue: unknown): Promise> { + const createFromOptionsMock = vi.fn(async (_fileset: unknown, options: Record) => ({ + generateResponse: (_prompt: string, callback: (partial: string, done: boolean) => void) => { + callback('hello', true); + }, + sizeInTokens: (text: string) => text.length, + })); + + configGetMock.mockReturnValue(configValue); + loadModelReaderWithCacheMock.mockResolvedValue({ + reader: new Uint8Array([1, 2, 3]), + totalBytes: 3, + fromCache: true, + cacheWritePromise: null, + }); + + vi.spyOn(LocalBrowserLLMProvider.prototype as never, 'getMediaPipeModule' as never).mockResolvedValue({ + FilesetResolver: { + forGenAiTasks: vi.fn(async () => ({})), + }, + LlmInference: { + createFromOptions: createFromOptionsMock, + }, + }); + + const provider = new LocalBrowserLLMProvider(); + const chunks: unknown[] = []; + for await (const chunk of provider.generateStream([])) { + chunks.push(chunk); + } + + expect(chunks.length).toBeGreaterThan(0); + return createFromOptionsMock.mock.calls[0][1] as Record; + } + + it('uses the configured maxTokens value', async () => { + const options = await runProviderAndCaptureOptions(65536); + expect(options.maxTokens).toBe(65536); + }); + + it('falls back to 32768 when config is invalid', async () => { + const options = await runProviderAndCaptureOptions(99999); + expect(options.maxTokens).toBe(32768); + }); +}); diff --git a/src/agent/llm/LocalBrowserLLMProvider.ts b/src/agent/llm/LocalBrowserLLMProvider.ts index 298e5dd..57970f8 100644 --- a/src/agent/llm/LocalBrowserLLMProvider.ts +++ b/src/agent/llm/LocalBrowserLLMProvider.ts @@ -9,9 +9,15 @@ import { parseToolCalls, stripToolProtocol, } from './gemmaToolProtocol'; -import { LOCAL_LLM_MODEL_FILENAME, LOCAL_LLM_MODEL_URL } from '../../util/localLLMConfig'; +import { + LOCAL_LLM_DEFAULT_CONTEXT_LENGTH, + LOCAL_LLM_MODEL_FILENAME, + LOCAL_LLM_MODEL_URL, + normalizeLocalLLMContextLength, +} from '../../util/localLLMConfig'; import { LocalLLMModelCache } from '../../util/localLLMModelCache'; import type { LLMProvider } from './LLMProvider'; +import { ConfigManager } from '../../core/config/ConfigManager'; type MediaPipeGenAI = { FilesetResolver: { @@ -60,6 +66,9 @@ export class LocalBrowserLLMProvider implements LLMProvider { return this.inference; } + const maxTokens = this.getConfiguredContextLength(); + console.log(`[localLLM] Initializing with max context length: ${maxTokens} tokens`); + const [{ FilesetResolver, LlmInference }, modelLoad] = await Promise.all([ this.getMediaPipeModule(), LocalLLMModelCache.loadModelReaderWithCache( @@ -86,7 +95,7 @@ export class LocalBrowserLLMProvider implements LLMProvider { modelAssetBuffer: modelLoad.reader, }, numResponses: 1, - maxTokens: 32768, + maxTokens, topK: 64, temperature: 1.0, }); @@ -112,6 +121,17 @@ export class LocalBrowserLLMProvider implements LLMProvider { return importMediaPipe(); } + private getConfiguredContextLength(): number { + try { + const configManager = ConfigManager.instance(); + return normalizeLocalLLMContextLength( + configManager.get('general.local_browser.context_length'), + ); + } catch { + return LOCAL_LLM_DEFAULT_CONTEXT_LENGTH; + } + } + private applyTemplate(message: { role: 'user' | 'model'; text: string }): string { const template = PROMPT_TEMPLATE[message.role]; return `${template.pre}${message.text}${template.post}`; diff --git a/src/components/ChatBox.tsx b/src/components/ChatBox.tsx index 4a1cdfd..7be2842 100644 --- a/src/components/ChatBox.tsx +++ b/src/components/ChatBox.tsx @@ -200,6 +200,7 @@ const ChatBox: React.FC = ({ isVisible }) => { const unsubscribe = configManager.addChangeListener((changedKeys) => { if ( changedKeys.includes('general.llm_provider') || + changedKeys.includes('general.local_browser.context_length') || changedKeys.some(k => k.startsWith('general.openai.')) || changedKeys.some(k => k.startsWith('general.openai_compatible.')) ) { diff --git a/src/components/settings/sections/GeneralSettings.test.tsx b/src/components/settings/sections/GeneralSettings.test.tsx new file mode 100644 index 0000000..d45197f --- /dev/null +++ b/src/components/settings/sections/GeneralSettings.test.tsx @@ -0,0 +1,120 @@ +import React from 'react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import GeneralSettings from './GeneralSettings'; + +const configState = new Map([ + ['general.llm_provider', 'local_browser'], + ['general.persist_api_keys_non_localhost', false], + ['general.openai.api_key', ''], + ['general.openai.model', 'gpt-5.4-mini'], + ['general.openai.flex', false], + ['general.gemini.api_key', ''], + ['general.gemini.model', 'gemini-2.5-flash'], + ['general.claude.api_key', ''], + ['general.claude.model', 'claude-sonnet-4.6'], + ['general.claude_openrouter.api_key', ''], + ['general.claude_openrouter.base_url', 'https://openrouter.ai/api/v1'], + ['general.claude_openrouter.model', 'anthropic/claude-sonnet-4.6'], + ['general.openai_compatible.api_key', ''], + ['general.openai_compatible.base_url', ''], + ['general.openai_compatible.model', ''], + ['general.local_browser.context_length', 65536], + ['general.soundfont.base_url', 'https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/'], + ['general.kgone.enabled', false], + ['general.kgone.base_url', 'http://127.0.0.1:8000'], +]); + +const configManagerMock = { + getIsInitialized: vi.fn(() => true), + initialize: vi.fn().mockResolvedValue(undefined), + get: vi.fn((key: string) => configState.get(key)), + set: vi.fn(async (key: string, value: unknown) => { + configState.set(key, value); + }), + isKGOneServerManaged: vi.fn(() => false), + isSoundfontServerManaged: vi.fn(() => false), +}; + +vi.mock('../../../core/config/ConfigManager', () => ({ + ConfigManager: { + instance: () => configManagerMock, + }, +})); + +vi.mock('../../../util/localLLMModelManager', () => ({ + LocalLLMModelManager: { + getState: () => ({ + isCached: false, + isChecking: false, + isDownloading: false, + isDeleting: false, + progressPercent: 0, + progressText: '', + error: '', + runtimeSupport: { + supported: true, + webgpuExposed: true, + crossOriginIsolated: true, + sharedArrayBufferAvailable: true, + secureContext: true, + reason: null, + }, + }), + subscribe: (listener: (state: unknown) => void) => { + listener({ + isCached: false, + isChecking: false, + isDownloading: false, + isDeleting: false, + progressPercent: 0, + progressText: '', + error: '', + runtimeSupport: { + supported: true, + webgpuExposed: true, + crossOriginIsolated: true, + sharedArrayBufferAvailable: true, + secureContext: true, + reason: null, + }, + }); + return () => {}; + }, + deleteCachedModel: vi.fn().mockResolvedValue(undefined), + }, +})); + +describe('GeneralSettings', () => { + beforeEach(() => { + configState.set('general.local_browser.context_length', 65536); + configManagerMock.get.mockClear(); + configManagerMock.set.mockClear(); + }); + + it('renders the local context length selector and VRAM hint', async () => { + render(); + + expect(await screen.findByText('Gemma 4 E4B Local Runtime')).toBeTruthy(); + expect(screen.getByLabelText('Context Length')).toBeTruthy(); + expect(screen.getByText(/require more VRAM/i)).toBeTruthy(); + }); + + it('initializes the local context length from config', async () => { + render(); + + const select = await screen.findByLabelText('Context Length'); + expect((select as HTMLSelectElement).value).toBe('65536'); + }); + + it('persists local context length changes', async () => { + render(); + + const select = await screen.findByLabelText('Context Length'); + fireEvent.change(select, { target: { value: '131072' } }); + + await waitFor(() => { + expect(configManagerMock.set).toHaveBeenCalledWith('general.local_browser.context_length', 131072); + }); + }); +}); diff --git a/src/components/settings/sections/GeneralSettings.tsx b/src/components/settings/sections/GeneralSettings.tsx index 8814ad7..b3edf73 100644 --- a/src/components/settings/sections/GeneralSettings.tsx +++ b/src/components/settings/sections/GeneralSettings.tsx @@ -1,7 +1,15 @@ import React, { useState, useEffect, useCallback, useMemo } from 'react'; import { ConfigManager } from '../../../core/config/ConfigManager'; import { LocalLLMModelManager, type LocalLLMModelState } from '../../../util/localLLMModelManager'; -import { LOCAL_LLM_DISPLAY_NAME, LOCAL_LLM_PROVIDER_KEY } from '../../../util/localLLMConfig'; +import { + formatLocalLLMContextLength, + LOCAL_LLM_CONTEXT_LENGTH_OPTIONS, + LOCAL_LLM_DEFAULT_CONTEXT_LENGTH, + LOCAL_LLM_DISPLAY_NAME, + LOCAL_LLM_PROVIDER_KEY, + normalizeLocalLLMContextLength, + type LocalLLMContextLength, +} from '../../../util/localLLMConfig'; const GeneralSettings: React.FC = () => { const [llmProvider, setLlmProvider] = useState(LOCAL_LLM_PROVIDER_KEY); @@ -24,6 +32,7 @@ const GeneralSettings: React.FC = () => { const [kgoneBaseUrl, setKgoneBaseUrl] = useState(''); const [kgoneServerManaged, setKgoneServerManaged] = useState(false); const [soundfontServerManaged, setSoundfontServerManaged] = useState(false); + const [localContextLength, setLocalContextLength] = useState(LOCAL_LLM_DEFAULT_CONTEXT_LENGTH); const [localModelState, setLocalModelState] = useState(LocalLLMModelManager.getState()); const configManager = ConfigManager.instance(); @@ -64,6 +73,7 @@ const GeneralSettings: React.FC = () => { setCompatibleKey((configManager.get('general.openai_compatible.api_key') as string) || ''); setCompatibleBaseUrl((configManager.get('general.openai_compatible.base_url') as string) || ''); setCompatibleModel((configManager.get('general.openai_compatible.model') as string) || ''); + setLocalContextLength(normalizeLocalLLMContextLength(configManager.get('general.local_browser.context_length'))); setSoundfontBaseUrl((configManager.get('general.soundfont.base_url') as string) || ''); setKgoneEnabled((configManager.get('general.kgone.enabled') as boolean) ?? false); setKgoneBaseUrl((configManager.get('general.kgone.base_url') as string) || ''); @@ -210,6 +220,18 @@ const GeneralSettings: React.FC = () => { } }; + const handleLocalContextLengthChange = async (value: string) => { + const parsed = Number(value); + const normalized = normalizeLocalLLMContextLength(parsed); + setLocalContextLength(normalized); + try { + await configManager.set('general.local_browser.context_length', normalized); + console.log('Local browser context length changed to:', normalized); + } catch (error) { + console.error('Failed to save local browser context length:', error); + } + }; + // NOTE: Gemini and Claude are not supported yet due to CORS issues. return (
@@ -279,6 +301,27 @@ const GeneralSettings: React.FC = () => {
+
+ + +
+ Larger context lengths require more VRAM and may also reduce performance as conversations become longer. +
+
+ {!localModelState.isCached && !localModelState.isDownloading && localModelState.runtimeSupport.supported && (
The local model downloads automatically the next time you chat with `Local LLM (Browser)`. diff --git a/src/constants/coreConstants.ts b/src/constants/coreConstants.ts index 1e03999..33f0f40 100644 --- a/src/constants/coreConstants.ts +++ b/src/constants/coreConstants.ts @@ -104,7 +104,7 @@ export const OPFS_CONSTANTS = { export const CONFIG_UPGRADER_CONSTANTS = { VERSION_KEY: '__config_version', - CURRENT_VERSION: 2, + CURRENT_VERSION: 3, }; export const URL_CONSTANTS = { diff --git a/src/core/config-upgrader/KGConfigUpgrader.ts b/src/core/config-upgrader/KGConfigUpgrader.ts index 22066fb..21e08d7 100644 --- a/src/core/config-upgrader/KGConfigUpgrader.ts +++ b/src/core/config-upgrader/KGConfigUpgrader.ts @@ -2,6 +2,7 @@ import { KGConfigStorage } from '../io/KGConfigStorage'; import { CONFIG_UPGRADER_CONSTANTS } from '../../constants/coreConstants'; import { upgradeConfigToV1 } from './upgradeConfigToV1'; import { upgradeConfigToV2 } from './upgradeConfigToV2'; +import { upgradeConfigToV3 } from './upgradeConfigToV3'; /** * KGConfigUpgrader — Orchestrates app-level migrations (e.g., storage backend changes). @@ -38,6 +39,10 @@ export class KGConfigUpgrader { await upgradeConfigToV2(); break; } + case 3: { + await upgradeConfigToV3(); + break; + } default: { throw new Error(`No config upgrader found for version ${nextVersion}`); } diff --git a/src/core/config-upgrader/upgradeConfigToV3.test.ts b/src/core/config-upgrader/upgradeConfigToV3.test.ts new file mode 100644 index 0000000..ea64dfb --- /dev/null +++ b/src/core/config-upgrader/upgradeConfigToV3.test.ts @@ -0,0 +1,88 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const configStore = new Map; lastModified: number }>(); + +vi.mock('../io/KGConfigStorage', () => ({ + KGConfigStorage: { + getInstance: () => ({ + getRaw: vi.fn(async (name: string) => configStore.get(name)?.data ?? null), + saveRaw: vi.fn(async (name: string, data: Record) => { + configStore.set(name, { name, data, lastModified: Date.now() }); + }), + }), + }, +})); + +import { upgradeConfigToV3 } from './upgradeConfigToV3'; + +describe('upgradeConfigToV3', () => { + beforeEach(() => { + configStore.clear(); + }); + + it('adds the default local browser context length when missing', async () => { + configStore.set('userConfig', { + name: 'userConfig', + data: { + general: { + llm_provider: 'local_browser', + }, + }, + lastModified: Date.now(), + }); + + await upgradeConfigToV3(); + + expect( + ((configStore.get('userConfig')?.data.general as Record).local_browser as Record).context_length, + ).toBe(32768); + }); + + it.each([32768, 65536, 131072])('preserves existing context length %s', async (existingValue) => { + configStore.set('userConfig', { + name: 'userConfig', + data: { + general: { + local_browser: { + context_length: existingValue, + }, + }, + }, + lastModified: Date.now(), + }); + + await upgradeConfigToV3(); + + expect( + ((configStore.get('userConfig')?.data.general as Record).local_browser as Record).context_length, + ).toBe(existingValue); + }); + + it('is a no-op when general is missing', async () => { + configStore.set('userConfig', { + name: 'userConfig', + data: {}, + lastModified: Date.now(), + }); + + await upgradeConfigToV3(); + + expect(configStore.get('userConfig')?.data).toEqual({}); + }); + + it('is a no-op when general is malformed', async () => { + configStore.set('userConfig', { + name: 'userConfig', + data: { + general: 'invalid', + }, + lastModified: Date.now(), + }); + + await upgradeConfigToV3(); + + expect(configStore.get('userConfig')?.data).toEqual({ + general: 'invalid', + }); + }); +}); diff --git a/src/core/config-upgrader/upgradeConfigToV3.ts b/src/core/config-upgrader/upgradeConfigToV3.ts new file mode 100644 index 0000000..22ffc63 --- /dev/null +++ b/src/core/config-upgrader/upgradeConfigToV3.ts @@ -0,0 +1,35 @@ +import { KGConfigStorage } from '../io/KGConfigStorage'; +import { LOCAL_LLM_DEFAULT_CONTEXT_LENGTH } from '../../util/localLLMConfig'; + +const CONFIG_KEY = 'userConfig'; + +export async function upgradeConfigToV3(): Promise { + const storage = KGConfigStorage.getInstance(); + const rawConfig = await storage.getRaw(CONFIG_KEY); + if (!rawConfig || typeof rawConfig !== 'object') { + return; + } + + const config = rawConfig as Record; + const general = config.general; + if (!general || typeof general !== 'object') { + return; + } + + const generalRecord = general as Record; + const localBrowser = generalRecord.local_browser; + + if (!localBrowser || typeof localBrowser !== 'object') { + generalRecord.local_browser = { context_length: LOCAL_LLM_DEFAULT_CONTEXT_LENGTH }; + await storage.saveRaw(CONFIG_KEY, config); + return; + } + + const localBrowserRecord = localBrowser as Record; + if ('context_length' in localBrowserRecord) { + return; + } + + localBrowserRecord.context_length = LOCAL_LLM_DEFAULT_CONTEXT_LENGTH; + await storage.saveRaw(CONFIG_KEY, config); +} diff --git a/src/core/config/ConfigManager.ts b/src/core/config/ConfigManager.ts index 118e048..88d50b2 100644 --- a/src/core/config/ConfigManager.ts +++ b/src/core/config/ConfigManager.ts @@ -8,6 +8,9 @@ interface AppConfig { language: string; llm_provider: 'local_browser' | 'openai' | 'gemini' | 'claude' | 'claude_openrouter' | 'openai_compatible'; persist_api_keys_non_localhost: boolean; + local_browser: { + context_length: 32768 | 65536 | 131072; + }; openai: { api_key: string; flex: boolean; @@ -208,6 +211,9 @@ export class ConfigManager { base_url: '', model: '' }, + local_browser: { + context_length: 32768 + }, soundfont: { base_url: 'https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/' }, diff --git a/src/util/localLLMConfig.test.ts b/src/util/localLLMConfig.test.ts new file mode 100644 index 0000000..0b6522b --- /dev/null +++ b/src/util/localLLMConfig.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import { + formatLocalLLMContextLength, + LOCAL_LLM_DEFAULT_CONTEXT_LENGTH, + normalizeLocalLLMContextLength, +} from './localLLMConfig'; + +describe('localLLMConfig', () => { + it('defaults invalid context lengths to 32768', () => { + expect(normalizeLocalLLMContextLength(undefined)).toBe(LOCAL_LLM_DEFAULT_CONTEXT_LENGTH); + expect(normalizeLocalLLMContextLength(12345)).toBe(LOCAL_LLM_DEFAULT_CONTEXT_LENGTH); + }); + + it('formats context lengths using k suffixes', () => { + expect(formatLocalLLMContextLength(32768)).toBe('32k'); + expect(formatLocalLLMContextLength(65536)).toBe('64k'); + expect(formatLocalLLMContextLength(131072)).toBe('128k'); + }); +}); diff --git a/src/util/localLLMConfig.ts b/src/util/localLLMConfig.ts index e2f632d..4b30060 100644 --- a/src/util/localLLMConfig.ts +++ b/src/util/localLLMConfig.ts @@ -6,6 +6,10 @@ export const LOCAL_LLM_DISPLAY_NAME = 'Gemma 4 E4B'; export const LOCAL_LLM_LEGACY_FILENAMES = [ 'gemma-3n-E4B-it-int4-Web.litertlm', ]; +export const LOCAL_LLM_CONTEXT_LENGTH_OPTIONS = [32768, 65536, 131072] as const; +export const LOCAL_LLM_DEFAULT_CONTEXT_LENGTH = 32768; + +export type LocalLLMContextLength = typeof LOCAL_LLM_CONTEXT_LENGTH_OPTIONS[number]; export interface LocalLLMRuntimeSupport { supported: boolean; @@ -40,3 +44,16 @@ export function detectLocalLLMRuntimeSupport(): LocalLLMRuntimeSupport { reason, }; } + +export function isLocalLLMContextLength(value: unknown): value is LocalLLMContextLength { + return typeof value === 'number' + && (LOCAL_LLM_CONTEXT_LENGTH_OPTIONS as readonly number[]).includes(value); +} + +export function normalizeLocalLLMContextLength(value: unknown): LocalLLMContextLength { + return isLocalLLMContextLength(value) ? value : LOCAL_LLM_DEFAULT_CONTEXT_LENGTH; +} + +export function formatLocalLLMContextLength(value: LocalLLMContextLength): string { + return `${Math.round(value / 1024)}k`; +}