From d3a2fe1f8d6836e34cf0ff722cc9d1170ea333a6 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 19 Dec 2025 11:44:58 -0800 Subject: [PATCH 1/5] feat: added MIDI keyboard support --- src/core/KGCore.ts | 13 +- src/core/midi-input/KGMidiInput.ts | 284 +++++++++++++++++++++++++++++ src/main.tsx | 43 ++++- 3 files changed, 329 insertions(+), 11 deletions(-) create mode 100644 src/core/midi-input/KGMidiInput.ts diff --git a/src/core/KGCore.ts b/src/core/KGCore.ts index 9c44685..6aea432 100644 --- a/src/core/KGCore.ts +++ b/src/core/KGCore.ts @@ -93,18 +93,23 @@ export class KGCore { if (this.isPlaying) { await this.stopPlaying(); } - + // Dispose audio interface const audioInterface = KGAudioInterface.instance(); await audioInterface.dispose(); - + + // Dispose MIDI input (dynamic import to avoid circular dependency) + const { KGMidiInput } = await import('./midi-input/KGMidiInput'); + const midiInput = KGMidiInput.instance(); + await midiInput.dispose(); + // Dispose config manager const configManager = ConfigManager.instance(); await configManager.dispose(); - + // Clear playback timer this.stopPlaybackUpdates(); - + console.log("KGCore resources disposed successfully"); } catch (error) { console.error("Error disposing KGCore resources:", error); diff --git a/src/core/midi-input/KGMidiInput.ts b/src/core/midi-input/KGMidiInput.ts new file mode 100644 index 0000000..3f93d7c --- /dev/null +++ b/src/core/midi-input/KGMidiInput.ts @@ -0,0 +1,284 @@ +import { KGAudioInterface } from '../audio-interface/KGAudioInterface'; +import { useProjectStore } from '../../stores/projectStore'; + +/** + * KGMidiInput - MIDI input manager for the DAW + * Implements the singleton pattern for global MIDI device management + * Handles Web MIDI API integration for keyboard input + */ +export class KGMidiInput { + // Private static instance for singleton pattern + private static _instance: KGMidiInput | null = null; + + // MIDI state + private midiAccess: MIDIAccess | null = null; + private isInitialized: boolean = false; + private connectedInputs: Map = new Map(); + + // Private constructor to prevent direct instantiation + private constructor() { + console.log("KGMidiInput initialized"); + } + + /** + * Get the singleton instance of KGMidiInput + * Creates the instance if it doesn't exist yet + */ + public static instance(): KGMidiInput { + if (!KGMidiInput._instance) { + KGMidiInput._instance = new KGMidiInput(); + } + return KGMidiInput._instance; + } + + /** + * Initialize the MIDI input manager + */ + public async initialize(): Promise { + if (this.isInitialized) { + return; + } + + try { + console.log("KGMidiInput ready for MIDI access request"); + this.isInitialized = true; + } catch (error) { + console.error("Failed to initialize MIDI input manager:", error); + throw error; + } + } + + /** + * Request MIDI access from browser + * This must be called after a user gesture (click, keydown, etc.) + */ + public async requestMIDIAccess(): Promise { + if (this.midiAccess) { + console.log("MIDI access already granted"); + return; + } + + try { + // Check if Web MIDI API is available + if (!navigator.requestMIDIAccess) { + throw new Error("Web MIDI API is not supported in this browser"); + } + + // Request MIDI access + this.midiAccess = await navigator.requestMIDIAccess(); + console.log("MIDI access granted"); + + // Set up device listeners + this.setupDeviceListeners(); + + // Connect to all existing inputs + this.connectToAllInputs(); + } catch (error) { + console.error("Failed to request MIDI access:", error); + throw error; + } + } + + /** + * Set up listeners for MIDI device connection/disconnection + */ + private setupDeviceListeners(): void { + if (!this.midiAccess) { + return; + } + + this.midiAccess.onstatechange = (event: MIDIConnectionEvent) => { + const port = event.port; + + if (port && port.type === "input") { + if (port.state === "connected") { + console.log(`MIDI device connected: ${port.name}`); + this.connectToInput(port as MIDIInput); + } else if (port.state === "disconnected") { + console.log(`MIDI device disconnected: ${port.name}`); + this.disconnectFromInput(port.id); + } + } + }; + } + + /** + * Connect to all available MIDI inputs + */ + private connectToAllInputs(): void { + if (!this.midiAccess) { + return; + } + + this.midiAccess.inputs.forEach((input) => { + this.connectToInput(input); + }); + + console.log(`Connected to ${this.connectedInputs.size} MIDI input device(s)`); + } + + /** + * Connect to a specific MIDI input + */ + private connectToInput(input: MIDIInput): void { + // Set up message handler + input.onmidimessage = (event: MIDIMessageEvent) => { + this.handleMIDIMessage(event); + }; + + // Store the input + this.connectedInputs.set(input.id, input); + + console.log(`Listening to MIDI input: ${input.name} (${input.id})`); + } + + /** + * Disconnect from a specific MIDI input + */ + private disconnectFromInput(inputId: string): void { + const input = this.connectedInputs.get(inputId); + if (input) { + input.onmidimessage = null; + } + this.connectedInputs.delete(inputId); + } + + /** + * Handle incoming MIDI messages + */ + private handleMIDIMessage(event: MIDIMessageEvent): void { + if (!event.data) { + return; + } + + const [status, pitch, velocity] = event.data; + + // Extract command (high nibble) and channel (low nibble) + const command = status & 0xf0; + const channel = status & 0x0f; + + // Note On: command = 0x90 (144) + if (command === 0x90 && velocity > 0) { + console.log(`MIDI Note On: pitch=${pitch}, velocity=${velocity}, channel=${channel}`); + this.triggerNoteOn(pitch, velocity); + } + // Note Off: command = 0x80 (128) or Note On with velocity 0 + else if (command === 0x80 || (command === 0x90 && velocity === 0)) { + console.log(`MIDI Note Off: pitch=${pitch}, channel=${channel}`); + this.triggerNoteOff(pitch); + } + // Control Change: command = 0xB0 (176) + else if (command === 0xb0) { + console.log(`MIDI Control Change: controller=${pitch}, value=${velocity}, channel=${channel}`); + // TODO: Handle control changes (modulation, sustain pedal, etc.) + } + // Pitch Bend: command = 0xE0 (224) + else if (command === 0xe0) { + const pitchBendValue = (velocity << 7) | pitch; + console.log(`MIDI Pitch Bend: value=${pitchBendValue}, channel=${channel}`); + // TODO: Handle pitch bend + } + } + + /** + * Trigger note on - play sound for MIDI note + */ + private triggerNoteOn(pitch: number, velocity: number): void { + try { + // Get the selected track ID from the store + const selectedTrackId = useProjectStore.getState().selectedTrackId; + + // Don't play if no track is selected + if (!selectedTrackId) { + console.log('No track selected - MIDI input ignored'); + return; + } + + // Get audio interface and start playing the note + const audioInterface = KGAudioInterface.instance(); + if (audioInterface.getIsInitialized()) { + // Try to start audio context if not started yet + if (!audioInterface.getIsAudioContextStarted()) { + audioInterface.startAudioContext().catch(() => { + // Silently fail if still not allowed - browser policy + }); + } + + // Trigger note attack if audio context is ready + if (audioInterface.getIsAudioContextStarted()) { + audioInterface.triggerNoteAttack(selectedTrackId, pitch, velocity); + console.log(`MIDI triggered note attack: pitch=${pitch}, velocity=${velocity}, track=${selectedTrackId}`); + } + } + } catch (error) { + console.error(`Error triggering MIDI note on (pitch ${pitch}):`, error); + } + } + + /** + * Trigger note off - stop sound for MIDI note + */ + private triggerNoteOff(pitch: number): void { + try { + // Get the selected track ID from the store + const selectedTrackId = useProjectStore.getState().selectedTrackId; + + // Don't try to release if no track is selected + if (!selectedTrackId) { + return; + } + + // Get audio interface and stop playing the note + const audioInterface = KGAudioInterface.instance(); + if (audioInterface.getIsInitialized() && audioInterface.getIsAudioContextStarted()) { + audioInterface.releaseNote(selectedTrackId, pitch); + console.log(`MIDI released note: pitch=${pitch}, track=${selectedTrackId}`); + } + } catch (error) { + console.error(`Error triggering MIDI note off (pitch ${pitch}):`, error); + } + } + + /** + * Clean up MIDI resources + */ + public async dispose(): Promise { + try { + // Disconnect from all inputs + this.connectedInputs.forEach((input, inputId) => { + this.disconnectFromInput(inputId); + }); + this.connectedInputs.clear(); + + // Clear MIDI access + if (this.midiAccess) { + this.midiAccess.onstatechange = null; + this.midiAccess = null; + } + + this.isInitialized = false; + + console.log("MIDI resources disposed successfully"); + } catch (error) { + console.error("Error disposing MIDI resources:", error); + } + } + + // ===== GETTERS ===== + + public getIsInitialized(): boolean { + return this.isInitialized; + } + + public getMIDIAccess(): MIDIAccess | null { + return this.midiAccess; + } + + public getConnectedInputs(): MIDIInput[] { + return Array.from(this.connectedInputs.values()); + } + + public getConnectedInputCount(): number { + return this.connectedInputs.size; + } +} diff --git a/src/main.tsx b/src/main.tsx index 2849187..65b9909 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -5,11 +5,15 @@ import './index.css'; import App from './App.tsx'; import { KGCore } from './core/KGCore'; import { KGAudioInterface } from './core/audio-interface/KGAudioInterface'; +import { KGMidiInput } from './core/midi-input/KGMidiInput'; import { KGDebugger } from './core/KGDebugger'; // Initialize KGCore instance await KGCore.instance().initialize(); +// Initialize KGMidiInput instance +await KGMidiInput.instance().initialize(); + // Attach debugger to global window in development mode if (import.meta.env.DEV) { (window as unknown as { KGDebugger: KGDebugger }).KGDebugger = KGDebugger.instance(); @@ -31,17 +35,42 @@ const tryStartAudioContext = async () => { console.log('Audio context start failed:', error); audioContextStarted = false; // Allow retry } - // Remove listeners after first attempt (whether successful or not) - document.removeEventListener('click', tryStartAudioContext); - document.removeEventListener('touchstart', tryStartAudioContext); - document.removeEventListener('keydown', tryStartAudioContext); } }; +// Request MIDI access on first user interaction +let midiAccessRequested = false; +const tryRequestMIDIAccess = async () => { + if (!midiAccessRequested) { + midiAccessRequested = true; + try { + const midiInput = KGMidiInput.instance(); + if (!midiInput.getMIDIAccess()) { + await midiInput.requestMIDIAccess(); + console.log('MIDI access granted on first user interaction'); + } + } catch (error) { + console.log('MIDI access failed:', error); + midiAccessRequested = false; // Allow retry + } + } +}; + +// Combined handler for first user interaction +const handleFirstInteraction = async () => { + await tryStartAudioContext(); + await tryRequestMIDIAccess(); + + // Remove listeners after first attempt + document.removeEventListener('click', handleFirstInteraction); + document.removeEventListener('touchstart', handleFirstInteraction); + document.removeEventListener('keydown', handleFirstInteraction); +}; + // Listen for first user interaction -document.addEventListener('click', tryStartAudioContext, { passive: true }); -document.addEventListener('touchstart', tryStartAudioContext, { passive: true }); -document.addEventListener('keydown', tryStartAudioContext, { passive: true }); +document.addEventListener('click', handleFirstInteraction, { passive: true }); +document.addEventListener('touchstart', handleFirstInteraction, { passive: true }); +document.addEventListener('keydown', handleFirstInteraction, { passive: true }); // Add event listener for beforeunload event window.addEventListener('beforeunload', (event) => { From 4b075e8979066c47f59687aec8644b0b9aed1191 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 19 Dec 2025 12:55:56 -0800 Subject: [PATCH 2/5] fix: reduce Tone.js context.lookAhead to reduce MIDI input latency --- src/core/audio-interface/KGAudioInterface.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/audio-interface/KGAudioInterface.ts b/src/core/audio-interface/KGAudioInterface.ts index a480145..33f2377 100644 --- a/src/core/audio-interface/KGAudioInterface.ts +++ b/src/core/audio-interface/KGAudioInterface.ts @@ -64,6 +64,9 @@ export class KGAudioInterface { } try { + // Reduce lookahead time to 0.05 seconds to improve MIDI input responsiveness + Tone.getContext().lookAhead = 0.05; + // Set up master gain for volume control this.masterGain = new Tone.Gain(this.masterVolume).toDestination(); From aadf9e06df9f46415a90193a485e885a1f7db963 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Sun, 21 Dec 2025 18:51:20 -0800 Subject: [PATCH 3/5] feat: added lookAhead option --- public/config.json | 3 +- .../settings/sections/BehaviorSettings.tsx | 64 ++++++++++++++++++- src/core/audio-interface/KGAudioInterface.ts | 22 +++++++ src/core/config/ConfigManager.ts | 4 +- 4 files changed, 90 insertions(+), 3 deletions(-) diff --git a/public/config.json b/public/config.json index 2c60bbc..8d398c7 100644 --- a/public/config.json +++ b/public/config.json @@ -62,7 +62,8 @@ "default_open": true }, "audio": { - "enable_audio_capture_for_screen_sharing": false + "enable_audio_capture_for_screen_sharing": false, + "lookahead_time": 0.05 }, "templates": { "custom_instructions": "" diff --git a/src/components/settings/sections/BehaviorSettings.tsx b/src/components/settings/sections/BehaviorSettings.tsx index 82fa4c5..02b7b25 100644 --- a/src/components/settings/sections/BehaviorSettings.tsx +++ b/src/components/settings/sections/BehaviorSettings.tsx @@ -1,9 +1,12 @@ import React, { useState, useEffect } from 'react'; import { ConfigManager } from '../../../core/config/ConfigManager'; +import { KGAudioInterface } from '../../../core/audio-interface/KGAudioInterface'; const BehaviorSettings: React.FC = () => { const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState(true); + const [audioLookaheadTime, setAudioLookaheadTime] = useState('50'); const [enableAudioCapture, setEnableAudioCapture] = useState(false); + const [validationErrors, setValidationErrors] = useState([]); const configManager = ConfigManager.instance(); @@ -15,6 +18,8 @@ const BehaviorSettings: React.FC = () => { } setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true); + const lookaheadTimeSeconds = (configManager.get('audio.lookahead_time') as number) ?? 0.05; + setAudioLookaheadTime(((lookaheadTimeSeconds * 1000).toFixed(0))); setEnableAudioCapture((configManager.get('audio.enable_audio_capture_for_screen_sharing') as boolean) ?? false); }; @@ -28,6 +33,36 @@ const BehaviorSettings: React.FC = () => { await configManager.set('chatbox.default_open', boolValue); }; + const handleAudioLookaheadTimeChange = async (value: string) => { + // Allow empty string, treat as 0 ms + const numValueMs = value === '' ? 0 : parseFloat(value); + const numValueSeconds = numValueMs / 1000; + const errors: string[] = []; + + // Validate the input + if (isNaN(numValueMs)) { + errors.push('Lookahead time must be a valid number'); + } else if (numValueSeconds < 0) { + errors.push('Lookahead time must be between 0 and 0.5 seconds (0-500ms)'); + } else if (numValueSeconds > 0.5) { + errors.push('Lookahead time must be between 0 and 0.5 seconds (0-500ms)'); + } + + setValidationErrors(errors); + + // Only apply if valid + if (errors.length === 0) { + setAudioLookaheadTime(value); + await configManager.set('audio.lookahead_time', numValueSeconds); + + // Apply the change immediately without restart + const audioInterface = KGAudioInterface.instance(); + audioInterface.setLookaheadTime(numValueSeconds); + + console.log(`Audio lookahead time changed to: ${numValueSeconds}s (${numValueMs}ms)`); + } + }; + const handleEnableAudioCaptureChange = async (value: string) => { const boolValue = value === 'yes'; setEnableAudioCapture(boolValue); @@ -61,7 +96,34 @@ const BehaviorSettings: React.FC = () => {

Audio

- + +
+ + handleAudioLookaheadTimeChange(e.target.value)} + min="0" + max="500" + step="1" + /> + {validationErrors.length > 0 && ( +
+ {validationErrors.map((error, index) => ( +
+ {error} +
+ ))} +
+ )} +
+ Audio scheduling lookahead time (0-500ms). Lower values (10-20ms) reduce MIDI input latency but may cause audio glitches on slower systems. Higher values (100ms+) are better for playback stability. Changes apply immediately without restart. +
+
+
+
+ + handlePlaybackDelayChange(e.target.value)} + min="0" + max="500" + step="1" + /> + {playbackDelayValidationErrors.length > 0 && ( +
+ {playbackDelayValidationErrors.map((error, index) => ( +
+ {error} +
+ ))} +
+ )} +
+ Playback will start with a short delay after pressing the start button (0-500ms). Increasing this value might help stabilize playback, especially for the first few ticks if the lookahead value is too low. Changes apply immediately without restart. +
+
+