feat: added support for real time MIDI CC input (CC1, CC2, CC7, CC11 -> velocity; CC64 -> sustain)

This commit is contained in:
Xiaohan-Tian
2026-05-06 22:58:49 -07:00
parent 0d9e2a9f9b
commit 6305983df8
7 changed files with 383 additions and 31 deletions
+67 -1
View File
@@ -1,5 +1,5 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { MockBufferSource, MockSampler } from '../../test/mocks/tone';
import { MockBufferSource, MockGain, MockSampler } from '../../test/mocks/tone';
vi.mock('tone', async () => {
const { ToneMock } = await import('../../test/mocks/tone');
@@ -32,6 +32,7 @@ describe('KGAudioBus live MIDI pitch bend', () => {
vi.clearAllMocks();
MockSampler.mockClear();
MockBufferSource.mockClear();
MockGain.mockClear();
const sampler = MockSampler();
createSamplerMock.mockResolvedValue(sampler);
@@ -77,4 +78,69 @@ describe('KGAudioBus live MIDI pitch bend', () => {
expect(source.playbackRate.setValueAtTime).toHaveBeenCalledWith(Math.pow(2, 2 / 12), 2);
});
it('updates held live MIDI note gain when expression changes', async () => {
const audioBus = await KGAudioBus.create('acoustic_grand_piano');
audioBus.triggerLiveMidiAttack(60, 0, 1);
const gainNode = MockGain.mock.results[0].value;
audioBus.setLiveMidiExpression(0.25);
expect(gainNode.gain.value).toBeCloseTo(0.25, 5);
});
it('applies current expression to newly triggered notes', async () => {
const audioBus = await KGAudioBus.create('acoustic_grand_piano');
audioBus.setLiveMidiExpression(0.4);
audioBus.triggerLiveMidiAttack(60, 0, 1);
const gainNode = MockGain.mock.results[0].value;
expect(MockGain).toHaveBeenCalledWith(0.4);
expect(gainNode.gain.value).toBeCloseTo(0.4, 5);
});
it('defers release while sustain is held and flushes on pedal up', async () => {
const audioBus = await KGAudioBus.create('acoustic_grand_piano');
audioBus.triggerLiveMidiAttack(60, 0, 0.5);
const source = MockBufferSource.mock.results[0].value;
audioBus.setLiveMidiSustain(true);
audioBus.releaseLiveMidiNote(60, 1.25);
expect(source.stop).not.toHaveBeenCalled();
audioBus.setLiveMidiSustain(false, 2);
expect(source.stop).toHaveBeenCalledWith(2);
});
it('keeps physically held notes sounding when sustain is released', async () => {
const audioBus = await KGAudioBus.create('acoustic_grand_piano');
audioBus.triggerLiveMidiAttack(60, 0, 0.5);
const source = MockBufferSource.mock.results[0].value;
audioBus.setLiveMidiSustain(true);
audioBus.setLiveMidiSustain(false, 2);
expect(source.stop).not.toHaveBeenCalled();
});
it('resets live CC state on releaseAll', async () => {
const audioBus = await KGAudioBus.create('acoustic_grand_piano');
audioBus.triggerLiveMidiAttack(60, 0, 0.5);
const source = MockBufferSource.mock.results[0].value;
const gainNode = MockGain.mock.results[0].value;
audioBus.setLiveMidiExpression(0.2);
audioBus.setLiveMidiSustain(true);
audioBus.releaseAll();
audioBus.triggerLiveMidiAttack(60, 0, 0.5);
expect(source.stop).toHaveBeenCalled();
expect(gainNode.dispose).toHaveBeenCalled();
expect(MockGain.mock.calls[1]?.[0]).toBe(1);
});
});
+84 -12
View File
@@ -9,7 +9,10 @@ import { KGToneSamplerFactory } from './KGToneSamplerFactory';
interface LiveMidiSource {
source: Tone.ToneBufferSource;
gainNode: Tone.Gain;
basePlaybackRate: number;
isPressed: boolean;
pendingSustainRelease: boolean;
}
/**
@@ -33,6 +36,8 @@ export class KGAudioBus {
private muted: boolean;
private solo: boolean;
private liveMidiPitchBend: number = 0;
private liveExpressionNormalized: number = 1;
private sustainPedalDown: boolean = false;
private liveMidiSources: Map<number, LiveMidiSource[]> = new Map();
// Audio processing chain (for future expansion)
@@ -199,15 +204,18 @@ export class KGAudioBus {
return;
}
const stopTime = time ?? Tone.now();
activeSources.forEach(({ source }) => {
try {
source.stop(stopTime);
} catch (error) {
console.error(`Error stopping live MIDI source for pitch ${pitch} on ${this.instrument}:`, error);
}
});
this.liveMidiSources.delete(pitch);
const liveSource = [...activeSources].reverse().find((entry) => entry.isPressed) ?? activeSources[activeSources.length - 1];
if (!liveSource) {
return;
}
liveSource.isPressed = false;
if (this.sustainPedalDown) {
liveSource.pendingSustainRelease = true;
return;
}
this.stopLiveMidiSource(pitch, liveSource, time ?? Tone.now());
} catch (error) {
console.error(`Error releasing live MIDI note ${pitch} on ${this.instrument}:`, error);
}
@@ -237,6 +245,30 @@ export class KGAudioBus {
this.setLiveMidiPitchBend(0);
}
public setLiveMidiExpression(normalizedValue: number): void {
this.liveExpressionNormalized = Math.max(0, Math.min(1, normalizedValue));
for (const activeSources of this.liveMidiSources.values()) {
activeSources.forEach(({ gainNode }) => {
this.setGainValue(gainNode, this.liveExpressionNormalized);
});
}
}
public setLiveMidiSustain(isDown: boolean, time?: number): void {
this.sustainPedalDown = isDown;
if (isDown) {
return;
}
const releaseTime = time ?? Tone.now();
for (const [pitch, activeSources] of this.liveMidiSources.entries()) {
[...activeSources]
.filter((entry) => !entry.isPressed && entry.pendingSustainRelease)
.forEach((entry) => this.stopLiveMidiSource(pitch, entry, releaseTime));
}
}
/**
* Release all currently playing notes
*/
@@ -244,16 +276,19 @@ export class KGAudioBus {
try {
this.sampler.releaseAll();
this.liveMidiSources.forEach((activeSources) => {
activeSources.forEach(({ source }) => {
activeSources.forEach(({ source, gainNode }) => {
try {
source.stop();
} catch (error) {
console.error(`Error stopping live MIDI source on ${this.instrument}:`, error);
}
gainNode.dispose();
});
});
this.liveMidiSources.clear();
this.resetLiveMidiPitchBend();
this.liveExpressionNormalized = 1;
this.sustainPedalDown = false;
} catch (error) {
console.error(`Error releasing all notes on ${this.instrument}:`, error);
}
@@ -498,17 +533,22 @@ export class KGAudioBus {
}
const basePlaybackRate = Math.pow(2, (pitch - closestPitch) / 12);
const gainNode = new Tone.Gain(this.liveExpressionNormalized);
const source = new Tone.ToneBufferSource({
url: buffer,
fadeIn: this.sampler.attack,
fadeOut: this.sampler.release,
curve: this.sampler.curve,
playbackRate: this.applyPitchBendToPlaybackRate(basePlaybackRate),
}).connect(this.sampler.output);
});
source.connect(gainNode);
gainNode.connect(this.sampler.output);
source.onended = () => {
const currentSources = this.liveMidiSources.get(pitch);
if (!currentSources) {
gainNode.dispose();
return;
}
@@ -518,10 +558,17 @@ export class KGAudioBus {
} else {
this.liveMidiSources.set(pitch, nextSources);
}
gainNode.dispose();
};
source.start(time, 0, duration ?? buffer.duration / basePlaybackRate, velocity ?? 1);
return { source, basePlaybackRate };
return {
source,
gainNode,
basePlaybackRate,
isPressed: true,
pendingSustainRelease: false,
};
}
public static applyNormalizedPitchBendToPlaybackRate(basePlaybackRate: number, normalizedBend: number): number {
@@ -575,4 +622,29 @@ export class KGAudioBus {
playbackRate.value = value;
}
private setGainValue(gainNode: Tone.Gain, value: number, time?: number): void {
if (time !== undefined && typeof gainNode.gain.setValueAtTime === 'function') {
gainNode.gain.setValueAtTime(value, time);
return;
}
gainNode.gain.value = value;
}
private stopLiveMidiSource(pitch: number, liveSource: LiveMidiSource, time: number): void {
try {
liveSource.pendingSustainRelease = false;
liveSource.source.stop(time);
} catch (error) {
console.error(`Error stopping live MIDI source for pitch ${pitch} on ${this.instrument}:`, error);
const remainingSources = this.liveMidiSources.get(pitch)?.filter((entry) => entry !== liveSource) ?? [];
if (remainingSources.length === 0) {
this.liveMidiSources.delete(pitch);
} else {
this.liveMidiSources.set(pitch, remainingSources);
}
liveSource.gainNode.dispose();
}
}
}
@@ -214,3 +214,33 @@ describe('KGAudioInterface preroll playback', () => {
expect(scheduledTimes).toContain(2);
});
});
describe('KGAudioInterface live MIDI CC forwarding', () => {
beforeEach(() => {
vi.clearAllMocks();
;(KGAudioInterface as unknown as { _instance: KGAudioInterface | null })._instance = null;
});
it('forwards live expression and sustain to the target audio bus', () => {
const audio = KGAudioInterface.instance();
const audioBus = {
setLiveMidiExpression: vi.fn(),
setLiveMidiSustain: vi.fn(),
};
;(audio as unknown as { trackAudioBuses: Map<string, unknown> }).trackAudioBuses.set('1', audioBus);
audio.setLiveMidiExpression('1', 0.5);
audio.setLiveMidiSustain('1', true, 2);
expect(audioBus.setLiveMidiExpression).toHaveBeenCalledWith(0.5);
expect(audioBus.setLiveMidiSustain).toHaveBeenCalledWith(true, 2);
});
it('safely ignores live expression and sustain when the bus is missing', () => {
const audio = KGAudioInterface.instance();
expect(() => audio.setLiveMidiExpression('missing', 0.5)).not.toThrow();
expect(() => audio.setLiveMidiSustain('missing', true)).not.toThrow();
});
});
@@ -914,6 +914,36 @@ export class KGAudioInterface {
}
}
public setLiveMidiExpression(trackId: string, normalizedValue: number): void {
try {
const audioBus = this.trackAudioBuses.get(trackId);
if (!audioBus) {
console.warn(`No audio bus found for track ${trackId}`);
return;
}
audioBus.setLiveMidiExpression(normalizedValue);
console.log(`Set live MIDI expression to ${normalizedValue} on track ${trackId}`);
} catch (error) {
console.error(`Error setting live MIDI expression for track ${trackId}:`, error);
}
}
public setLiveMidiSustain(trackId: string, isDown: boolean, time?: number): void {
try {
const audioBus = this.trackAudioBuses.get(trackId);
if (!audioBus) {
console.warn(`No audio bus found for track ${trackId}`);
return;
}
audioBus.setLiveMidiSustain(isDown, time);
console.log(`Set live MIDI sustain to ${isDown} on track ${trackId}`);
} catch (error) {
console.error(`Error setting live MIDI sustain for track ${trackId}:`, error);
}
}
/**
* Clear all scheduled events
*/
+78 -5
View File
@@ -1,4 +1,6 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { KGAudioTrack } from '../track/KGAudioTrack';
import { KGMidiTrack } from '../track/KGMidiTrack';
const { getStateMock, audioInterfaceMock } = vi.hoisted(() => ({
getStateMock: vi.fn(),
@@ -9,6 +11,8 @@ const { getStateMock, audioInterfaceMock } = vi.hoisted(() => ({
triggerLiveMidiNoteAttack: vi.fn(),
releaseLiveMidiNote: vi.fn(),
setLiveMidiPitchBend: vi.fn(),
setLiveMidiExpression: vi.fn(),
setLiveMidiSustain: vi.fn(),
},
}));
@@ -29,7 +33,10 @@ import { KGMidiInput } from './KGMidiInput';
describe('KGMidiInput pitch bend', () => {
beforeEach(() => {
vi.clearAllMocks();
getStateMock.mockReturnValue({ selectedTrackId: 'track-1' });
getStateMock.mockReturnValue({
selectedTrackId: '1',
tracks: [new KGMidiTrack('Track 1', 1)],
});
audioInterfaceMock.getIsInitialized.mockReturnValue(true);
audioInterfaceMock.getIsAudioContextStarted.mockReturnValue(true);
audioInterfaceMock.startAudioContext.mockResolvedValue(undefined);
@@ -44,8 +51,24 @@ describe('KGMidiInput pitch bend', () => {
midiInput.handleMIDIMessage({ data: new Uint8Array([0x90, 60, 100]) });
midiInput.handleMIDIMessage({ data: new Uint8Array([0x80, 60, 0]) });
expect(audioInterfaceMock.triggerLiveMidiNoteAttack).toHaveBeenCalledWith('track-1', 60, 100);
expect(audioInterfaceMock.releaseLiveMidiNote).toHaveBeenCalledWith('track-1', 60);
expect(audioInterfaceMock.triggerLiveMidiNoteAttack).toHaveBeenCalledWith('1', 60, 100);
expect(audioInterfaceMock.releaseLiveMidiNote).toHaveBeenCalledWith('1', 60);
});
it('latches live note ownership to the note-on track', () => {
const midiInput = KGMidiInput.instance() as unknown as {
handleMIDIMessage: (event: { data: Uint8Array }) => void;
};
midiInput.handleMIDIMessage({ data: new Uint8Array([0x90, 60, 100]) });
getStateMock.mockReturnValue({
selectedTrackId: '2',
tracks: [new KGMidiTrack('Track 1', 1), new KGMidiTrack('Track 2', 2)],
});
midiInput.handleMIDIMessage({ data: new Uint8Array([0x80, 60, 0]) });
expect(audioInterfaceMock.releaseLiveMidiNote).toHaveBeenCalledWith('1', 60);
});
it('normalizes MIDI pitch bend and forwards it to the selected track', () => {
@@ -56,9 +79,59 @@ describe('KGMidiInput pitch bend', () => {
midiInput.handleMIDIMessage({ data: new Uint8Array([0xe0, 0x00, 0x40]) });
midiInput.handleMIDIMessage({ data: new Uint8Array([0xe0, 0x7f, 0x7f]) });
expect(audioInterfaceMock.setLiveMidiPitchBend).toHaveBeenNthCalledWith(1, 'track-1', 0);
expect(audioInterfaceMock.setLiveMidiPitchBend).toHaveBeenNthCalledWith(1, '1', 0);
expect(audioInterfaceMock.setLiveMidiPitchBend).toHaveBeenCalledTimes(2);
expect(audioInterfaceMock.setLiveMidiPitchBend.mock.calls[1]?.[0]).toBe('track-1');
expect(audioInterfaceMock.setLiveMidiPitchBend.mock.calls[1]?.[0]).toBe('1');
expect(audioInterfaceMock.setLiveMidiPitchBend.mock.calls[1]?.[1]).toBeCloseTo(8191 / 8192, 5);
});
it('maps supported CC messages to live expression and sustain for standard pedals', () => {
const midiInput = KGMidiInput.instance() as unknown as {
handleMIDIMessage: (event: { data: Uint8Array }) => void;
};
midiInput.handleMIDIMessage({ data: new Uint8Array([0xb0, 0x01, 0x20]) });
midiInput.handleMIDIMessage({ data: new Uint8Array([0xb0, 0x02, 0x30]) });
midiInput.handleMIDIMessage({ data: new Uint8Array([0xb0, 0x07, 0x40]) });
midiInput.handleMIDIMessage({ data: new Uint8Array([0xb0, 0x0b, 0x50]) });
midiInput.handleMIDIMessage({ data: new Uint8Array([0xb0, 0x40, 0x7f]) });
midiInput.handleMIDIMessage({ data: new Uint8Array([0xb0, 0x40, 0x00]) });
expect(audioInterfaceMock.setLiveMidiExpression).toHaveBeenCalledTimes(4);
expect(audioInterfaceMock.setLiveMidiExpression).toHaveBeenNthCalledWith(1, '1', 0x20 / 127);
expect(audioInterfaceMock.setLiveMidiExpression).toHaveBeenNthCalledWith(4, '1', 0x50 / 127);
expect(audioInterfaceMock.setLiveMidiSustain).toHaveBeenNthCalledWith(1, '1', true);
expect(audioInterfaceMock.setLiveMidiSustain).toHaveBeenNthCalledWith(2, '1', false);
});
it('calibrates inverted sustain pedals from the first observed CC64 message', () => {
const midiInput = KGMidiInput.instance() as unknown as {
handleMIDIMessage: (event: { data: Uint8Array }) => void;
};
midiInput.handleMIDIMessage({ data: new Uint8Array([0xb0, 0x40, 0x00]) });
midiInput.handleMIDIMessage({ data: new Uint8Array([0xb0, 0x40, 0x7f]) });
expect(audioInterfaceMock.setLiveMidiSustain).toHaveBeenNthCalledWith(1, '1', true);
expect(audioInterfaceMock.setLiveMidiSustain).toHaveBeenNthCalledWith(2, '1', false);
});
it('ignores live MIDI input when the selected track is not a MIDI track', () => {
getStateMock.mockReturnValue({
selectedTrackId: '1',
tracks: [new KGAudioTrack('Audio Track', 1)],
});
const midiInput = KGMidiInput.instance() as unknown as {
handleMIDIMessage: (event: { data: Uint8Array }) => void;
};
midiInput.handleMIDIMessage({ data: new Uint8Array([0x90, 60, 100]) });
midiInput.handleMIDIMessage({ data: new Uint8Array([0xe0, 0x00, 0x40]) });
midiInput.handleMIDIMessage({ data: new Uint8Array([0xb0, 0x40, 0x7f]) });
expect(audioInterfaceMock.triggerLiveMidiNoteAttack).not.toHaveBeenCalled();
expect(audioInterfaceMock.setLiveMidiPitchBend).not.toHaveBeenCalled();
expect(audioInterfaceMock.setLiveMidiSustain).not.toHaveBeenCalled();
});
});
+91 -11
View File
@@ -1,5 +1,6 @@
import { KGAudioInterface } from '../audio-interface/KGAudioInterface';
import { useProjectStore } from '../../stores/projectStore';
import { KGMidiTrack } from '../track/KGMidiTrack';
/**
* KGMidiInput - MIDI input manager for the DAW
@@ -9,6 +10,12 @@ import { useProjectStore } from '../../stores/projectStore';
export class KGMidiInput {
private static readonly PITCH_BEND_CENTER = 8192;
private static readonly PITCH_BEND_MAX_OFFSET = 8192;
private static readonly CONTROL_CHANGE_MODULATION = 1;
private static readonly CONTROL_CHANGE_BREATH = 2;
private static readonly CONTROL_CHANGE_CHANNEL_VOLUME = 7;
private static readonly CONTROL_CHANGE_EXPRESSION = 11;
private static readonly CONTROL_CHANGE_SUSTAIN = 64;
private static readonly SUSTAIN_ON_THRESHOLD = 64;
// Private static instance for singleton pattern
private static _instance: KGMidiInput | null = null;
@@ -22,6 +29,8 @@ export class KGMidiInput {
private onRecordNoteOn: ((pitch: number, velocity: number) => void) | null = null;
private onRecordNoteOff: ((pitch: number) => void) | null = null;
private onRecordPitchBend: ((value: number) => void) | null = null;
private liveNoteTrackOwnership: Map<number, string[]> = new Map();
private sustainPolarityInverted: boolean | null = null;
// Private constructor to prevent direct instantiation
private constructor() {
@@ -180,7 +189,7 @@ export class KGMidiInput {
// Control Change: command = 0xB0 (176)
else if (command === 0xb0) {
console.log(`MIDI Control Change: controller=${pitch}, value=${velocity}, channel=${channel}`);
// TODO: Handle control changes (modulation, sustain pedal, etc.)
this.handleControlChange(pitch, velocity);
}
// Pitch Bend: command = 0xE0 (224)
else if (command === 0xe0) {
@@ -196,12 +205,8 @@ export class KGMidiInput {
*/
private triggerNoteOn(pitch: number, velocity: number): void {
try {
// Get the selected track ID from the store
const selectedTrackId = useProjectStore.getState().selectedTrackId;
// Don't play if no track is selected
const selectedTrackId = this.getSelectedMidiTrackId();
if (!selectedTrackId) {
console.log('No track selected - MIDI input ignored');
return;
}
@@ -217,6 +222,9 @@ export class KGMidiInput {
// Trigger note attack if audio context is ready
if (audioInterface.getIsAudioContextStarted()) {
const latchedTracks = this.liveNoteTrackOwnership.get(pitch) ?? [];
latchedTracks.push(selectedTrackId);
this.liveNoteTrackOwnership.set(pitch, latchedTracks);
audioInterface.triggerLiveMidiNoteAttack(selectedTrackId, pitch, velocity);
console.log(`MIDI triggered note attack: pitch=${pitch}, velocity=${velocity}, track=${selectedTrackId}`);
}
@@ -231,10 +239,7 @@ export class KGMidiInput {
*/
private triggerNoteOff(pitch: number): void {
try {
// Get the selected track ID from the store
const selectedTrackId = useProjectStore.getState().selectedTrackId;
// Don't try to release if no track is selected
const selectedTrackId = this.consumeLatchedTrackIdForPitch(pitch);
if (!selectedTrackId) {
return;
}
@@ -252,7 +257,7 @@ export class KGMidiInput {
private triggerPitchBend(normalizedBend: number): void {
try {
const selectedTrackId = useProjectStore.getState().selectedTrackId;
const selectedTrackId = this.getSelectedMidiTrackId();
if (!selectedTrackId) {
return;
}
@@ -271,6 +276,79 @@ export class KGMidiInput {
return Math.max(-1, Math.min(1, normalizedBend));
}
private handleControlChange(controller: number, value: number): void {
const selectedTrackId = this.getSelectedMidiTrackId();
if (!selectedTrackId) {
return;
}
const audioInterface = KGAudioInterface.instance();
if (!audioInterface.getIsInitialized() || !audioInterface.getIsAudioContextStarted()) {
return;
}
if (controller === KGMidiInput.CONTROL_CHANGE_SUSTAIN) {
audioInterface.setLiveMidiSustain(
selectedTrackId,
this.normalizeSustainPedalValue(value)
);
return;
}
if (
controller === KGMidiInput.CONTROL_CHANGE_MODULATION ||
controller === KGMidiInput.CONTROL_CHANGE_BREATH ||
controller === KGMidiInput.CONTROL_CHANGE_CHANNEL_VOLUME ||
controller === KGMidiInput.CONTROL_CHANGE_EXPRESSION
) {
audioInterface.setLiveMidiExpression(selectedTrackId, value / 127);
}
}
private getSelectedMidiTrackId(): string | null {
const { selectedTrackId, tracks } = useProjectStore.getState();
if (!selectedTrackId) {
console.log('No track selected - MIDI input ignored');
return null;
}
const selectedTrack = tracks.find((track) => track.getId().toString() === selectedTrackId);
if (!(selectedTrack instanceof KGMidiTrack)) {
console.log(`Selected track ${selectedTrackId} is not a MIDI track - MIDI input ignored`);
return null;
}
return selectedTrackId;
}
private consumeLatchedTrackIdForPitch(pitch: number): string | null {
const latchedTracks = this.liveNoteTrackOwnership.get(pitch);
if (!latchedTracks || latchedTracks.length === 0) {
return null;
}
const trackId = latchedTracks.pop() ?? null;
if (latchedTracks.length === 0) {
this.liveNoteTrackOwnership.delete(pitch);
} else {
this.liveNoteTrackOwnership.set(pitch, latchedTracks);
}
return trackId;
}
private normalizeSustainPedalValue(value: number): boolean {
const rawPressed = value >= KGMidiInput.SUSTAIN_ON_THRESHOLD;
if (this.sustainPolarityInverted === null) {
// Assume the pedal starts released. The first observed sustain CC therefore
// represents a press-down gesture and reveals whether the device is inverted.
this.sustainPolarityInverted = !rawPressed;
}
return this.sustainPolarityInverted ? !rawPressed : rawPressed;
}
/**
* Clean up MIDI resources
*/
@@ -289,6 +367,8 @@ export class KGMidiInput {
}
this.isInitialized = false;
this.liveNoteTrackOwnership.clear();
this.sustainPolarityInverted = null;
console.log("MIDI resources disposed successfully");
} catch (error) {