3dtours/KGStudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: added pitch bend support

Browse Source
This commit is contained in:
Xiaohan-Tian
2026-05-05 21:28:32 -07:00
parent 8f453f8f4c
commit 98cccf5c7a
28 changed files with 1704 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/audio-interface/KGAudioBus.test.ts
+69
View File
@@ -0,0 +1,69 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { MockBufferSource, MockSampler } from '../../test/mocks/tone';
vi.mock('tone', async () => {
const { ToneMock } = await import('../../test/mocks/tone');
return ToneMock;
});
const getToneAudioBuffersMock = vi.fn();
const createSamplerMock = vi.fn();
vi.mock('./KGToneBuffersPool', () => ({
KGToneBuffersPool: {
instance: () => ({
getToneAudioBuffers: getToneAudioBuffersMock,
}),
},
}));
vi.mock('./KGToneSamplerFactory', () => ({
KGToneSamplerFactory: {
instance: () => ({
createSampler: createSamplerMock,
}),
},
}));
import { KGAudioBus } from './KGAudioBus';
describe('KGAudioBus live MIDI pitch bend', () => {
beforeEach(() => {
vi.clearAllMocks();
MockSampler.mockClear();
MockBufferSource.mockClear();
const sampler = MockSampler();
createSamplerMock.mockResolvedValue(sampler);
getToneAudioBuffersMock.mockResolvedValue({
loaded: true,
has: (key: string) => key === 'C4',
get: (key: string) => key === 'C4' ? { duration: 1 } : undefined,
});
});
it('retunes held live MIDI notes when pitch bend changes', async () => {
const audioBus = await KGAudioBus.create('acoustic_grand_piano');
audioBus.triggerLiveMidiAttack(60, 0, 1);
expect(MockBufferSource).toHaveBeenCalledTimes(1);
const source = MockBufferSource.mock.results[0].value;
expect(source.start).toHaveBeenCalledWith(0, 0, 1, 1);
expect(source.playbackRate.value).toBeCloseTo(1, 5);
audioBus.setLiveMidiPitchBend(1);
expect(source.playbackRate.value).toBeCloseTo(Math.pow(2, 2 / 12), 5);
});
it('stops active live MIDI sources on release and reset paths', async () => {
const audioBus = await KGAudioBus.create('acoustic_grand_piano');
audioBus.triggerLiveMidiAttack(60, 0, 0.5);
const source = MockBufferSource.mock.results[0].value;
audioBus.releaseLiveMidiNote(60, 1.25);
expect(source.stop).toHaveBeenCalledWith(1.25);
});
});
src/core/audio-interface/KGAudioBus.ts
+196 -5
View File
@@ -1,24 +1,39 @@
import * as Tone from 'tone';
import { AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants';
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
import type { InstrumentType } from '../track/KGMidiTrack';
import { KGToneBuffersPool } from './KGToneBuffersPool';
import { KGToneSamplerFactory } from './KGToneSamplerFactory';
// InstrumentType is defined in KGMidiTrack and re-used here
interface LiveMidiSource {
source: Tone.ToneBufferSource;
basePlaybackRate: number;
}
/**
* KGAudioBus - Represents a complete audio bus for a track
* Replaces the separate trackSynths, trackInstruments, trackVolumes, trackMuted, trackSolo maps
* Each instance manages a single track's audio processing chain
*/
export class KGAudioBus {
// Fixed at +/-2 semitones for now. Future work: make this user-configurable
// or honor MIDI RPN 0,0 (Pitch Bend Sensitivity).
private static readonly LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES = 2;
private static readonly LIVE_MIDI_NOTE_NAMES = ['C', 'Db', 'D', 'Eb', 'E', 'F', 'Gb', 'G', 'Ab', 'A', 'Bb', 'B'];
// Core audio components
private sampler: Tone.Sampler;
private audioBuffers: Tone.ToneAudioBuffers;
private instrument: InstrumentType;
// Audio properties
private volume: number;
private muted: boolean;
private solo: boolean;
private liveMidiPitchBend: number = 0;
private liveMidiSources: Map<number, LiveMidiSource[]> = new Map();
// Audio processing chain (for future expansion)
// private gain: Tone.Gain;
@@ -29,12 +44,14 @@ export class KGAudioBus {
*/
private constructor(
sampler: Tone.Sampler,
audioBuffers: Tone.ToneAudioBuffers,
instrument: InstrumentType,
volume: number,
muted: boolean,
solo: boolean
) {
this.sampler = sampler;
this.audioBuffers = audioBuffers;
this.instrument = instrument;
this.volume = volume;
this.muted = muted;
@@ -60,11 +77,15 @@ export class KGAudioBus {
console.log(`Creating KGAudioBus for ${instrument}...`);
// Create the sampler using the factory
const samplerFactory = KGToneSamplerFactory.instance();
const sampler = await samplerFactory.createSampler(String(instrument));
const samplerFactory = KGToneSamplerFactory.instance();
const buffersPool = KGToneBuffersPool.instance();
const [sampler, audioBuffers] = await Promise.all([
samplerFactory.createSampler(String(instrument)),
buffersPool.getToneAudioBuffers(String(instrument)),
]);
// Create the audio bus instance
const audioBus = new KGAudioBus(sampler, instrument, volume, muted, solo);
const audioBus = new KGAudioBus(sampler, audioBuffers, instrument, volume, muted, solo);
console.log(`KGAudioBus created successfully for ${instrument}`);
return audioBus;
@@ -117,6 +138,42 @@ export class KGAudioBus {
}
}
/**
* Trigger note attack for live MIDI keyboard monitoring.
* This path tracks the underlying buffer sources so pitch bend can retune held notes.
*/
public triggerLiveMidiAttack(
pitch: number,
time?: number,
velocity?: number
): void {
this.triggerPitchBendAwareAttack(pitch, time, velocity);
}
public triggerPitchBendAwareAttack(
pitch: number,
time?: number,
velocity?: number,
duration?: number
): void {
if (!this.shouldPlay()) {
return;
}
try {
const liveSource = this.createPitchBendAwareSource(pitch, time, velocity, duration);
if (!liveSource) {
return;
}
const activeSources = this.liveMidiSources.get(pitch) ?? [];
activeSources.push(liveSource);
this.liveMidiSources.set(pitch, activeSources);
} catch (error) {
console.error(`Error triggering live MIDI attack for pitch ${pitch} on ${this.instrument}:`, error);
}
}
/**
* Release a specific note
* Used for ending sustained notes like piano key releases
@@ -132,12 +189,61 @@ export class KGAudioBus {
}
}
/**
* Release a live MIDI note and forget any active bent sources tied to that pitch.
*/
public releaseLiveMidiNote(pitch: number, time?: number): void {
try {
const activeSources = this.liveMidiSources.get(pitch);
if (!activeSources || activeSources.length === 0) {
return;
}
const stopTime = time ?? Tone.now();
activeSources.forEach(({ source }) => {
try {
source.stop(stopTime);
} catch (error) {
console.error(`Error stopping live MIDI source for pitch ${pitch} on ${this.instrument}:`, error);
}
});
this.liveMidiSources.delete(pitch);
} catch (error) {
console.error(`Error releasing live MIDI note ${pitch} on ${this.instrument}:`, error);
}
}
public setLiveMidiPitchBend(normalizedBend: number): void {
this.liveMidiPitchBend = Math.max(-1, Math.min(1, normalizedBend));
for (const activeSources of this.liveMidiSources.values()) {
activeSources.forEach(({ source, basePlaybackRate }) => {
source.playbackRate.value = this.applyPitchBendToPlaybackRate(basePlaybackRate);
});
}
}
public resetLiveMidiPitchBend(): void {
this.setLiveMidiPitchBend(0);
}
/**
* Release all currently playing notes
*/
public releaseAll(): void {
try {
this.sampler.releaseAll();
this.liveMidiSources.forEach((activeSources) => {
activeSources.forEach(({ source }) => {
try {
source.stop();
} catch (error) {
console.error(`Error stopping live MIDI source on ${this.instrument}:`, error);
}
});
});
this.liveMidiSources.clear();
this.resetLiveMidiPitchBend();
} catch (error) {
console.error(`Error releasing all notes on ${this.instrument}:`, error);
}
@@ -206,12 +312,20 @@ export class KGAudioBus {
try {
console.log(`Changing instrument from ${this.instrument} to ${newInstrument}...`);
this.releaseAll();
// Dispose of the current sampler
this.sampler.dispose();
// Create new sampler with new instrument
const samplerFactory = KGToneSamplerFactory.instance();
this.sampler = await samplerFactory.createSampler(String(newInstrument));
const buffersPool = KGToneBuffersPool.instance();
const [sampler, audioBuffers] = await Promise.all([
samplerFactory.createSampler(String(newInstrument)),
buffersPool.getToneAudioBuffers(String(newInstrument)),
]);
this.sampler = sampler;
this.audioBuffers = audioBuffers;
this.instrument = newInstrument;
// Restore volume settings
@@ -269,6 +383,7 @@ export class KGAudioBus {
*/
public dispose(): void {
try {
this.releaseAll();
this.sampler.dispose();
console.log(`Disposed KGAudioBus for ${this.instrument}`);
} catch (error) {
@@ -352,4 +467,80 @@ export class KGAudioBus {
solo: this.solo
};
}
}
private createPitchBendAwareSource(
pitch: number,
time?: number,
velocity?: number,
duration?: number
): LiveMidiSource | null {
const closestPitch = this.findClosestBufferedPitch(pitch);
if (closestPitch === null) {
console.warn(`No audio buffer found for live MIDI pitch ${pitch} on ${this.instrument}`);
return null;
}
const bufferKey = this.midiPitchToBufferKey(closestPitch);
const buffer = this.audioBuffers.get(bufferKey);
if (!buffer) {
console.warn(`Missing audio buffer ${bufferKey} for ${this.instrument}`);
return null;
}
const basePlaybackRate = Math.pow(2, (pitch - closestPitch) / 12);
const source = new Tone.ToneBufferSource({
url: buffer,
fadeIn: this.sampler.attack,
fadeOut: this.sampler.release,
curve: this.sampler.curve,
playbackRate: this.applyPitchBendToPlaybackRate(basePlaybackRate),
}).connect(this.sampler.output);
source.onended = () => {
const currentSources = this.liveMidiSources.get(pitch);
if (!currentSources) {
return;
}
const nextSources = currentSources.filter((entry) => entry.source !== source);
if (nextSources.length === 0) {
this.liveMidiSources.delete(pitch);
} else {
this.liveMidiSources.set(pitch, nextSources);
}
};
source.start(time, 0, duration ?? buffer.duration / basePlaybackRate, velocity ?? 1);
return { source, basePlaybackRate };
}
private applyPitchBendToPlaybackRate(basePlaybackRate: number): number {
const bendSemitones = this.liveMidiPitchBend * KGAudioBus.LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES;
return basePlaybackRate * Math.pow(2, bendSemitones / 12);
}
private findClosestBufferedPitch(targetPitch: number): number | null {
const [minPitch, maxPitch] = FLUIDR3_INSTRUMENT_MAP[this.instrument]?.pitchRange || [21, 108];
const boundedPitch = Math.max(minPitch, Math.min(maxPitch, targetPitch));
for (let offset = 0; offset <= 96; offset++) {
const upwardPitch = boundedPitch + offset;
if (upwardPitch <= maxPitch && this.audioBuffers.has(this.midiPitchToBufferKey(upwardPitch))) {
return upwardPitch;
}
const downwardPitch = boundedPitch - offset;
if (downwardPitch >= minPitch && this.audioBuffers.has(this.midiPitchToBufferKey(downwardPitch))) {
return downwardPitch;
}
}
return null;
}
private midiPitchToBufferKey(pitch: number): string {
const octave = Math.floor((pitch - 12) / 12);
const noteIndex = (pitch - 12) % 12;
return `${KGAudioBus.LIVE_MIDI_NOTE_NAMES[noteIndex]}${octave}`;
}
}
src/core/audio-interface/KGAudioInterface.ts
+132 -27
View File
@@ -1,8 +1,9 @@
import type { KGProject } from '../KGProject';
import type { KGMidiNote } from '../midi/KGMidiNote';
import type { KGMidiPitchBend } from '../midi/KGMidiPitchBend';
import { TIME_CONSTANTS, AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants';
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
import { pitchToNoteNameString } from '../../util/midiUtil';
import { midiPitchBendToNormalized, pitchToNoteNameString } from '../../util/midiUtil';
import * as Tone from 'tone';
import { KGAudioBus } from './KGAudioBus';
import { KGAudioPlayerBus } from './KGAudioPlayerBus';
@@ -399,6 +400,7 @@ export class KGAudioInterface {
// Clear any existing scheduled events
this.clearScheduledEvents();
this.clearDelayedTransportStart();
this.trackAudioBuses.forEach(audioBus => audioBus.resetLiveMidiPitchBend());
console.log("Preparing playback");
@@ -476,20 +478,31 @@ export class KGAudioInterface {
// Schedule MIDI track events
if (audioBus && track.getType() === 'MIDI') {
const trackPitchBends: Array<{ pitchBend: KGMidiPitchBend; absoluteBeat: number }> = [];
const trackNotes: Array<{ note: KGMidiNote; absoluteStartBeat: number; absoluteEndBeat: number }> = [];
track.getRegions().forEach(region => {
console.log(`Region ${region.getId().toString()}: type: ${region.getCurrentType()}`);
if (region.getCurrentType() === 'KGMidiRegion') {
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[] };
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[]; getPitchBends: () => KGMidiPitchBend[] };
const regionStartBeat = region.getStartFromBeat();
if (midiRegion.getPitchBends) {
midiRegion.getPitchBends().forEach((pitchBend: KGMidiPitchBend) => {
trackPitchBends.push({
pitchBend,
absoluteBeat: regionStartBeat + pitchBend.getBeat(),
});
});
}
// Get notes from region (assuming it has a getNotes method)
if (midiRegion.getNotes) {
midiRegion.getNotes().forEach((note: KGMidiNote) => {
// Calculate absolute note timing in beats (note position + region start position)
const regionStartBeat = region.getStartFromBeat();
const noteStartBeat = note.getStartBeat() + regionStartBeat;
const noteEndBeat = note.getEndBeat() + regionStartBeat;
const noteDurationBeats = note.getEndBeat() - note.getStartBeat();
// Skip notes outside loop range when looping
if (noteStartBeat >= scheduleEndBeat || noteEndBeat <= scheduleStartBeat) {
@@ -500,33 +513,69 @@ export class KGAudioInterface {
if (noteStartBeat < startPosition) {
return; // Skip notes that would have already finished before playback starts
}
// Convert beats to Tone.js time format for scheduling
const noteStartTime = this.beatsToToneTime(noteStartBeat);
const noteDuration = this.beatsToToneTime(noteDurationBeats);
// Convert MIDI note number to note name
const noteName = pitchToNoteNameString(note.getPitch());
const velocity = note.getVelocity() / 127; // Normalize to 0-1
console.log(
`Scheduling note ${noteName} at beat ${Number(noteStartBeat.toFixed ? noteStartBeat.toFixed(3) : noteStartBeat.toLocaleString(undefined, {maximumFractionDigits: 3}))}, Tone time: ${Number(Number(noteStartTime).toFixed(3))}, duration: ${Number(Number(noteDuration).toFixed(3))}, delay: ${playbackDelay}s`
);
// Schedule the note with delay offset
const eventId = Tone.Transport.schedule((time) => {
// Check if track should play considering solo logic
const hasSoloedTracks = this.hasSoloedTracks();
if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) {
audioBus.triggerAttackRelease(noteName, noteDuration, time + playbackDelay, velocity);
}
}, noteStartTime);
this.scheduledEvents.add(eventId);
trackNotes.push({
note,
absoluteStartBeat: noteStartBeat,
absoluteEndBeat: noteEndBeat,
});
});
}
}
});
const boundedTrackPitchBends = trackPitchBends
.filter(({ absoluteBeat }) => absoluteBeat >= scheduleStartBeat && absoluteBeat < scheduleEndBeat)
.sort((a, b) => a.absoluteBeat - b.absoluteBeat);
const initialPitchBend = [...boundedTrackPitchBends]
.reverse()
.find(({ absoluteBeat }) => absoluteBeat <= startPosition);
audioBus.setLiveMidiPitchBend(midiPitchBendToNormalized(initialPitchBend?.pitchBend.getValue() ?? 8192));
if (isLooping && !boundedTrackPitchBends.some(({ absoluteBeat }) => absoluteBeat === scheduleStartBeat)) {
const eventId = Tone.Transport.schedule((time) => {
const hasSoloedTracks = this.hasSoloedTracks();
if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) {
audioBus.setLiveMidiPitchBend(0);
}
}, this.beatsToToneTime(scheduleStartBeat));
this.scheduledEvents.add(eventId);
}
boundedTrackPitchBends.forEach(({ pitchBend, absoluteBeat }) => {
if (absoluteBeat < startPosition) {
return;
}
const eventId = Tone.Transport.schedule(() => {
const hasSoloedTracks = this.hasSoloedTracks();
if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) {
audioBus.setLiveMidiPitchBend(midiPitchBendToNormalized(pitchBend.getValue()));
}
}, this.beatsToToneTime(absoluteBeat));
this.scheduledEvents.add(eventId);
});
trackNotes.forEach(({ note, absoluteStartBeat, absoluteEndBeat }) => {
const noteDurationBeats = absoluteEndBeat - absoluteStartBeat;
const noteStartTime = this.beatsToToneTime(absoluteStartBeat);
const noteDuration = this.beatsToToneTime(noteDurationBeats);
const velocity = note.getVelocity() / 127;
const noteName = pitchToNoteNameString(note.getPitch());
console.log(
`Scheduling note ${noteName} at beat ${Number(absoluteStartBeat.toFixed ? absoluteStartBeat.toFixed(3) : absoluteStartBeat.toLocaleString(undefined, {maximumFractionDigits: 3}))}, Tone time: ${Number(Number(noteStartTime).toFixed(3))}, duration: ${Number(Number(noteDuration).toFixed(3))}, delay: ${playbackDelay}s`
);
const eventId = Tone.Transport.schedule((time) => {
const hasSoloedTracks = this.hasSoloedTracks();
if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) {
audioBus.triggerPitchBendAwareAttack(note.getPitch(), time + playbackDelay, velocity, Tone.Time(noteDuration).toSeconds());
}
}, noteStartTime);
this.scheduledEvents.add(eventId);
});
}
// Schedule audio/wav track events
@@ -771,6 +820,31 @@ export class KGAudioInterface {
}
}
/**
* Trigger note attack for live MIDI keyboard monitoring.
* Unlike piano-roll audition, this path tracks active sources so pitch bend can retune them.
*/
public triggerLiveMidiNoteAttack(trackId: string, pitch: number, velocity: number = 127, time?: number): void {
try {
const audioBus = this.trackAudioBuses.get(trackId);
if (!audioBus) {
console.warn(`No audio bus found for track ${trackId}`);
return;
}
const normalizedVelocity = velocity / 127;
const triggerTime = time ?? Tone.now();
const hasSoloedTracks = this.hasSoloedTracks();
if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) {
audioBus.triggerLiveMidiAttack(pitch, triggerTime, normalizedVelocity);
console.log(`Triggered live MIDI attack for pitch ${pitch} on track ${trackId}`);
}
} catch (error) {
console.error(`Error triggering live MIDI note attack for track ${trackId}:`, error);
}
}
/**
* Release a specific note
* Used for piano key release
@@ -793,6 +867,37 @@ export class KGAudioInterface {
}
}
public releaseLiveMidiNote(trackId: string, pitch: number, time?: number): void {
try {
const audioBus = this.trackAudioBuses.get(trackId);
if (!audioBus) {
console.warn(`No audio bus found for track ${trackId}`);
return;
}
const releaseTime = time ?? Tone.now();
audioBus.releaseLiveMidiNote(pitch, releaseTime);
console.log(`Released live MIDI note ${pitch} on track ${trackId}`);
} catch (error) {
console.error(`Error releasing live MIDI note for track ${trackId}:`, error);
}
}
public setLiveMidiPitchBend(trackId: string, normalizedBend: number): void {
try {
const audioBus = this.trackAudioBuses.get(trackId);
if (!audioBus) {
console.warn(`No audio bus found for track ${trackId}`);
return;
}
audioBus.setLiveMidiPitchBend(normalizedBend);
console.log(`Set live MIDI pitch bend to ${normalizedBend} on track ${trackId}`);
} catch (error) {
console.error(`Error setting live MIDI pitch bend for track ${trackId}:`, error);
}
}
/**
* Clear all scheduled events
*/