feat: added pitch bend automation (linear interpolation)

This commit is contained in:
Xiaohan-Tian
2026-05-06 18:25:14 -07:00
parent fc1a044e98
commit 738628fc76
18 changed files with 963 additions and 76 deletions
+1
View File
@@ -75,6 +75,7 @@
"audio": { "audio": {
"enable_audio_capture_for_screen_sharing": false, "enable_audio_capture_for_screen_sharing": false,
"lookahead_time": 0.05, "lookahead_time": 0.05,
"midi_automation_interpolation_interval_ms": 10,
"playback_delay": 0.2, "playback_delay": 0.2,
"recording_offset": 0 "recording_offset": 0
}, },
+114
View File
@@ -0,0 +1,114 @@
import { act, render, screen } from '@testing-library/react';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
let mockState = { isPreparingPlayback: false };
vi.mock('./stores/projectStore', () => ({
useProjectStore: (selector?: (state: typeof mockState) => unknown) => (
selector ? selector(mockState) : mockState
),
}));
vi.mock('./hooks/useGlobalKeyboardHandler', () => ({
useGlobalKeyboardHandler: () => undefined,
}));
vi.mock('./components/Toolbar', () => ({ default: () => null }));
vi.mock('./components/StatusBar', () => ({ default: () => null }));
vi.mock('./components/MainContent', () => ({ default: () => null }));
vi.mock('./components/InstrumentSelection', () => ({ default: () => null }));
vi.mock('./components/ChatBox', () => ({ default: () => null }));
vi.mock('./components/KGOnePanel', () => ({ default: () => null }));
vi.mock('./components/ListEventPanel', () => ({ default: () => null }));
vi.mock('./components/settings', () => ({ SettingsPanel: () => null }));
vi.mock('./core/audio-interface/KGToneBuffersPool', () => ({
KGToneBuffersPool: {
instance: () => ({
getActiveLoadCount: () => 0,
addLoadingListener: () => undefined,
removeLoadingListener: () => undefined,
}),
},
}));
vi.mock('./core/audio-interface/KGOfflineRenderer', () => ({
KGOfflineRenderer: {
instance: () => ({
addRenderingListener: () => undefined,
removeRenderingListener: () => undefined,
}),
},
}));
vi.mock('./core/KGCore', () => ({
KGCore: {
instance: () => ({
getIsMigrating: () => false,
setMigrationStateChangeCallback: () => undefined,
}),
},
}));
import { PlaybackPreparationOverlayContainer } from './App';
describe('PlaybackPreparationOverlayContainer', () => {
beforeEach(() => {
vi.useFakeTimers();
mockState = { isPreparingPlayback: false };
});
afterEach(() => {
vi.useRealTimers();
});
it('does not appear if preparation finishes before the delay', () => {
const { rerender } = render(<PlaybackPreparationOverlayContainer />);
mockState = { isPreparingPlayback: true };
rerender(<PlaybackPreparationOverlayContainer />);
act(() => {
vi.advanceTimersByTime(100);
});
mockState = { isPreparingPlayback: false };
rerender(<PlaybackPreparationOverlayContainer />);
act(() => {
vi.advanceTimersByTime(100);
});
expect(screen.queryByText('Preparing playback...')).not.toBeInTheDocument();
});
it('appears after 150ms while preparation is still in progress', () => {
const { rerender } = render(<PlaybackPreparationOverlayContainer />);
mockState = { isPreparingPlayback: true };
rerender(<PlaybackPreparationOverlayContainer />);
act(() => {
vi.advanceTimersByTime(149);
});
expect(screen.queryByText('Preparing playback...')).not.toBeInTheDocument();
act(() => {
vi.advanceTimersByTime(1);
});
expect(screen.getByText('Preparing playback...')).toBeInTheDocument();
});
it('hides once preparation completes after becoming visible', () => {
const { rerender } = render(<PlaybackPreparationOverlayContainer />);
mockState = { isPreparingPlayback: true };
rerender(<PlaybackPreparationOverlayContainer />);
act(() => {
vi.advanceTimersByTime(150);
});
expect(screen.getByText('Preparing playback...')).toBeInTheDocument();
mockState = { isPreparingPlayback: false };
rerender(<PlaybackPreparationOverlayContainer />);
expect(screen.queryByText('Preparing playback...')).not.toBeInTheDocument();
});
});
+42
View File
@@ -185,6 +185,9 @@ function App() {
{/* Bounce/Render Overlay */} {/* Bounce/Render Overlay */}
<BounceOverlayContainer /> <BounceOverlayContainer />
{/* Playback Preparation Overlay */}
<PlaybackPreparationOverlayContainer />
</div> </div>
); );
} }
@@ -295,3 +298,42 @@ const BounceOverlayContainer: React.FC = () => {
/> />
); );
}; };
const PLAYBACK_PREPARATION_OVERLAY_DELAY_MS = 150;
export const PlaybackPreparationOverlayContainer: React.FC = () => {
const isPreparingPlayback = useProjectStore(state => state.isPreparingPlayback);
const [visible, setVisible] = useState<boolean>(false);
const timeoutRef = useRef<number | null>(null);
useEffectReact(() => {
if (isPreparingPlayback) {
if (timeoutRef.current === null) {
timeoutRef.current = window.setTimeout(() => {
setVisible(true);
timeoutRef.current = null;
}, PLAYBACK_PREPARATION_OVERLAY_DELAY_MS);
}
} else {
if (timeoutRef.current !== null) {
clearTimeout(timeoutRef.current);
timeoutRef.current = null;
}
setVisible(false);
}
return () => {
if (timeoutRef.current !== null) {
clearTimeout(timeoutRef.current);
timeoutRef.current = null;
}
};
}, [isPreparingPlayback]);
return (
<LoadingOverlay
visible={visible}
message="Preparing playback..."
/>
);
};
+2 -2
View File
@@ -42,7 +42,7 @@ const Toolbar: React.FC = () => {
projectName, setProjectName, projectName, setProjectName,
savedProjectName, setSavedProjectName, savedProjectName, setSavedProjectName,
bpm, timeSignature, keySignature, setStatus, bpm, timeSignature, keySignature, setStatus,
isPlaying, startPlaying, stopTransport, setPlayheadPosition, isPlaying, isPreparingPlayback, startPlaying, stopTransport, setPlayheadPosition,
currentTime, setBpm, setTimeSignature, setKeySignature, currentTime, setBpm, setTimeSignature, setKeySignature,
maxBars, setMaxBars, maxBars, setMaxBars,
barWidthMultiplier, setBarWidthMultiplier, barWidthMultiplier, setBarWidthMultiplier,
@@ -1110,7 +1110,7 @@ const Toolbar: React.FC = () => {
<div className="toolbar-separator"></div> <div className="toolbar-separator"></div>
<button title="Back to beginning" className="button-back-to-beginning" onClick={handleBackToBeginningClick}><FaStepBackward /></button> <button title="Back to beginning" className="button-back-to-beginning" onClick={handleBackToBeginningClick}><FaStepBackward /></button>
{!isPlaying ? ( {!isPlaying ? (
<button title="Play" className="button-play" onClick={handlePlayClick}><FaPlay /></button> <button title="Play" className="button-play" onClick={handlePlayClick} disabled={isPreparingPlayback}><FaPlay /></button>
) : ( ) : (
<button title="Pause" className="button-pause" onClick={handlePauseClick}><FaPause /></button> <button title="Pause" className="button-pause" onClick={handlePauseClick}><FaPause /></button>
)} )}
@@ -11,6 +11,7 @@ const BehaviorSettings: React.FC = () => {
const [spectrogramHeightResolution, setSpectrogramHeightResolution] = useState<SpectrogramHeightResolution>(3); const [spectrogramHeightResolution, setSpectrogramHeightResolution] = useState<SpectrogramHeightResolution>(3);
const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState<boolean>(true); const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState<boolean>(true);
const [audioLookaheadTime, setAudioLookaheadTime] = useState<string>('50'); const [audioLookaheadTime, setAudioLookaheadTime] = useState<string>('50');
const [midiAutomationInterpolationIntervalMs, setMidiAutomationInterpolationIntervalMs] = useState<number>(10);
const [playbackDelay, setPlaybackDelay] = useState<string>('200'); const [playbackDelay, setPlaybackDelay] = useState<string>('200');
const [recordingOffset, setRecordingOffset] = useState<string>('0'); const [recordingOffset, setRecordingOffset] = useState<string>('0');
const [enableAudioCapture, setEnableAudioCapture] = useState<boolean>(false); const [enableAudioCapture, setEnableAudioCapture] = useState<boolean>(false);
@@ -34,6 +35,9 @@ const BehaviorSettings: React.FC = () => {
setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true); setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true);
const lookaheadTimeSeconds = (configManager.get('audio.lookahead_time') as number) ?? 0.05; const lookaheadTimeSeconds = (configManager.get('audio.lookahead_time') as number) ?? 0.05;
setAudioLookaheadTime(((lookaheadTimeSeconds * 1000).toFixed(0))); setAudioLookaheadTime(((lookaheadTimeSeconds * 1000).toFixed(0)));
setMidiAutomationInterpolationIntervalMs(
(configManager.get('audio.midi_automation_interpolation_interval_ms') as number) ?? 10
);
const playbackDelaySeconds = (configManager.get('audio.playback_delay') as number) ?? 0.2; const playbackDelaySeconds = (configManager.get('audio.playback_delay') as number) ?? 0.2;
setPlaybackDelay(((playbackDelaySeconds * 1000).toFixed(0))); setPlaybackDelay(((playbackDelaySeconds * 1000).toFixed(0)));
const recordingOffsetSeconds = (configManager.get('audio.recording_offset') as number) ?? 0; const recordingOffsetSeconds = (configManager.get('audio.recording_offset') as number) ?? 0;
@@ -120,6 +124,12 @@ const BehaviorSettings: React.FC = () => {
} }
}; };
const handleMidiAutomationInterpolationIntervalChange = async (value: string) => {
const nextValue = parseInt(value, 10);
setMidiAutomationInterpolationIntervalMs(nextValue);
await configManager.set('audio.midi_automation_interpolation_interval_ms', nextValue);
};
const handleRecordingOffsetChange = async (value: string) => { const handleRecordingOffsetChange = async (value: string) => {
const numValueMs = value === '' ? 0 : parseFloat(value); const numValueMs = value === '' ? 0 : parseFloat(value);
const numValueSeconds = numValueMs / 1000; const numValueSeconds = numValueMs / 1000;
@@ -197,13 +207,13 @@ const BehaviorSettings: React.FC = () => {
<div className="settings-group"> <div className="settings-group">
<h4>Chat Box</h4> <h4>Chat Box</h4>
<div className="settings-item"> <div className="settings-item">
<label className="settings-label"> <label className="settings-label">
Open at Start Up Open at Start Up
</label> </label>
<select <select
className="settings-select" className="settings-select"
value={chatboxDefaultOpen ? 'yes' : 'no'} value={chatboxDefaultOpen ? 'yes' : 'no'}
onChange={(e) => handleChatboxDefaultOpenChange(e.target.value)} onChange={(e) => handleChatboxDefaultOpenChange(e.target.value)}
> >
@@ -270,6 +280,24 @@ const BehaviorSettings: React.FC = () => {
</div> </div>
</div> </div>
<div className="settings-item">
<label className="settings-label">
MIDI Automation Interpolation
</label>
<select
className="settings-select"
value={midiAutomationInterpolationIntervalMs}
onChange={(e) => handleMidiAutomationInterpolationIntervalChange(e.target.value)}
>
<option value="20">Low-end (20 ms)</option>
<option value="10">Balanced (10 ms)</option>
<option value="5">High quality (5 ms)</option>
</select>
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
Controls how densely MIDI automation are baked for playback and bounce. Smaller intervals sound smoother but schedule more events.
</div>
</div>
<div className="settings-item"> <div className="settings-item">
<label className="settings-label"> <label className="settings-label">
MIDI Input Latency (ms) MIDI Input Latency (ms)
@@ -301,8 +329,8 @@ const BehaviorSettings: React.FC = () => {
<label className="settings-label"> <label className="settings-label">
Capture Audio for Screen Sharing Capture Audio for Screen Sharing
</label> </label>
<select <select
className="settings-select" className="settings-select"
value={enableAudioCapture ? 'yes' : 'no'} value={enableAudioCapture ? 'yes' : 'no'}
onChange={(e) => handleEnableAudioCaptureChange(e.target.value)} onChange={(e) => handleEnableAudioCaptureChange(e.target.value)}
> >
@@ -310,7 +338,7 @@ const BehaviorSettings: React.FC = () => {
<option value="yes">Yes</option> <option value="yes">Yes</option>
</select> </select>
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}> <div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
Restart KGStudio (refresh the page) to take effect. Enable this option when KGStudio's audio cannot be captured during screen sharing in video calls (e.g., Zoom, Teams). This creates an additional audio stream that screen capture applications can detect. Restart KGStudio (refresh the page) to take effect. Enable this option when KGStudio's audio cannot be captured during screen sharing in video calls (e.g., Zoom, Teams). This creates an additional audio stream that screen capture applications can detect.
<br /> <br />
<b>It is important to make sure when screen sharing in Zoom, the "Share Sound" option is enabled.</b> <b>It is important to make sure when screen sharing in Zoom, the "Share Sound" option is enabled.</b>
</div> </div>
@@ -66,4 +66,15 @@ describe('KGAudioBus live MIDI pitch bend', () => {
audioBus.releaseLiveMidiNote(60, 1.25); audioBus.releaseLiveMidiNote(60, 1.25);
expect(source.stop).toHaveBeenCalledWith(1.25); expect(source.stop).toHaveBeenCalledWith(1.25);
}); });
it('uses timed playback-rate automation when scheduling pitch bend updates', async () => {
const audioBus = await KGAudioBus.create('acoustic_grand_piano');
audioBus.triggerLiveMidiAttack(60, 0, 1);
const source = MockBufferSource.mock.results[0].value;
audioBus.scheduleLiveMidiPitchBend(1, 2);
expect(source.playbackRate.setValueAtTime).toHaveBeenCalledWith(Math.pow(2, 2 / 12), 2);
});
}); });
+43 -11
View File
@@ -20,7 +20,7 @@ interface LiveMidiSource {
export class KGAudioBus { export class KGAudioBus {
// Fixed at +/-2 semitones for now. Future work: make this user-configurable // Fixed at +/-2 semitones for now. Future work: make this user-configurable
// or honor MIDI RPN 0,0 (Pitch Bend Sensitivity). // or honor MIDI RPN 0,0 (Pitch Bend Sensitivity).
private static readonly LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES = 2; public static readonly LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES = 2;
private static readonly LIVE_MIDI_NOTE_NAMES = ['C', 'Db', 'D', 'Eb', 'E', 'F', 'Gb', 'G', 'Ab', 'A', 'Bb', 'B']; private static readonly LIVE_MIDI_NOTE_NAMES = ['C', 'Db', 'D', 'Eb', 'E', 'F', 'Gb', 'G', 'Ab', 'A', 'Bb', 'B'];
// Core audio components // Core audio components
@@ -218,7 +218,17 @@ export class KGAudioBus {
for (const activeSources of this.liveMidiSources.values()) { for (const activeSources of this.liveMidiSources.values()) {
activeSources.forEach(({ source, basePlaybackRate }) => { activeSources.forEach(({ source, basePlaybackRate }) => {
source.playbackRate.value = this.applyPitchBendToPlaybackRate(basePlaybackRate); this.setPlaybackRateValue(source, this.applyPitchBendToPlaybackRate(basePlaybackRate));
});
}
}
public scheduleLiveMidiPitchBend(normalizedBend: number, time: number): void {
this.liveMidiPitchBend = Math.max(-1, Math.min(1, normalizedBend));
for (const activeSources of this.liveMidiSources.values()) {
activeSources.forEach(({ source, basePlaybackRate }) => {
this.setPlaybackRateValue(source, this.applyPitchBendToPlaybackRate(basePlaybackRate), time);
}); });
} }
} }
@@ -474,13 +484,13 @@ export class KGAudioBus {
velocity?: number, velocity?: number,
duration?: number duration?: number
): LiveMidiSource | null { ): LiveMidiSource | null {
const closestPitch = this.findClosestBufferedPitch(pitch); const closestPitch = KGAudioBus.findClosestBufferedPitch(this.instrument, this.audioBuffers, pitch);
if (closestPitch === null) { if (closestPitch === null) {
console.warn(`No audio buffer found for live MIDI pitch ${pitch} on ${this.instrument}`); console.warn(`No audio buffer found for live MIDI pitch ${pitch} on ${this.instrument}`);
return null; return null;
} }
const bufferKey = this.midiPitchToBufferKey(closestPitch); const bufferKey = KGAudioBus.midiPitchToBufferKey(closestPitch);
const buffer = this.audioBuffers.get(bufferKey); const buffer = this.audioBuffers.get(bufferKey);
if (!buffer) { if (!buffer) {
console.warn(`Missing audio buffer ${bufferKey} for ${this.instrument}`); console.warn(`Missing audio buffer ${bufferKey} for ${this.instrument}`);
@@ -514,23 +524,31 @@ export class KGAudioBus {
return { source, basePlaybackRate }; return { source, basePlaybackRate };
} }
private applyPitchBendToPlaybackRate(basePlaybackRate: number): number { public static applyNormalizedPitchBendToPlaybackRate(basePlaybackRate: number, normalizedBend: number): number {
const bendSemitones = this.liveMidiPitchBend * KGAudioBus.LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES; const bendSemitones = normalizedBend * KGAudioBus.LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES;
return basePlaybackRate * Math.pow(2, bendSemitones / 12); return basePlaybackRate * Math.pow(2, bendSemitones / 12);
} }
private findClosestBufferedPitch(targetPitch: number): number | null { private applyPitchBendToPlaybackRate(basePlaybackRate: number): number {
const [minPitch, maxPitch] = FLUIDR3_INSTRUMENT_MAP[this.instrument]?.pitchRange || [21, 108]; return KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, this.liveMidiPitchBend);
}
public static findClosestBufferedPitch(
instrument: InstrumentType,
audioBuffers: Tone.ToneAudioBuffers,
targetPitch: number
): number | null {
const [minPitch, maxPitch] = FLUIDR3_INSTRUMENT_MAP[instrument]?.pitchRange || [21, 108];
const boundedPitch = Math.max(minPitch, Math.min(maxPitch, targetPitch)); const boundedPitch = Math.max(minPitch, Math.min(maxPitch, targetPitch));
for (let offset = 0; offset <= 96; offset++) { for (let offset = 0; offset <= 96; offset++) {
const upwardPitch = boundedPitch + offset; const upwardPitch = boundedPitch + offset;
if (upwardPitch <= maxPitch && this.audioBuffers.has(this.midiPitchToBufferKey(upwardPitch))) { if (upwardPitch <= maxPitch && audioBuffers.has(KGAudioBus.midiPitchToBufferKey(upwardPitch))) {
return upwardPitch; return upwardPitch;
} }
const downwardPitch = boundedPitch - offset; const downwardPitch = boundedPitch - offset;
if (downwardPitch >= minPitch && this.audioBuffers.has(this.midiPitchToBufferKey(downwardPitch))) { if (downwardPitch >= minPitch && audioBuffers.has(KGAudioBus.midiPitchToBufferKey(downwardPitch))) {
return downwardPitch; return downwardPitch;
} }
} }
@@ -538,9 +556,23 @@ export class KGAudioBus {
return null; return null;
} }
private midiPitchToBufferKey(pitch: number): string { public static midiPitchToBufferKey(pitch: number): string {
const octave = Math.floor((pitch - 12) / 12); const octave = Math.floor((pitch - 12) / 12);
const noteIndex = (pitch - 12) % 12; const noteIndex = (pitch - 12) % 12;
return `${KGAudioBus.LIVE_MIDI_NOTE_NAMES[noteIndex]}${octave}`; return `${KGAudioBus.LIVE_MIDI_NOTE_NAMES[noteIndex]}${octave}`;
} }
private setPlaybackRateValue(source: Tone.ToneBufferSource, value: number, time?: number): void {
const playbackRate = source.playbackRate as unknown as {
value: number;
setValueAtTime?: (nextValue: number, nextTime: number) => void;
};
if (time !== undefined && typeof playbackRate.setValueAtTime === 'function') {
playbackRate.setValueAtTime(value, time);
return;
}
playbackRate.value = value;
}
} }
@@ -1,5 +1,5 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'; import { beforeEach, describe, expect, it, vi } from 'vitest';
import { createMockProject } from '../../test/utils/mock-data'; import { createMockMidiPitchBend, createMockMidiRegion, createMockMidiTrack, createMockProject } from '../../test/utils/mock-data';
import { MockTransport } from '../../test/mocks/tone'; import { MockTransport } from '../../test/mocks/tone';
vi.mock('tone', async () => { vi.mock('tone', async () => {
@@ -22,6 +22,7 @@ vi.mock('../config/ConfigManager', () => ({
import { KGCore } from '../KGCore'; import { KGCore } from '../KGCore';
import { ConfigManager } from '../config/ConfigManager'; import { ConfigManager } from '../config/ConfigManager';
import { KGAudioInterface } from './KGAudioInterface'; import { KGAudioInterface } from './KGAudioInterface';
import { MIDI_PITCH_BEND_CENTER, midiPitchBendToNormalized } from '../../util/midiUtil';
describe('KGAudioInterface preroll playback', () => { describe('KGAudioInterface preroll playback', () => {
beforeEach(() => { beforeEach(() => {
@@ -50,6 +51,7 @@ describe('KGAudioInterface preroll playback', () => {
get: (key: string) => { get: (key: string) => {
if (key === 'audio.playback_delay') return 0.2; if (key === 'audio.playback_delay') return 0.2;
if (key === 'audio.lookahead_time') return 0.05; if (key === 'audio.lookahead_time') return 0.05;
if (key === 'audio.midi_automation_interpolation_interval_ms') return 250;
return null; return null;
}, },
} as unknown as ConfigManager) } as unknown as ConfigManager)
@@ -112,4 +114,79 @@ describe('KGAudioInterface preroll playback', () => {
expect(MockTransport.position).toBe(6); expect(MockTransport.position).toBe(6);
}); });
it('schedules baked pitch bend points instead of authored step changes only', () => {
const region = createMockMidiRegion({
pitchBends: [
createMockMidiPitchBend({ id: 'bend-1', beat: 0, value: MIDI_PITCH_BEND_CENTER }),
createMockMidiPitchBend({ id: 'bend-2', beat: 1, value: 0 }),
],
});
const track = createMockMidiTrack({ id: 1, regions: [region] });
const project = createMockProject({ tracks: [track] });
const audio = KGAudioInterface.instance();
const audioBus = {
resetLiveMidiPitchBend: vi.fn(),
setLiveMidiPitchBend: vi.fn(),
scheduleLiveMidiPitchBend: vi.fn(),
shouldPlayWithSolo: vi.fn().mockReturnValue(true),
};
;(audio as unknown as { trackAudioBuses: Map<string, unknown> }).trackAudioBuses.set('1', audioBus);
audio.preparePlayback(project, 0);
const scheduledTimes = MockTransport.schedule.mock.calls.map(([, time]) => time);
expect(scheduledTimes).toContain(0.25);
expect(scheduledTimes).toContain(0.5);
});
it('computes the initial interpolated bend for non-zero playback starts', () => {
const region = createMockMidiRegion({
pitchBends: [
createMockMidiPitchBend({ id: 'bend-1', beat: 0, value: MIDI_PITCH_BEND_CENTER }),
createMockMidiPitchBend({ id: 'bend-2', beat: 4, value: 0 }),
],
});
const track = createMockMidiTrack({ id: 1, regions: [region] });
const project = createMockProject({ tracks: [track] });
const audio = KGAudioInterface.instance();
const audioBus = {
resetLiveMidiPitchBend: vi.fn(),
setLiveMidiPitchBend: vi.fn(),
scheduleLiveMidiPitchBend: vi.fn(),
shouldPlayWithSolo: vi.fn().mockReturnValue(true),
};
;(audio as unknown as { trackAudioBuses: Map<string, unknown> }).trackAudioBuses.set('1', audioBus);
audio.preparePlayback(project, 2);
expect(audioBus.setLiveMidiPitchBend).toHaveBeenCalledWith(midiPitchBendToNormalized(4096));
});
it('adds a loop-start re-anchor for interpolated pitch bend state', () => {
const region = createMockMidiRegion({
pitchBends: [
createMockMidiPitchBend({ id: 'bend-1', beat: 2, value: 0 }),
createMockMidiPitchBend({ id: 'bend-2', beat: 6, value: MIDI_PITCH_BEND_CENTER }),
],
});
const track = createMockMidiTrack({ id: 1, regions: [region] });
const project = createMockProject({ tracks: [track] });
project.setIsLooping(true);
project.setLoopingRange([1, 1]);
const audio = KGAudioInterface.instance();
const audioBus = {
resetLiveMidiPitchBend: vi.fn(),
setLiveMidiPitchBend: vi.fn(),
scheduleLiveMidiPitchBend: vi.fn(),
shouldPlayWithSolo: vi.fn().mockReturnValue(true),
};
;(audio as unknown as { trackAudioBuses: Map<string, unknown> }).trackAudioBuses.set('1', audioBus);
audio.preparePlayback(project, 5);
const scheduledTimes = MockTransport.schedule.mock.calls.map(([, time]) => time);
expect(scheduledTimes).toContain(2);
});
}); });
+49 -33
View File
@@ -3,7 +3,16 @@ import type { KGMidiNote } from '../midi/KGMidiNote';
import type { KGMidiPitchBend } from '../midi/KGMidiPitchBend'; import type { KGMidiPitchBend } from '../midi/KGMidiPitchBend';
import { TIME_CONSTANTS, AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants'; import { TIME_CONSTANTS, AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants';
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants'; import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
import { midiPitchBendToNormalized, pitchToNoteNameString } from '../../util/midiUtil'; import {
MIDI_PITCH_BEND_CENTER,
midiPitchBendToNormalized,
pitchToNoteNameString,
} from '../../util/midiUtil';
import {
bakeMidiAutomationPointsInWindow,
collectRegionMidiAutomationPoints,
resolveMidiAutomationValueAtBeat,
} from '../../util/midiAutomationUtil';
import * as Tone from 'tone'; import * as Tone from 'tone';
import { KGAudioBus } from './KGAudioBus'; import { KGAudioBus } from './KGAudioBus';
import { KGAudioPlayerBus } from './KGAudioPlayerBus'; import { KGAudioPlayerBus } from './KGAudioPlayerBus';
@@ -473,30 +482,35 @@ export class KGAudioInterface {
project.getTracks().forEach(track => { project.getTracks().forEach(track => {
const trackId = track.getId().toString(); const trackId = track.getId().toString();
const audioBus = this.trackAudioBuses.get(trackId); const audioBus = this.trackAudioBuses.get(trackId);
const interpolationIntervalMs = (configManager.get('audio.midi_automation_interpolation_interval_ms') as number) ?? 10;
console.log(`Track ${trackId} has audio bus: ${audioBus ? 'true' : 'false'}; type: ${track.getType()}`); console.log(`Track ${trackId} has audio bus: ${audioBus ? 'true' : 'false'}; type: ${track.getType()}`);
// Schedule MIDI track events // Schedule MIDI track events
if (audioBus && track.getType() === 'MIDI') { if (audioBus && track.getType() === 'MIDI') {
const trackPitchBends: Array<{ pitchBend: KGMidiPitchBend; absoluteBeat: number }> = [];
const trackNotes: Array<{ note: KGMidiNote; absoluteStartBeat: number; absoluteEndBeat: number }> = []; const trackNotes: Array<{ note: KGMidiNote; absoluteStartBeat: number; absoluteEndBeat: number }> = [];
const trackPitchBends = collectRegionMidiAutomationPoints(
track.getRegions()
.filter(region => region.getCurrentType() === 'KGMidiRegion')
.map(region => {
const midiRegion = region as unknown as { getPitchBends: () => KGMidiPitchBend[] };
return {
startBeat: region.getStartFromBeat(),
points: midiRegion.getPitchBends().map(pitchBend => ({
beat: pitchBend.getBeat(),
value: pitchBend.getValue(),
})),
};
})
);
track.getRegions().forEach(region => { track.getRegions().forEach(region => {
console.log(`Region ${region.getId().toString()}: type: ${region.getCurrentType()}`); console.log(`Region ${region.getId().toString()}: type: ${region.getCurrentType()}`);
if (region.getCurrentType() === 'KGMidiRegion') { if (region.getCurrentType() === 'KGMidiRegion') {
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[]; getPitchBends: () => KGMidiPitchBend[] }; const midiRegion = region as unknown as { getNotes: () => KGMidiNote[] };
const regionStartBeat = region.getStartFromBeat(); const regionStartBeat = region.getStartFromBeat();
if (midiRegion.getPitchBends) {
midiRegion.getPitchBends().forEach((pitchBend: KGMidiPitchBend) => {
trackPitchBends.push({
pitchBend,
absoluteBeat: regionStartBeat + pitchBend.getBeat(),
});
});
}
// Get notes from region (assuming it has a getNotes method) // Get notes from region (assuming it has a getNotes method)
if (midiRegion.getNotes) { if (midiRegion.getNotes) {
midiRegion.getNotes().forEach((note: KGMidiNote) => { midiRegion.getNotes().forEach((note: KGMidiNote) => {
@@ -523,35 +537,37 @@ export class KGAudioInterface {
} }
}); });
const boundedTrackPitchBends = trackPitchBends const initialPitchBendBeat = isLooping ? Math.max(startPosition, scheduleStartBeat) : startPosition;
.filter(({ absoluteBeat }) => absoluteBeat >= scheduleStartBeat && absoluteBeat < scheduleEndBeat) const initialPitchBendValue = resolveMidiAutomationValueAtBeat(
.sort((a, b) => a.absoluteBeat - b.absoluteBeat); trackPitchBends,
const initialPitchBend = [...boundedTrackPitchBends] initialPitchBendBeat,
.reverse() MIDI_PITCH_BEND_CENTER
.find(({ absoluteBeat }) => absoluteBeat <= startPosition); );
audioBus.setLiveMidiPitchBend(midiPitchBendToNormalized(initialPitchBend?.pitchBend.getValue() ?? 8192)); audioBus.setLiveMidiPitchBend(midiPitchBendToNormalized(initialPitchBendValue));
if (isLooping && !boundedTrackPitchBends.some(({ absoluteBeat }) => absoluteBeat === scheduleStartBeat)) { const pitchBendWindowStartBeat = isLooping ? scheduleStartBeat : Math.max(startPosition, 0);
const eventId = Tone.Transport.schedule((time) => { const bakedTrackPitchBends = bakeMidiAutomationPointsInWindow(
const hasSoloedTracks = this.hasSoloedTracks(); trackPitchBends,
if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) { pitchBendWindowStartBeat,
audioBus.setLiveMidiPitchBend(0); scheduleEndBeat,
} {
}, this.beatsToToneTime(scheduleStartBeat)); maxIntervalMs: interpolationIntervalMs,
this.scheduledEvents.add(eventId); bpm: project.getBpm(),
} defaultValue: MIDI_PITCH_BEND_CENTER,
}
);
boundedTrackPitchBends.forEach(({ pitchBend, absoluteBeat }) => { bakedTrackPitchBends.forEach(({ beat, value }) => {
if (absoluteBeat < startPosition) { if (!isLooping && beat <= pitchBendWindowStartBeat) {
return; return;
} }
const eventId = Tone.Transport.schedule(() => { const eventId = Tone.Transport.schedule((time) => {
const hasSoloedTracks = this.hasSoloedTracks(); const hasSoloedTracks = this.hasSoloedTracks();
if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) { if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) {
audioBus.setLiveMidiPitchBend(midiPitchBendToNormalized(pitchBend.getValue())); audioBus.scheduleLiveMidiPitchBend(midiPitchBendToNormalized(value), time);
} }
}, this.beatsToToneTime(absoluteBeat)); }, this.beatsToToneTime(beat));
this.scheduledEvents.add(eventId); this.scheduledEvents.add(eventId);
}); });
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest'; import { describe, it, expect } from 'vitest';
import { encodeWav, getOfflineTrackGain, getOfflineTrackVolumeDb } from './KGOfflineRenderer'; import { applyOfflinePitchBendAutomation, encodeWav, getOfflineTrackGain, getOfflineTrackVolumeDb } from './KGOfflineRenderer';
/** /**
* Create a minimal AudioBuffer-like object for testing. * Create a minimal AudioBuffer-like object for testing.
@@ -164,3 +164,32 @@ describe('offline track volume conversion', () => {
expect(getOfflineTrackGain(-60, false)).toBe(0); expect(getOfflineTrackGain(-60, false)).toBe(0);
}); });
}); });
describe('offline pitch bend automation', () => {
it('applies baked pitch bends to source playback rate automation', () => {
const source = {
playbackRate: {
value: 1,
setValueAtTime: (..._args: unknown[]) => undefined,
},
} as unknown as Parameters<typeof applyOfflinePitchBendAutomation>[0];
const calls: Array<[number, number]> = [];
source.playbackRate.setValueAtTime = ((value: number, time: number) => {
calls.push([value, time]);
return source.playbackRate as never;
}) as typeof source.playbackRate.setValueAtTime;
applyOfflinePitchBendAutomation(
source,
1,
[{ beat: 1, value: 0 }],
0,
0.5
);
expect(calls).toHaveLength(1);
expect(calls[0][1]).toBe(0.5);
expect(calls[0][0]).toBeCloseTo(Math.pow(2, -2 / 12), 5);
});
});
+137 -9
View File
@@ -1,13 +1,24 @@
import * as Tone from 'tone'; import * as Tone from 'tone';
import type { KGProject } from '../KGProject'; import type { KGProject } from '../KGProject';
import type { KGMidiNote } from '../midi/KGMidiNote'; import type { KGMidiNote } from '../midi/KGMidiNote';
import type { KGMidiPitchBend } from '../midi/KGMidiPitchBend';
import type { KGAudioRegion } from '../region/KGAudioRegion'; import type { KGAudioRegion } from '../region/KGAudioRegion';
import type { InstrumentType } from '../track/KGMidiTrack';
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants'; import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
import { AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants'; import { AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants';
import { pitchToNoteNameString } from '../../util/midiUtil'; import { MIDI_PITCH_BEND_CENTER, midiPitchBendToNormalized } from '../../util/midiUtil';
import {
bakeMidiAutomationPointsInWindow,
collectRegionMidiAutomationPoints,
resolveMidiAutomationValueAtBeat,
type BakedMidiAutomationPoint,
type MidiAutomationPoint,
} from '../../util/midiAutomationUtil';
import { KGToneBuffersPool } from './KGToneBuffersPool'; import { KGToneBuffersPool } from './KGToneBuffersPool';
import { KGToneSamplerFactory } from './KGToneSamplerFactory'; import { KGToneSamplerFactory } from './KGToneSamplerFactory';
import { KGAudioInterface } from './KGAudioInterface'; import { KGAudioInterface } from './KGAudioInterface';
import { KGAudioBus } from './KGAudioBus';
import { ConfigManager } from '../config/ConfigManager';
import { Mp3Encoder } from '@breezystack/lamejs'; import { Mp3Encoder } from '@breezystack/lamejs';
export interface RenderOptions { export interface RenderOptions {
@@ -104,7 +115,7 @@ export class KGOfflineRenderer {
// Pre-collect all the data we need before entering the offline context // Pre-collect all the data we need before entering the offline context
const midiTrackData: Array<{ const midiTrackData: Array<{
trackId: string; trackId: string;
instrumentName: string; instrumentName: InstrumentType;
volume: number; volume: number;
muted: boolean; muted: boolean;
solo: boolean; solo: boolean;
@@ -112,6 +123,7 @@ export class KGOfflineRenderer {
startBeat: number; startBeat: number;
notes: Array<{ startBeat: number; endBeat: number; durationBeats: number; pitch: number; velocity: number }>; notes: Array<{ startBeat: number; endBeat: number; durationBeats: number; pitch: number; velocity: number }>;
}>; }>;
pitchBends: MidiAutomationPoint[];
}> = []; }> = [];
const audioTrackData: Array<{ const audioTrackData: Array<{
@@ -130,13 +142,14 @@ export class KGOfflineRenderer {
}> = []; }> = [];
let hasSoloedTracks = false; let hasSoloedTracks = false;
const interpolationIntervalMs = (ConfigManager.instance().get('audio.midi_automation_interpolation_interval_ms') as number) ?? 10;
for (const track of tracks) { for (const track of tracks) {
const trackId = track.getId().toString(); const trackId = track.getId().toString();
if (track.getType() === 'MIDI') { if (track.getType() === 'MIDI') {
const midiTrack = track as unknown as { getInstrument: () => string }; const midiTrack = track as unknown as { getInstrument: () => InstrumentType };
const instrumentName = String(midiTrack.getInstrument()); const instrumentName = midiTrack.getInstrument();
// Get live bus state for volume/mute/solo via public getters // Get live bus state for volume/mute/solo via public getters
const volume = audioInterface.getTrackVolume(trackId); const volume = audioInterface.getTrackVolume(trackId);
@@ -147,7 +160,7 @@ export class KGOfflineRenderer {
const regions: typeof midiTrackData[0]['regions'] = []; const regions: typeof midiTrackData[0]['regions'] = [];
for (const region of track.getRegions()) { for (const region of track.getRegions()) {
if (region.getCurrentType() === 'KGMidiRegion') { if (region.getCurrentType() === 'KGMidiRegion') {
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[] }; const midiRegion = region as unknown as { getNotes: () => KGMidiNote[]; getPitchBends: () => KGMidiPitchBend[] };
if (midiRegion.getNotes) { if (midiRegion.getNotes) {
const notes = midiRegion.getNotes().map(note => ({ const notes = midiRegion.getNotes().map(note => ({
startBeat: note.getStartBeat() + region.getStartFromBeat(), startBeat: note.getStartBeat() + region.getStartFromBeat(),
@@ -160,8 +173,22 @@ export class KGOfflineRenderer {
} }
} }
} }
const pitchBends = collectRegionMidiAutomationPoints(
track.getRegions()
.filter(region => region.getCurrentType() === 'KGMidiRegion')
.map(region => {
const midiRegion = region as unknown as { getPitchBends: () => KGMidiPitchBend[] };
return {
startBeat: region.getStartFromBeat(),
points: midiRegion.getPitchBends().map(pitchBend => ({
beat: pitchBend.getBeat(),
value: pitchBend.getValue(),
})),
};
})
);
midiTrackData.push({ trackId, instrumentName, volume, muted, solo, regions }); midiTrackData.push({ trackId, instrumentName, volume, muted, solo, regions, pitchBends });
} else if (track.getType() === 'Wave') { } else if (track.getType() === 'Wave') {
const volume = audioInterface.getTrackVolume(trackId); const volume = audioInterface.getTrackVolume(trackId);
const muted = audioInterface.getTrackMuted(trackId); const muted = audioInterface.getTrackMuted(trackId);
@@ -241,7 +268,7 @@ export class KGOfflineRenderer {
const promise = (async () => { const promise = (async () => {
try { try {
// Get cached buffers from pool // Get cached buffers from pool
const audioBuffers = await KGToneBuffersPool.instance().getToneAudioBuffers(trackInfo.instrumentName); const audioBuffers = await KGToneBuffersPool.instance().getToneAudioBuffers(String(trackInfo.instrumentName));
const pitchRange = FLUIDR3_INSTRUMENT_MAP[trackInfo.instrumentName]?.pitchRange || [21, 108]; const pitchRange = FLUIDR3_INSTRUMENT_MAP[trackInfo.instrumentName]?.pitchRange || [21, 108];
const urlMap = KGToneSamplerFactory.instance().convertBuffersToUrls(audioBuffers, pitchRange); const urlMap = KGToneSamplerFactory.instance().convertBuffersToUrls(audioBuffers, pitchRange);
@@ -258,6 +285,16 @@ export class KGOfflineRenderer {
// Track volumes are stored in dB across the app, with 0 meaning unity gain. // Track volumes are stored in dB across the app, with 0 meaning unity gain.
sampler.volume.value = getOfflineTrackVolumeDb(trackInfo.volume, trackInfo.muted); sampler.volume.value = getOfflineTrackVolumeDb(trackInfo.volume, trackInfo.muted);
sampler.connect(masterGain); sampler.connect(masterGain);
const bakedTrackPitchBends = bakeMidiAutomationPointsInWindow(
trackInfo.pitchBends,
renderStartBeat,
renderEndBeat,
{
maxIntervalMs: interpolationIntervalMs,
bpm: project.getBpm(),
defaultValue: MIDI_PITCH_BEND_CENTER,
}
);
// Schedule all notes for this track // Schedule all notes for this track
for (const regionInfo of trackInfo.regions) { for (const regionInfo of trackInfo.regions) {
@@ -268,11 +305,32 @@ export class KGOfflineRenderer {
const offsetBeat = note.startBeat - renderStartBeat; const offsetBeat = note.startBeat - renderStartBeat;
const noteStartTime = offsetBeat * secondsPerBeat; const noteStartTime = offsetBeat * secondsPerBeat;
const noteDuration = note.durationBeats * secondsPerBeat; const noteDuration = note.durationBeats * secondsPerBeat;
const noteName = pitchToNoteNameString(note.pitch);
const velocity = note.velocity / 127; const velocity = note.velocity / 127;
const initialNormalizedPitchBend = midiPitchBendToNormalized(
resolveMidiAutomationValueAtBeat(trackInfo.pitchBends, note.startBeat, MIDI_PITCH_BEND_CENTER)
);
const offlineSource = createOfflinePitchBendAwareSource(
sampler,
audioBuffers,
trackInfo.instrumentName,
note.pitch,
initialNormalizedPitchBend
);
if (!offlineSource) {
continue;
}
const { source, basePlaybackRate } = offlineSource;
applyOfflinePitchBendAutomation(
source,
basePlaybackRate,
bakedTrackPitchBends.filter(point => point.beat > note.startBeat && point.beat < note.endBeat),
renderStartBeat,
secondsPerBeat
);
context.transport.schedule((time) => { context.transport.schedule((time) => {
sampler.triggerAttackRelease(noteName, noteDuration, time, velocity); source.start(time, 0, noteDuration, velocity);
}, noteStartTime); }, noteStartTime);
} }
} }
@@ -418,6 +476,76 @@ function shouldPlay(trackInfo: { muted: boolean; solo: boolean }, hasSoloedTrack
return true; return true;
} }
function setOfflinePlaybackRate(
source: Tone.ToneBufferSource,
value: number,
time: number
): void {
const playbackRate = source.playbackRate as unknown as {
value: number;
setValueAtTime?: (nextValue: number, nextTime: number) => void;
};
if (typeof playbackRate.setValueAtTime === 'function') {
playbackRate.setValueAtTime(value, time);
return;
}
playbackRate.value = value;
}
function createOfflinePitchBendAwareSource(
sampler: Tone.Sampler,
audioBuffers: Tone.ToneAudioBuffers,
instrumentName: InstrumentType,
pitch: number,
initialNormalizedPitchBend: number
): { source: Tone.ToneBufferSource; basePlaybackRate: number } | null {
const closestPitch = KGAudioBus.findClosestBufferedPitch(instrumentName, audioBuffers, pitch);
if (closestPitch === null) {
return null;
}
const bufferKey = KGAudioBus.midiPitchToBufferKey(closestPitch);
const buffer = audioBuffers.get(bufferKey);
if (!buffer) {
return null;
}
const basePlaybackRate = Math.pow(2, (pitch - closestPitch) / 12);
const source = new Tone.ToneBufferSource({
url: buffer,
fadeIn: sampler.attack,
fadeOut: sampler.release,
curve: sampler.curve,
playbackRate: KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, initialNormalizedPitchBend),
}).connect(sampler.output);
setOfflinePlaybackRate(
source,
KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, initialNormalizedPitchBend),
0
);
return { source, basePlaybackRate };
}
export function applyOfflinePitchBendAutomation(
source: Tone.ToneBufferSource,
basePlaybackRate: number,
bakedPitchBends: BakedMidiAutomationPoint[],
renderStartBeat: number,
secondsPerBeat: number
): void {
bakedPitchBends.forEach(point => {
const automationTime = (point.beat - renderStartBeat) * secondsPerBeat;
setOfflinePlaybackRate(
source,
KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, midiPitchBendToNormalized(point.value)),
automationTime
);
});
}
export function getOfflineTrackVolumeDb(volumeDb: number, muted: boolean): number { export function getOfflineTrackVolumeDb(volumeDb: number, muted: boolean): number {
const isSilent = muted || volumeDb <= AUDIO_INTERFACE_CONSTANTS.MIN_TRACK_VOLUME_DB; const isSilent = muted || volumeDb <= AUDIO_INTERFACE_CONSTANTS.MIN_TRACK_VOLUME_DB;
return isSilent ? -Infinity : volumeDb; return isSilent ? -Infinity : volumeDb;
+2
View File
@@ -79,6 +79,7 @@ interface AppConfig {
audio: { audio: {
enable_audio_capture_for_screen_sharing: boolean; enable_audio_capture_for_screen_sharing: boolean;
lookahead_time: number; lookahead_time: number;
midi_automation_interpolation_interval_ms: number;
playback_delay: number; playback_delay: number;
recording_offset: number; recording_offset: number;
}; };
@@ -253,6 +254,7 @@ export class ConfigManager {
audio: { audio: {
enable_audio_capture_for_screen_sharing: false, enable_audio_capture_for_screen_sharing: false,
lookahead_time: 0.05, lookahead_time: 0.05,
midi_automation_interpolation_interval_ms: 10,
playback_delay: 0.2, playback_delay: 0.2,
recording_offset: 0 recording_offset: 0
}, },
+78
View File
@@ -31,6 +31,8 @@ const mockCore = {
getStatus: () => 'Ready', getStatus: () => 'Ready',
getPlayheadPosition: () => 0, getPlayheadPosition: () => 0,
getIsPlaying: () => false, getIsPlaying: () => false,
startPlaying: vi.fn().mockResolvedValue(undefined),
stopPlaying: vi.fn().mockResolvedValue(undefined),
}; };
vi.mock('../core/KGCore', () => ({ vi.mock('../core/KGCore', () => ({
@@ -51,6 +53,10 @@ vi.mock('../core/config/ConfigManager', () => ({
describe('projectStore piano roll state', () => { describe('projectStore piano roll state', () => {
beforeEach(() => { beforeEach(() => {
vi.resetModules(); vi.resetModules();
mockCore.startPlaying.mockReset();
mockCore.startPlaying.mockResolvedValue(undefined);
mockCore.stopPlaying.mockReset();
mockCore.stopPlaying.mockResolvedValue(undefined);
}); });
it('clears hybrid state when opening a MIDI region', async () => { it('clears hybrid state when opening a MIDI region', async () => {
@@ -75,4 +81,76 @@ describe('projectStore piano roll state', () => {
expect(state.activeRegionId).toBe('midi-b'); expect(state.activeRegionId).toBe('midi-b');
expect(state.hybridAudioRegionId).toBeNull(); expect(state.hybridAudioRegionId).toBeNull();
}); });
it('tracks playback preparation around startPlaying success', async () => {
let resolveStart: (() => void) | null = null;
mockCore.startPlaying.mockImplementationOnce(() => new Promise<void>((resolve) => {
resolveStart = resolve;
}));
const { useProjectStore } = await import('./projectStore');
const startPromise = useProjectStore.getState().startPlaying();
expect(useProjectStore.getState().isPreparingPlayback).toBe(true);
await act(async () => {
resolveStart?.();
await startPromise;
});
const state = useProjectStore.getState();
expect(state.isPreparingPlayback).toBe(false);
expect(state.isPlaying).toBe(true);
});
it('clears playback preparation if startPlaying fails', async () => {
mockCore.startPlaying.mockRejectedValueOnce(new Error('prepare failed'));
const { useProjectStore } = await import('./projectStore');
await expect(useProjectStore.getState().startPlaying()).rejects.toThrow('prepare failed');
expect(useProjectStore.getState().isPreparingPlayback).toBe(false);
});
it('ignores repeated startPlaying calls while already preparing', async () => {
let resolveStart: (() => void) | null = null;
mockCore.startPlaying.mockImplementationOnce(() => new Promise<void>((resolve) => {
resolveStart = resolve;
}));
const { useProjectStore } = await import('./projectStore');
const firstStart = useProjectStore.getState().startPlaying();
const secondStart = useProjectStore.getState().startPlaying();
expect(mockCore.startPlaying).toHaveBeenCalledTimes(1);
await act(async () => {
resolveStart?.();
await Promise.all([firstStart, secondStart]);
});
});
it('clears playback preparation when stopTransport is called during prepare', async () => {
let resolveStart: (() => void) | null = null;
mockCore.startPlaying.mockImplementationOnce(() => new Promise<void>((resolve) => {
resolveStart = resolve;
}));
const { useProjectStore } = await import('./projectStore');
const startPromise = useProjectStore.getState().startPlaying();
expect(useProjectStore.getState().isPreparingPlayback).toBe(true);
await act(async () => {
await useProjectStore.getState().stopTransport();
});
expect(useProjectStore.getState().isPreparingPlayback).toBe(false);
await act(async () => {
resolveStart?.();
await startPromise;
});
});
}); });
+40 -10
View File
@@ -68,6 +68,7 @@ interface ProjectState {
loopingRange: [number, number]; // [startBar, endBar] - bar indices (0-based) loopingRange: [number, number]; // [startBar, endBar] - bar indices (0-based)
playheadPosition: number; // in beats playheadPosition: number; // in beats
isPlaying: boolean; isPlaying: boolean;
isPreparingPlayback: boolean;
autoScrollEnabled: boolean; autoScrollEnabled: boolean;
currentTime: string; // formatted time string currentTime: string; // formatted time string
@@ -341,6 +342,7 @@ export const useProjectStore = create<ProjectState>((set, get) => {
loopingRange: currentProject.getLoopingRange(), loopingRange: currentProject.getLoopingRange(),
playheadPosition: KGCore.instance().getPlayheadPosition(), playheadPosition: KGCore.instance().getPlayheadPosition(),
isPlaying: KGCore.instance().getIsPlaying(), isPlaying: KGCore.instance().getIsPlaying(),
isPreparingPlayback: false,
autoScrollEnabled: true, autoScrollEnabled: true,
currentTime: beatsToTimeString(KGCore.instance().getPlayheadPosition(), currentProject.getBpm(), currentProject.getTimeSignature()), currentTime: beatsToTimeString(KGCore.instance().getPlayheadPosition(), currentProject.getBpm(), currentProject.getTimeSignature()),
@@ -854,13 +856,26 @@ export const useProjectStore = create<ProjectState>((set, get) => {
}, },
startPlaying: async () => { startPlaying: async () => {
await KGCore.instance().startPlaying(); if (get().isPreparingPlayback) {
set({ isPlaying: true, autoScrollEnabled: true }); return;
}
set({ isPreparingPlayback: true });
try {
await KGCore.instance().startPlaying();
set({ isPlaying: true, autoScrollEnabled: true });
} finally {
set({ isPreparingPlayback: false });
}
}, },
stopPlaying: async () => { stopPlaying: async () => {
await KGCore.instance().stopPlaying(); try {
set({ isPlaying: false }); await KGCore.instance().stopPlaying();
set({ isPlaying: false });
} finally {
set({ isPreparingPlayback: false });
}
}, },
stopTransport: async () => { stopTransport: async () => {
@@ -868,7 +883,11 @@ export const useProjectStore = create<ProjectState>((set, get) => {
await get().stopRecording(); await get().stopRecording();
return; return;
} }
await get().stopPlaying(); try {
await get().stopPlaying();
} finally {
set({ isPreparingPlayback: false });
}
}, },
startRecording: async () => { startRecording: async () => {
@@ -944,10 +963,15 @@ export const useProjectStore = create<ProjectState>((set, get) => {
: playheadPosition - timeSignature.numerator; : playheadPosition - timeSignature.numerator;
setPlayheadPosition(recordingStartBeat); setPlayheadPosition(recordingStartBeat);
await KGCore.instance().startPlaying({ set({ isPreparingPlayback: true });
preserveLoopPreroll: projectLooping, try {
}); await KGCore.instance().startPlaying({
set({ isPlaying: true, autoScrollEnabled: true }); preserveLoopPreroll: projectLooping,
});
set({ isPlaying: true, autoScrollEnabled: true });
} finally {
set({ isPreparingPlayback: false });
}
}, },
stopRecording: async () => { stopRecording: async () => {
@@ -1002,7 +1026,13 @@ export const useProjectStore = create<ProjectState>((set, get) => {
await stopPlaying(); await stopPlaying();
setPlayheadPosition(recordingOriginalPlayhead); setPlayheadPosition(recordingOriginalPlayhead);
set({ isRecording: false, recordingNotes: [], recordingPitchBends: [], recordingTargetRegionId: null }); set({
isRecording: false,
isPreparingPlayback: false,
recordingNotes: [],
recordingPitchBends: [],
recordingTargetRegionId: null
});
_lastRecordedPitchBendValue = null; _lastRecordedPitchBendValue = null;
}, },
+3 -3
View File
@@ -23,7 +23,7 @@ export const mockSampler = {
}; };
export const mockBufferSource = { export const mockBufferSource = {
playbackRate: { value: 1 }, playbackRate: { value: 1, setValueAtTime: vi.fn() },
connect: vi.fn(), connect: vi.fn(),
start: vi.fn(), start: vi.fn(),
stop: vi.fn(), stop: vi.fn(),
@@ -50,7 +50,7 @@ export const mockTone = {
BufferSource: vi.fn().mockImplementation(() => { BufferSource: vi.fn().mockImplementation(() => {
const instance = { const instance = {
...mockBufferSource, ...mockBufferSource,
playbackRate: { value: 1 }, playbackRate: { value: 1, setValueAtTime: vi.fn() },
}; };
instance.connect.mockImplementation(() => instance); instance.connect.mockImplementation(() => instance);
instance.start.mockImplementation(() => instance); instance.start.mockImplementation(() => instance);
@@ -60,7 +60,7 @@ export const mockTone = {
ToneBufferSource: vi.fn().mockImplementation(() => { ToneBufferSource: vi.fn().mockImplementation(() => {
const instance = { const instance = {
...mockBufferSource, ...mockBufferSource,
playbackRate: { value: 1 }, playbackRate: { value: 1, setValueAtTime: vi.fn() },
}; };
instance.connect.mockImplementation(() => instance); instance.connect.mockImplementation(() => instance);
instance.start.mockImplementation(() => instance); instance.start.mockImplementation(() => instance);
+1
View File
@@ -28,6 +28,7 @@ export const MockBufferSource = vi.fn().mockImplementation((options?: { playback
const instance = { const instance = {
playbackRate: { playbackRate: {
value: options?.playbackRate ?? 1, value: options?.playbackRate ?? 1,
setValueAtTime: vi.fn(),
}, },
connect: vi.fn(() => instance), connect: vi.fn(() => instance),
start: vi.fn(() => instance), start: vi.fn(() => instance),
+127
View File
@@ -0,0 +1,127 @@
import { describe, expect, it } from 'vitest';
import {
bakeMidiAutomationPointsInWindow,
collectRegionMidiAutomationPoints,
normalizeMidiAutomationPoints,
resolveMidiAutomationValueAtBeat,
type MidiAutomationPoint,
} from './midiAutomationUtil';
const defaultOptions = {
maxIntervalMs: 10,
bpm: 120,
defaultValue: 8192,
};
describe('midiAutomationUtil', () => {
it('normalizes points and keeps the last duplicate beat', () => {
const points: MidiAutomationPoint[] = [
{ beat: 2, value: 1000 },
{ beat: 1, value: 2000 },
{ beat: 2, value: 3000 },
];
expect(normalizeMidiAutomationPoints(points)).toEqual([
{ beat: 1, value: 2000 },
{ beat: 2, value: 3000 },
]);
});
it('holds the default value before the first point', () => {
const points = [{ beat: 4, value: 0 }];
expect(resolveMidiAutomationValueAtBeat(points, 2, 8192)).toBe(8192);
});
it('interpolates linearly between two points', () => {
const points = [
{ beat: 0, value: 8192 },
{ beat: 4, value: 0 },
];
expect(resolveMidiAutomationValueAtBeat(points, 2, 8192)).toBe(4096);
});
it('holds the last value after the final point', () => {
const points = [{ beat: 1, value: 2048 }];
expect(resolveMidiAutomationValueAtBeat(points, 8, 8192)).toBe(2048);
});
it('collects region-local points into absolute beats', () => {
const collected = collectRegionMidiAutomationPoints([
{ startBeat: 4, points: [{ beat: 0.5, value: 7000 }] },
{ startBeat: 1, points: [{ beat: 0.25, value: 6000 }] },
]);
expect(collected).toEqual([
{ beat: 1.25, value: 6000 },
{ beat: 4.5, value: 7000 },
]);
});
it('bakes dense points according to the requested ms interval', () => {
const points = [
{ beat: 0, value: 8192 },
{ beat: 1, value: 0 },
];
expect(bakeMidiAutomationPointsInWindow(points, 0, 1, { ...defaultOptions, maxIntervalMs: 20 })).toHaveLength(25);
expect(bakeMidiAutomationPointsInWindow(points, 0, 1, { ...defaultOptions, maxIntervalMs: 10 })).toHaveLength(50);
expect(bakeMidiAutomationPointsInWindow(points, 0, 1, { ...defaultOptions, maxIntervalMs: 5 })).toHaveLength(100);
});
it('treats flat segments as holds without adding interior baked points', () => {
const points = [
{ beat: 0, value: 4096 },
{ beat: 4, value: 4096 },
];
expect(bakeMidiAutomationPointsInWindow(points, 0, 4, { ...defaultOptions, maxIntervalMs: 10 })).toEqual([
{ beat: 0, value: 4096 },
]);
});
it('collapses consecutive baked points with the same value', () => {
const points = [
{ beat: 0.5, value: 8192 },
{ beat: 1, value: 4096 },
{ beat: 2, value: 4096 },
{ beat: 3, value: 0 },
];
expect(bakeMidiAutomationPointsInWindow(points, 0, 4, { ...defaultOptions, maxIntervalMs: 500 })).toEqual([
{ beat: 0, value: 8192 },
{ beat: 1, value: 4096 },
{ beat: 3, value: 0 },
]);
});
it('keeps later changing segments after skipping a flat segment', () => {
const points = [
{ beat: 0, value: 4096 },
{ beat: 2, value: 4096 },
{ beat: 4, value: 0 },
];
const baked = bakeMidiAutomationPointsInWindow(points, 0, 4, { ...defaultOptions, maxIntervalMs: 500 });
expect(baked).toEqual([
{ beat: 0, value: 4096 },
{ beat: 3, value: 2048 },
]);
});
it('adds a window anchor and preserves the correct loop boundary value', () => {
const points = [
{ beat: 2, value: 0 },
{ beat: 6, value: 8192 },
];
const baked = bakeMidiAutomationPointsInWindow(points, 4, 8, { ...defaultOptions, maxIntervalMs: 500 });
expect(baked[0]).toEqual({ beat: 4, value: 4096 });
expect(baked.some(point => point.beat === 5 && point.value === 6144)).toBe(true);
expect(baked.some(point => point.beat === 6 && point.value === 8192)).toBe(true);
});
});
+171
View File
@@ -0,0 +1,171 @@
export interface MidiAutomationPoint {
beat: number;
value: number;
}
export interface BakedMidiAutomationPoint {
beat: number;
value: number;
}
export interface MidiAutomationBakeOptions {
maxIntervalMs: number;
bpm: number;
defaultValue: number;
}
const BEAT_EPSILON = 1e-9;
function appendPoint(points: BakedMidiAutomationPoint[], nextPoint: BakedMidiAutomationPoint): void {
const lastPoint = points[points.length - 1];
if (!lastPoint) {
points.push(nextPoint);
return;
}
if (Math.abs(lastPoint.beat - nextPoint.beat) <= BEAT_EPSILON) {
points[points.length - 1] = nextPoint;
return;
}
if (lastPoint.value === nextPoint.value) {
return;
}
points.push(nextPoint);
}
function interpolateBetweenPoints(startPoint: MidiAutomationPoint, endPoint: MidiAutomationPoint, beat: number): number {
if (Math.abs(endPoint.beat - startPoint.beat) <= BEAT_EPSILON) {
return endPoint.value;
}
const progress = (beat - startPoint.beat) / (endPoint.beat - startPoint.beat);
return startPoint.value + ((endPoint.value - startPoint.value) * progress);
}
function getMaxIntervalBeats(options: MidiAutomationBakeOptions): number {
if (!Number.isFinite(options.maxIntervalMs) || options.maxIntervalMs <= 0) {
return Number.POSITIVE_INFINITY;
}
return (options.maxIntervalMs / 1000) * (options.bpm / 60);
}
export function normalizeMidiAutomationPoints(points: MidiAutomationPoint[]): MidiAutomationPoint[] {
const sortedPoints = [...points]
.filter(point => Number.isFinite(point.beat) && Number.isFinite(point.value))
.sort((a, b) => a.beat - b.beat);
const normalizedPoints: MidiAutomationPoint[] = [];
sortedPoints.forEach(point => {
const lastPoint = normalizedPoints[normalizedPoints.length - 1];
if (lastPoint && Math.abs(lastPoint.beat - point.beat) <= BEAT_EPSILON) {
normalizedPoints[normalizedPoints.length - 1] = point;
return;
}
normalizedPoints.push(point);
});
return normalizedPoints;
}
export function collectRegionMidiAutomationPoints(
regions: Array<{ startBeat: number; points: MidiAutomationPoint[] }>
): MidiAutomationPoint[] {
return normalizeMidiAutomationPoints(
regions.flatMap(region => region.points.map(point => ({
beat: region.startBeat + point.beat,
value: point.value,
})))
);
}
export function resolveMidiAutomationValueAtBeat(
points: MidiAutomationPoint[],
beat: number,
defaultValue: number
): number {
const normalizedPoints = normalizeMidiAutomationPoints(points);
if (normalizedPoints.length === 0) {
return defaultValue;
}
let previousPoint: MidiAutomationPoint | null = null;
for (const point of normalizedPoints) {
if (beat < point.beat) {
if (!previousPoint) {
return defaultValue;
}
return interpolateBetweenPoints(previousPoint, point, beat);
}
previousPoint = point;
}
return previousPoint?.value ?? defaultValue;
}
export function bakeMidiAutomationPointsInWindow(
points: MidiAutomationPoint[],
windowStartBeat: number,
windowEndBeat: number,
options: MidiAutomationBakeOptions
): BakedMidiAutomationPoint[] {
const normalizedPoints = normalizeMidiAutomationPoints(points);
const anchorPoint = {
beat: windowStartBeat,
value: resolveMidiAutomationValueAtBeat(normalizedPoints, windowStartBeat, options.defaultValue),
};
if (windowEndBeat <= windowStartBeat) {
return [anchorPoint];
}
const bakedPoints: BakedMidiAutomationPoint[] = [anchorPoint];
if (normalizedPoints.length === 0) {
return bakedPoints;
}
const firstPoint = normalizedPoints[0];
if (windowStartBeat < firstPoint.beat && firstPoint.beat < windowEndBeat) {
appendPoint(bakedPoints, { beat: firstPoint.beat, value: firstPoint.value });
}
const maxIntervalBeats = getMaxIntervalBeats(options);
for (let index = 0; index < normalizedPoints.length - 1; index += 1) {
const startPoint = normalizedPoints[index];
const endPoint = normalizedPoints[index + 1];
const overlapStartBeat = Math.max(windowStartBeat, startPoint.beat);
const overlapEndBeat = Math.min(windowEndBeat, endPoint.beat);
if (overlapEndBeat - overlapStartBeat <= BEAT_EPSILON) {
continue;
}
if (startPoint.value === endPoint.value) {
continue;
}
const segmentLengthBeats = overlapEndBeat - overlapStartBeat;
const segmentCount = Number.isFinite(maxIntervalBeats)
? Math.max(1, Math.ceil(segmentLengthBeats / maxIntervalBeats))
: 1;
for (let segmentIndex = 1; segmentIndex <= segmentCount; segmentIndex += 1) {
const beat = overlapStartBeat + ((segmentLengthBeats * segmentIndex) / segmentCount);
if (beat >= windowEndBeat - BEAT_EPSILON) {
continue;
}
appendPoint(bakedPoints, {
beat,
value: interpolateBetweenPoints(startPoint, endPoint, beat),
});
}
}
return bakedPoints;
}