feat: added pitch bend automation (linear interpolation)
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
import { act, render, screen } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
let mockState = { isPreparingPlayback: false };
|
||||
|
||||
vi.mock('./stores/projectStore', () => ({
|
||||
useProjectStore: (selector?: (state: typeof mockState) => unknown) => (
|
||||
selector ? selector(mockState) : mockState
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock('./hooks/useGlobalKeyboardHandler', () => ({
|
||||
useGlobalKeyboardHandler: () => undefined,
|
||||
}));
|
||||
|
||||
vi.mock('./components/Toolbar', () => ({ default: () => null }));
|
||||
vi.mock('./components/StatusBar', () => ({ default: () => null }));
|
||||
vi.mock('./components/MainContent', () => ({ default: () => null }));
|
||||
vi.mock('./components/InstrumentSelection', () => ({ default: () => null }));
|
||||
vi.mock('./components/ChatBox', () => ({ default: () => null }));
|
||||
vi.mock('./components/KGOnePanel', () => ({ default: () => null }));
|
||||
vi.mock('./components/ListEventPanel', () => ({ default: () => null }));
|
||||
vi.mock('./components/settings', () => ({ SettingsPanel: () => null }));
|
||||
vi.mock('./core/audio-interface/KGToneBuffersPool', () => ({
|
||||
KGToneBuffersPool: {
|
||||
instance: () => ({
|
||||
getActiveLoadCount: () => 0,
|
||||
addLoadingListener: () => undefined,
|
||||
removeLoadingListener: () => undefined,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
vi.mock('./core/audio-interface/KGOfflineRenderer', () => ({
|
||||
KGOfflineRenderer: {
|
||||
instance: () => ({
|
||||
addRenderingListener: () => undefined,
|
||||
removeRenderingListener: () => undefined,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
vi.mock('./core/KGCore', () => ({
|
||||
KGCore: {
|
||||
instance: () => ({
|
||||
getIsMigrating: () => false,
|
||||
setMigrationStateChangeCallback: () => undefined,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
import { PlaybackPreparationOverlayContainer } from './App';
|
||||
|
||||
describe('PlaybackPreparationOverlayContainer', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
mockState = { isPreparingPlayback: false };
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('does not appear if preparation finishes before the delay', () => {
|
||||
const { rerender } = render(<PlaybackPreparationOverlayContainer />);
|
||||
|
||||
mockState = { isPreparingPlayback: true };
|
||||
rerender(<PlaybackPreparationOverlayContainer />);
|
||||
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(100);
|
||||
});
|
||||
|
||||
mockState = { isPreparingPlayback: false };
|
||||
rerender(<PlaybackPreparationOverlayContainer />);
|
||||
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(100);
|
||||
});
|
||||
|
||||
expect(screen.queryByText('Preparing playback...')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('appears after 150ms while preparation is still in progress', () => {
|
||||
const { rerender } = render(<PlaybackPreparationOverlayContainer />);
|
||||
|
||||
mockState = { isPreparingPlayback: true };
|
||||
rerender(<PlaybackPreparationOverlayContainer />);
|
||||
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(149);
|
||||
});
|
||||
expect(screen.queryByText('Preparing playback...')).not.toBeInTheDocument();
|
||||
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(1);
|
||||
});
|
||||
expect(screen.getByText('Preparing playback...')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('hides once preparation completes after becoming visible', () => {
|
||||
const { rerender } = render(<PlaybackPreparationOverlayContainer />);
|
||||
|
||||
mockState = { isPreparingPlayback: true };
|
||||
rerender(<PlaybackPreparationOverlayContainer />);
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(150);
|
||||
});
|
||||
expect(screen.getByText('Preparing playback...')).toBeInTheDocument();
|
||||
|
||||
mockState = { isPreparingPlayback: false };
|
||||
rerender(<PlaybackPreparationOverlayContainer />);
|
||||
|
||||
expect(screen.queryByText('Preparing playback...')).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
+42
@@ -185,6 +185,9 @@ function App() {
|
||||
|
||||
{/* Bounce/Render Overlay */}
|
||||
<BounceOverlayContainer />
|
||||
|
||||
{/* Playback Preparation Overlay */}
|
||||
<PlaybackPreparationOverlayContainer />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -295,3 +298,42 @@ const BounceOverlayContainer: React.FC = () => {
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const PLAYBACK_PREPARATION_OVERLAY_DELAY_MS = 150;
|
||||
|
||||
export const PlaybackPreparationOverlayContainer: React.FC = () => {
|
||||
const isPreparingPlayback = useProjectStore(state => state.isPreparingPlayback);
|
||||
const [visible, setVisible] = useState<boolean>(false);
|
||||
const timeoutRef = useRef<number | null>(null);
|
||||
|
||||
useEffectReact(() => {
|
||||
if (isPreparingPlayback) {
|
||||
if (timeoutRef.current === null) {
|
||||
timeoutRef.current = window.setTimeout(() => {
|
||||
setVisible(true);
|
||||
timeoutRef.current = null;
|
||||
}, PLAYBACK_PREPARATION_OVERLAY_DELAY_MS);
|
||||
}
|
||||
} else {
|
||||
if (timeoutRef.current !== null) {
|
||||
clearTimeout(timeoutRef.current);
|
||||
timeoutRef.current = null;
|
||||
}
|
||||
setVisible(false);
|
||||
}
|
||||
|
||||
return () => {
|
||||
if (timeoutRef.current !== null) {
|
||||
clearTimeout(timeoutRef.current);
|
||||
timeoutRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [isPreparingPlayback]);
|
||||
|
||||
return (
|
||||
<LoadingOverlay
|
||||
visible={visible}
|
||||
message="Preparing playback..."
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -42,7 +42,7 @@ const Toolbar: React.FC = () => {
|
||||
projectName, setProjectName,
|
||||
savedProjectName, setSavedProjectName,
|
||||
bpm, timeSignature, keySignature, setStatus,
|
||||
isPlaying, startPlaying, stopTransport, setPlayheadPosition,
|
||||
isPlaying, isPreparingPlayback, startPlaying, stopTransport, setPlayheadPosition,
|
||||
currentTime, setBpm, setTimeSignature, setKeySignature,
|
||||
maxBars, setMaxBars,
|
||||
barWidthMultiplier, setBarWidthMultiplier,
|
||||
@@ -1110,7 +1110,7 @@ const Toolbar: React.FC = () => {
|
||||
<div className="toolbar-separator"></div>
|
||||
<button title="Back to beginning" className="button-back-to-beginning" onClick={handleBackToBeginningClick}><FaStepBackward /></button>
|
||||
{!isPlaying ? (
|
||||
<button title="Play" className="button-play" onClick={handlePlayClick}><FaPlay /></button>
|
||||
<button title="Play" className="button-play" onClick={handlePlayClick} disabled={isPreparingPlayback}><FaPlay /></button>
|
||||
) : (
|
||||
<button title="Pause" className="button-pause" onClick={handlePauseClick}><FaPause /></button>
|
||||
)}
|
||||
|
||||
@@ -11,6 +11,7 @@ const BehaviorSettings: React.FC = () => {
|
||||
const [spectrogramHeightResolution, setSpectrogramHeightResolution] = useState<SpectrogramHeightResolution>(3);
|
||||
const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState<boolean>(true);
|
||||
const [audioLookaheadTime, setAudioLookaheadTime] = useState<string>('50');
|
||||
const [midiAutomationInterpolationIntervalMs, setMidiAutomationInterpolationIntervalMs] = useState<number>(10);
|
||||
const [playbackDelay, setPlaybackDelay] = useState<string>('200');
|
||||
const [recordingOffset, setRecordingOffset] = useState<string>('0');
|
||||
const [enableAudioCapture, setEnableAudioCapture] = useState<boolean>(false);
|
||||
@@ -34,6 +35,9 @@ const BehaviorSettings: React.FC = () => {
|
||||
setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true);
|
||||
const lookaheadTimeSeconds = (configManager.get('audio.lookahead_time') as number) ?? 0.05;
|
||||
setAudioLookaheadTime(((lookaheadTimeSeconds * 1000).toFixed(0)));
|
||||
setMidiAutomationInterpolationIntervalMs(
|
||||
(configManager.get('audio.midi_automation_interpolation_interval_ms') as number) ?? 10
|
||||
);
|
||||
const playbackDelaySeconds = (configManager.get('audio.playback_delay') as number) ?? 0.2;
|
||||
setPlaybackDelay(((playbackDelaySeconds * 1000).toFixed(0)));
|
||||
const recordingOffsetSeconds = (configManager.get('audio.recording_offset') as number) ?? 0;
|
||||
@@ -120,6 +124,12 @@ const BehaviorSettings: React.FC = () => {
|
||||
}
|
||||
};
|
||||
|
||||
const handleMidiAutomationInterpolationIntervalChange = async (value: string) => {
|
||||
const nextValue = parseInt(value, 10);
|
||||
setMidiAutomationInterpolationIntervalMs(nextValue);
|
||||
await configManager.set('audio.midi_automation_interpolation_interval_ms', nextValue);
|
||||
};
|
||||
|
||||
const handleRecordingOffsetChange = async (value: string) => {
|
||||
const numValueMs = value === '' ? 0 : parseFloat(value);
|
||||
const numValueSeconds = numValueMs / 1000;
|
||||
@@ -197,13 +207,13 @@ const BehaviorSettings: React.FC = () => {
|
||||
|
||||
<div className="settings-group">
|
||||
<h4>Chat Box</h4>
|
||||
|
||||
|
||||
<div className="settings-item">
|
||||
<label className="settings-label">
|
||||
Open at Start Up
|
||||
</label>
|
||||
<select
|
||||
className="settings-select"
|
||||
<select
|
||||
className="settings-select"
|
||||
value={chatboxDefaultOpen ? 'yes' : 'no'}
|
||||
onChange={(e) => handleChatboxDefaultOpenChange(e.target.value)}
|
||||
>
|
||||
@@ -270,6 +280,24 @@ const BehaviorSettings: React.FC = () => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="settings-item">
|
||||
<label className="settings-label">
|
||||
MIDI Automation Interpolation
|
||||
</label>
|
||||
<select
|
||||
className="settings-select"
|
||||
value={midiAutomationInterpolationIntervalMs}
|
||||
onChange={(e) => handleMidiAutomationInterpolationIntervalChange(e.target.value)}
|
||||
>
|
||||
<option value="20">Low-end (20 ms)</option>
|
||||
<option value="10">Balanced (10 ms)</option>
|
||||
<option value="5">High quality (5 ms)</option>
|
||||
</select>
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
|
||||
Controls how densely MIDI automation are baked for playback and bounce. Smaller intervals sound smoother but schedule more events.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="settings-item">
|
||||
<label className="settings-label">
|
||||
MIDI Input Latency (ms)
|
||||
@@ -301,8 +329,8 @@ const BehaviorSettings: React.FC = () => {
|
||||
<label className="settings-label">
|
||||
Capture Audio for Screen Sharing
|
||||
</label>
|
||||
<select
|
||||
className="settings-select"
|
||||
<select
|
||||
className="settings-select"
|
||||
value={enableAudioCapture ? 'yes' : 'no'}
|
||||
onChange={(e) => handleEnableAudioCaptureChange(e.target.value)}
|
||||
>
|
||||
@@ -310,7 +338,7 @@ const BehaviorSettings: React.FC = () => {
|
||||
<option value="yes">Yes</option>
|
||||
</select>
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
|
||||
Restart KGStudio (refresh the page) to take effect. Enable this option when KGStudio's audio cannot be captured during screen sharing in video calls (e.g., Zoom, Teams). This creates an additional audio stream that screen capture applications can detect.
|
||||
Restart KGStudio (refresh the page) to take effect. Enable this option when KGStudio's audio cannot be captured during screen sharing in video calls (e.g., Zoom, Teams). This creates an additional audio stream that screen capture applications can detect.
|
||||
<br />
|
||||
<b>It is important to make sure when screen sharing in Zoom, the "Share Sound" option is enabled.</b>
|
||||
</div>
|
||||
|
||||
@@ -66,4 +66,15 @@ describe('KGAudioBus live MIDI pitch bend', () => {
|
||||
audioBus.releaseLiveMidiNote(60, 1.25);
|
||||
expect(source.stop).toHaveBeenCalledWith(1.25);
|
||||
});
|
||||
|
||||
it('uses timed playback-rate automation when scheduling pitch bend updates', async () => {
|
||||
const audioBus = await KGAudioBus.create('acoustic_grand_piano');
|
||||
|
||||
audioBus.triggerLiveMidiAttack(60, 0, 1);
|
||||
|
||||
const source = MockBufferSource.mock.results[0].value;
|
||||
audioBus.scheduleLiveMidiPitchBend(1, 2);
|
||||
|
||||
expect(source.playbackRate.setValueAtTime).toHaveBeenCalledWith(Math.pow(2, 2 / 12), 2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -20,7 +20,7 @@ interface LiveMidiSource {
|
||||
export class KGAudioBus {
|
||||
// Fixed at +/-2 semitones for now. Future work: make this user-configurable
|
||||
// or honor MIDI RPN 0,0 (Pitch Bend Sensitivity).
|
||||
private static readonly LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES = 2;
|
||||
public static readonly LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES = 2;
|
||||
private static readonly LIVE_MIDI_NOTE_NAMES = ['C', 'Db', 'D', 'Eb', 'E', 'F', 'Gb', 'G', 'Ab', 'A', 'Bb', 'B'];
|
||||
|
||||
// Core audio components
|
||||
@@ -218,7 +218,17 @@ export class KGAudioBus {
|
||||
|
||||
for (const activeSources of this.liveMidiSources.values()) {
|
||||
activeSources.forEach(({ source, basePlaybackRate }) => {
|
||||
source.playbackRate.value = this.applyPitchBendToPlaybackRate(basePlaybackRate);
|
||||
this.setPlaybackRateValue(source, this.applyPitchBendToPlaybackRate(basePlaybackRate));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public scheduleLiveMidiPitchBend(normalizedBend: number, time: number): void {
|
||||
this.liveMidiPitchBend = Math.max(-1, Math.min(1, normalizedBend));
|
||||
|
||||
for (const activeSources of this.liveMidiSources.values()) {
|
||||
activeSources.forEach(({ source, basePlaybackRate }) => {
|
||||
this.setPlaybackRateValue(source, this.applyPitchBendToPlaybackRate(basePlaybackRate), time);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -474,13 +484,13 @@ export class KGAudioBus {
|
||||
velocity?: number,
|
||||
duration?: number
|
||||
): LiveMidiSource | null {
|
||||
const closestPitch = this.findClosestBufferedPitch(pitch);
|
||||
const closestPitch = KGAudioBus.findClosestBufferedPitch(this.instrument, this.audioBuffers, pitch);
|
||||
if (closestPitch === null) {
|
||||
console.warn(`No audio buffer found for live MIDI pitch ${pitch} on ${this.instrument}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const bufferKey = this.midiPitchToBufferKey(closestPitch);
|
||||
const bufferKey = KGAudioBus.midiPitchToBufferKey(closestPitch);
|
||||
const buffer = this.audioBuffers.get(bufferKey);
|
||||
if (!buffer) {
|
||||
console.warn(`Missing audio buffer ${bufferKey} for ${this.instrument}`);
|
||||
@@ -514,23 +524,31 @@ export class KGAudioBus {
|
||||
return { source, basePlaybackRate };
|
||||
}
|
||||
|
||||
private applyPitchBendToPlaybackRate(basePlaybackRate: number): number {
|
||||
const bendSemitones = this.liveMidiPitchBend * KGAudioBus.LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES;
|
||||
public static applyNormalizedPitchBendToPlaybackRate(basePlaybackRate: number, normalizedBend: number): number {
|
||||
const bendSemitones = normalizedBend * KGAudioBus.LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES;
|
||||
return basePlaybackRate * Math.pow(2, bendSemitones / 12);
|
||||
}
|
||||
|
||||
private findClosestBufferedPitch(targetPitch: number): number | null {
|
||||
const [minPitch, maxPitch] = FLUIDR3_INSTRUMENT_MAP[this.instrument]?.pitchRange || [21, 108];
|
||||
private applyPitchBendToPlaybackRate(basePlaybackRate: number): number {
|
||||
return KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, this.liveMidiPitchBend);
|
||||
}
|
||||
|
||||
public static findClosestBufferedPitch(
|
||||
instrument: InstrumentType,
|
||||
audioBuffers: Tone.ToneAudioBuffers,
|
||||
targetPitch: number
|
||||
): number | null {
|
||||
const [minPitch, maxPitch] = FLUIDR3_INSTRUMENT_MAP[instrument]?.pitchRange || [21, 108];
|
||||
const boundedPitch = Math.max(minPitch, Math.min(maxPitch, targetPitch));
|
||||
|
||||
for (let offset = 0; offset <= 96; offset++) {
|
||||
const upwardPitch = boundedPitch + offset;
|
||||
if (upwardPitch <= maxPitch && this.audioBuffers.has(this.midiPitchToBufferKey(upwardPitch))) {
|
||||
if (upwardPitch <= maxPitch && audioBuffers.has(KGAudioBus.midiPitchToBufferKey(upwardPitch))) {
|
||||
return upwardPitch;
|
||||
}
|
||||
|
||||
const downwardPitch = boundedPitch - offset;
|
||||
if (downwardPitch >= minPitch && this.audioBuffers.has(this.midiPitchToBufferKey(downwardPitch))) {
|
||||
if (downwardPitch >= minPitch && audioBuffers.has(KGAudioBus.midiPitchToBufferKey(downwardPitch))) {
|
||||
return downwardPitch;
|
||||
}
|
||||
}
|
||||
@@ -538,9 +556,23 @@ export class KGAudioBus {
|
||||
return null;
|
||||
}
|
||||
|
||||
private midiPitchToBufferKey(pitch: number): string {
|
||||
public static midiPitchToBufferKey(pitch: number): string {
|
||||
const octave = Math.floor((pitch - 12) / 12);
|
||||
const noteIndex = (pitch - 12) % 12;
|
||||
return `${KGAudioBus.LIVE_MIDI_NOTE_NAMES[noteIndex]}${octave}`;
|
||||
}
|
||||
|
||||
private setPlaybackRateValue(source: Tone.ToneBufferSource, value: number, time?: number): void {
|
||||
const playbackRate = source.playbackRate as unknown as {
|
||||
value: number;
|
||||
setValueAtTime?: (nextValue: number, nextTime: number) => void;
|
||||
};
|
||||
|
||||
if (time !== undefined && typeof playbackRate.setValueAtTime === 'function') {
|
||||
playbackRate.setValueAtTime(value, time);
|
||||
return;
|
||||
}
|
||||
|
||||
playbackRate.value = value;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { createMockProject } from '../../test/utils/mock-data';
|
||||
import { createMockMidiPitchBend, createMockMidiRegion, createMockMidiTrack, createMockProject } from '../../test/utils/mock-data';
|
||||
import { MockTransport } from '../../test/mocks/tone';
|
||||
|
||||
vi.mock('tone', async () => {
|
||||
@@ -22,6 +22,7 @@ vi.mock('../config/ConfigManager', () => ({
|
||||
import { KGCore } from '../KGCore';
|
||||
import { ConfigManager } from '../config/ConfigManager';
|
||||
import { KGAudioInterface } from './KGAudioInterface';
|
||||
import { MIDI_PITCH_BEND_CENTER, midiPitchBendToNormalized } from '../../util/midiUtil';
|
||||
|
||||
describe('KGAudioInterface preroll playback', () => {
|
||||
beforeEach(() => {
|
||||
@@ -50,6 +51,7 @@ describe('KGAudioInterface preroll playback', () => {
|
||||
get: (key: string) => {
|
||||
if (key === 'audio.playback_delay') return 0.2;
|
||||
if (key === 'audio.lookahead_time') return 0.05;
|
||||
if (key === 'audio.midi_automation_interpolation_interval_ms') return 250;
|
||||
return null;
|
||||
},
|
||||
} as unknown as ConfigManager)
|
||||
@@ -112,4 +114,79 @@ describe('KGAudioInterface preroll playback', () => {
|
||||
|
||||
expect(MockTransport.position).toBe(6);
|
||||
});
|
||||
|
||||
it('schedules baked pitch bend points instead of authored step changes only', () => {
|
||||
const region = createMockMidiRegion({
|
||||
pitchBends: [
|
||||
createMockMidiPitchBend({ id: 'bend-1', beat: 0, value: MIDI_PITCH_BEND_CENTER }),
|
||||
createMockMidiPitchBend({ id: 'bend-2', beat: 1, value: 0 }),
|
||||
],
|
||||
});
|
||||
const track = createMockMidiTrack({ id: 1, regions: [region] });
|
||||
const project = createMockProject({ tracks: [track] });
|
||||
const audio = KGAudioInterface.instance();
|
||||
const audioBus = {
|
||||
resetLiveMidiPitchBend: vi.fn(),
|
||||
setLiveMidiPitchBend: vi.fn(),
|
||||
scheduleLiveMidiPitchBend: vi.fn(),
|
||||
shouldPlayWithSolo: vi.fn().mockReturnValue(true),
|
||||
};
|
||||
|
||||
;(audio as unknown as { trackAudioBuses: Map<string, unknown> }).trackAudioBuses.set('1', audioBus);
|
||||
audio.preparePlayback(project, 0);
|
||||
|
||||
const scheduledTimes = MockTransport.schedule.mock.calls.map(([, time]) => time);
|
||||
expect(scheduledTimes).toContain(0.25);
|
||||
expect(scheduledTimes).toContain(0.5);
|
||||
});
|
||||
|
||||
it('computes the initial interpolated bend for non-zero playback starts', () => {
|
||||
const region = createMockMidiRegion({
|
||||
pitchBends: [
|
||||
createMockMidiPitchBend({ id: 'bend-1', beat: 0, value: MIDI_PITCH_BEND_CENTER }),
|
||||
createMockMidiPitchBend({ id: 'bend-2', beat: 4, value: 0 }),
|
||||
],
|
||||
});
|
||||
const track = createMockMidiTrack({ id: 1, regions: [region] });
|
||||
const project = createMockProject({ tracks: [track] });
|
||||
const audio = KGAudioInterface.instance();
|
||||
const audioBus = {
|
||||
resetLiveMidiPitchBend: vi.fn(),
|
||||
setLiveMidiPitchBend: vi.fn(),
|
||||
scheduleLiveMidiPitchBend: vi.fn(),
|
||||
shouldPlayWithSolo: vi.fn().mockReturnValue(true),
|
||||
};
|
||||
|
||||
;(audio as unknown as { trackAudioBuses: Map<string, unknown> }).trackAudioBuses.set('1', audioBus);
|
||||
audio.preparePlayback(project, 2);
|
||||
|
||||
expect(audioBus.setLiveMidiPitchBend).toHaveBeenCalledWith(midiPitchBendToNormalized(4096));
|
||||
});
|
||||
|
||||
it('adds a loop-start re-anchor for interpolated pitch bend state', () => {
|
||||
const region = createMockMidiRegion({
|
||||
pitchBends: [
|
||||
createMockMidiPitchBend({ id: 'bend-1', beat: 2, value: 0 }),
|
||||
createMockMidiPitchBend({ id: 'bend-2', beat: 6, value: MIDI_PITCH_BEND_CENTER }),
|
||||
],
|
||||
});
|
||||
const track = createMockMidiTrack({ id: 1, regions: [region] });
|
||||
const project = createMockProject({ tracks: [track] });
|
||||
project.setIsLooping(true);
|
||||
project.setLoopingRange([1, 1]);
|
||||
|
||||
const audio = KGAudioInterface.instance();
|
||||
const audioBus = {
|
||||
resetLiveMidiPitchBend: vi.fn(),
|
||||
setLiveMidiPitchBend: vi.fn(),
|
||||
scheduleLiveMidiPitchBend: vi.fn(),
|
||||
shouldPlayWithSolo: vi.fn().mockReturnValue(true),
|
||||
};
|
||||
|
||||
;(audio as unknown as { trackAudioBuses: Map<string, unknown> }).trackAudioBuses.set('1', audioBus);
|
||||
audio.preparePlayback(project, 5);
|
||||
|
||||
const scheduledTimes = MockTransport.schedule.mock.calls.map(([, time]) => time);
|
||||
expect(scheduledTimes).toContain(2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,7 +3,16 @@ import type { KGMidiNote } from '../midi/KGMidiNote';
|
||||
import type { KGMidiPitchBend } from '../midi/KGMidiPitchBend';
|
||||
import { TIME_CONSTANTS, AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants';
|
||||
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
|
||||
import { midiPitchBendToNormalized, pitchToNoteNameString } from '../../util/midiUtil';
|
||||
import {
|
||||
MIDI_PITCH_BEND_CENTER,
|
||||
midiPitchBendToNormalized,
|
||||
pitchToNoteNameString,
|
||||
} from '../../util/midiUtil';
|
||||
import {
|
||||
bakeMidiAutomationPointsInWindow,
|
||||
collectRegionMidiAutomationPoints,
|
||||
resolveMidiAutomationValueAtBeat,
|
||||
} from '../../util/midiAutomationUtil';
|
||||
import * as Tone from 'tone';
|
||||
import { KGAudioBus } from './KGAudioBus';
|
||||
import { KGAudioPlayerBus } from './KGAudioPlayerBus';
|
||||
@@ -473,30 +482,35 @@ export class KGAudioInterface {
|
||||
project.getTracks().forEach(track => {
|
||||
const trackId = track.getId().toString();
|
||||
const audioBus = this.trackAudioBuses.get(trackId);
|
||||
const interpolationIntervalMs = (configManager.get('audio.midi_automation_interpolation_interval_ms') as number) ?? 10;
|
||||
|
||||
console.log(`Track ${trackId} has audio bus: ${audioBus ? 'true' : 'false'}; type: ${track.getType()}`);
|
||||
|
||||
// Schedule MIDI track events
|
||||
if (audioBus && track.getType() === 'MIDI') {
|
||||
const trackPitchBends: Array<{ pitchBend: KGMidiPitchBend; absoluteBeat: number }> = [];
|
||||
const trackNotes: Array<{ note: KGMidiNote; absoluteStartBeat: number; absoluteEndBeat: number }> = [];
|
||||
const trackPitchBends = collectRegionMidiAutomationPoints(
|
||||
track.getRegions()
|
||||
.filter(region => region.getCurrentType() === 'KGMidiRegion')
|
||||
.map(region => {
|
||||
const midiRegion = region as unknown as { getPitchBends: () => KGMidiPitchBend[] };
|
||||
return {
|
||||
startBeat: region.getStartFromBeat(),
|
||||
points: midiRegion.getPitchBends().map(pitchBend => ({
|
||||
beat: pitchBend.getBeat(),
|
||||
value: pitchBend.getValue(),
|
||||
})),
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
track.getRegions().forEach(region => {
|
||||
console.log(`Region ${region.getId().toString()}: type: ${region.getCurrentType()}`);
|
||||
|
||||
if (region.getCurrentType() === 'KGMidiRegion') {
|
||||
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[]; getPitchBends: () => KGMidiPitchBend[] };
|
||||
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[] };
|
||||
const regionStartBeat = region.getStartFromBeat();
|
||||
|
||||
if (midiRegion.getPitchBends) {
|
||||
midiRegion.getPitchBends().forEach((pitchBend: KGMidiPitchBend) => {
|
||||
trackPitchBends.push({
|
||||
pitchBend,
|
||||
absoluteBeat: regionStartBeat + pitchBend.getBeat(),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Get notes from region (assuming it has a getNotes method)
|
||||
if (midiRegion.getNotes) {
|
||||
midiRegion.getNotes().forEach((note: KGMidiNote) => {
|
||||
@@ -523,35 +537,37 @@ export class KGAudioInterface {
|
||||
}
|
||||
});
|
||||
|
||||
const boundedTrackPitchBends = trackPitchBends
|
||||
.filter(({ absoluteBeat }) => absoluteBeat >= scheduleStartBeat && absoluteBeat < scheduleEndBeat)
|
||||
.sort((a, b) => a.absoluteBeat - b.absoluteBeat);
|
||||
const initialPitchBend = [...boundedTrackPitchBends]
|
||||
.reverse()
|
||||
.find(({ absoluteBeat }) => absoluteBeat <= startPosition);
|
||||
audioBus.setLiveMidiPitchBend(midiPitchBendToNormalized(initialPitchBend?.pitchBend.getValue() ?? 8192));
|
||||
const initialPitchBendBeat = isLooping ? Math.max(startPosition, scheduleStartBeat) : startPosition;
|
||||
const initialPitchBendValue = resolveMidiAutomationValueAtBeat(
|
||||
trackPitchBends,
|
||||
initialPitchBendBeat,
|
||||
MIDI_PITCH_BEND_CENTER
|
||||
);
|
||||
audioBus.setLiveMidiPitchBend(midiPitchBendToNormalized(initialPitchBendValue));
|
||||
|
||||
if (isLooping && !boundedTrackPitchBends.some(({ absoluteBeat }) => absoluteBeat === scheduleStartBeat)) {
|
||||
const eventId = Tone.Transport.schedule((time) => {
|
||||
const hasSoloedTracks = this.hasSoloedTracks();
|
||||
if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) {
|
||||
audioBus.setLiveMidiPitchBend(0);
|
||||
}
|
||||
}, this.beatsToToneTime(scheduleStartBeat));
|
||||
this.scheduledEvents.add(eventId);
|
||||
}
|
||||
const pitchBendWindowStartBeat = isLooping ? scheduleStartBeat : Math.max(startPosition, 0);
|
||||
const bakedTrackPitchBends = bakeMidiAutomationPointsInWindow(
|
||||
trackPitchBends,
|
||||
pitchBendWindowStartBeat,
|
||||
scheduleEndBeat,
|
||||
{
|
||||
maxIntervalMs: interpolationIntervalMs,
|
||||
bpm: project.getBpm(),
|
||||
defaultValue: MIDI_PITCH_BEND_CENTER,
|
||||
}
|
||||
);
|
||||
|
||||
boundedTrackPitchBends.forEach(({ pitchBend, absoluteBeat }) => {
|
||||
if (absoluteBeat < startPosition) {
|
||||
bakedTrackPitchBends.forEach(({ beat, value }) => {
|
||||
if (!isLooping && beat <= pitchBendWindowStartBeat) {
|
||||
return;
|
||||
}
|
||||
|
||||
const eventId = Tone.Transport.schedule(() => {
|
||||
const eventId = Tone.Transport.schedule((time) => {
|
||||
const hasSoloedTracks = this.hasSoloedTracks();
|
||||
if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) {
|
||||
audioBus.setLiveMidiPitchBend(midiPitchBendToNormalized(pitchBend.getValue()));
|
||||
audioBus.scheduleLiveMidiPitchBend(midiPitchBendToNormalized(value), time);
|
||||
}
|
||||
}, this.beatsToToneTime(absoluteBeat));
|
||||
}, this.beatsToToneTime(beat));
|
||||
|
||||
this.scheduledEvents.add(eventId);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { encodeWav, getOfflineTrackGain, getOfflineTrackVolumeDb } from './KGOfflineRenderer';
|
||||
import { applyOfflinePitchBendAutomation, encodeWav, getOfflineTrackGain, getOfflineTrackVolumeDb } from './KGOfflineRenderer';
|
||||
|
||||
/**
|
||||
* Create a minimal AudioBuffer-like object for testing.
|
||||
@@ -164,3 +164,32 @@ describe('offline track volume conversion', () => {
|
||||
expect(getOfflineTrackGain(-60, false)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('offline pitch bend automation', () => {
|
||||
it('applies baked pitch bends to source playback rate automation', () => {
|
||||
const source = {
|
||||
playbackRate: {
|
||||
value: 1,
|
||||
setValueAtTime: (..._args: unknown[]) => undefined,
|
||||
},
|
||||
} as unknown as Parameters<typeof applyOfflinePitchBendAutomation>[0];
|
||||
|
||||
const calls: Array<[number, number]> = [];
|
||||
source.playbackRate.setValueAtTime = ((value: number, time: number) => {
|
||||
calls.push([value, time]);
|
||||
return source.playbackRate as never;
|
||||
}) as typeof source.playbackRate.setValueAtTime;
|
||||
|
||||
applyOfflinePitchBendAutomation(
|
||||
source,
|
||||
1,
|
||||
[{ beat: 1, value: 0 }],
|
||||
0,
|
||||
0.5
|
||||
);
|
||||
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0][1]).toBe(0.5);
|
||||
expect(calls[0][0]).toBeCloseTo(Math.pow(2, -2 / 12), 5);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,13 +1,24 @@
|
||||
import * as Tone from 'tone';
|
||||
import type { KGProject } from '../KGProject';
|
||||
import type { KGMidiNote } from '../midi/KGMidiNote';
|
||||
import type { KGMidiPitchBend } from '../midi/KGMidiPitchBend';
|
||||
import type { KGAudioRegion } from '../region/KGAudioRegion';
|
||||
import type { InstrumentType } from '../track/KGMidiTrack';
|
||||
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
|
||||
import { AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants';
|
||||
import { pitchToNoteNameString } from '../../util/midiUtil';
|
||||
import { MIDI_PITCH_BEND_CENTER, midiPitchBendToNormalized } from '../../util/midiUtil';
|
||||
import {
|
||||
bakeMidiAutomationPointsInWindow,
|
||||
collectRegionMidiAutomationPoints,
|
||||
resolveMidiAutomationValueAtBeat,
|
||||
type BakedMidiAutomationPoint,
|
||||
type MidiAutomationPoint,
|
||||
} from '../../util/midiAutomationUtil';
|
||||
import { KGToneBuffersPool } from './KGToneBuffersPool';
|
||||
import { KGToneSamplerFactory } from './KGToneSamplerFactory';
|
||||
import { KGAudioInterface } from './KGAudioInterface';
|
||||
import { KGAudioBus } from './KGAudioBus';
|
||||
import { ConfigManager } from '../config/ConfigManager';
|
||||
import { Mp3Encoder } from '@breezystack/lamejs';
|
||||
|
||||
export interface RenderOptions {
|
||||
@@ -104,7 +115,7 @@ export class KGOfflineRenderer {
|
||||
// Pre-collect all the data we need before entering the offline context
|
||||
const midiTrackData: Array<{
|
||||
trackId: string;
|
||||
instrumentName: string;
|
||||
instrumentName: InstrumentType;
|
||||
volume: number;
|
||||
muted: boolean;
|
||||
solo: boolean;
|
||||
@@ -112,6 +123,7 @@ export class KGOfflineRenderer {
|
||||
startBeat: number;
|
||||
notes: Array<{ startBeat: number; endBeat: number; durationBeats: number; pitch: number; velocity: number }>;
|
||||
}>;
|
||||
pitchBends: MidiAutomationPoint[];
|
||||
}> = [];
|
||||
|
||||
const audioTrackData: Array<{
|
||||
@@ -130,13 +142,14 @@ export class KGOfflineRenderer {
|
||||
}> = [];
|
||||
|
||||
let hasSoloedTracks = false;
|
||||
const interpolationIntervalMs = (ConfigManager.instance().get('audio.midi_automation_interpolation_interval_ms') as number) ?? 10;
|
||||
|
||||
for (const track of tracks) {
|
||||
const trackId = track.getId().toString();
|
||||
|
||||
if (track.getType() === 'MIDI') {
|
||||
const midiTrack = track as unknown as { getInstrument: () => string };
|
||||
const instrumentName = String(midiTrack.getInstrument());
|
||||
const midiTrack = track as unknown as { getInstrument: () => InstrumentType };
|
||||
const instrumentName = midiTrack.getInstrument();
|
||||
|
||||
// Get live bus state for volume/mute/solo via public getters
|
||||
const volume = audioInterface.getTrackVolume(trackId);
|
||||
@@ -147,7 +160,7 @@ export class KGOfflineRenderer {
|
||||
const regions: typeof midiTrackData[0]['regions'] = [];
|
||||
for (const region of track.getRegions()) {
|
||||
if (region.getCurrentType() === 'KGMidiRegion') {
|
||||
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[] };
|
||||
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[]; getPitchBends: () => KGMidiPitchBend[] };
|
||||
if (midiRegion.getNotes) {
|
||||
const notes = midiRegion.getNotes().map(note => ({
|
||||
startBeat: note.getStartBeat() + region.getStartFromBeat(),
|
||||
@@ -160,8 +173,22 @@ export class KGOfflineRenderer {
|
||||
}
|
||||
}
|
||||
}
|
||||
const pitchBends = collectRegionMidiAutomationPoints(
|
||||
track.getRegions()
|
||||
.filter(region => region.getCurrentType() === 'KGMidiRegion')
|
||||
.map(region => {
|
||||
const midiRegion = region as unknown as { getPitchBends: () => KGMidiPitchBend[] };
|
||||
return {
|
||||
startBeat: region.getStartFromBeat(),
|
||||
points: midiRegion.getPitchBends().map(pitchBend => ({
|
||||
beat: pitchBend.getBeat(),
|
||||
value: pitchBend.getValue(),
|
||||
})),
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
midiTrackData.push({ trackId, instrumentName, volume, muted, solo, regions });
|
||||
midiTrackData.push({ trackId, instrumentName, volume, muted, solo, regions, pitchBends });
|
||||
} else if (track.getType() === 'Wave') {
|
||||
const volume = audioInterface.getTrackVolume(trackId);
|
||||
const muted = audioInterface.getTrackMuted(trackId);
|
||||
@@ -241,7 +268,7 @@ export class KGOfflineRenderer {
|
||||
const promise = (async () => {
|
||||
try {
|
||||
// Get cached buffers from pool
|
||||
const audioBuffers = await KGToneBuffersPool.instance().getToneAudioBuffers(trackInfo.instrumentName);
|
||||
const audioBuffers = await KGToneBuffersPool.instance().getToneAudioBuffers(String(trackInfo.instrumentName));
|
||||
const pitchRange = FLUIDR3_INSTRUMENT_MAP[trackInfo.instrumentName]?.pitchRange || [21, 108];
|
||||
const urlMap = KGToneSamplerFactory.instance().convertBuffersToUrls(audioBuffers, pitchRange);
|
||||
|
||||
@@ -258,6 +285,16 @@ export class KGOfflineRenderer {
|
||||
// Track volumes are stored in dB across the app, with 0 meaning unity gain.
|
||||
sampler.volume.value = getOfflineTrackVolumeDb(trackInfo.volume, trackInfo.muted);
|
||||
sampler.connect(masterGain);
|
||||
const bakedTrackPitchBends = bakeMidiAutomationPointsInWindow(
|
||||
trackInfo.pitchBends,
|
||||
renderStartBeat,
|
||||
renderEndBeat,
|
||||
{
|
||||
maxIntervalMs: interpolationIntervalMs,
|
||||
bpm: project.getBpm(),
|
||||
defaultValue: MIDI_PITCH_BEND_CENTER,
|
||||
}
|
||||
);
|
||||
|
||||
// Schedule all notes for this track
|
||||
for (const regionInfo of trackInfo.regions) {
|
||||
@@ -268,11 +305,32 @@ export class KGOfflineRenderer {
|
||||
const offsetBeat = note.startBeat - renderStartBeat;
|
||||
const noteStartTime = offsetBeat * secondsPerBeat;
|
||||
const noteDuration = note.durationBeats * secondsPerBeat;
|
||||
const noteName = pitchToNoteNameString(note.pitch);
|
||||
const velocity = note.velocity / 127;
|
||||
const initialNormalizedPitchBend = midiPitchBendToNormalized(
|
||||
resolveMidiAutomationValueAtBeat(trackInfo.pitchBends, note.startBeat, MIDI_PITCH_BEND_CENTER)
|
||||
);
|
||||
const offlineSource = createOfflinePitchBendAwareSource(
|
||||
sampler,
|
||||
audioBuffers,
|
||||
trackInfo.instrumentName,
|
||||
note.pitch,
|
||||
initialNormalizedPitchBend
|
||||
);
|
||||
if (!offlineSource) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const { source, basePlaybackRate } = offlineSource;
|
||||
applyOfflinePitchBendAutomation(
|
||||
source,
|
||||
basePlaybackRate,
|
||||
bakedTrackPitchBends.filter(point => point.beat > note.startBeat && point.beat < note.endBeat),
|
||||
renderStartBeat,
|
||||
secondsPerBeat
|
||||
);
|
||||
|
||||
context.transport.schedule((time) => {
|
||||
sampler.triggerAttackRelease(noteName, noteDuration, time, velocity);
|
||||
source.start(time, 0, noteDuration, velocity);
|
||||
}, noteStartTime);
|
||||
}
|
||||
}
|
||||
@@ -418,6 +476,76 @@ function shouldPlay(trackInfo: { muted: boolean; solo: boolean }, hasSoloedTrack
|
||||
return true;
|
||||
}
|
||||
|
||||
function setOfflinePlaybackRate(
|
||||
source: Tone.ToneBufferSource,
|
||||
value: number,
|
||||
time: number
|
||||
): void {
|
||||
const playbackRate = source.playbackRate as unknown as {
|
||||
value: number;
|
||||
setValueAtTime?: (nextValue: number, nextTime: number) => void;
|
||||
};
|
||||
|
||||
if (typeof playbackRate.setValueAtTime === 'function') {
|
||||
playbackRate.setValueAtTime(value, time);
|
||||
return;
|
||||
}
|
||||
|
||||
playbackRate.value = value;
|
||||
}
|
||||
|
||||
function createOfflinePitchBendAwareSource(
|
||||
sampler: Tone.Sampler,
|
||||
audioBuffers: Tone.ToneAudioBuffers,
|
||||
instrumentName: InstrumentType,
|
||||
pitch: number,
|
||||
initialNormalizedPitchBend: number
|
||||
): { source: Tone.ToneBufferSource; basePlaybackRate: number } | null {
|
||||
const closestPitch = KGAudioBus.findClosestBufferedPitch(instrumentName, audioBuffers, pitch);
|
||||
if (closestPitch === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const bufferKey = KGAudioBus.midiPitchToBufferKey(closestPitch);
|
||||
const buffer = audioBuffers.get(bufferKey);
|
||||
if (!buffer) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const basePlaybackRate = Math.pow(2, (pitch - closestPitch) / 12);
|
||||
const source = new Tone.ToneBufferSource({
|
||||
url: buffer,
|
||||
fadeIn: sampler.attack,
|
||||
fadeOut: sampler.release,
|
||||
curve: sampler.curve,
|
||||
playbackRate: KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, initialNormalizedPitchBend),
|
||||
}).connect(sampler.output);
|
||||
setOfflinePlaybackRate(
|
||||
source,
|
||||
KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, initialNormalizedPitchBend),
|
||||
0
|
||||
);
|
||||
|
||||
return { source, basePlaybackRate };
|
||||
}
|
||||
|
||||
export function applyOfflinePitchBendAutomation(
|
||||
source: Tone.ToneBufferSource,
|
||||
basePlaybackRate: number,
|
||||
bakedPitchBends: BakedMidiAutomationPoint[],
|
||||
renderStartBeat: number,
|
||||
secondsPerBeat: number
|
||||
): void {
|
||||
bakedPitchBends.forEach(point => {
|
||||
const automationTime = (point.beat - renderStartBeat) * secondsPerBeat;
|
||||
setOfflinePlaybackRate(
|
||||
source,
|
||||
KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, midiPitchBendToNormalized(point.value)),
|
||||
automationTime
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function getOfflineTrackVolumeDb(volumeDb: number, muted: boolean): number {
|
||||
const isSilent = muted || volumeDb <= AUDIO_INTERFACE_CONSTANTS.MIN_TRACK_VOLUME_DB;
|
||||
return isSilent ? -Infinity : volumeDb;
|
||||
|
||||
@@ -79,6 +79,7 @@ interface AppConfig {
|
||||
audio: {
|
||||
enable_audio_capture_for_screen_sharing: boolean;
|
||||
lookahead_time: number;
|
||||
midi_automation_interpolation_interval_ms: number;
|
||||
playback_delay: number;
|
||||
recording_offset: number;
|
||||
};
|
||||
@@ -253,6 +254,7 @@ export class ConfigManager {
|
||||
audio: {
|
||||
enable_audio_capture_for_screen_sharing: false,
|
||||
lookahead_time: 0.05,
|
||||
midi_automation_interpolation_interval_ms: 10,
|
||||
playback_delay: 0.2,
|
||||
recording_offset: 0
|
||||
},
|
||||
|
||||
@@ -31,6 +31,8 @@ const mockCore = {
|
||||
getStatus: () => 'Ready',
|
||||
getPlayheadPosition: () => 0,
|
||||
getIsPlaying: () => false,
|
||||
startPlaying: vi.fn().mockResolvedValue(undefined),
|
||||
stopPlaying: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
vi.mock('../core/KGCore', () => ({
|
||||
@@ -51,6 +53,10 @@ vi.mock('../core/config/ConfigManager', () => ({
|
||||
describe('projectStore piano roll state', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
mockCore.startPlaying.mockReset();
|
||||
mockCore.startPlaying.mockResolvedValue(undefined);
|
||||
mockCore.stopPlaying.mockReset();
|
||||
mockCore.stopPlaying.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
it('clears hybrid state when opening a MIDI region', async () => {
|
||||
@@ -75,4 +81,76 @@ describe('projectStore piano roll state', () => {
|
||||
expect(state.activeRegionId).toBe('midi-b');
|
||||
expect(state.hybridAudioRegionId).toBeNull();
|
||||
});
|
||||
|
||||
it('tracks playback preparation around startPlaying success', async () => {
|
||||
let resolveStart: (() => void) | null = null;
|
||||
mockCore.startPlaying.mockImplementationOnce(() => new Promise<void>((resolve) => {
|
||||
resolveStart = resolve;
|
||||
}));
|
||||
|
||||
const { useProjectStore } = await import('./projectStore');
|
||||
|
||||
const startPromise = useProjectStore.getState().startPlaying();
|
||||
expect(useProjectStore.getState().isPreparingPlayback).toBe(true);
|
||||
|
||||
await act(async () => {
|
||||
resolveStart?.();
|
||||
await startPromise;
|
||||
});
|
||||
|
||||
const state = useProjectStore.getState();
|
||||
expect(state.isPreparingPlayback).toBe(false);
|
||||
expect(state.isPlaying).toBe(true);
|
||||
});
|
||||
|
||||
it('clears playback preparation if startPlaying fails', async () => {
|
||||
mockCore.startPlaying.mockRejectedValueOnce(new Error('prepare failed'));
|
||||
|
||||
const { useProjectStore } = await import('./projectStore');
|
||||
|
||||
await expect(useProjectStore.getState().startPlaying()).rejects.toThrow('prepare failed');
|
||||
expect(useProjectStore.getState().isPreparingPlayback).toBe(false);
|
||||
});
|
||||
|
||||
it('ignores repeated startPlaying calls while already preparing', async () => {
|
||||
let resolveStart: (() => void) | null = null;
|
||||
mockCore.startPlaying.mockImplementationOnce(() => new Promise<void>((resolve) => {
|
||||
resolveStart = resolve;
|
||||
}));
|
||||
|
||||
const { useProjectStore } = await import('./projectStore');
|
||||
|
||||
const firstStart = useProjectStore.getState().startPlaying();
|
||||
const secondStart = useProjectStore.getState().startPlaying();
|
||||
|
||||
expect(mockCore.startPlaying).toHaveBeenCalledTimes(1);
|
||||
|
||||
await act(async () => {
|
||||
resolveStart?.();
|
||||
await Promise.all([firstStart, secondStart]);
|
||||
});
|
||||
});
|
||||
|
||||
it('clears playback preparation when stopTransport is called during prepare', async () => {
|
||||
let resolveStart: (() => void) | null = null;
|
||||
mockCore.startPlaying.mockImplementationOnce(() => new Promise<void>((resolve) => {
|
||||
resolveStart = resolve;
|
||||
}));
|
||||
|
||||
const { useProjectStore } = await import('./projectStore');
|
||||
|
||||
const startPromise = useProjectStore.getState().startPlaying();
|
||||
expect(useProjectStore.getState().isPreparingPlayback).toBe(true);
|
||||
|
||||
await act(async () => {
|
||||
await useProjectStore.getState().stopTransport();
|
||||
});
|
||||
|
||||
expect(useProjectStore.getState().isPreparingPlayback).toBe(false);
|
||||
|
||||
await act(async () => {
|
||||
resolveStart?.();
|
||||
await startPromise;
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+40
-10
@@ -68,6 +68,7 @@ interface ProjectState {
|
||||
loopingRange: [number, number]; // [startBar, endBar] - bar indices (0-based)
|
||||
playheadPosition: number; // in beats
|
||||
isPlaying: boolean;
|
||||
isPreparingPlayback: boolean;
|
||||
autoScrollEnabled: boolean;
|
||||
currentTime: string; // formatted time string
|
||||
|
||||
@@ -341,6 +342,7 @@ export const useProjectStore = create<ProjectState>((set, get) => {
|
||||
loopingRange: currentProject.getLoopingRange(),
|
||||
playheadPosition: KGCore.instance().getPlayheadPosition(),
|
||||
isPlaying: KGCore.instance().getIsPlaying(),
|
||||
isPreparingPlayback: false,
|
||||
autoScrollEnabled: true,
|
||||
currentTime: beatsToTimeString(KGCore.instance().getPlayheadPosition(), currentProject.getBpm(), currentProject.getTimeSignature()),
|
||||
|
||||
@@ -854,13 +856,26 @@ export const useProjectStore = create<ProjectState>((set, get) => {
|
||||
},
|
||||
|
||||
startPlaying: async () => {
|
||||
await KGCore.instance().startPlaying();
|
||||
set({ isPlaying: true, autoScrollEnabled: true });
|
||||
if (get().isPreparingPlayback) {
|
||||
return;
|
||||
}
|
||||
|
||||
set({ isPreparingPlayback: true });
|
||||
try {
|
||||
await KGCore.instance().startPlaying();
|
||||
set({ isPlaying: true, autoScrollEnabled: true });
|
||||
} finally {
|
||||
set({ isPreparingPlayback: false });
|
||||
}
|
||||
},
|
||||
|
||||
stopPlaying: async () => {
|
||||
await KGCore.instance().stopPlaying();
|
||||
set({ isPlaying: false });
|
||||
try {
|
||||
await KGCore.instance().stopPlaying();
|
||||
set({ isPlaying: false });
|
||||
} finally {
|
||||
set({ isPreparingPlayback: false });
|
||||
}
|
||||
},
|
||||
|
||||
stopTransport: async () => {
|
||||
@@ -868,7 +883,11 @@ export const useProjectStore = create<ProjectState>((set, get) => {
|
||||
await get().stopRecording();
|
||||
return;
|
||||
}
|
||||
await get().stopPlaying();
|
||||
try {
|
||||
await get().stopPlaying();
|
||||
} finally {
|
||||
set({ isPreparingPlayback: false });
|
||||
}
|
||||
},
|
||||
|
||||
startRecording: async () => {
|
||||
@@ -944,10 +963,15 @@ export const useProjectStore = create<ProjectState>((set, get) => {
|
||||
: playheadPosition - timeSignature.numerator;
|
||||
|
||||
setPlayheadPosition(recordingStartBeat);
|
||||
await KGCore.instance().startPlaying({
|
||||
preserveLoopPreroll: projectLooping,
|
||||
});
|
||||
set({ isPlaying: true, autoScrollEnabled: true });
|
||||
set({ isPreparingPlayback: true });
|
||||
try {
|
||||
await KGCore.instance().startPlaying({
|
||||
preserveLoopPreroll: projectLooping,
|
||||
});
|
||||
set({ isPlaying: true, autoScrollEnabled: true });
|
||||
} finally {
|
||||
set({ isPreparingPlayback: false });
|
||||
}
|
||||
},
|
||||
|
||||
stopRecording: async () => {
|
||||
@@ -1002,7 +1026,13 @@ export const useProjectStore = create<ProjectState>((set, get) => {
|
||||
|
||||
await stopPlaying();
|
||||
setPlayheadPosition(recordingOriginalPlayhead);
|
||||
set({ isRecording: false, recordingNotes: [], recordingPitchBends: [], recordingTargetRegionId: null });
|
||||
set({
|
||||
isRecording: false,
|
||||
isPreparingPlayback: false,
|
||||
recordingNotes: [],
|
||||
recordingPitchBends: [],
|
||||
recordingTargetRegionId: null
|
||||
});
|
||||
_lastRecordedPitchBendValue = null;
|
||||
},
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ export const mockSampler = {
|
||||
};
|
||||
|
||||
export const mockBufferSource = {
|
||||
playbackRate: { value: 1 },
|
||||
playbackRate: { value: 1, setValueAtTime: vi.fn() },
|
||||
connect: vi.fn(),
|
||||
start: vi.fn(),
|
||||
stop: vi.fn(),
|
||||
@@ -50,7 +50,7 @@ export const mockTone = {
|
||||
BufferSource: vi.fn().mockImplementation(() => {
|
||||
const instance = {
|
||||
...mockBufferSource,
|
||||
playbackRate: { value: 1 },
|
||||
playbackRate: { value: 1, setValueAtTime: vi.fn() },
|
||||
};
|
||||
instance.connect.mockImplementation(() => instance);
|
||||
instance.start.mockImplementation(() => instance);
|
||||
@@ -60,7 +60,7 @@ export const mockTone = {
|
||||
ToneBufferSource: vi.fn().mockImplementation(() => {
|
||||
const instance = {
|
||||
...mockBufferSource,
|
||||
playbackRate: { value: 1 },
|
||||
playbackRate: { value: 1, setValueAtTime: vi.fn() },
|
||||
};
|
||||
instance.connect.mockImplementation(() => instance);
|
||||
instance.start.mockImplementation(() => instance);
|
||||
|
||||
@@ -28,6 +28,7 @@ export const MockBufferSource = vi.fn().mockImplementation((options?: { playback
|
||||
const instance = {
|
||||
playbackRate: {
|
||||
value: options?.playbackRate ?? 1,
|
||||
setValueAtTime: vi.fn(),
|
||||
},
|
||||
connect: vi.fn(() => instance),
|
||||
start: vi.fn(() => instance),
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
bakeMidiAutomationPointsInWindow,
|
||||
collectRegionMidiAutomationPoints,
|
||||
normalizeMidiAutomationPoints,
|
||||
resolveMidiAutomationValueAtBeat,
|
||||
type MidiAutomationPoint,
|
||||
} from './midiAutomationUtil';
|
||||
|
||||
const defaultOptions = {
|
||||
maxIntervalMs: 10,
|
||||
bpm: 120,
|
||||
defaultValue: 8192,
|
||||
};
|
||||
|
||||
describe('midiAutomationUtil', () => {
|
||||
it('normalizes points and keeps the last duplicate beat', () => {
|
||||
const points: MidiAutomationPoint[] = [
|
||||
{ beat: 2, value: 1000 },
|
||||
{ beat: 1, value: 2000 },
|
||||
{ beat: 2, value: 3000 },
|
||||
];
|
||||
|
||||
expect(normalizeMidiAutomationPoints(points)).toEqual([
|
||||
{ beat: 1, value: 2000 },
|
||||
{ beat: 2, value: 3000 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('holds the default value before the first point', () => {
|
||||
const points = [{ beat: 4, value: 0 }];
|
||||
|
||||
expect(resolveMidiAutomationValueAtBeat(points, 2, 8192)).toBe(8192);
|
||||
});
|
||||
|
||||
it('interpolates linearly between two points', () => {
|
||||
const points = [
|
||||
{ beat: 0, value: 8192 },
|
||||
{ beat: 4, value: 0 },
|
||||
];
|
||||
|
||||
expect(resolveMidiAutomationValueAtBeat(points, 2, 8192)).toBe(4096);
|
||||
});
|
||||
|
||||
it('holds the last value after the final point', () => {
|
||||
const points = [{ beat: 1, value: 2048 }];
|
||||
|
||||
expect(resolveMidiAutomationValueAtBeat(points, 8, 8192)).toBe(2048);
|
||||
});
|
||||
|
||||
it('collects region-local points into absolute beats', () => {
|
||||
const collected = collectRegionMidiAutomationPoints([
|
||||
{ startBeat: 4, points: [{ beat: 0.5, value: 7000 }] },
|
||||
{ startBeat: 1, points: [{ beat: 0.25, value: 6000 }] },
|
||||
]);
|
||||
|
||||
expect(collected).toEqual([
|
||||
{ beat: 1.25, value: 6000 },
|
||||
{ beat: 4.5, value: 7000 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('bakes dense points according to the requested ms interval', () => {
|
||||
const points = [
|
||||
{ beat: 0, value: 8192 },
|
||||
{ beat: 1, value: 0 },
|
||||
];
|
||||
|
||||
expect(bakeMidiAutomationPointsInWindow(points, 0, 1, { ...defaultOptions, maxIntervalMs: 20 })).toHaveLength(25);
|
||||
expect(bakeMidiAutomationPointsInWindow(points, 0, 1, { ...defaultOptions, maxIntervalMs: 10 })).toHaveLength(50);
|
||||
expect(bakeMidiAutomationPointsInWindow(points, 0, 1, { ...defaultOptions, maxIntervalMs: 5 })).toHaveLength(100);
|
||||
});
|
||||
|
||||
it('treats flat segments as holds without adding interior baked points', () => {
|
||||
const points = [
|
||||
{ beat: 0, value: 4096 },
|
||||
{ beat: 4, value: 4096 },
|
||||
];
|
||||
|
||||
expect(bakeMidiAutomationPointsInWindow(points, 0, 4, { ...defaultOptions, maxIntervalMs: 10 })).toEqual([
|
||||
{ beat: 0, value: 4096 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('collapses consecutive baked points with the same value', () => {
|
||||
const points = [
|
||||
{ beat: 0.5, value: 8192 },
|
||||
{ beat: 1, value: 4096 },
|
||||
{ beat: 2, value: 4096 },
|
||||
{ beat: 3, value: 0 },
|
||||
];
|
||||
|
||||
expect(bakeMidiAutomationPointsInWindow(points, 0, 4, { ...defaultOptions, maxIntervalMs: 500 })).toEqual([
|
||||
{ beat: 0, value: 8192 },
|
||||
{ beat: 1, value: 4096 },
|
||||
{ beat: 3, value: 0 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps later changing segments after skipping a flat segment', () => {
|
||||
const points = [
|
||||
{ beat: 0, value: 4096 },
|
||||
{ beat: 2, value: 4096 },
|
||||
{ beat: 4, value: 0 },
|
||||
];
|
||||
|
||||
const baked = bakeMidiAutomationPointsInWindow(points, 0, 4, { ...defaultOptions, maxIntervalMs: 500 });
|
||||
|
||||
expect(baked).toEqual([
|
||||
{ beat: 0, value: 4096 },
|
||||
{ beat: 3, value: 2048 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('adds a window anchor and preserves the correct loop boundary value', () => {
|
||||
const points = [
|
||||
{ beat: 2, value: 0 },
|
||||
{ beat: 6, value: 8192 },
|
||||
];
|
||||
|
||||
const baked = bakeMidiAutomationPointsInWindow(points, 4, 8, { ...defaultOptions, maxIntervalMs: 500 });
|
||||
|
||||
expect(baked[0]).toEqual({ beat: 4, value: 4096 });
|
||||
expect(baked.some(point => point.beat === 5 && point.value === 6144)).toBe(true);
|
||||
expect(baked.some(point => point.beat === 6 && point.value === 8192)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
export interface MidiAutomationPoint {
|
||||
beat: number;
|
||||
value: number;
|
||||
}
|
||||
|
||||
export interface BakedMidiAutomationPoint {
|
||||
beat: number;
|
||||
value: number;
|
||||
}
|
||||
|
||||
export interface MidiAutomationBakeOptions {
|
||||
maxIntervalMs: number;
|
||||
bpm: number;
|
||||
defaultValue: number;
|
||||
}
|
||||
|
||||
const BEAT_EPSILON = 1e-9;
|
||||
|
||||
function appendPoint(points: BakedMidiAutomationPoint[], nextPoint: BakedMidiAutomationPoint): void {
|
||||
const lastPoint = points[points.length - 1];
|
||||
if (!lastPoint) {
|
||||
points.push(nextPoint);
|
||||
return;
|
||||
}
|
||||
|
||||
if (Math.abs(lastPoint.beat - nextPoint.beat) <= BEAT_EPSILON) {
|
||||
points[points.length - 1] = nextPoint;
|
||||
return;
|
||||
}
|
||||
|
||||
if (lastPoint.value === nextPoint.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
points.push(nextPoint);
|
||||
}
|
||||
|
||||
function interpolateBetweenPoints(startPoint: MidiAutomationPoint, endPoint: MidiAutomationPoint, beat: number): number {
|
||||
if (Math.abs(endPoint.beat - startPoint.beat) <= BEAT_EPSILON) {
|
||||
return endPoint.value;
|
||||
}
|
||||
|
||||
const progress = (beat - startPoint.beat) / (endPoint.beat - startPoint.beat);
|
||||
return startPoint.value + ((endPoint.value - startPoint.value) * progress);
|
||||
}
|
||||
|
||||
function getMaxIntervalBeats(options: MidiAutomationBakeOptions): number {
|
||||
if (!Number.isFinite(options.maxIntervalMs) || options.maxIntervalMs <= 0) {
|
||||
return Number.POSITIVE_INFINITY;
|
||||
}
|
||||
|
||||
return (options.maxIntervalMs / 1000) * (options.bpm / 60);
|
||||
}
|
||||
|
||||
export function normalizeMidiAutomationPoints(points: MidiAutomationPoint[]): MidiAutomationPoint[] {
|
||||
const sortedPoints = [...points]
|
||||
.filter(point => Number.isFinite(point.beat) && Number.isFinite(point.value))
|
||||
.sort((a, b) => a.beat - b.beat);
|
||||
const normalizedPoints: MidiAutomationPoint[] = [];
|
||||
|
||||
sortedPoints.forEach(point => {
|
||||
const lastPoint = normalizedPoints[normalizedPoints.length - 1];
|
||||
if (lastPoint && Math.abs(lastPoint.beat - point.beat) <= BEAT_EPSILON) {
|
||||
normalizedPoints[normalizedPoints.length - 1] = point;
|
||||
return;
|
||||
}
|
||||
|
||||
normalizedPoints.push(point);
|
||||
});
|
||||
|
||||
return normalizedPoints;
|
||||
}
|
||||
|
||||
export function collectRegionMidiAutomationPoints(
|
||||
regions: Array<{ startBeat: number; points: MidiAutomationPoint[] }>
|
||||
): MidiAutomationPoint[] {
|
||||
return normalizeMidiAutomationPoints(
|
||||
regions.flatMap(region => region.points.map(point => ({
|
||||
beat: region.startBeat + point.beat,
|
||||
value: point.value,
|
||||
})))
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveMidiAutomationValueAtBeat(
|
||||
points: MidiAutomationPoint[],
|
||||
beat: number,
|
||||
defaultValue: number
|
||||
): number {
|
||||
const normalizedPoints = normalizeMidiAutomationPoints(points);
|
||||
if (normalizedPoints.length === 0) {
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
let previousPoint: MidiAutomationPoint | null = null;
|
||||
for (const point of normalizedPoints) {
|
||||
if (beat < point.beat) {
|
||||
if (!previousPoint) {
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
return interpolateBetweenPoints(previousPoint, point, beat);
|
||||
}
|
||||
|
||||
previousPoint = point;
|
||||
}
|
||||
|
||||
return previousPoint?.value ?? defaultValue;
|
||||
}
|
||||
|
||||
export function bakeMidiAutomationPointsInWindow(
|
||||
points: MidiAutomationPoint[],
|
||||
windowStartBeat: number,
|
||||
windowEndBeat: number,
|
||||
options: MidiAutomationBakeOptions
|
||||
): BakedMidiAutomationPoint[] {
|
||||
const normalizedPoints = normalizeMidiAutomationPoints(points);
|
||||
const anchorPoint = {
|
||||
beat: windowStartBeat,
|
||||
value: resolveMidiAutomationValueAtBeat(normalizedPoints, windowStartBeat, options.defaultValue),
|
||||
};
|
||||
|
||||
if (windowEndBeat <= windowStartBeat) {
|
||||
return [anchorPoint];
|
||||
}
|
||||
|
||||
const bakedPoints: BakedMidiAutomationPoint[] = [anchorPoint];
|
||||
if (normalizedPoints.length === 0) {
|
||||
return bakedPoints;
|
||||
}
|
||||
|
||||
const firstPoint = normalizedPoints[0];
|
||||
if (windowStartBeat < firstPoint.beat && firstPoint.beat < windowEndBeat) {
|
||||
appendPoint(bakedPoints, { beat: firstPoint.beat, value: firstPoint.value });
|
||||
}
|
||||
|
||||
const maxIntervalBeats = getMaxIntervalBeats(options);
|
||||
for (let index = 0; index < normalizedPoints.length - 1; index += 1) {
|
||||
const startPoint = normalizedPoints[index];
|
||||
const endPoint = normalizedPoints[index + 1];
|
||||
const overlapStartBeat = Math.max(windowStartBeat, startPoint.beat);
|
||||
const overlapEndBeat = Math.min(windowEndBeat, endPoint.beat);
|
||||
|
||||
if (overlapEndBeat - overlapStartBeat <= BEAT_EPSILON) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (startPoint.value === endPoint.value) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const segmentLengthBeats = overlapEndBeat - overlapStartBeat;
|
||||
const segmentCount = Number.isFinite(maxIntervalBeats)
|
||||
? Math.max(1, Math.ceil(segmentLengthBeats / maxIntervalBeats))
|
||||
: 1;
|
||||
|
||||
for (let segmentIndex = 1; segmentIndex <= segmentCount; segmentIndex += 1) {
|
||||
const beat = overlapStartBeat + ((segmentLengthBeats * segmentIndex) / segmentCount);
|
||||
if (beat >= windowEndBeat - BEAT_EPSILON) {
|
||||
continue;
|
||||
}
|
||||
|
||||
appendPoint(bakedPoints, {
|
||||
beat,
|
||||
value: interpolateBetweenPoints(startPoint, endPoint, beat),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return bakedPoints;
|
||||
}
|
||||
Reference in New Issue
Block a user