feat: added pitch bend automation (linear interpolation)
This commit is contained in:
@@ -66,4 +66,15 @@ describe('KGAudioBus live MIDI pitch bend', () => {
|
||||
audioBus.releaseLiveMidiNote(60, 1.25);
|
||||
expect(source.stop).toHaveBeenCalledWith(1.25);
|
||||
});
|
||||
|
||||
it('uses timed playback-rate automation when scheduling pitch bend updates', async () => {
|
||||
const audioBus = await KGAudioBus.create('acoustic_grand_piano');
|
||||
|
||||
audioBus.triggerLiveMidiAttack(60, 0, 1);
|
||||
|
||||
const source = MockBufferSource.mock.results[0].value;
|
||||
audioBus.scheduleLiveMidiPitchBend(1, 2);
|
||||
|
||||
expect(source.playbackRate.setValueAtTime).toHaveBeenCalledWith(Math.pow(2, 2 / 12), 2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -20,7 +20,7 @@ interface LiveMidiSource {
|
||||
export class KGAudioBus {
|
||||
// Fixed at +/-2 semitones for now. Future work: make this user-configurable
|
||||
// or honor MIDI RPN 0,0 (Pitch Bend Sensitivity).
|
||||
private static readonly LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES = 2;
|
||||
public static readonly LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES = 2;
|
||||
private static readonly LIVE_MIDI_NOTE_NAMES = ['C', 'Db', 'D', 'Eb', 'E', 'F', 'Gb', 'G', 'Ab', 'A', 'Bb', 'B'];
|
||||
|
||||
// Core audio components
|
||||
@@ -218,7 +218,17 @@ export class KGAudioBus {
|
||||
|
||||
for (const activeSources of this.liveMidiSources.values()) {
|
||||
activeSources.forEach(({ source, basePlaybackRate }) => {
|
||||
source.playbackRate.value = this.applyPitchBendToPlaybackRate(basePlaybackRate);
|
||||
this.setPlaybackRateValue(source, this.applyPitchBendToPlaybackRate(basePlaybackRate));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public scheduleLiveMidiPitchBend(normalizedBend: number, time: number): void {
|
||||
this.liveMidiPitchBend = Math.max(-1, Math.min(1, normalizedBend));
|
||||
|
||||
for (const activeSources of this.liveMidiSources.values()) {
|
||||
activeSources.forEach(({ source, basePlaybackRate }) => {
|
||||
this.setPlaybackRateValue(source, this.applyPitchBendToPlaybackRate(basePlaybackRate), time);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -474,13 +484,13 @@ export class KGAudioBus {
|
||||
velocity?: number,
|
||||
duration?: number
|
||||
): LiveMidiSource | null {
|
||||
const closestPitch = this.findClosestBufferedPitch(pitch);
|
||||
const closestPitch = KGAudioBus.findClosestBufferedPitch(this.instrument, this.audioBuffers, pitch);
|
||||
if (closestPitch === null) {
|
||||
console.warn(`No audio buffer found for live MIDI pitch ${pitch} on ${this.instrument}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const bufferKey = this.midiPitchToBufferKey(closestPitch);
|
||||
const bufferKey = KGAudioBus.midiPitchToBufferKey(closestPitch);
|
||||
const buffer = this.audioBuffers.get(bufferKey);
|
||||
if (!buffer) {
|
||||
console.warn(`Missing audio buffer ${bufferKey} for ${this.instrument}`);
|
||||
@@ -514,23 +524,31 @@ export class KGAudioBus {
|
||||
return { source, basePlaybackRate };
|
||||
}
|
||||
|
||||
private applyPitchBendToPlaybackRate(basePlaybackRate: number): number {
|
||||
const bendSemitones = this.liveMidiPitchBend * KGAudioBus.LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES;
|
||||
public static applyNormalizedPitchBendToPlaybackRate(basePlaybackRate: number, normalizedBend: number): number {
|
||||
const bendSemitones = normalizedBend * KGAudioBus.LIVE_MIDI_PITCH_BEND_RANGE_SEMITONES;
|
||||
return basePlaybackRate * Math.pow(2, bendSemitones / 12);
|
||||
}
|
||||
|
||||
private findClosestBufferedPitch(targetPitch: number): number | null {
|
||||
const [minPitch, maxPitch] = FLUIDR3_INSTRUMENT_MAP[this.instrument]?.pitchRange || [21, 108];
|
||||
private applyPitchBendToPlaybackRate(basePlaybackRate: number): number {
|
||||
return KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, this.liveMidiPitchBend);
|
||||
}
|
||||
|
||||
public static findClosestBufferedPitch(
|
||||
instrument: InstrumentType,
|
||||
audioBuffers: Tone.ToneAudioBuffers,
|
||||
targetPitch: number
|
||||
): number | null {
|
||||
const [minPitch, maxPitch] = FLUIDR3_INSTRUMENT_MAP[instrument]?.pitchRange || [21, 108];
|
||||
const boundedPitch = Math.max(minPitch, Math.min(maxPitch, targetPitch));
|
||||
|
||||
for (let offset = 0; offset <= 96; offset++) {
|
||||
const upwardPitch = boundedPitch + offset;
|
||||
if (upwardPitch <= maxPitch && this.audioBuffers.has(this.midiPitchToBufferKey(upwardPitch))) {
|
||||
if (upwardPitch <= maxPitch && audioBuffers.has(KGAudioBus.midiPitchToBufferKey(upwardPitch))) {
|
||||
return upwardPitch;
|
||||
}
|
||||
|
||||
const downwardPitch = boundedPitch - offset;
|
||||
if (downwardPitch >= minPitch && this.audioBuffers.has(this.midiPitchToBufferKey(downwardPitch))) {
|
||||
if (downwardPitch >= minPitch && audioBuffers.has(KGAudioBus.midiPitchToBufferKey(downwardPitch))) {
|
||||
return downwardPitch;
|
||||
}
|
||||
}
|
||||
@@ -538,9 +556,23 @@ export class KGAudioBus {
|
||||
return null;
|
||||
}
|
||||
|
||||
private midiPitchToBufferKey(pitch: number): string {
|
||||
public static midiPitchToBufferKey(pitch: number): string {
|
||||
const octave = Math.floor((pitch - 12) / 12);
|
||||
const noteIndex = (pitch - 12) % 12;
|
||||
return `${KGAudioBus.LIVE_MIDI_NOTE_NAMES[noteIndex]}${octave}`;
|
||||
}
|
||||
|
||||
private setPlaybackRateValue(source: Tone.ToneBufferSource, value: number, time?: number): void {
|
||||
const playbackRate = source.playbackRate as unknown as {
|
||||
value: number;
|
||||
setValueAtTime?: (nextValue: number, nextTime: number) => void;
|
||||
};
|
||||
|
||||
if (time !== undefined && typeof playbackRate.setValueAtTime === 'function') {
|
||||
playbackRate.setValueAtTime(value, time);
|
||||
return;
|
||||
}
|
||||
|
||||
playbackRate.value = value;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { createMockProject } from '../../test/utils/mock-data';
|
||||
import { createMockMidiPitchBend, createMockMidiRegion, createMockMidiTrack, createMockProject } from '../../test/utils/mock-data';
|
||||
import { MockTransport } from '../../test/mocks/tone';
|
||||
|
||||
vi.mock('tone', async () => {
|
||||
@@ -22,6 +22,7 @@ vi.mock('../config/ConfigManager', () => ({
|
||||
import { KGCore } from '../KGCore';
|
||||
import { ConfigManager } from '../config/ConfigManager';
|
||||
import { KGAudioInterface } from './KGAudioInterface';
|
||||
import { MIDI_PITCH_BEND_CENTER, midiPitchBendToNormalized } from '../../util/midiUtil';
|
||||
|
||||
describe('KGAudioInterface preroll playback', () => {
|
||||
beforeEach(() => {
|
||||
@@ -50,6 +51,7 @@ describe('KGAudioInterface preroll playback', () => {
|
||||
get: (key: string) => {
|
||||
if (key === 'audio.playback_delay') return 0.2;
|
||||
if (key === 'audio.lookahead_time') return 0.05;
|
||||
if (key === 'audio.midi_automation_interpolation_interval_ms') return 250;
|
||||
return null;
|
||||
},
|
||||
} as unknown as ConfigManager)
|
||||
@@ -112,4 +114,79 @@ describe('KGAudioInterface preroll playback', () => {
|
||||
|
||||
expect(MockTransport.position).toBe(6);
|
||||
});
|
||||
|
||||
it('schedules baked pitch bend points instead of authored step changes only', () => {
|
||||
const region = createMockMidiRegion({
|
||||
pitchBends: [
|
||||
createMockMidiPitchBend({ id: 'bend-1', beat: 0, value: MIDI_PITCH_BEND_CENTER }),
|
||||
createMockMidiPitchBend({ id: 'bend-2', beat: 1, value: 0 }),
|
||||
],
|
||||
});
|
||||
const track = createMockMidiTrack({ id: 1, regions: [region] });
|
||||
const project = createMockProject({ tracks: [track] });
|
||||
const audio = KGAudioInterface.instance();
|
||||
const audioBus = {
|
||||
resetLiveMidiPitchBend: vi.fn(),
|
||||
setLiveMidiPitchBend: vi.fn(),
|
||||
scheduleLiveMidiPitchBend: vi.fn(),
|
||||
shouldPlayWithSolo: vi.fn().mockReturnValue(true),
|
||||
};
|
||||
|
||||
;(audio as unknown as { trackAudioBuses: Map<string, unknown> }).trackAudioBuses.set('1', audioBus);
|
||||
audio.preparePlayback(project, 0);
|
||||
|
||||
const scheduledTimes = MockTransport.schedule.mock.calls.map(([, time]) => time);
|
||||
expect(scheduledTimes).toContain(0.25);
|
||||
expect(scheduledTimes).toContain(0.5);
|
||||
});
|
||||
|
||||
it('computes the initial interpolated bend for non-zero playback starts', () => {
|
||||
const region = createMockMidiRegion({
|
||||
pitchBends: [
|
||||
createMockMidiPitchBend({ id: 'bend-1', beat: 0, value: MIDI_PITCH_BEND_CENTER }),
|
||||
createMockMidiPitchBend({ id: 'bend-2', beat: 4, value: 0 }),
|
||||
],
|
||||
});
|
||||
const track = createMockMidiTrack({ id: 1, regions: [region] });
|
||||
const project = createMockProject({ tracks: [track] });
|
||||
const audio = KGAudioInterface.instance();
|
||||
const audioBus = {
|
||||
resetLiveMidiPitchBend: vi.fn(),
|
||||
setLiveMidiPitchBend: vi.fn(),
|
||||
scheduleLiveMidiPitchBend: vi.fn(),
|
||||
shouldPlayWithSolo: vi.fn().mockReturnValue(true),
|
||||
};
|
||||
|
||||
;(audio as unknown as { trackAudioBuses: Map<string, unknown> }).trackAudioBuses.set('1', audioBus);
|
||||
audio.preparePlayback(project, 2);
|
||||
|
||||
expect(audioBus.setLiveMidiPitchBend).toHaveBeenCalledWith(midiPitchBendToNormalized(4096));
|
||||
});
|
||||
|
||||
it('adds a loop-start re-anchor for interpolated pitch bend state', () => {
|
||||
const region = createMockMidiRegion({
|
||||
pitchBends: [
|
||||
createMockMidiPitchBend({ id: 'bend-1', beat: 2, value: 0 }),
|
||||
createMockMidiPitchBend({ id: 'bend-2', beat: 6, value: MIDI_PITCH_BEND_CENTER }),
|
||||
],
|
||||
});
|
||||
const track = createMockMidiTrack({ id: 1, regions: [region] });
|
||||
const project = createMockProject({ tracks: [track] });
|
||||
project.setIsLooping(true);
|
||||
project.setLoopingRange([1, 1]);
|
||||
|
||||
const audio = KGAudioInterface.instance();
|
||||
const audioBus = {
|
||||
resetLiveMidiPitchBend: vi.fn(),
|
||||
setLiveMidiPitchBend: vi.fn(),
|
||||
scheduleLiveMidiPitchBend: vi.fn(),
|
||||
shouldPlayWithSolo: vi.fn().mockReturnValue(true),
|
||||
};
|
||||
|
||||
;(audio as unknown as { trackAudioBuses: Map<string, unknown> }).trackAudioBuses.set('1', audioBus);
|
||||
audio.preparePlayback(project, 5);
|
||||
|
||||
const scheduledTimes = MockTransport.schedule.mock.calls.map(([, time]) => time);
|
||||
expect(scheduledTimes).toContain(2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,7 +3,16 @@ import type { KGMidiNote } from '../midi/KGMidiNote';
|
||||
import type { KGMidiPitchBend } from '../midi/KGMidiPitchBend';
|
||||
import { TIME_CONSTANTS, AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants';
|
||||
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
|
||||
import { midiPitchBendToNormalized, pitchToNoteNameString } from '../../util/midiUtil';
|
||||
import {
|
||||
MIDI_PITCH_BEND_CENTER,
|
||||
midiPitchBendToNormalized,
|
||||
pitchToNoteNameString,
|
||||
} from '../../util/midiUtil';
|
||||
import {
|
||||
bakeMidiAutomationPointsInWindow,
|
||||
collectRegionMidiAutomationPoints,
|
||||
resolveMidiAutomationValueAtBeat,
|
||||
} from '../../util/midiAutomationUtil';
|
||||
import * as Tone from 'tone';
|
||||
import { KGAudioBus } from './KGAudioBus';
|
||||
import { KGAudioPlayerBus } from './KGAudioPlayerBus';
|
||||
@@ -473,30 +482,35 @@ export class KGAudioInterface {
|
||||
project.getTracks().forEach(track => {
|
||||
const trackId = track.getId().toString();
|
||||
const audioBus = this.trackAudioBuses.get(trackId);
|
||||
const interpolationIntervalMs = (configManager.get('audio.midi_automation_interpolation_interval_ms') as number) ?? 10;
|
||||
|
||||
console.log(`Track ${trackId} has audio bus: ${audioBus ? 'true' : 'false'}; type: ${track.getType()}`);
|
||||
|
||||
// Schedule MIDI track events
|
||||
if (audioBus && track.getType() === 'MIDI') {
|
||||
const trackPitchBends: Array<{ pitchBend: KGMidiPitchBend; absoluteBeat: number }> = [];
|
||||
const trackNotes: Array<{ note: KGMidiNote; absoluteStartBeat: number; absoluteEndBeat: number }> = [];
|
||||
const trackPitchBends = collectRegionMidiAutomationPoints(
|
||||
track.getRegions()
|
||||
.filter(region => region.getCurrentType() === 'KGMidiRegion')
|
||||
.map(region => {
|
||||
const midiRegion = region as unknown as { getPitchBends: () => KGMidiPitchBend[] };
|
||||
return {
|
||||
startBeat: region.getStartFromBeat(),
|
||||
points: midiRegion.getPitchBends().map(pitchBend => ({
|
||||
beat: pitchBend.getBeat(),
|
||||
value: pitchBend.getValue(),
|
||||
})),
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
track.getRegions().forEach(region => {
|
||||
console.log(`Region ${region.getId().toString()}: type: ${region.getCurrentType()}`);
|
||||
|
||||
if (region.getCurrentType() === 'KGMidiRegion') {
|
||||
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[]; getPitchBends: () => KGMidiPitchBend[] };
|
||||
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[] };
|
||||
const regionStartBeat = region.getStartFromBeat();
|
||||
|
||||
if (midiRegion.getPitchBends) {
|
||||
midiRegion.getPitchBends().forEach((pitchBend: KGMidiPitchBend) => {
|
||||
trackPitchBends.push({
|
||||
pitchBend,
|
||||
absoluteBeat: regionStartBeat + pitchBend.getBeat(),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Get notes from region (assuming it has a getNotes method)
|
||||
if (midiRegion.getNotes) {
|
||||
midiRegion.getNotes().forEach((note: KGMidiNote) => {
|
||||
@@ -523,35 +537,37 @@ export class KGAudioInterface {
|
||||
}
|
||||
});
|
||||
|
||||
const boundedTrackPitchBends = trackPitchBends
|
||||
.filter(({ absoluteBeat }) => absoluteBeat >= scheduleStartBeat && absoluteBeat < scheduleEndBeat)
|
||||
.sort((a, b) => a.absoluteBeat - b.absoluteBeat);
|
||||
const initialPitchBend = [...boundedTrackPitchBends]
|
||||
.reverse()
|
||||
.find(({ absoluteBeat }) => absoluteBeat <= startPosition);
|
||||
audioBus.setLiveMidiPitchBend(midiPitchBendToNormalized(initialPitchBend?.pitchBend.getValue() ?? 8192));
|
||||
const initialPitchBendBeat = isLooping ? Math.max(startPosition, scheduleStartBeat) : startPosition;
|
||||
const initialPitchBendValue = resolveMidiAutomationValueAtBeat(
|
||||
trackPitchBends,
|
||||
initialPitchBendBeat,
|
||||
MIDI_PITCH_BEND_CENTER
|
||||
);
|
||||
audioBus.setLiveMidiPitchBend(midiPitchBendToNormalized(initialPitchBendValue));
|
||||
|
||||
if (isLooping && !boundedTrackPitchBends.some(({ absoluteBeat }) => absoluteBeat === scheduleStartBeat)) {
|
||||
const eventId = Tone.Transport.schedule((time) => {
|
||||
const hasSoloedTracks = this.hasSoloedTracks();
|
||||
if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) {
|
||||
audioBus.setLiveMidiPitchBend(0);
|
||||
}
|
||||
}, this.beatsToToneTime(scheduleStartBeat));
|
||||
this.scheduledEvents.add(eventId);
|
||||
}
|
||||
const pitchBendWindowStartBeat = isLooping ? scheduleStartBeat : Math.max(startPosition, 0);
|
||||
const bakedTrackPitchBends = bakeMidiAutomationPointsInWindow(
|
||||
trackPitchBends,
|
||||
pitchBendWindowStartBeat,
|
||||
scheduleEndBeat,
|
||||
{
|
||||
maxIntervalMs: interpolationIntervalMs,
|
||||
bpm: project.getBpm(),
|
||||
defaultValue: MIDI_PITCH_BEND_CENTER,
|
||||
}
|
||||
);
|
||||
|
||||
boundedTrackPitchBends.forEach(({ pitchBend, absoluteBeat }) => {
|
||||
if (absoluteBeat < startPosition) {
|
||||
bakedTrackPitchBends.forEach(({ beat, value }) => {
|
||||
if (!isLooping && beat <= pitchBendWindowStartBeat) {
|
||||
return;
|
||||
}
|
||||
|
||||
const eventId = Tone.Transport.schedule(() => {
|
||||
const eventId = Tone.Transport.schedule((time) => {
|
||||
const hasSoloedTracks = this.hasSoloedTracks();
|
||||
if (audioBus.shouldPlayWithSolo(hasSoloedTracks)) {
|
||||
audioBus.setLiveMidiPitchBend(midiPitchBendToNormalized(pitchBend.getValue()));
|
||||
audioBus.scheduleLiveMidiPitchBend(midiPitchBendToNormalized(value), time);
|
||||
}
|
||||
}, this.beatsToToneTime(absoluteBeat));
|
||||
}, this.beatsToToneTime(beat));
|
||||
|
||||
this.scheduledEvents.add(eventId);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { encodeWav, getOfflineTrackGain, getOfflineTrackVolumeDb } from './KGOfflineRenderer';
|
||||
import { applyOfflinePitchBendAutomation, encodeWav, getOfflineTrackGain, getOfflineTrackVolumeDb } from './KGOfflineRenderer';
|
||||
|
||||
/**
|
||||
* Create a minimal AudioBuffer-like object for testing.
|
||||
@@ -164,3 +164,32 @@ describe('offline track volume conversion', () => {
|
||||
expect(getOfflineTrackGain(-60, false)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('offline pitch bend automation', () => {
|
||||
it('applies baked pitch bends to source playback rate automation', () => {
|
||||
const source = {
|
||||
playbackRate: {
|
||||
value: 1,
|
||||
setValueAtTime: (..._args: unknown[]) => undefined,
|
||||
},
|
||||
} as unknown as Parameters<typeof applyOfflinePitchBendAutomation>[0];
|
||||
|
||||
const calls: Array<[number, number]> = [];
|
||||
source.playbackRate.setValueAtTime = ((value: number, time: number) => {
|
||||
calls.push([value, time]);
|
||||
return source.playbackRate as never;
|
||||
}) as typeof source.playbackRate.setValueAtTime;
|
||||
|
||||
applyOfflinePitchBendAutomation(
|
||||
source,
|
||||
1,
|
||||
[{ beat: 1, value: 0 }],
|
||||
0,
|
||||
0.5
|
||||
);
|
||||
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0][1]).toBe(0.5);
|
||||
expect(calls[0][0]).toBeCloseTo(Math.pow(2, -2 / 12), 5);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,13 +1,24 @@
|
||||
import * as Tone from 'tone';
|
||||
import type { KGProject } from '../KGProject';
|
||||
import type { KGMidiNote } from '../midi/KGMidiNote';
|
||||
import type { KGMidiPitchBend } from '../midi/KGMidiPitchBend';
|
||||
import type { KGAudioRegion } from '../region/KGAudioRegion';
|
||||
import type { InstrumentType } from '../track/KGMidiTrack';
|
||||
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
|
||||
import { AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants';
|
||||
import { pitchToNoteNameString } from '../../util/midiUtil';
|
||||
import { MIDI_PITCH_BEND_CENTER, midiPitchBendToNormalized } from '../../util/midiUtil';
|
||||
import {
|
||||
bakeMidiAutomationPointsInWindow,
|
||||
collectRegionMidiAutomationPoints,
|
||||
resolveMidiAutomationValueAtBeat,
|
||||
type BakedMidiAutomationPoint,
|
||||
type MidiAutomationPoint,
|
||||
} from '../../util/midiAutomationUtil';
|
||||
import { KGToneBuffersPool } from './KGToneBuffersPool';
|
||||
import { KGToneSamplerFactory } from './KGToneSamplerFactory';
|
||||
import { KGAudioInterface } from './KGAudioInterface';
|
||||
import { KGAudioBus } from './KGAudioBus';
|
||||
import { ConfigManager } from '../config/ConfigManager';
|
||||
import { Mp3Encoder } from '@breezystack/lamejs';
|
||||
|
||||
export interface RenderOptions {
|
||||
@@ -104,7 +115,7 @@ export class KGOfflineRenderer {
|
||||
// Pre-collect all the data we need before entering the offline context
|
||||
const midiTrackData: Array<{
|
||||
trackId: string;
|
||||
instrumentName: string;
|
||||
instrumentName: InstrumentType;
|
||||
volume: number;
|
||||
muted: boolean;
|
||||
solo: boolean;
|
||||
@@ -112,6 +123,7 @@ export class KGOfflineRenderer {
|
||||
startBeat: number;
|
||||
notes: Array<{ startBeat: number; endBeat: number; durationBeats: number; pitch: number; velocity: number }>;
|
||||
}>;
|
||||
pitchBends: MidiAutomationPoint[];
|
||||
}> = [];
|
||||
|
||||
const audioTrackData: Array<{
|
||||
@@ -130,13 +142,14 @@ export class KGOfflineRenderer {
|
||||
}> = [];
|
||||
|
||||
let hasSoloedTracks = false;
|
||||
const interpolationIntervalMs = (ConfigManager.instance().get('audio.midi_automation_interpolation_interval_ms') as number) ?? 10;
|
||||
|
||||
for (const track of tracks) {
|
||||
const trackId = track.getId().toString();
|
||||
|
||||
if (track.getType() === 'MIDI') {
|
||||
const midiTrack = track as unknown as { getInstrument: () => string };
|
||||
const instrumentName = String(midiTrack.getInstrument());
|
||||
const midiTrack = track as unknown as { getInstrument: () => InstrumentType };
|
||||
const instrumentName = midiTrack.getInstrument();
|
||||
|
||||
// Get live bus state for volume/mute/solo via public getters
|
||||
const volume = audioInterface.getTrackVolume(trackId);
|
||||
@@ -147,7 +160,7 @@ export class KGOfflineRenderer {
|
||||
const regions: typeof midiTrackData[0]['regions'] = [];
|
||||
for (const region of track.getRegions()) {
|
||||
if (region.getCurrentType() === 'KGMidiRegion') {
|
||||
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[] };
|
||||
const midiRegion = region as unknown as { getNotes: () => KGMidiNote[]; getPitchBends: () => KGMidiPitchBend[] };
|
||||
if (midiRegion.getNotes) {
|
||||
const notes = midiRegion.getNotes().map(note => ({
|
||||
startBeat: note.getStartBeat() + region.getStartFromBeat(),
|
||||
@@ -160,8 +173,22 @@ export class KGOfflineRenderer {
|
||||
}
|
||||
}
|
||||
}
|
||||
const pitchBends = collectRegionMidiAutomationPoints(
|
||||
track.getRegions()
|
||||
.filter(region => region.getCurrentType() === 'KGMidiRegion')
|
||||
.map(region => {
|
||||
const midiRegion = region as unknown as { getPitchBends: () => KGMidiPitchBend[] };
|
||||
return {
|
||||
startBeat: region.getStartFromBeat(),
|
||||
points: midiRegion.getPitchBends().map(pitchBend => ({
|
||||
beat: pitchBend.getBeat(),
|
||||
value: pitchBend.getValue(),
|
||||
})),
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
midiTrackData.push({ trackId, instrumentName, volume, muted, solo, regions });
|
||||
midiTrackData.push({ trackId, instrumentName, volume, muted, solo, regions, pitchBends });
|
||||
} else if (track.getType() === 'Wave') {
|
||||
const volume = audioInterface.getTrackVolume(trackId);
|
||||
const muted = audioInterface.getTrackMuted(trackId);
|
||||
@@ -241,7 +268,7 @@ export class KGOfflineRenderer {
|
||||
const promise = (async () => {
|
||||
try {
|
||||
// Get cached buffers from pool
|
||||
const audioBuffers = await KGToneBuffersPool.instance().getToneAudioBuffers(trackInfo.instrumentName);
|
||||
const audioBuffers = await KGToneBuffersPool.instance().getToneAudioBuffers(String(trackInfo.instrumentName));
|
||||
const pitchRange = FLUIDR3_INSTRUMENT_MAP[trackInfo.instrumentName]?.pitchRange || [21, 108];
|
||||
const urlMap = KGToneSamplerFactory.instance().convertBuffersToUrls(audioBuffers, pitchRange);
|
||||
|
||||
@@ -258,6 +285,16 @@ export class KGOfflineRenderer {
|
||||
// Track volumes are stored in dB across the app, with 0 meaning unity gain.
|
||||
sampler.volume.value = getOfflineTrackVolumeDb(trackInfo.volume, trackInfo.muted);
|
||||
sampler.connect(masterGain);
|
||||
const bakedTrackPitchBends = bakeMidiAutomationPointsInWindow(
|
||||
trackInfo.pitchBends,
|
||||
renderStartBeat,
|
||||
renderEndBeat,
|
||||
{
|
||||
maxIntervalMs: interpolationIntervalMs,
|
||||
bpm: project.getBpm(),
|
||||
defaultValue: MIDI_PITCH_BEND_CENTER,
|
||||
}
|
||||
);
|
||||
|
||||
// Schedule all notes for this track
|
||||
for (const regionInfo of trackInfo.regions) {
|
||||
@@ -268,11 +305,32 @@ export class KGOfflineRenderer {
|
||||
const offsetBeat = note.startBeat - renderStartBeat;
|
||||
const noteStartTime = offsetBeat * secondsPerBeat;
|
||||
const noteDuration = note.durationBeats * secondsPerBeat;
|
||||
const noteName = pitchToNoteNameString(note.pitch);
|
||||
const velocity = note.velocity / 127;
|
||||
const initialNormalizedPitchBend = midiPitchBendToNormalized(
|
||||
resolveMidiAutomationValueAtBeat(trackInfo.pitchBends, note.startBeat, MIDI_PITCH_BEND_CENTER)
|
||||
);
|
||||
const offlineSource = createOfflinePitchBendAwareSource(
|
||||
sampler,
|
||||
audioBuffers,
|
||||
trackInfo.instrumentName,
|
||||
note.pitch,
|
||||
initialNormalizedPitchBend
|
||||
);
|
||||
if (!offlineSource) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const { source, basePlaybackRate } = offlineSource;
|
||||
applyOfflinePitchBendAutomation(
|
||||
source,
|
||||
basePlaybackRate,
|
||||
bakedTrackPitchBends.filter(point => point.beat > note.startBeat && point.beat < note.endBeat),
|
||||
renderStartBeat,
|
||||
secondsPerBeat
|
||||
);
|
||||
|
||||
context.transport.schedule((time) => {
|
||||
sampler.triggerAttackRelease(noteName, noteDuration, time, velocity);
|
||||
source.start(time, 0, noteDuration, velocity);
|
||||
}, noteStartTime);
|
||||
}
|
||||
}
|
||||
@@ -418,6 +476,76 @@ function shouldPlay(trackInfo: { muted: boolean; solo: boolean }, hasSoloedTrack
|
||||
return true;
|
||||
}
|
||||
|
||||
function setOfflinePlaybackRate(
|
||||
source: Tone.ToneBufferSource,
|
||||
value: number,
|
||||
time: number
|
||||
): void {
|
||||
const playbackRate = source.playbackRate as unknown as {
|
||||
value: number;
|
||||
setValueAtTime?: (nextValue: number, nextTime: number) => void;
|
||||
};
|
||||
|
||||
if (typeof playbackRate.setValueAtTime === 'function') {
|
||||
playbackRate.setValueAtTime(value, time);
|
||||
return;
|
||||
}
|
||||
|
||||
playbackRate.value = value;
|
||||
}
|
||||
|
||||
function createOfflinePitchBendAwareSource(
|
||||
sampler: Tone.Sampler,
|
||||
audioBuffers: Tone.ToneAudioBuffers,
|
||||
instrumentName: InstrumentType,
|
||||
pitch: number,
|
||||
initialNormalizedPitchBend: number
|
||||
): { source: Tone.ToneBufferSource; basePlaybackRate: number } | null {
|
||||
const closestPitch = KGAudioBus.findClosestBufferedPitch(instrumentName, audioBuffers, pitch);
|
||||
if (closestPitch === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const bufferKey = KGAudioBus.midiPitchToBufferKey(closestPitch);
|
||||
const buffer = audioBuffers.get(bufferKey);
|
||||
if (!buffer) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const basePlaybackRate = Math.pow(2, (pitch - closestPitch) / 12);
|
||||
const source = new Tone.ToneBufferSource({
|
||||
url: buffer,
|
||||
fadeIn: sampler.attack,
|
||||
fadeOut: sampler.release,
|
||||
curve: sampler.curve,
|
||||
playbackRate: KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, initialNormalizedPitchBend),
|
||||
}).connect(sampler.output);
|
||||
setOfflinePlaybackRate(
|
||||
source,
|
||||
KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, initialNormalizedPitchBend),
|
||||
0
|
||||
);
|
||||
|
||||
return { source, basePlaybackRate };
|
||||
}
|
||||
|
||||
export function applyOfflinePitchBendAutomation(
|
||||
source: Tone.ToneBufferSource,
|
||||
basePlaybackRate: number,
|
||||
bakedPitchBends: BakedMidiAutomationPoint[],
|
||||
renderStartBeat: number,
|
||||
secondsPerBeat: number
|
||||
): void {
|
||||
bakedPitchBends.forEach(point => {
|
||||
const automationTime = (point.beat - renderStartBeat) * secondsPerBeat;
|
||||
setOfflinePlaybackRate(
|
||||
source,
|
||||
KGAudioBus.applyNormalizedPitchBendToPlaybackRate(basePlaybackRate, midiPitchBendToNormalized(point.value)),
|
||||
automationTime
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function getOfflineTrackVolumeDb(volumeDb: number, muted: boolean): number {
|
||||
const isSilent = muted || volumeDb <= AUDIO_INTERFACE_CONSTANTS.MIN_TRACK_VOLUME_DB;
|
||||
return isSilent ? -Infinity : volumeDb;
|
||||
|
||||
Reference in New Issue
Block a user