diff --git a/public/test-data/audio-to-midi-test01.mp3 b/public/test-data/audio-to-midi-test01.mp3 new file mode 100644 index 0000000..ff72a49 Binary files /dev/null and b/public/test-data/audio-to-midi-test01.mp3 differ diff --git a/src/components/common/DialogProvider.css b/src/components/common/DialogProvider.css index 903e79b..ab49e9a 100644 --- a/src/components/common/DialogProvider.css +++ b/src/components/common/DialogProvider.css @@ -153,6 +153,17 @@ gap: 16px; } +.dialog-two-column-row { + display: flex; + align-items: flex-start; + gap: 12px; +} + +.dialog-half-width-group { + flex: 1 1 0; + min-width: 0; +} + .dialog-hint-card { margin-top: 12px; background-color: #252525; @@ -221,6 +232,27 @@ accent-color: #5a9fd4; } +.dialog-compact-checkbox-row { + flex: 1 1 0; + min-width: 0; +} + +.dialog-compact-checkbox-row span { + line-height: 1.3; +} + +.dialog-compact-input { + margin-top: 0; +} + +.dialog-inline-sep { + color: #b0b0b0; + font-size: 13px; + font-weight: 500; + line-height: 1; + flex: 0 0 auto; +} + .dialog-footer { display: flex; justify-content: flex-end; diff --git a/src/components/common/DialogProvider.test.tsx b/src/components/common/DialogProvider.test.tsx index 68af2b2..2f38353 100644 --- a/src/components/common/DialogProvider.test.tsx +++ b/src/components/common/DialogProvider.test.tsx @@ -19,6 +19,7 @@ vi.mock('../../core/config/ConfigManager', () => ({ import DialogProvider from './DialogProvider'; import { + showAudioToMidiOptions, showChoice, showChordDetectionOptions, showMidiChordDetectionOptions, @@ -170,6 +171,123 @@ describe('DialogProvider MIDI chord detection dialog', () => { }); }); +describe('DialogProvider audio-to-MIDI dialog', () => { + it('opens with the expected defaults and resolves cancel to null', async () => { + let resolved: unknown = 'pending'; + + render( + + + , + ); + + fireEvent.click(screen.getByRole('button', { name: 'Open audio-to-midi' })); + + expect(screen.getByText('Convert to MIDI')).toBeInTheDocument(); + expect(screen.getByText('Experimental Feature')).toBeInTheDocument(); + expect(screen.getByText(/best results, switch to spectrogram view/i)).toBeInTheDocument(); + expect(screen.getByLabelText('Monophonic only (v1)')).toBeChecked(); + expect(screen.getByLabelText('Monophonic only (v1)')).toBeDisabled(); + expect(screen.getByLabelText('Use current Floor dB')).toBeChecked(); + expect(screen.getByLabelText('Manual Floor dB')).toBeDisabled(); + expect(screen.getByLabelText('Convert loop-range audio clip only')).toBeChecked(); + expect(screen.getByLabelText('Convert loop-range audio clip only')).toBeDisabled(); + + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })); + finishDialogCloseAnimation(); + + await waitFor(() => expect(resolved).toBeNull()); + }); + + it('returns adjusted conversion options when confirmed', async () => { + let resolved: unknown = null; + + render( + + + , + ); + + fireEvent.click(screen.getByRole('button', { name: 'Open audio-to-midi' })); + fireEvent.click(screen.getByLabelText('Use current Floor dB')); + fireEvent.change(screen.getByLabelText('Manual Floor dB'), { target: { value: '-16' } }); + fireEvent.change(screen.getByLabelText('Pitch range start'), { target: { value: '21' } }); + fireEvent.change(screen.getByLabelText('Pitch range end'), { target: { value: '38' } }); + fireEvent.change(screen.getByLabelText('Quantize note start'), { target: { value: '1/8' } }); + fireEvent.change(screen.getByLabelText('Quantize note length'), { target: { value: '1/8' } }); + fireEvent.click(screen.getByLabelText('Convert loop-range audio clip only')); + fireEvent.click(screen.getByLabelText('Group adjacent pitches to the strongest bin')); + fireEvent.change(screen.getByLabelText('Target MIDI track'), { target: { value: 'track-2' } }); + fireEvent.click(screen.getByRole('button', { name: 'OK' })); + finishDialogCloseAnimation(); + + await waitFor(() => expect(resolved).toEqual({ + monophonic: true, + useCurrentFloorDb: false, + manualFloorDb: -16, + pitchRangeStart: 21, + pitchRangeEnd: 38, + quantizeNoteStart: '1/8', + quantizeNoteLength: '1/8', + convertLoopRangeOnly: false, + groupAdjacentPitchesToHighest: false, + targetTrackId: 'track-2', + })); + }); +}); + describe('DialogProvider tempo detection dialog', () => { it('opens the tempo detection modal with the expected defaults and resolves cancel to null', async () => { let resolved: unknown = 'pending'; diff --git a/src/components/common/DialogProvider.tsx b/src/components/common/DialogProvider.tsx index ac2bdc7..cd161ca 100644 --- a/src/components/common/DialogProvider.tsx +++ b/src/components/common/DialogProvider.tsx @@ -2,9 +2,11 @@ import React, { useState, useCallback, useRef } from 'react'; import './DialogProvider.css'; import { FaTimes } from 'react-icons/fa'; import { ConfigManager } from '../../core/config/ConfigManager'; +import { KGPianoRollState } from '../../core/state/KGPianoRollState'; import { useI18n } from '../../i18n/useI18n'; import { registerDialogFns } from '../../util/dialogUtil'; import type { + AudioToMidiOptionsResult, ChoiceOption, ChordDetectionOptionsResult, ConfirmOptions, @@ -16,7 +18,7 @@ import type { } from '../../util/dialogUtil'; interface DialogInfo { - type: 'alert' | 'confirm' | 'prompt' | 'timesig' | 'choice' | 'chord-detection' | 'midi-chord-detection' | 'tempo-detection' | 'tempo-apply'; + type: 'alert' | 'confirm' | 'prompt' | 'timesig' | 'choice' | 'chord-detection' | 'midi-chord-detection' | 'tempo-detection' | 'tempo-apply' | 'audio-to-midi'; message: string; options?: ConfirmOptions | PromptOptions; defaultValue?: string; @@ -25,6 +27,9 @@ interface DialogInfo { defaultChordDetectionOptions?: ChordDetectionOptionsResult; defaultMidiChordDetectionOptions?: MidiChordDetectionOptionsResult; defaultTempoDetectionOptions?: TempoDetectionOptionsResult; + defaultAudioToMidiOptions?: AudioToMidiOptionsResult; + audioToMidiTargetTracks?: ChoiceOption[]; + audioToMidiLoopModeEnabled?: boolean; } const DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS: ChordDetectionOptionsResult = { @@ -45,6 +50,19 @@ const DEFAULT_TEMPO_DETECTION_OPTIONS: TempoDetectionOptionsResult = { maxTempo: 180, }; +const DEFAULT_AUDIO_TO_MIDI_OPTIONS: AudioToMidiOptionsResult = { + monophonic: true, + useCurrentFloorDb: true, + manualFloorDb: -25, + pitchRangeStart: 12, + pitchRangeEnd: 107, + quantizeNoteStart: '1/16', + quantizeNoteLength: '1/16', + convertLoopRangeOnly: true, + groupAdjacentPitchesToHighest: true, + targetTrackId: '', +}; + const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { const { t } = useI18n(); const [dialog, setDialog] = useState(null); @@ -55,6 +73,8 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = const [chordDetectionOptions, setChordDetectionOptions] = useState(DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS); const [midiChordDetectionOptions, setMidiChordDetectionOptions] = useState(DEFAULT_MIDI_CHORD_DETECTION_OPTIONS); const [tempoDetectionOptions, setTempoDetectionOptions] = useState(DEFAULT_TEMPO_DETECTION_OPTIONS); + const [audioToMidiOptions, setAudioToMidiOptions] = useState(DEFAULT_AUDIO_TO_MIDI_OPTIONS); + const [audioToMidiTargetTracks, setAudioToMidiTargetTracks] = useState([]); const [autoAlignRegionToBeat, setAutoAlignRegionToBeat] = useState(false); // eslint-disable-next-line @typescript-eslint/no-explicit-any const resolveRef = useRef<((value: any) => void) | null>(null); @@ -142,6 +162,30 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = }); }, []); + const openAudioToMidiOptions = useCallback(( + message: string, + targetTracks: ChoiceOption[], + loopModeEnabled: boolean, + defaultValue?: AudioToMidiOptionsResult, + ): Promise => { + return new Promise((resolve) => { + resolveRef.current = resolve; + const fallbackTrackId = targetTracks[0]?.value ?? ''; + setAudioToMidiTargetTracks(targetTracks); + setAudioToMidiOptions({ + ...(defaultValue ?? DEFAULT_AUDIO_TO_MIDI_OPTIONS), + targetTrackId: defaultValue?.targetTrackId || fallbackTrackId, + }); + setDialog({ + type: 'audio-to-midi', + message, + defaultAudioToMidiOptions: defaultValue, + audioToMidiTargetTracks: targetTracks, + audioToMidiLoopModeEnabled: loopModeEnabled, + }); + }); + }, []); + const close = useCallback((value: unknown) => { pendingValueRef.current = value; setIsClosing(true); @@ -158,6 +202,8 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = setChordDetectionOptions(DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS); setMidiChordDetectionOptions(DEFAULT_MIDI_CHORD_DETECTION_OPTIONS); setTempoDetectionOptions(DEFAULT_TEMPO_DETECTION_OPTIONS); + setAudioToMidiOptions(DEFAULT_AUDIO_TO_MIDI_OPTIONS); + setAudioToMidiTargetTracks([]); setAutoAlignRegionToBeat(false); if (resolveRef.current) { resolveRef.current(pendingValueRef.current); @@ -180,6 +226,7 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = openMidiChordDetectionOptions, openTempoDetectionOptions, openTempoApply, + openAudioToMidiOptions, ); } @@ -195,6 +242,7 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = const isMidiChordDetection = dialog.type === 'midi-chord-detection'; const isTempoDetection = dialog.type === 'tempo-detection'; const isTempoApply = dialog.type === 'tempo-apply'; + const isAudioToMidi = dialog.type === 'audio-to-midi'; const promptOptions = isPrompt ? (dialog.options as PromptOptions | undefined) : undefined; const isKGOneEnabled = (ConfigManager.instance().get('general.kgone.enabled') as boolean | undefined) ?? false; @@ -203,10 +251,12 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = : isTimeSig ? t('dialog.title.timeSignature') : isTempoDetection - ? t('dialog.title.tempoDetection') - : isTempoApply - ? t('dialog.title.applyTempo') - : (isChordDetection || isMidiChordDetection) + ? t('dialog.title.tempoDetection') + : isTempoApply + ? t('dialog.title.applyTempo') + : isAudioToMidi + ? t('dialog.title.audioToMidi') + : (isChordDetection || isMidiChordDetection) ? t('dialog.title.chordDetection') : isPrompt ? t('dialog.title.input') @@ -218,11 +268,11 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = const handleOverlayClick = (e: React.MouseEvent) => { if (e.target === e.currentTarget && mouseDownOnOverlay.current) { - close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice || isChordDetection || isTempoDetection || isTempoApply) ? null : false); + close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice || isChordDetection || isMidiChordDetection || isTempoDetection || isTempoApply || isAudioToMidi) ? null : false); } }; - const handleCancel = () => close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice || isChordDetection || isTempoDetection || isTempoApply) ? null : false); + const handleCancel = () => close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice || isChordDetection || isMidiChordDetection || isTempoDetection || isTempoApply || isAudioToMidi) ? null : false); const handleConfirm = () => { if (isAlert) { close(undefined); return; } @@ -243,6 +293,10 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = close(tempoDetectionOptions); return; } + if (isAudioToMidi) { + close(audioToMidiOptions); + return; + } if (isTempoApply) { close({ action: dialog.choices?.[dialog.choices.length - 1]?.value ?? '', @@ -274,14 +328,30 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = setTempoDetectionOptions(current => ({ ...current, [key]: value })); }; + const updateAudioToMidiOption = ( + key: K, + value: AudioToMidiOptionsResult[K], + ) => { + setAudioToMidiOptions(current => ({ ...current, [key]: value })); + }; + const detectionHintText = isChordDetection ? t('dialog.chordHint.audio') : isMidiChordDetection ? t('dialog.chordHint.midi') : isTempoDetection ? t('dialog.chordHint.tempo') + : isAudioToMidi + ? t('dialog.audioToMidiHint') : null; + const audioToMidiPitchOptions = Array.from({ length: 96 }, (_, index) => { + const pitch = 12 + index; + const noteNames = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B']; + const noteName = `${noteNames[pitch % 12]}${Math.floor(pitch / 12) - 1}`; + return { value: String(pitch), label: noteName }; + }); + return ( <> {children} @@ -298,7 +368,9 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
-

{dialog.message}

+ {(!isAudioToMidi || dialog.message.trim().length > 0) && ( +

{dialog.message}

+ )} {detectionHintText && (
{t('dialog.experimentalFeature')}
@@ -347,6 +419,160 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = />
)} + {isAudioToMidi && ( +
+
+ + +
+
+
+ + {audioToMidiOptions.manualFloorDb} dB +
+ updateAudioToMidiOption('manualFloorDb', Number(e.target.value))} + disabled={audioToMidiOptions.useCurrentFloorDb} + /> +
+
+
+ +
+
+ + {t('dialog.to')} + +
+
+
+
+
+ +
+ +
+
+
+ +
+ +
+
+ + +
+
+ +
+ +
+
+ )} {isChordDetection && (
@@ -545,14 +771,14 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = diff --git a/src/components/common/index.ts b/src/components/common/index.ts index d9ba3a7..38fa196 100644 --- a/src/components/common/index.ts +++ b/src/components/common/index.ts @@ -7,5 +7,22 @@ export { default as OpenProjectModal } from './OpenProjectModal'; export { default as DialogProvider } from './DialogProvider'; export { default as TrackCreateDialog } from './TrackCreateDialog'; export { default as FloatingPopup } from './FloatingPopup'; -export { showAlert, showChordDetectionOptions, showConfirm, showPrompt, showTempoApply, showTempoDetectionOptions, showTimeSigPrompt } from '../../util/dialogUtil'; -export type { ChordDetectionOptionsResult, ConfirmOptions, PromptOptions, TempoApplyResult, TempoDetectionOptionsResult, TimeSigResult } from '../../util/dialogUtil'; +export { + showAlert, + showAudioToMidiOptions, + showChordDetectionOptions, + showConfirm, + showPrompt, + showTempoApply, + showTempoDetectionOptions, + showTimeSigPrompt, +} from '../../util/dialogUtil'; +export type { + AudioToMidiOptionsResult, + ChordDetectionOptionsResult, + ConfirmOptions, + PromptOptions, + TempoApplyResult, + TempoDetectionOptionsResult, + TimeSigResult, +} from '../../util/dialogUtil'; diff --git a/src/components/event-list-panel/RegionEventListTab.tsx b/src/components/event-list-panel/RegionEventListTab.tsx index 561f831..b411d08 100644 --- a/src/components/event-list-panel/RegionEventListTab.tsx +++ b/src/components/event-list-panel/RegionEventListTab.tsx @@ -843,7 +843,9 @@ const RegionEventListTab: React.FC = ({ activeMidiRegio ? lastSelectedNote.getEndBeat() - lastSelectedNote.getStartBeat() : KGPianoRollState.instance().getLastEditedNoteLength(); const defaultPitch = lastSelectedNote ? lastSelectedNote.getPitch() : noteNameToPitch('C4'); - const defaultVelocity = lastSelectedNote ? lastSelectedNote.getVelocity() : 127; + const defaultVelocity = lastSelectedNote + ? lastSelectedNote.getVelocity() + : KGPianoRollState.instance().getLastEditedNoteVelocity(); const command = new CreateNoteCommand( activeMidiRegion.getId(), @@ -855,6 +857,7 @@ const RegionEventListTab: React.FC = ({ activeMidiRegio KGCore.instance().executeCommand(command); KGPianoRollState.instance().setLastEditedNoteLength(defaultLength); + KGPianoRollState.instance().setLastEditedNoteVelocity(defaultVelocity); const createdNote = command.getCreatedNote(); if (createdNote) { createdNote.select(); diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index 16b9cae..b13e4d6 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -10,6 +10,7 @@ import PianoRollHeader from './PianoRollHeader'; import PianoRollToolbar from './PianoRollToolbar'; import NoteAttributeBar from './NoteAttributeBar'; import PianoRollContent from './PianoRollContent'; +import { LoadingOverlay } from '../common'; import { KGCore } from '../../core/KGCore'; import { KGMidiNote } from '../../core/midi/KGMidiNote'; import { KGMidiTrack, type InstrumentType } from '../../core/track/KGMidiTrack'; @@ -21,11 +22,18 @@ import { type PianoRollSnapValue, } from '../../core/state/KGPianoRollState'; import { ConfigManager } from '../../core/config/ConfigManager'; -import { beatsToBar } from '../../util/midiUtil'; -import { ReplaceChordRegionsInRangeCommand, UpdateRegionCommand } from '../../core/commands'; +import { beatsToBar, type RawMidiNote } from '../../util/midiUtil'; +import { ImportMidiClipCommand, ReplaceChordRegionsInRangeCommand, UpdateRegionCommand } from '../../core/commands'; import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface'; import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage'; -import { showAlert, showChordDetectionOptions, showMidiChordDetectionOptions, showTempoApply, showTempoDetectionOptions } from '../../util/dialogUtil'; +import { + showAlert, + showAudioToMidiOptions, + showChordDetectionOptions, + showMidiChordDetectionOptions, + showTempoApply, + showTempoDetectionOptions, +} from '../../util/dialogUtil'; import { matchesKeyboardShortcut } from '../../util/osUtil'; import { resolveChordGuideItems } from '../../util/chordGuideDataUtil'; import { @@ -59,6 +67,7 @@ import { type MidiChordDetectionOptions, } from '../../util/midiChordDetection'; import type { AudioChordDetectionWorkerMessage } from '../../workers/audioChordDetectionWorker'; +import type { AudioToMidiWorkerResult } from '../../workers/audioToMidiWorker'; import type { PianoRollAutomationType } from './pianoRollAutomation'; import type { SheetMeasureMetric } from './sheetNotationTypes'; import { getSheetPlayheadPixel, getSheetQuantizationOptions, parseSheetQuantization } from './sheetNotation'; @@ -70,6 +79,11 @@ import { } from './pianoRollViewport'; import { getNextChordGuideSelection, resolveChordGuideContext, type ChordGuideFunction } from './chordGuideUtil'; import { useI18n } from '../../i18n/useI18n'; +import { + buildAudioToMidiAnalysisSpan, + convertDetectedAudioNotesToRawMidiNotes, +} from '../../util/audioToMidi'; +import { TrackType } from '../../core/track/KGTrack'; interface PianoRollProps { onClose: () => void; @@ -84,6 +98,13 @@ interface PianoRollProps { projectName?: string; } +type AudioToMidiWorkerHandle = { + onmessage: ((event: MessageEvent) => void) | null; + onerror: ((this: AbstractWorker, ev: ErrorEvent) => unknown) | null; + postMessage: Worker['postMessage']; + terminate: () => void; +}; + const PianoRoll: React.FC = ({ onClose, regionId, @@ -101,7 +122,32 @@ const PianoRoll: React.FC = ({ const isAudioWaveform = currentMode === 'audio-waveform'; const isAudioOnly = isAudioWaveform || isSpectrogram; const isHybrid = currentMode === 'hybrid'; - const { maxBars, tracks, updateTrack, updateRegionProperties, timeSignature, showChatBox, showKGOnePanel, showEventListPanel, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm, pianoRollScrollRequest, selectedNoteIds, selectedRegionIds, automationRedrawVersion, refreshProjectState, setBpm } = useProjectStore(); + const { + maxBars, + tracks, + updateTrack, + updateRegionProperties, + timeSignature, + showChatBox, + showKGOnePanel, + showEventListPanel, + showInstrumentSelection, + keySignature, + selectedMode, + setSelectedMode, + playheadPosition, + isPlaying, + autoScrollEnabled, + bpm, + pianoRollScrollRequest, + selectedNoteIds, + selectedRegionIds, + automationRedrawVersion, + refreshProjectState, + setBpm, + isLooping, + loopingRange, + } = useProjectStore(); const { t } = useI18n(); // Tool state for piano roll @@ -115,6 +161,7 @@ const PianoRoll: React.FC = ({ const [isDetectingChords, setIsDetectingChords] = useState(false); const [detectChordProgressPercent, setDetectChordProgressPercent] = useState(0); const [isDetectingTempo, setIsDetectingTempo] = useState(false); + const [isConvertingToMidi, setIsConvertingToMidi] = useState(false); // Piano roll zoom (1x–8x); updates --region-grid-beat-width CSS variable const [pianoRollZoom, setPianoRollZoom] = useState(() => KGPianoRollState.instance().getPianoRollZoom()); @@ -164,6 +211,14 @@ const PianoRoll: React.FC = ({ return tracks.find(track => track.getId().toString() === activeRegion.getTrackId()) ?? null; }, [activeRegion, tracks]); + const availableMidiTracks = useMemo( + () => tracks.filter(track => track.getType() === TrackType.MIDI), + [tracks], + ); + const sourceAudioTrack = useMemo( + () => audioRegion ? tracks.find(track => track.getId().toString() === audioRegion.getTrackId()) ?? null : null, + [audioRegion, tracks], + ); const activeInstrument = useMemo(() => ( parentMidiTrack instanceof KGMidiTrack ? parentMidiTrack.getInstrument() : 'acoustic_grand_piano' ), [parentMidiTrack]); @@ -514,6 +569,165 @@ const PianoRoll: React.FC = ({ } }, [activeEditableRegionId, selectedRegionIds, tracks, updateRegionProperties]); + const getMonoPcmForAudioRegion = useCallback(async () => { + if (!audioRegion || !projectName || !trackId) { + throw new Error('Audio-to-MIDI conversion requires an active audio region with track and project context.'); + } + + let audioBuffer = KGAudioInterface.instance().getAudioBuffer(trackId, audioRegion.getAudioFileId()); + if (!audioBuffer) { + const rawBuffer = await KGAudioFileStorage.loadAudioFile(projectName, audioRegion.getAudioFileId()); + const audioContext = Tone.getContext().rawContext as AudioContext; + audioBuffer = await audioContext.decodeAudioData(rawBuffer); + } + + const monoPcm = new Float32Array(audioBuffer.length); + for (let channelIndex = 0; channelIndex < audioBuffer.numberOfChannels; channelIndex++) { + const channelData = audioBuffer.getChannelData(channelIndex); + for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) { + monoPcm[sampleIndex] += channelData[sampleIndex] / audioBuffer.numberOfChannels; + } + } + + return { + pcm: monoPcm, + sampleRate: audioBuffer.sampleRate, + }; + }, [audioRegion, projectName, trackId]); + + const handleConvertToMidi = useCallback(async () => { + if (!audioRegion) { + await showAlert('Open an audio region before converting it to MIDI.'); + return; + } + + if (!projectName || !trackId) { + await showAlert('Open an audio region in the piano roll before converting it to MIDI.'); + return; + } + + if (availableMidiTracks.length === 0) { + return; + } + + const targetTrackChoices = availableMidiTracks.map(track => ({ + label: track.getName(), + value: track.getId().toString(), + })); + const dialogResult = await showAudioToMidiOptions( + '', + targetTrackChoices, + isLooping, + { + monophonic: true, + useCurrentFloorDb: true, + manualFloorDb: spectrogramThresholdDb, + pitchRangeStart: 12, + pitchRangeEnd: 107, + quantizeNoteStart: '1/16', + quantizeNoteLength: '1/16', + convertLoopRangeOnly: true, + groupAdjacentPitchesToHighest: true, + targetTrackId: targetTrackChoices[0]?.value ?? '', + }, + ); + if (!dialogResult) { + return; + } + + const project = KGCore.instance().getCurrentProject(); + const analysisSpan = buildAudioToMidiAnalysisSpan(project, audioRegion, { + loopModeEnabled: isLooping, + convertLoopRangeOnly: dialogResult.convertLoopRangeOnly, + loopingRange, + }); + if (!analysisSpan) { + await showAlert('The loop range does not overlap with the selected audio region.'); + return; + } + + const targetTrack = availableMidiTracks.find(track => track.getId().toString() === dialogResult.targetTrackId); + if (!targetTrack) { + await showAlert('Please choose a valid MIDI target track.'); + return; + } + + const floorDb = dialogResult.useCurrentFloorDb + ? spectrogramThresholdDb + : dialogResult.manualFloorDb; + + let worker: AudioToMidiWorkerHandle | null = null; + setIsConvertingToMidi(true); + + try { + const { pcm, sampleRate } = await getMonoPcmForAudioRegion(); + const detectedNotes = await new Promise((resolve, reject) => { + worker = new Worker( + new URL('../../workers/audioToMidiWorker.ts', import.meta.url), + { type: 'module' }, + ) as AudioToMidiWorkerHandle; + + worker.onmessage = (event: MessageEvent) => { + if (event.data.type !== 'result') { + return; + } + + resolve(convertDetectedAudioNotesToRawMidiNotes( + project, + analysisSpan, + event.data.notes, + { + quantizeNoteStart: dialogResult.quantizeNoteStart as PianoRollQuantizePositionValue, + quantizeNoteLength: dialogResult.quantizeNoteLength as PianoRollQuantizeLengthValue, + }, + )); + }; + worker.onerror = () => reject(new Error('Audio-to-MIDI worker failed.')); + worker.postMessage({ + pcm, + sampleRate, + startSeconds: analysisSpan.startSeconds, + endSeconds: analysisSpan.endSeconds, + floorDb, + pitchRangeStart: dialogResult.pitchRangeStart, + pitchRangeEnd: dialogResult.pitchRangeEnd, + groupAdjacentPitchesToHighest: dialogResult.groupAdjacentPitchesToHighest, + }, [pcm.buffer]); + }); + + const regionLengthBeats = analysisSpan.regionEndBeat - analysisSpan.regionStartBeat; + const importCommand = new ImportMidiClipCommand( + targetTrack.getId().toString(), + targetTrack.getTrackIndex(), + analysisSpan.regionStartBeat, + regionLengthBeats, + detectedNotes, + `Converted from ${sourceAudioTrack?.getName() ?? audioRegion.getName()}`, + ); + KGCore.instance().executeCommand(importCommand); + refreshProjectState(); + } catch (error) { + console.error('Error converting audio to MIDI:', error); + await showAlert('Failed to convert this audio region to MIDI.'); + } finally { + const cleanupWorker = worker as AudioToMidiWorkerHandle | null; + cleanupWorker?.terminate(); + setIsConvertingToMidi(false); + } + }, [ + audioRegion, + availableMidiTracks, + getMonoPcmForAudioRegion, + isLooping, + loopingRange, + projectName, + refreshProjectState, + sourceAudioTrack, + spectrogramThresholdDb, + t, + trackId, + ]); + const handleDetectChords = useCallback(async () => { if (!audioRegion && !activeRegion) { await showAlert('Open a MIDI or audio region before detecting chords.'); @@ -1619,6 +1833,8 @@ const PianoRoll: React.FC = ({ detectingChords={isDetectingChords} onDetectTempo={audioRegion ? handleDetectTempo : undefined} detectingTempo={isDetectingTempo} + onConvertToMidi={audioRegion ? handleConvertToMidi : undefined} + convertToMidiDisabled={availableMidiTracks.length === 0} selectedRegionColor={selectedRegionColor} onRegionColorSelect={activeEditableRegionId ? (color) => { void handleRegionColorSelect(color); } : undefined} /> @@ -1669,6 +1885,7 @@ const PianoRoll: React.FC = ({ >
+
); }; diff --git a/src/components/piano-roll/PianoRollToolbar.test.tsx b/src/components/piano-roll/PianoRollToolbar.test.tsx index 2623a81..4f1fbd9 100644 --- a/src/components/piano-roll/PianoRollToolbar.test.tsx +++ b/src/components/piano-roll/PianoRollToolbar.test.tsx @@ -325,6 +325,46 @@ describe('PianoRollToolbar', () => { expect(screen.queryByText('Detect tempo...')).not.toBeInTheDocument(); }); + it('shows the convert to MIDI action as the last audio option and triggers it', () => { + const onConvertToMidi = vi.fn(); + + renderWithLocale( + + ); + + fireEvent.click(screen.getByTitle('More options')); + const options = screen.getAllByText(/Detect chords...|Convert to MIDI.../); + expect(options[options.length - 1]).toHaveTextContent('Convert to MIDI...'); + fireEvent.click(screen.getByText('Convert to MIDI...')); + + expect(onConvertToMidi).toHaveBeenCalledTimes(1); + }); + + it('disables the convert to MIDI action when no MIDI tracks are available', () => { + const onConvertToMidi = vi.fn(); + + renderWithLocale( + + ); + + fireEvent.click(screen.getByTitle('More options')); + const convertItem = screen.getByText('Convert to MIDI...'); + expect(convertItem).toHaveAttribute('aria-disabled', 'true'); + fireEvent.click(convertItem); + expect(onConvertToMidi).not.toHaveBeenCalled(); + }); + it('shows only the sheet controls when sheet mode is enabled', () => { renderWithLocale( void | Promise; detectingTempo?: boolean; + onConvertToMidi?: () => void | Promise; + convertToMidiDisabled?: boolean; selectedRegionColor?: string; onRegionColorSelect?: (color: string | null) => void; } @@ -94,6 +96,8 @@ const PianoRollToolbar: React.FC = ({ detectingChords = false, onDetectTempo, detectingTempo = false, + onConvertToMidi, + convertToMidiDisabled = false, selectedRegionColor, onRegionColorSelect, }) => { @@ -102,7 +106,7 @@ const PianoRollToolbar: React.FC = ({ const showAudioOnlyControls = mode === 'audio-waveform' && !sheetMusicViewEnabled; const showSpectrogramOnlyControls = mode === 'spectrogram' && !sheetMusicViewEnabled; const showSpecControls = !sheetMusicViewEnabled && (mode === 'spectrogram' || mode === 'hybrid'); - const showSpecMenu = !sheetMusicViewEnabled && (!!onDetectChords || !!onDetectTempo); + const showSpecMenu = !sheetMusicViewEnabled && (!!onDetectChords || !!onDetectTempo || !!onConvertToMidi); const automationOptions = React.useMemo(() => getTranslatedAutomationOptions(t), [t]); const snapOptions = React.useMemo( () => KGPianoRollState.SNAP_OPTIONS.map(option => ({ label: t(option.labelKey), value: option.value })), @@ -447,6 +451,22 @@ const PianoRollToolbar: React.FC = ({ {detectingTempo ? t('pianoRoll.detectingTempo') : t('pianoRoll.detectTempo')}
)} + {onConvertToMidi && ( +
{ + if (convertToMidiDisabled) { + return; + } + setShowRegionColorPalette(false); + setShowMoreMenu(false); + void onConvertToMidi(); + }} + aria-disabled={convertToMidiDisabled} + > + {t('pianoRoll.convertToMidi')} +
+ )} )} diff --git a/src/components/piano-roll/SpectrogramCanvas.tsx b/src/components/piano-roll/SpectrogramCanvas.tsx index bdcee5b..93c7058 100644 --- a/src/components/piano-roll/SpectrogramCanvas.tsx +++ b/src/components/piano-roll/SpectrogramCanvas.tsx @@ -5,6 +5,7 @@ import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface'; import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage'; import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker'; import { + SPECTROGRAM_FULL_SEMITONES, getSpectrogramVisibleBinRange, normalizeSpectrogramHeightResolution, SPECTROGRAM_VISIBLE_SEMITONES, @@ -52,6 +53,25 @@ function hotColormap(v: number): [number, number, number] { return COLORMAP_STOPS[COLORMAP_STOPS.length - 1][1]; } +function smoothSpectrogramVertically( + source: Float32Array, + timeSteps: number, + pitchBins: number, +): Float32Array { + const smoothed = new Float32Array(source.length); + + for (let col = 0; col < timeSteps; col++) { + for (let row = 0; row < pitchBins; row++) { + const center = source[col * pitchBins + row]; + const above = row > 0 ? source[col * pitchBins + row - 1] : center; + const below = row < pitchBins - 1 ? source[col * pitchBins + row + 1] : center; + smoothed[col * pitchBins + row] = above * 0.2 + center * 0.6 + below * 0.2; + } + } + + return smoothed; +} + const SpectrogramCanvas: React.FC = ({ audioRegion, trackId, @@ -102,18 +122,21 @@ const SpectrogramCanvas: React.FC = ({ // Convert dB threshold to linear: values below this → black const linearThreshold = Math.pow(10, thresholdDb / 20); - const visibleRange = getSpectrogramVisibleBinRange(heightResolution); + const analysisResolution = result.pitchBins / SPECTROGRAM_FULL_SEMITONES; + const visibleRange = getSpectrogramVisibleBinRange(analysisResolution); const visiblePitchBins = visibleRange.end - visibleRange.start; // 1. Paint at natural spectrogram resolution onto an offscreen canvas. // Result data is low-to-high pitch; draw only the visible C0-B7 window, reversed for display. - const offscreen = document.createElement('canvas'); - offscreen.width = result.timeSteps; - offscreen.height = visiblePitchBins; - const offCtx = offscreen.getContext('2d'); - if (!offCtx) return; + const smoothedData = smoothSpectrogramVertically(result.data, result.timeSteps, result.pitchBins); - const imgData = offCtx.createImageData(result.timeSteps, visiblePitchBins); + const sourceCanvas = document.createElement('canvas'); + sourceCanvas.width = result.timeSteps; + sourceCanvas.height = visiblePitchBins; + const sourceCtx = sourceCanvas.getContext('2d'); + if (!sourceCtx) return; + + const imgData = sourceCtx.createImageData(result.timeSteps, visiblePitchBins); const pixels = imgData.data; for (let row = 0; row < visiblePitchBins; row++) { @@ -122,7 +145,7 @@ const SpectrogramCanvas: React.FC = ({ const idx = (row * result.timeSteps + col) * 4; pixels[idx + 3] = 255; // always opaque - const raw = result.data[col * result.pitchBins + sourceRow]; + const raw = smoothedData[col * result.pitchBins + sourceRow]; // Hard threshold: values below noise floor → 0 (black) // Re-scale surviving range to [0,1] then apply power curve @@ -139,14 +162,24 @@ const SpectrogramCanvas: React.FC = ({ } } - offCtx.putImageData(imgData, 0, 0); + sourceCtx.putImageData(imgData, 0, 0); - // 2. Stretch onto the full canvas — browser bilinear filter smooths between bins. + const verticallyScaledCanvas = document.createElement('canvas'); + verticallyScaledCanvas.width = result.timeSteps; + verticallyScaledCanvas.height = canvasHeight; + const verticallyScaledCtx = verticallyScaledCanvas.getContext('2d'); + if (!verticallyScaledCtx) return; + + // Scale only along the pitch axis so note starts/stops stay crisp in time. + verticallyScaledCtx.imageSmoothingEnabled = true; + verticallyScaledCtx.imageSmoothingQuality = 'high'; + verticallyScaledCtx.drawImage(sourceCanvas, 0, 0, result.timeSteps, canvasHeight); + + // 2. Stretch onto the full canvas horizontally without temporal blur. canvas.width = canvasWidth; canvas.height = canvasHeight; - ctx.imageSmoothingEnabled = true; - ctx.imageSmoothingQuality = 'high'; - ctx.drawImage(offscreen, 0, 0, canvasWidth, canvasHeight); + ctx.imageSmoothingEnabled = false; + ctx.drawImage(verticallyScaledCanvas, 0, 0, canvasWidth, canvasHeight); // Store natural width and apply current zoom as CSS stretch (no pixel recompute on zoom) naturalWidthRef.current = canvasWidth; diff --git a/src/core/KGProject.ts b/src/core/KGProject.ts index 2a9e997..52d9fcc 100644 --- a/src/core/KGProject.ts +++ b/src/core/KGProject.ts @@ -62,6 +62,10 @@ export class KGProject { @WithDefault(false) private isMetronomeEnabled: boolean = false; + @Expose() + @WithDefault(0) + private playheadPosition: number = 0; + @Expose() @WithDefault(0) private projectStructureVersion: number = 0; @@ -97,7 +101,7 @@ export class KGProject { private globalTracks: KGGlobalTrack[] = createDefaultGlobalTracks(); // Constructor - constructor(name: string = RESERVED_PROJECT_NAME, maxBars: number = 32, currentBars: number = 0, bpm: number = TIME_CONSTANTS.DEFAULT_BPM, timeSignature: TimeSignature = { numerator: 4, denominator: 4 }, keySignature: KeySignature = "C major", selectedMode: string = "ionian", isLooping: boolean = false, loopingRange: [number, number] = [0, 0], barWidthMultiplier: number = 2, tracks: KGTrack[] = [], projectStructureVersion: number = KGProject.CURRENT_PROJECT_STRUCTURE_VERSION, pianoRollZoom: number = 1, globalTracks: KGGlobalTrack[] = createDefaultGlobalTracks(), showGlobalTracks: boolean = false, isMetronomeEnabled: boolean = false) { + constructor(name: string = RESERVED_PROJECT_NAME, maxBars: number = 32, currentBars: number = 0, bpm: number = TIME_CONSTANTS.DEFAULT_BPM, timeSignature: TimeSignature = { numerator: 4, denominator: 4 }, keySignature: KeySignature = "C major", selectedMode: string = "ionian", isLooping: boolean = false, loopingRange: [number, number] = [0, 0], barWidthMultiplier: number = 2, tracks: KGTrack[] = [], projectStructureVersion: number = KGProject.CURRENT_PROJECT_STRUCTURE_VERSION, pianoRollZoom: number = 1, globalTracks: KGGlobalTrack[] = createDefaultGlobalTracks(), showGlobalTracks: boolean = false, isMetronomeEnabled: boolean = false, playheadPosition: number = 0) { this.name = name; this.maxBars = maxBars; this.currentBars = currentBars; @@ -114,6 +118,7 @@ export class KGProject { this.globalTracks = globalTracks; this.showGlobalTracks = showGlobalTracks; this.isMetronomeEnabled = isMetronomeEnabled; + this.playheadPosition = playheadPosition; } // Getters @@ -210,10 +215,18 @@ export class KGProject { return this.isMetronomeEnabled; } + public getPlayheadPosition(): number { + return this.playheadPosition; + } + public setIsMetronomeEnabled(isMetronomeEnabled: boolean): void { this.isMetronomeEnabled = isMetronomeEnabled; } + public setPlayheadPosition(playheadPosition: number): void { + this.playheadPosition = playheadPosition; + } + public getIsLooping(): boolean { return this.isLooping; } diff --git a/src/core/io/KGProjectStorage.test.ts b/src/core/io/KGProjectStorage.test.ts index 2dcc4ee..aa49fa5 100644 --- a/src/core/io/KGProjectStorage.test.ts +++ b/src/core/io/KGProjectStorage.test.ts @@ -175,6 +175,38 @@ describe('KGProjectStorage', () => { expect(loaded!.getPianoRollZoom()).toBe(5); }); + it('preserves playhead position when saving and loading', async () => { + const project = createTestProject('Playhead Song'); + project.setPlayheadPosition(18.5); + + await storage.save('Playhead Song', project); + + const loaded = await storage.load('Playhead Song'); + + expect(loaded).not.toBeNull(); + expect(loaded!.getPlayheadPosition()).toBe(18.5); + }); + + it('defaults playhead position to 0 when loading older project data without the field', async () => { + const project = createTestProject('Legacy Song'); + await storage.save('Legacy Song', project); + + const projectsDir = await mockRoot.getDirectoryHandle('projects'); + const projectDir = await projectsDir.getDirectoryHandle('Legacy Song'); + const projectHandle = await projectDir.getFileHandle('project.json'); + const legacyPayload = JSON.parse(await (await projectHandle.getFile()).text()) as Record; + delete legacyPayload.playheadPosition; + + const writable = await projectHandle.createWritable(); + await writable.write(JSON.stringify(legacyPayload, null, 2)); + await writable.close(); + + const loaded = await storage.load('Legacy Song'); + + expect(loaded).not.toBeNull(); + expect(loaded!.getPlayheadPosition()).toBe(0); + }); + it('preserves persisted global-track visibility and metronome state when saving and loading', async () => { const project = createTestProject('Toggle Song'); project.setShowGlobalTracks(true); diff --git a/src/core/state/KGPianoRollState.ts b/src/core/state/KGPianoRollState.ts index e52a7d0..4e74757 100644 --- a/src/core/state/KGPianoRollState.ts +++ b/src/core/state/KGPianoRollState.ts @@ -55,6 +55,7 @@ export class KGPianoRollState { private activeTool: string = "pointer"; private currentSnap: PianoRollSnapValue = PIANO_ROLL_NO_SNAP; private lastEditedNoteLength: number = 1; // Default to 1 beat + private lastEditedNoteVelocity: number = 127; private currentMode: string = "ionian"; // Default mode private automationViewEnabled: boolean = false; private currentAutomationType: string = "pitch-bend"; @@ -108,6 +109,14 @@ export class KGPianoRollState { this.lastEditedNoteLength = length; } + public getLastEditedNoteVelocity(): number { + return this.lastEditedNoteVelocity; + } + + public setLastEditedNoteVelocity(velocity: number): void { + this.lastEditedNoteVelocity = velocity; + } + public getCurrentMode(): string { return this.currentMode; } diff --git a/src/hooks/useNoteOperations.test.ts b/src/hooks/useNoteOperations.test.ts index d0a3c6f..dcc9c15 100644 --- a/src/hooks/useNoteOperations.test.ts +++ b/src/hooks/useNoteOperations.test.ts @@ -59,10 +59,22 @@ vi.mock('../stores/projectStore', () => ({ }), })); +vi.mock('../core/audio-interface/KGAudioInterface', () => ({ + KGAudioInterface: { + instance: () => ({ + getIsInitialized: () => false, + getIsAudioContextStarted: () => false, + startAudioContext: vi.fn(), + triggerNote: vi.fn(), + }), + }, +})); + import { useNoteOperations } from './useNoteOperations'; import { KGCore } from '../core/KGCore'; import { KGPianoRollState } from '../core/state/KGPianoRollState'; -import { MoveNotesCommand, ResizeNotesCommand } from '../core/commands'; +import { CreateNoteCommand, MoveNotesCommand, ResizeNotesCommand } from '../core/commands'; +import { CreateNotesCommand } from '../core/commands/note/CreateNotesCommand'; import { useProjectStore } from '../stores/projectStore'; import { createMockMidiNote, createMockMidiRegion, createMockMidiTrack } from '../test/utils/mock-data'; @@ -91,20 +103,157 @@ describe('useNoteOperations', () => { KGPianoRollState.instance().setActiveTool('pointer'); KGPianoRollState.instance().setCurrentSnap('1/4'); + KGPianoRollState.instance().setLastEditedNoteLength(1); + KGPianoRollState.instance().setLastEditedNoteVelocity(127); + KGPianoRollState.instance().setCurrentMatchingChords([]); + KGPianoRollState.instance().setCurrentSelectedChordIndex(0); + KGPianoRollState.instance().setCurrentChordCursorPitch(null); }); - const renderNoteOperations = (activeRegion: ReturnType, track = createMockMidiTrack({ id: 1, regions: [activeRegion] }), updateTrack = vi.fn()) => { + const createPianoGridRef = () => ({ + current: { + getBoundingClientRect: () => ({ + left: 0, + top: 0, + right: 800, + bottom: 600, + width: 800, + height: 600, + x: 0, + y: 0, + toJSON: () => ({}), + }), + } as HTMLDivElement, + }); + + const createGridClickEvent = (overrides: Partial = {}): React.MouseEvent => ({ + clientX: 80, + clientY: 120, + ctrlKey: false, + metaKey: false, + shiftKey: false, + altKey: false, + ...overrides, + } as React.MouseEvent); + + const renderNoteOperations = ( + activeRegion: ReturnType, + track = createMockMidiTrack({ id: 1, regions: [activeRegion] }), + updateTrack = vi.fn(), + pianoGridRef = createPianoGridRef(), + ) => { const hook = renderHook(() => useNoteOperations({ activeRegion, timeSignature: { numerator: 4, denominator: 4 }, updateTrack, tracks: [track], - pianoGridRef: { current: null }, + pianoGridRef, })); - return { ...hook, track, updateTrack }; + return { ...hook, track, updateTrack, pianoGridRef }; }; + it('uses the last selected note velocity when creating after deselecting', () => { + const selectedNote = createMockMidiNote({ id: 'note-a', velocity: 91, pitch: 60 }); + const activeRegion = createMockMidiRegion({ + id: 'region-1', + trackId: '1', + notes: [selectedNote], + }); + + selectedNote.select(); + KGCore.instance().addSelectedItem(selectedNote); + KGPianoRollState.instance().setLastEditedNoteVelocity(selectedNote.getVelocity()); + KGCore.instance().clearSelectedItems(); + + const { result } = renderNoteOperations(activeRegion); + + act(() => { + result.current.handleGridDoubleClick(createGridClickEvent()); + }); + + expect(coreState.executeCommand).toHaveBeenCalledTimes(1); + const createCommand = coreState.executeCommand.mock.calls[0][0] as CreateNoteCommand & { velocity: number }; + expect(createCommand).toBeInstanceOf(CreateNoteCommand); + expect(createCommand.velocity).toBe(91); + }); + + it('uses the most recently selected note velocity after deselecting a multi-selection', () => { + const noteA = createMockMidiNote({ id: 'note-a', velocity: 40, pitch: 60 }); + const noteB = createMockMidiNote({ id: 'note-b', velocity: 105, pitch: 64 }); + const activeRegion = createMockMidiRegion({ + id: 'region-1', + trackId: '1', + notes: [noteA, noteB], + }); + + noteA.select(); + noteB.select(); + KGCore.instance().addSelectedItem(noteA); + KGCore.instance().addSelectedItem(noteB); + KGPianoRollState.instance().setLastEditedNoteVelocity(noteB.getVelocity()); + KGCore.instance().clearSelectedItems(); + + const { result } = renderNoteOperations(activeRegion); + + act(() => { + result.current.handleGridDoubleClick(createGridClickEvent()); + }); + + expect(coreState.executeCommand).toHaveBeenCalledTimes(1); + const createCommand = coreState.executeCommand.mock.calls[0][0] as CreateNoteCommand & { velocity: number }; + expect(createCommand).toBeInstanceOf(CreateNoteCommand); + expect(createCommand.velocity).toBe(105); + }); + + it('falls back to velocity 127 when creating a manual note with no selection', () => { + const activeRegion = createMockMidiRegion({ + id: 'region-1', + trackId: '1', + notes: [], + }); + + const { result } = renderNoteOperations(activeRegion); + + act(() => { + result.current.handleGridDoubleClick(createGridClickEvent()); + }); + + expect(coreState.executeCommand).toHaveBeenCalledTimes(1); + const createCommand = coreState.executeCommand.mock.calls[0][0] as CreateNoteCommand & { velocity: number }; + expect(createCommand).toBeInstanceOf(CreateNoteCommand); + expect(createCommand.velocity).toBe(127); + }); + + it('applies the cached velocity to every note in manual chord creation after deselecting', () => { + const selectedNote = createMockMidiNote({ id: 'note-a', velocity: 73, pitch: 60 }); + const activeRegion = createMockMidiRegion({ + id: 'region-1', + trackId: '1', + notes: [selectedNote], + }); + + selectedNote.select(); + KGCore.instance().addSelectedItem(selectedNote); + KGPianoRollState.instance().setLastEditedNoteVelocity(selectedNote.getVelocity()); + KGCore.instance().clearSelectedItems(); + KGPianoRollState.instance().setCurrentMatchingChords([[0, 4, 7]]); + KGPianoRollState.instance().setCurrentSelectedChordIndex(0); + KGPianoRollState.instance().setCurrentChordCursorPitch(72); + + const { result } = renderNoteOperations(activeRegion); + + act(() => { + result.current.handleGridDoubleClick(createGridClickEvent({ clientY: 706 })); + }); + + expect(coreState.executeCommand).toHaveBeenCalledTimes(1); + const createCommand = coreState.executeCommand.mock.calls[0][0] as CreateNotesCommand; + expect(createCommand).toBeInstanceOf(CreateNotesCommand); + expect(createCommand.getNoteCreationData()).toHaveLength(3); + expect(createCommand.getNoteCreationData().every(note => note.velocity === 73)).toBe(true); + }); + it('selects the grabbed note before resizing when it was not part of the current selection', () => { const noteA = createMockMidiNote({ id: 'note-a', startBeat: 0, endBeat: 1, pitch: 60 }); const noteB = createMockMidiNote({ id: 'note-b', startBeat: 1, endBeat: 2, pitch: 62 }); diff --git a/src/hooks/useNoteOperations.ts b/src/hooks/useNoteOperations.ts index 63dcaac..2cac957 100644 --- a/src/hooks/useNoteOperations.ts +++ b/src/hooks/useNoteOperations.ts @@ -192,10 +192,10 @@ export const useNoteOperations = ({ // Calculate note timing relative to the region const regionStartBeat = activeRegion.getStartFromBeat(); const noteStartBeat = beatNumber - regionStartBeat; // relative beat position - const lastEditedLength = KGPianoRollState.instance().getLastEditedNoteLength(); - const noteEndBeat = noteStartBeat + lastEditedLength; // Use last edited note length - const velocity = 127; // Maximum velocity const pianoRollState = KGPianoRollState.instance(); + const lastEditedLength = pianoRollState.getLastEditedNoteLength(); + const noteEndBeat = noteStartBeat + lastEditedLength; // Use last edited note length + const velocity = pianoRollState.getLastEditedNoteVelocity(); const matchingChordPitches = pianoRollState.getCurrentMatchingChords(); const selectedChordIndex = pianoRollState.getCurrentSelectedChordIndex(); const cursorChordPitch = pianoRollState.getCurrentChordCursorPitch(); @@ -333,6 +333,7 @@ export const useNoteOperations = ({ note.select(); core.addSelectedItem(note); KGPianoRollState.instance().setLastEditedNoteLength(note.getEndBeat() - note.getStartBeat()); + KGPianoRollState.instance().setLastEditedNoteVelocity(note.getVelocity()); const track = tracks.find(t => t.getId().toString() === activeRegion.getTrackId()); if (track) { diff --git a/src/hooks/useNoteSelection.ts b/src/hooks/useNoteSelection.ts index a372c2a..c686418 100644 --- a/src/hooks/useNoteSelection.ts +++ b/src/hooks/useNoteSelection.ts @@ -108,6 +108,7 @@ export const useNoteSelection = ({ // Update last edited note length const noteLength = note.getEndBeat() - note.getStartBeat(); KGPianoRollState.instance().setLastEditedNoteLength(noteLength); + KGPianoRollState.instance().setLastEditedNoteVelocity(note.getVelocity()); if (DEBUG_MODE.PIANO_ROLL) { console.log(`Added note to selection: ${noteId}`); @@ -139,6 +140,7 @@ export const useNoteSelection = ({ // Update last edited note length const noteLength = note.getEndBeat() - note.getStartBeat(); KGPianoRollState.instance().setLastEditedNoteLength(noteLength); + KGPianoRollState.instance().setLastEditedNoteVelocity(note.getVelocity()); if (DEBUG_MODE.PIANO_ROLL) { console.log(`Selected note (replacing previous selection): ${noteId}`); @@ -416,6 +418,7 @@ export const useNoteSelection = ({ const noteLength = closestNote.getEndBeat() - closestNote.getStartBeat(); KGPianoRollState.instance().setLastEditedNoteLength(noteLength); + KGPianoRollState.instance().setLastEditedNoteVelocity(closestNote.getVelocity()); if (DEBUG_MODE.PIANO_ROLL) { console.log(`Updated last edited note length to ${noteLength} from closest note: ${closestNote.getId()}`); diff --git a/src/i18n/messages/en_us.ts b/src/i18n/messages/en_us.ts index ba27541..342c5bd 100644 --- a/src/i18n/messages/en_us.ts +++ b/src/i18n/messages/en_us.ts @@ -193,6 +193,7 @@ export const enUsMessages: TranslationMessages = { 'dialog.title.tempoDetection': 'Tempo Detection', 'dialog.title.applyTempo': 'Apply Tempo', 'dialog.title.chordDetection': 'Chord Detection', + 'dialog.title.audioToMidi': 'Convert to MIDI', 'dialog.title.input': 'Input', 'dialog.title.confirm': 'Confirm', 'dialog.close': 'Close dialog', @@ -203,6 +204,7 @@ export const enUsMessages: TranslationMessages = { 'dialog.chordHint.audio': 'Chord analysis is still experimental. Harmonic content, arrangement density, and transient-heavy material can affect accuracy. For more reliable results, start with the default settings, then refine sensitivity and stability until the detected harmony best matches the musical phrasing.', 'dialog.chordHint.midi': 'Chord analysis is still experimental. Voicing density, overlaps, and ornamental notes can influence the result. For more reliable chord labels, begin with the default settings, then adjust note suppression and harmonic focus to match the musical role of the passage.', 'dialog.chordHint.tempo': 'Tempo analysis is still experimental. Rubato phrasing, sparse transients, and layered percussion can reduce accuracy. Start with the default BPM range, then narrow the analysis window to the most plausible tempo span for the material if the first pass is not musically convincing.', + 'dialog.audioToMidiHint': 'Audio-to-MIDI conversion is still experimental. For best results, switch to spectrogram view, adjust Floor until only the true note ridges remain visible and most ghost pitches disappear, then enable Use current Floor dB before running the conversion.', 'dialog.sourceHint.kgone': 'For the most dependable chord labels, analyze a stem with vocals and percussion reduced or removed. If K.G.One Music Studio server integration is available, run Separator with the "Vocal, Drums, Bass, Guitar, Piano, and Others" model and use the Piano or Others stem for analysis.', 'dialog.sourceHint.local': 'For the most dependable chord labels, analyze a stem with vocals and percussion reduced or removed. If you are using the local separator, choose the "Vocal, Drums, Bass, and Others" model and use the Others stem for analysis.', 'dialog.label.sensitivity': 'Sensitivity', @@ -219,7 +221,18 @@ export const enUsMessages: TranslationMessages = { 'dialog.label.minimumBpm': 'Minimum BPM', 'dialog.label.maximumBpm': 'Maximum BPM', 'dialog.label.autoAlignRegionToBeat': 'Auto-align region to beat', + 'dialog.label.monophonicOnly': 'Monophonic only (v1)', + 'dialog.label.useCurrentFloorDb': 'Use current Floor dB', + 'dialog.label.manualFloorDb': 'Manual Floor dB', + 'dialog.label.pitchRange': 'Pitch range', + 'dialog.label.quantizeNoteStart': 'Quantize note start', + 'dialog.label.quantizeNoteLength': 'Quantize note length', + 'dialog.label.convertLoopRangeOnly': 'Convert loop-range audio clip only', + 'dialog.label.groupAdjacentPitchesToHighest': 'Group adjacent pitches to the strongest bin', + 'dialog.label.targetTrack': 'Target MIDI track', + 'dialog.to': 'to', 'dialog.message.chordDetection': 'Tune audio chord detection settings before processing.', + 'dialog.message.audioToMidi': 'Tune audio-to-MIDI conversion settings before processing.', 'dialog.message.tempoDetection': 'Tune audio tempo detection settings before processing.', 'dialog.message.tempoApply': 'Detected tempo: {bpm} BPM. Choose how to apply it.\n\nUpdate Current Tempo changes the active tempo at this clip location. Insert Tempo Change adds a new tempo region at the nearest bar before the clip starts.', 'dialog.action.updateCurrentTempo': 'Update Current Tempo', @@ -354,6 +367,7 @@ export const enUsMessages: TranslationMessages = { 'pianoRoll.detectingChords': 'Detecting chords...', 'pianoRoll.detectTempo': 'Detect tempo...', 'pianoRoll.detectingTempo': 'Detecting tempo...', + 'pianoRoll.convertToMidi': 'Convert to MIDI...', 'percussion.short.35': 'Ac.Bass', 'percussion.short.36': 'BassDrum', 'percussion.short.37': 'SideStick', diff --git a/src/i18n/messages/fr_fr.ts b/src/i18n/messages/fr_fr.ts index 4bd5ec3..f609080 100644 --- a/src/i18n/messages/fr_fr.ts +++ b/src/i18n/messages/fr_fr.ts @@ -190,6 +190,7 @@ export const frFrMessages: TranslationMessages = { 'dialog.title.tempoDetection': 'Détection du tempo', 'dialog.title.applyTempo': 'Appliquer le tempo', 'dialog.title.chordDetection': 'Détection d\'accords', + 'dialog.title.audioToMidi': 'Convertir en MIDI', 'dialog.title.input': 'Saisie', 'dialog.title.confirm': 'Confirmation', 'dialog.close': 'Fermer la boîte de dialogue', @@ -200,6 +201,7 @@ export const frFrMessages: TranslationMessages = { 'dialog.chordHint.audio': 'L\'analyse d\'accords reste expérimentale. Le contenu harmonique, la densité de l\'arrangement et les matériaux riches en transitoires peuvent affecter la précision. Pour de meilleurs résultats, commencez avec les réglages par défaut puis affinez la sensibilité et la stabilité jusqu\'à ce que l\'harmonie détectée corresponde à la phrase musicale.', 'dialog.chordHint.midi': 'L\'analyse d\'accords reste expérimentale. La densité des voicings, les chevauchements et les notes d\'ornement peuvent influencer le résultat. Pour des étiquettes d\'accord plus fiables, partez des réglages par défaut puis ajustez la suppression des notes courtes et le focus harmonique selon le rôle musical du passage.', 'dialog.chordHint.tempo': 'L\'analyse du tempo reste expérimentale. Le rubato, les transitoires clairsemées et les couches de percussions peuvent réduire la précision. Commencez avec la plage de BPM par défaut, puis resserrez la fenêtre d\'analyse vers la zone de tempo la plus plausible si le premier résultat n\'est pas convaincant musicalement.', + 'dialog.audioToMidiHint': 'La conversion audio-vers-MIDI reste expérimentale. Pour de meilleurs résultats, passez en vue spectrogramme, ajustez Floor jusqu\'à ne laisser visibles que les vraies notes et à faire disparaître autant que possible les hauteurs fantômes, puis activez Utiliser le Floor dB actuel avant la conversion.', 'dialog.sourceHint.kgone': 'Pour des étiquettes d\'accord plus fiables, analysez de préférence un stem où les voix et les percussions ont été atténuées ou retirées. Si l\'intégration du serveur K.G.One Music Studio est disponible, lancez Separator avec le modèle "Vocal, Drums, Bass, Guitar, Piano, and Others" et utilisez de préférence le stem Piano ou Others.', 'dialog.sourceHint.local': 'Pour des étiquettes d\'accord plus fiables, analysez de préférence un stem où les voix et les percussions ont été atténuées ou retirées. Si vous utilisez le séparateur local, choisissez le modèle "Vocal, Drums, Bass, and Others" et analysez le stem Others.', 'dialog.label.sensitivity': 'Sensibilité', @@ -216,7 +218,18 @@ export const frFrMessages: TranslationMessages = { 'dialog.label.minimumBpm': 'BPM minimum', 'dialog.label.maximumBpm': 'BPM maximum', 'dialog.label.autoAlignRegionToBeat': 'Aligner automatiquement la région sur le temps', + 'dialog.label.monophonicOnly': 'Monophonique uniquement (v1)', + 'dialog.label.useCurrentFloorDb': 'Utiliser le Floor dB actuel', + 'dialog.label.manualFloorDb': 'Floor dB manuel', + 'dialog.label.pitchRange': 'Plage de hauteurs', + 'dialog.label.quantizeNoteStart': 'Quantifier le début des notes', + 'dialog.label.quantizeNoteLength': 'Quantifier la durée des notes', + 'dialog.label.convertLoopRangeOnly': 'Convertir uniquement l\'audio dans la plage de boucle', + 'dialog.label.groupAdjacentPitchesToHighest': 'Regrouper les hauteurs adjacentes vers le bin le plus fort', + 'dialog.label.targetTrack': 'Piste MIDI cible', + 'dialog.to': 'à', 'dialog.message.chordDetection': 'Ajustez les réglages de détection d\'accords audio avant le traitement.', + 'dialog.message.audioToMidi': 'Ajustez les réglages de conversion audio-vers-MIDI avant le traitement.', 'dialog.message.tempoDetection': 'Ajustez les réglages de détection du tempo audio avant le traitement.', 'dialog.message.tempoApply': 'Tempo détecté : {bpm} BPM. Choisissez comment l\'appliquer.\n\nMettre à jour le tempo courant modifie le tempo actif à l\'emplacement de ce clip. Insérer un changement de tempo ajoute une nouvelle région de tempo à la mesure la plus proche avant le début du clip.', 'dialog.action.updateCurrentTempo': 'Mettre à jour le tempo courant', @@ -332,6 +345,7 @@ export const frFrMessages: TranslationMessages = { 'pianoRoll.detectingChords': 'Détection des accords...', 'pianoRoll.detectTempo': 'Détecter le tempo...', 'pianoRoll.detectingTempo': 'Détection du tempo...', + 'pianoRoll.convertToMidi': 'Convertir en MIDI...', 'toolbar.export.kgstudio': 'Exporter au format KGStudio', 'toolbar.export.midi': 'Exporter au format MIDI', 'toolbar.export.wav': 'Exporter en WAV', diff --git a/src/i18n/messages/zh_cn.ts b/src/i18n/messages/zh_cn.ts index e02dbb6..aaec418 100644 --- a/src/i18n/messages/zh_cn.ts +++ b/src/i18n/messages/zh_cn.ts @@ -191,6 +191,7 @@ export const zhCnMessages: TranslationMessages = { 'dialog.title.tempoDetection': '速度检测', 'dialog.title.applyTempo': '应用速度', 'dialog.title.chordDetection': '和弦检测', + 'dialog.title.audioToMidi': '转换为 MIDI', 'dialog.title.input': '输入', 'dialog.title.confirm': '确认', 'dialog.close': '关闭对话框', @@ -201,6 +202,7 @@ export const zhCnMessages: TranslationMessages = { 'dialog.chordHint.audio': '和弦分析仍属实验性功能。和声内容、编配密度以及瞬态较强的素材都可能影响准确率。建议先使用默认设置，再逐步微调灵敏度和稳定性，让检测结果更贴近实际乐句。', 'dialog.chordHint.midi': '和弦分析仍属实验性功能。和弦堆叠密度、重叠以及装饰音都会影响结果。建议先使用默认设置，再调整短音抑制和和声关注方式，以匹配该段音乐的和声作用。', 'dialog.chordHint.tempo': '速度分析仍属实验性功能。自由速度、瞬态稀疏以及多层打击乐都可能降低准确率。建议先使用默认 BPM 范围，如果第一次结果不理想，再把分析范围收窄到更可能的速度区间。', + 'dialog.audioToMidiHint': 'Audio-to-MIDI 转换仍属实验性功能。建议先切换到频谱视图，调整 Floor，直到只剩下真正的音符轮廓而大部分鬼影音高消失，再启用使用当前 Floor dB 后执行转换。', 'dialog.sourceHint.kgone': '为了获得更可靠的和弦标签，建议分析已经削弱或移除了人声和打击乐的 stem。如果已启用 K.G.One Music Studio 服务器集成，请使用带有 “Vocal, Drums, Bass, Guitar, Piano, and Others” 的 Separator 模型，并优先分析 Piano 或 Others stem。', 'dialog.sourceHint.local': '为了获得更可靠的和弦标签，建议分析已经削弱或移除了人声和打击乐的 stem。如果您使用本地分轨器，请选择 “Vocal, Drums, Bass, and Others” 模型，并使用 Others stem 进行分析。', 'dialog.label.sensitivity': '灵敏度', @@ -217,7 +219,18 @@ export const zhCnMessages: TranslationMessages = { 'dialog.label.minimumBpm': '最小 BPM', 'dialog.label.maximumBpm': '最大 BPM', 'dialog.label.autoAlignRegionToBeat': '自动将片段对齐到拍点', + 'dialog.label.monophonicOnly': '仅限单音 (v1)', + 'dialog.label.useCurrentFloorDb': '使用当前 Floor dB', + 'dialog.label.manualFloorDb': '手动 Floor dB', + 'dialog.label.pitchRange': '音高范围', + 'dialog.label.quantizeNoteStart': '量化音符起点', + 'dialog.label.quantizeNoteLength': '量化音符长度', + 'dialog.label.convertLoopRangeOnly': '仅转换 loop 范围内的音频片段', + 'dialog.label.groupAdjacentPitchesToHighest': '将相邻音高归并到最强的音高格', + 'dialog.label.targetTrack': '目标 MIDI 轨', + 'dialog.to': '到', 'dialog.message.chordDetection': '在处理前调整音频和弦检测设置。', + 'dialog.message.audioToMidi': '在处理前调整 Audio-to-MIDI 转换设置。', 'dialog.message.tempoDetection': '在处理前调整音频速度检测设置。', 'dialog.message.tempoApply': '检测到速度：{bpm} BPM。请选择应用方式。\n\n"更新当前速度"将修改此片段位置的有效速度。"插入速度变更"将在片段起始点前最近的小节处添加新的速度区域。', 'dialog.action.updateCurrentTempo': '更新当前速度', @@ -352,6 +365,7 @@ export const zhCnMessages: TranslationMessages = { 'pianoRoll.detectingChords': '正在检测和弦...', 'pianoRoll.detectTempo': '检测速度...', 'pianoRoll.detectingTempo': '正在检测速度...', + 'pianoRoll.convertToMidi': '转换为 MIDI...', 'percussion.short.35': '原底鼓', 'percussion.short.36': '底鼓', 'percussion.short.37': '边击', diff --git a/src/i18n/messages/zh_hk.ts b/src/i18n/messages/zh_hk.ts index b61c6d1..a02672d 100644 --- a/src/i18n/messages/zh_hk.ts +++ b/src/i18n/messages/zh_hk.ts @@ -191,6 +191,7 @@ export const zhHkMessages: TranslationMessages = { 'dialog.title.tempoDetection': '速度檢測', 'dialog.title.applyTempo': '應用速度', 'dialog.title.chordDetection': '和弦檢測', + 'dialog.title.audioToMidi': '轉換為 MIDI', 'dialog.title.input': '輸入', 'dialog.title.confirm': '確認', 'dialog.close': '關閉對話框', @@ -201,6 +202,7 @@ export const zhHkMessages: TranslationMessages = { 'dialog.chordHint.audio': '和弦分析仍屬實驗性功能。和聲內容、編配密度以及瞬態較強的素材都可能影響準確率。建議先使用預設設定，再逐步微調靈敏度和穩定性，讓檢測結果更貼近實際樂句。', 'dialog.chordHint.midi': '和弦分析仍屬實驗性功能。和弦堆疊密度、重疊以及裝飾音都會影響結果。建議先使用預設設定，再調整短音抑制和和聲關注方式，以配合該段音樂的和聲作用。', 'dialog.chordHint.tempo': '速度分析仍屬實驗性功能。自由速度、瞬態稀疏以及多層打擊樂都可能降低準確率。建議先使用預設 BPM 範圍，如果第一次結果不理想，再把分析範圍收窄到更可能的速度區間。', + 'dialog.audioToMidiHint': 'Audio-to-MIDI 轉換仍屬實驗性功能。建議先切換到頻譜視圖，調整 Floor，直到只剩下真正的音符輪廓而大部分鬼影音高消失，再啟用使用目前 Floor dB 後執行轉換。', 'dialog.sourceHint.kgone': '為了獲得更可靠的和弦標籤，建議分析已經削弱或移除了人聲和打擊樂的 stem。如果已啟用 K.G.One Music Studio 伺服器整合，請使用帶有 "Vocal, Drums, Bass, Guitar, Piano, and Others" 的 Separator 模型，並優先分析 Piano 或 Others stem。', 'dialog.sourceHint.local': '為了獲得更可靠的和弦標籤，建議分析已經削弱或移除了人聲和打擊樂的 stem。如果您使用本地分軌器，請選擇 "Vocal, Drums, Bass, and Others" 模型，並使用 Others stem 進行分析。', 'dialog.label.sensitivity': '靈敏度', @@ -217,7 +219,18 @@ export const zhHkMessages: TranslationMessages = { 'dialog.label.minimumBpm': '最小 BPM', 'dialog.label.maximumBpm': '最大 BPM', 'dialog.label.autoAlignRegionToBeat': '自動將片段對齊到拍點', + 'dialog.label.monophonicOnly': '僅限單音 (v1)', + 'dialog.label.useCurrentFloorDb': '使用目前 Floor dB', + 'dialog.label.manualFloorDb': '手動 Floor dB', + 'dialog.label.pitchRange': '音高範圍', + 'dialog.label.quantizeNoteStart': '量化音符起點', + 'dialog.label.quantizeNoteLength': '量化音符長度', + 'dialog.label.convertLoopRangeOnly': '僅轉換 loop 範圍內的音訊片段', + 'dialog.label.groupAdjacentPitchesToHighest': '將相鄰音高群組到最強的音高格', + 'dialog.label.targetTrack': '目標 MIDI 軌', + 'dialog.to': '至', 'dialog.message.chordDetection': '在處理前調整音訊和弦檢測設定。', + 'dialog.message.audioToMidi': '在處理前調整 Audio-to-MIDI 轉換設定。', 'dialog.message.tempoDetection': '在處理前調整音訊速度檢測設定。', 'dialog.message.tempoApply': '檢測到速度：{bpm} BPM。請選擇應用方式。\n\n"更新當前速度"會修改此片段位置的有效速度。"插入速度變更"會在片段起始點前最近的小節加入新的速度區域。', 'dialog.action.updateCurrentTempo': '更新當前速度', @@ -352,6 +365,7 @@ export const zhHkMessages: TranslationMessages = { 'pianoRoll.detectingChords': '正在檢測和弦...', 'pianoRoll.detectTempo': '檢測速度...', 'pianoRoll.detectingTempo': '正在檢測速度...', + 'pianoRoll.convertToMidi': '轉換為 MIDI...', 'percussion.short.35': '原底鼓', 'percussion.short.36': '底鼓', 'percussion.short.37': '邊擊', diff --git a/src/stores/projectStore.test.ts b/src/stores/projectStore.test.ts index 7afff29..c389d82 100644 --- a/src/stores/projectStore.test.ts +++ b/src/stores/projectStore.test.ts @@ -4,6 +4,8 @@ import { KGTrack } from '../core/track/KGTrack'; import { KGMidiTrack } from '../core/track/KGMidiTrack'; import { KGAudioTrack } from '../core/track/KGAudioTrack'; import { KGAudioRegion } from '../core/region/KGAudioRegion'; +import { KGMidiRegion } from '../core/region/KGMidiRegion'; +import { KGMidiNote } from '../core/midi/KGMidiNote'; import { createDefaultGlobalTracks } from '../core/global-track'; import { getAudioRegionDisplayLengthBeats } from '../util/globalTrackUtil'; @@ -35,6 +37,9 @@ let mockTracks: KGTrack[] = [new KGMidiTrack('Track 1', 0, 'acoustic_grand_piano let mockIsMetronomeEnabled = false; let mockShowGlobalTracks = false; let mockPlayheadPosition = 0; +let mockSelectedItems: Array<{ getId: () => string; select: () => void; deselect: () => void; isSelected: () => boolean }> = []; +let mockCopiedItems: Array<{ getId: () => string }> = []; +const selectionChangedCallbacks: Array<() => void> = []; const mockProject = { getTimeSignature: () => ({ numerator: 4, denominator: 4 }), getMaxBars: () => 32, @@ -90,8 +95,11 @@ const mockCore = { setPlayheadUpdateCallback: vi.fn(), setPlaybackStateChangeCallback: vi.fn(), setLoopBoundaryReachedCallback: vi.fn(), - getSelectedItems: () => [], - onSelectionChanged: vi.fn(), + getSelectedItems: () => mockSelectedItems, + getCopiedItems: () => mockCopiedItems, + onSelectionChanged: vi.fn((callback: () => void) => { + selectionChangedCallbacks.push(callback); + }), canUndo: () => false, canRedo: () => false, getUndoDescription: () => '', @@ -100,7 +108,16 @@ const mockCore = { executeCommand: vi.fn(), undo: vi.fn(() => true), redo: vi.fn(() => true), - clearSelectedItems: vi.fn(), + clearSelectedItems: vi.fn(() => { + mockSelectedItems = []; + selectionChangedCallbacks.forEach(callback => callback()); + }), + addSelectedItems: vi.fn((items: Array<{ getId: () => string; select: () => void; deselect: () => void; isSelected: () => boolean }>) => { + const incomingIds = new Set(items.map(item => item.getId())); + mockSelectedItems = mockSelectedItems.filter(item => !incomingIds.has(item.getId())); + mockSelectedItems.push(...items); + selectionChangedCallbacks.forEach(callback => callback()); + }), getStatus: () => 'Ready', setStatus: vi.fn(), getPlayheadPosition: () => mockPlayheadPosition, @@ -197,6 +214,12 @@ describe('projectStore piano roll state', () => { audioStorageMocks.loadAudioFile.mockReset(); toneMocks.decodeAudioData.mockReset(); toneMocks.toneBufferSet.mockReset(); + mockSelectedItems = []; + mockCopiedItems = []; + selectionChangedCallbacks.length = 0; + mockCore.onSelectionChanged.mockClear(); + mockCore.clearSelectedItems.mockClear(); + mockCore.addSelectedItems.mockClear(); mockIsMetronomeEnabled = false; mockShowGlobalTracks = false; mockProject.setIsMetronomeEnabled.mockClear(); @@ -737,4 +760,81 @@ describe('projectStore piano roll state', () => { expect(state.maxBars).toBe(16); expect(state.playheadPosition).toBe(64); }); + + it('selects only the newly pasted notes after pasting into the active MIDI region', async () => { + const { KGMidiTrack: TestMidiTrack } = await import('../core/track/KGMidiTrack'); + const { KGMidiRegion: TestMidiRegion } = await import('../core/region/KGMidiRegion'); + const { KGMidiNote: TestMidiNote } = await import('../core/midi/KGMidiNote'); + const track = new TestMidiTrack('Track 1', 1, 'acoustic_grand_piano'); + track.setTrackIndex(0); + const region = new TestMidiRegion('region-1', '1', 0, 'Region 1', 0, 16); + const existingSelectedNote = new TestMidiNote('existing-note', 0, 1, 60, 100); + existingSelectedNote.select(); + region.addNote(existingSelectedNote); + track.setRegions([region]); + + mockTracks = [track]; + currentProject = { + ...mockProject, + getTracks: () => mockTracks, + } as typeof mockProject; + + mockSelectedItems = [existingSelectedNote]; + mockCopiedItems = [ + new TestMidiNote('copied-a', 2, 3, 64, 110), + new TestMidiNote('copied-b', 3, 4, 67, 120), + ]; + mockCore.executeCommand.mockImplementation((command: { execute: () => void }) => command.execute()); + + const { useProjectStore } = await import('./projectStore'); + + act(() => { + useProjectStore.getState().pasteNotesToActiveRegion(region.getId(), 8); + }); + + const pastedNotes = region.getNotes().filter(note => note.getId() !== existingSelectedNote.getId()); + const pastedNoteIds = pastedNotes.map(note => note.getId()); + const state = useProjectStore.getState(); + + expect(mockCore.executeCommand).toHaveBeenCalledTimes(1); + expect(existingSelectedNote.isSelected()).toBe(false); + expect(pastedNotes).toHaveLength(2); + expect(pastedNotes.every(note => note.isSelected())).toBe(true); + expect(mockCore.clearSelectedItems).toHaveBeenCalledTimes(1); + expect(mockCore.addSelectedItems).toHaveBeenCalledWith(pastedNotes); + expect(state.selectedNoteIds).toEqual(pastedNoteIds); + }); + + it('keeps selection unchanged when note paste has no clipboard notes', async () => { + const { KGMidiTrack: TestMidiTrack } = await import('../core/track/KGMidiTrack'); + const { KGMidiRegion: TestMidiRegion } = await import('../core/region/KGMidiRegion'); + const { KGMidiNote: TestMidiNote } = await import('../core/midi/KGMidiNote'); + const track = new TestMidiTrack('Track 1', 1, 'acoustic_grand_piano'); + track.setTrackIndex(0); + const region = new TestMidiRegion('region-1', '1', 0, 'Region 1', 0, 16); + const existingSelectedNote = new TestMidiNote('existing-note', 0, 1, 60, 100); + existingSelectedNote.select(); + region.addNote(existingSelectedNote); + track.setRegions([region]); + + mockTracks = [track]; + currentProject = { + ...mockProject, + getTracks: () => mockTracks, + } as typeof mockProject; + + mockSelectedItems = [existingSelectedNote]; + mockCopiedItems = []; + + const { useProjectStore } = await import('./projectStore'); + + act(() => { + useProjectStore.getState().pasteNotesToActiveRegion(region.getId(), 8); + }); + + expect(mockCore.executeCommand).not.toHaveBeenCalled(); + expect(mockCore.clearSelectedItems).not.toHaveBeenCalled(); + expect(mockCore.addSelectedItems).not.toHaveBeenCalled(); + expect(existingSelectedNote.isSelected()).toBe(true); + }); }); diff --git a/src/stores/projectStore.ts b/src/stores/projectStore.ts index 11e3a72..2d69bb0 100644 --- a/src/stores/projectStore.ts +++ b/src/stores/projectStore.ts @@ -62,6 +62,11 @@ function formatCurrentTime(project: KGProject, beat: number): string { return beatsToTimeString(beat, bpmForLegacyFormatting, project.getTimeSignature()); } +function clampPlayheadPosition(project: KGProject, position: number): number { + const maxBeat = project.getMaxBars() * project.getTimeSignature().numerator; + return Math.max(0, Math.min(position, maxBeat)); +} + function getProjectGlobalTracks(project: KGProject): KGGlobalTrack[] { return (project.getGlobalTracks?.() ?? []) as KGGlobalTrack[]; } @@ -923,8 +928,6 @@ export const useProjectStore = create((set, get) => { loadProject: async (project: KGProject | null = null, savedName?: string) => { try { - const { setPlayheadPosition } = get(); - // Upgrade incoming project data to latest structure version (only when provided explicitly) if (project) { project = upgradeProjectToLatest(project); @@ -944,15 +947,15 @@ export const useProjectStore = create((set, get) => { KGCore.instance().executeCommand(addDefaultTrackCommand); } - // Reset playhead to 0 when loading a project - setPlayheadPosition(0); - // Get project properties const maxBars = projectToLoad.getMaxBars(); const timeSignature = projectToLoad.getTimeSignature(); const bpm = projectToLoad.getBpm(); const keySignature = projectToLoad.getKeySignature(); const tracks = projectToLoad.getTracks(); + const restoredPlayheadPosition = clampPlayheadPosition(projectToLoad, projectToLoad.getPlayheadPosition()); + projectToLoad.setPlayheadPosition(restoredPlayheadPosition); + KGCore.instance().setPlayheadPosition(restoredPlayheadPosition); // Setup audio synths for all tracks const audioInterface = KGAudioInterface.instance(); @@ -1036,8 +1039,9 @@ export const useProjectStore = create((set, get) => { recordingAudioPreviewPeaks: [], recordingAudioPreviewCurrentBeat: 0, recordingAudioPreviewFileName: null, - playheadPosition: 0, // Ensure store state is also updated - currentTime: formatCurrentTime(projectToLoad, 0) // Reset time display + playheadPosition: restoredPlayheadPosition, + currentTime: formatCurrentTime(projectToLoad, restoredPlayheadPosition), + mainContentScrollRequest: restoredPlayheadPosition, }); // After loading a project, auto-select the first track and open Instrument Selection @@ -1052,6 +1056,7 @@ export const useProjectStore = create((set, get) => { // Reset piano roll state for new/loaded project KGPianoRollState.instance().setLastEditedNoteLength(1); + KGPianoRollState.instance().setLastEditedNoteVelocity(127); KGPianoRollState.instance().setPianoRollZoom(projectToLoad.getPianoRollZoom()); // Add a status message @@ -1064,11 +1069,13 @@ export const useProjectStore = create((set, get) => { }, setPlayheadPosition: (position: number) => { - const { bpm, timeSignature } = get(); - KGCore.instance().setPlayheadPosition(position); + const project = KGCore.instance().getCurrentProject(); + const clampedPosition = clampPlayheadPosition(project, position); + project.setPlayheadPosition(clampedPosition); + KGCore.instance().setPlayheadPosition(clampedPosition); set({ - playheadPosition: position, - currentTime: formatCurrentTime(KGCore.instance().getCurrentProject(), position) + playheadPosition: clampedPosition, + currentTime: formatCurrentTime(project, clampedPosition) }); }, @@ -1904,7 +1911,26 @@ export const useProjectStore = create((set, get) => { } try { - KGCore.instance().executeCommand(command); + const core = KGCore.instance(); + core.executeCommand(command); + + const createdNoteIds = new Set(command.getCreatedNotes().map(note => note.noteId)); + const targetRegion = command.getTargetRegion(); + const createdNotes = targetRegion + ? targetRegion.getNotes().filter(note => createdNoteIds.has(note.getId())) + : []; + + if (createdNotes.length > 0) { + core.getSelectedItems().forEach(item => { + item.deselect(); + }); + core.clearSelectedItems(); + + createdNotes.forEach(note => { + note.select(); + }); + core.addSelectedItems(createdNotes); + } // Update the store to trigger re-render const { tracks } = get(); diff --git a/src/test/integration/audio-to-midi.integration.test.ts b/src/test/integration/audio-to-midi.integration.test.ts new file mode 100644 index 0000000..4da0c1a --- /dev/null +++ b/src/test/integration/audio-to-midi.integration.test.ts @@ -0,0 +1,67 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { pitchToNoteNameString } from '../../util/midiUtil'; +import { detectMonophonicNotesFromAudio } from '../../util/audioToMidi'; + +const FIXTURE_PATH = path.resolve(process.cwd(), 'public/test-data/audio-to-midi-test01.mp3'); +const ffmpegAvailable = spawnSync('ffmpeg', ['-version'], { stdio: 'ignore' }).status === 0; +const runIfFfmpeg = ffmpegAvailable ? it : it.skip; + +function decodeMp3ToMonoPcm(pathname: string): { sampleRate: number; pcm: Float32Array } { + const wav = execFileSync( + 'ffmpeg', + ['-v', 'error', '-i', pathname, '-ac', '1', '-f', 'wav', 'pipe:1'], + { maxBuffer: 64 * 1024 * 1024 }, + ); + + const readString = (offset: number, length: number) => wav.toString('ascii', offset, offset + length); + if (readString(0, 4) !== 'RIFF' || readString(8, 4) !== 'WAVE') { + throw new Error('ffmpeg did not return a RIFF/WAVE stream'); + } + + let offset = 12; + let sampleRate = 44100; + let pcmData = Buffer.alloc(0); + + while (offset + 8 <= wav.length) { + const chunkId = readString(offset, 4); + const chunkSize = wav.readUInt32LE(offset + 4); + const chunkStart = offset + 8; + + if (chunkId === 'fmt ') { + sampleRate = wav.readUInt32LE(chunkStart + 4); + } else if (chunkId === 'data') { + pcmData = wav.subarray(chunkStart, chunkStart + chunkSize); + break; + } + + offset = chunkStart + chunkSize + (chunkSize % 2); + } + + const pcm = new Float32Array(pcmData.length / 2); + for (let sampleIndex = 0; sampleIndex < pcm.length; sampleIndex++) { + pcm[sampleIndex] = pcmData.readInt16LE(sampleIndex * 2) / 32768; + } + + return { sampleRate, pcm }; +} + +describe('audio-to-MIDI fixture regression', () => { + runIfFfmpeg('extracts exactly A1 then D2 from the dedicated mp3 fixture at -16 dB', () => { + const { sampleRate, pcm } = decodeMp3ToMonoPcm(FIXTURE_PATH); + const notes = detectMonophonicNotesFromAudio({ + pcm, + sampleRate, + startSeconds: 0, + endSeconds: pcm.length / sampleRate, + floorDb: -16, + pitchRangeStart: 12, + pitchRangeEnd: 107, + groupAdjacentPitchesToHighest: true, + }); + + expect(notes).toHaveLength(2); + expect(notes.map(note => pitchToNoteNameString(note.pitch))).toEqual(['A1', 'D2']); + }); +}); diff --git a/src/test/integration/store/project-store-sync.integration.test.ts b/src/test/integration/store/project-store-sync.integration.test.ts index 27461c0..1ae05d6 100644 --- a/src/test/integration/store/project-store-sync.integration.test.ts +++ b/src/test/integration/store/project-store-sync.integration.test.ts @@ -247,6 +247,7 @@ describe('Project Store Synchronization Integration Tests', () => { // Verify store state updated const storeState = useProjectStore.getState(); expect(storeState.playheadPosition).toBe(newPosition); + expect(testProject.getPlayheadPosition()).toBe(newPosition); // Verify formatted time string was updated expect(storeState.currentTime).toBeDefined(); @@ -632,6 +633,7 @@ describe('Project Store Synchronization Integration Tests', () => { newProject.setTimeSignature({ numerator: 6, denominator: 8 }); newProject.setKeySignature('D major'); newProject.setMaxBars(48); + newProject.setPlayheadPosition(17.5); // Add a track with region and notes const track = new KGMidiTrack('Loaded Track', 0, 'violin'); @@ -654,11 +656,14 @@ describe('Project Store Synchronization Integration Tests', () => { expect(storeState.timeSignature).toEqual({ numerator: 6, denominator: 8 }); expect(storeState.keySignature).toBe('D major'); expect(storeState.maxBars).toBe(48); + expect(storeState.playheadPosition).toBe(17.5); + expect(storeState.mainContentScrollRequest).toBe(17.5); expect(storeState.tracks).toHaveLength(1); // Verify core model is updated const core = KGCore.instance(); expect(core.getCurrentProject()?.getName()).toBe('New Loaded Project'); + expect(core.getCurrentProject()?.getPlayheadPosition()).toBe(17.5); // Verify CSS properties were updated const timeSignatureCSS = getComputedStyle(document.documentElement).getPropertyValue('--time-signature-numerator'); @@ -667,5 +672,24 @@ describe('Project Store Synchronization Integration Tests', () => { const maxBarsCSS = getComputedStyle(document.documentElement).getPropertyValue('--max-number-of-bars'); expect(maxBarsCSS.trim()).toBe('48'); }); + + it('should clamp restored playhead position when loading beyond the project end', async () => { + const { loadProject } = useProjectStore.getState(); + const newProject = new KGProject('Clamped Loaded Project'); + newProject.setTimeSignature({ numerator: 4, denominator: 4 }); + newProject.setMaxBars(8); + newProject.setPlayheadPosition(100); + + await act(async () => { + await loadProject(newProject); + }); + + const clampedBeat = 32; + const storeState = useProjectStore.getState(); + expect(storeState.playheadPosition).toBe(clampedBeat); + expect(storeState.mainContentScrollRequest).toBe(clampedBeat); + expect(newProject.getPlayheadPosition()).toBe(clampedBeat); + expect(KGCore.instance().getCurrentProject().getPlayheadPosition()).toBe(clampedBeat); + }); }); }); diff --git a/src/util/audioToMidi.test.ts b/src/util/audioToMidi.test.ts new file mode 100644 index 0000000..3610f6d --- /dev/null +++ b/src/util/audioToMidi.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from 'vitest'; +import { KGProject } from '../core/KGProject'; +import { KGAudioRegion } from '../core/region/KGAudioRegion'; +import { + buildAudioToMidiAnalysisSpan, + convertDetectedAudioNotesToRawMidiNotes, + detectMonophonicNotesFromAudio, +} from './audioToMidi'; + +function createSineWavePcm( + frequency: number, + durationSeconds: number, + sampleRate: number, + amplitude: number = 0.8, +): Float32Array { + const sampleCount = Math.floor(durationSeconds * sampleRate); + const pcm = new Float32Array(sampleCount); + for (let i = 0; i < sampleCount; i++) { + pcm[i] = amplitude * Math.sin((2 * Math.PI * frequency * i) / sampleRate); + } + return pcm; +} + +describe('audioToMidi analysis span helpers', () => { + it('uses the exact source audio region bounds when loop mode is off', () => { + const project = new KGProject('Test', 32, 0, 120); + const region = new KGAudioRegion('audio-1', '1', 0, 'Audio', 8, 4, 'file-1', 'test.wav', 10, 1.5); + + const span = buildAudioToMidiAnalysisSpan(project, region, { + loopModeEnabled: false, + convertLoopRangeOnly: true, + loopingRange: [0, 0], + }); + + expect(span).toEqual({ + regionStartBeat: 8, + regionEndBeat: 12, + startSeconds: 1.5, + endSeconds: 3.5, + }); + }); + + it('uses the exact loop overlap when loop-only conversion is enabled', () => { + const project = new KGProject('Test', 32, 0, 120); + const region = new KGAudioRegion('audio-1', '1', 0, 'Audio', 8, 8, 'file-1', 'test.wav', 10, 0.5); + + const span = buildAudioToMidiAnalysisSpan(project, region, { + loopModeEnabled: true, + convertLoopRangeOnly: true, + loopingRange: [3, 4], + }); + + expect(span).toEqual({ + regionStartBeat: 12, + regionEndBeat: 16, + startSeconds: 2.5, + endSeconds: 4.5, + }); + }); +}); + +describe('audioToMidi note conversion helpers', () => { + it('drops raw notes shorter than the selected quantized note length and quantizes survivors', () => { + const project = new KGProject('Test', 32, 0, 120); + const rawNotes = convertDetectedAudioNotesToRawMidiNotes( + project, + { + regionStartBeat: 8, + regionEndBeat: 12, + startSeconds: 0, + endSeconds: 2, + }, + [ + { startOffsetSeconds: 0.0, endOffsetSeconds: 0.24, pitch: 33, heat: 0.9 }, + { startOffsetSeconds: 0.5, endOffsetSeconds: 1.45, pitch: 38, heat: 0.7 }, + ], + { + quantizeNoteStart: '1/16', + quantizeNoteLength: '1/4', + }, + ); + + expect(rawNotes).toEqual([ + { + startBeat: 1, + endBeat: 3, + pitch: 38, + velocity: 102, + }, + ]); + }); +}); + +describe('audioToMidi synthetic detection', () => { + it('detects sequential monophonic notes from a simple sine-wave fixture', () => { + const sampleRate = 44100; + const a1 = createSineWavePcm(55, 0.7, sampleRate); + const silence = new Float32Array(Math.floor(0.08 * sampleRate)); + const d2 = createSineWavePcm(73.41619197935188, 0.7, sampleRate); + const pcm = new Float32Array(a1.length + silence.length + d2.length); + pcm.set(a1, 0); + pcm.set(silence, a1.length); + pcm.set(d2, a1.length + silence.length); + + const notes = detectMonophonicNotesFromAudio({ + pcm, + sampleRate, + startSeconds: 0, + endSeconds: pcm.length / sampleRate, + floorDb: -30, + pitchRangeStart: 21, + pitchRangeEnd: 38, + groupAdjacentPitchesToHighest: true, + }); + + expect(notes.map(note => note.pitch)).toEqual([33, 38]); + }); +}); diff --git a/src/util/audioToMidi.ts b/src/util/audioToMidi.ts new file mode 100644 index 0000000..f048309 --- /dev/null +++ b/src/util/audioToMidi.ts @@ -0,0 +1,161 @@ +import { KGProject } from '../core/KGProject'; +import { KGAudioRegion } from '../core/region/KGAudioRegion'; +import type { + PianoRollQuantizeLengthValue, + PianoRollQuantizePositionValue, +} from '../core/state/KGPianoRollState'; +import type { RawMidiNote } from './midiUtil'; +import { + beatRangeToSeconds, + beatToSeconds, + secondsToBeat, +} from './globalTrackUtil'; + +export { + detectMonophonicNotesFromAudio, + type AudioToMidiDetectedNote, + type AudioToMidiDetectionRequest, +} from './audioToMidiCore'; + +import type { AudioToMidiDetectedNote } from './audioToMidiCore'; + +export interface AudioToMidiAnalysisSpan { + regionStartBeat: number; + regionEndBeat: number; + startSeconds: number; + endSeconds: number; +} + +export interface AudioToMidiConversionOptions { + floorDb: number; + pitchRangeStart: number; + pitchRangeEnd: number; + quantizeNoteStart: PianoRollQuantizePositionValue; + quantizeNoteLength: PianoRollQuantizeLengthValue; + groupAdjacentPitchesToHighest: boolean; +} + +export function buildAudioToMidiAnalysisSpan( + project: KGProject, + audioRegion: KGAudioRegion, + options: { + loopModeEnabled: boolean; + convertLoopRangeOnly: boolean; + loopingRange: [number, number]; + }, +): AudioToMidiAnalysisSpan | null { + const regionStartBeat = audioRegion.getStartFromBeat(); + const regionEndBeat = regionStartBeat + audioRegion.getLength(); + if (regionEndBeat <= regionStartBeat) { + return null; + } + + if (options.loopModeEnabled && options.convertLoopRangeOnly) { + const beatsPerBar = project.getTimeSignature().numerator; + const loopStartBeat = options.loopingRange[0] * beatsPerBar; + const loopEndBeat = (options.loopingRange[1] + 1) * beatsPerBar; + const overlapStartBeat = Math.max(regionStartBeat, loopStartBeat); + const overlapEndBeat = Math.min(regionEndBeat, loopEndBeat); + if (overlapEndBeat <= overlapStartBeat) { + return null; + } + + return { + regionStartBeat: overlapStartBeat, + regionEndBeat: overlapEndBeat, + startSeconds: audioRegion.getClipStartOffsetSeconds() + beatRangeToSeconds(project, regionStartBeat, overlapStartBeat), + endSeconds: audioRegion.getClipStartOffsetSeconds() + beatRangeToSeconds(project, regionStartBeat, overlapEndBeat), + }; + } + + return { + regionStartBeat, + regionEndBeat, + startSeconds: audioRegion.getClipStartOffsetSeconds(), + endSeconds: audioRegion.getClipStartOffsetSeconds() + beatRangeToSeconds(project, regionStartBeat, regionEndBeat), + }; +} + +export function quantizationValueToBeats(value: string): number { + const denominator = Number.parseInt(value.split('/')[1] ?? '', 10); + if (!Number.isFinite(denominator) || denominator <= 0) { + throw new Error(`Invalid quantization value: ${value}`); + } + + return 4 / denominator; +} + +function clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); +} + +export function convertDetectedAudioNotesToRawMidiNotes( + project: KGProject, + span: AudioToMidiAnalysisSpan, + detectedNotes: AudioToMidiDetectedNote[], + options: Pick, +): RawMidiNote[] { + const regionStartAbsoluteSeconds = beatToSeconds(project, span.regionStartBeat); + const regionLengthBeats = Math.max(0, span.regionEndBeat - span.regionStartBeat); + const quantizedStartStep = quantizationValueToBeats(options.quantizeNoteStart); + const quantizedLengthStep = quantizationValueToBeats(options.quantizeNoteLength); + + const converted = detectedNotes + .map(note => { + const startBeatAbsolute = secondsToBeat(project, regionStartAbsoluteSeconds + note.startOffsetSeconds); + const endBeatAbsolute = secondsToBeat(project, regionStartAbsoluteSeconds + note.endOffsetSeconds); + const rawStartBeat = startBeatAbsolute - span.regionStartBeat; + const rawEndBeat = endBeatAbsolute - span.regionStartBeat; + const rawDurationBeats = rawEndBeat - rawStartBeat; + if (rawDurationBeats < quantizedLengthStep) { + return null; + } + + const quantizedStartBeat = Math.round(rawStartBeat / quantizedStartStep) * quantizedStartStep; + const quantizedDurationBeats = Math.max( + quantizedLengthStep, + Math.round(rawDurationBeats / quantizedLengthStep) * quantizedLengthStep, + ); + const quantizedEndBeat = quantizedStartBeat + quantizedDurationBeats; + if (quantizedStartBeat >= regionLengthBeats) { + return null; + } + + const clampedStartBeat = clamp(quantizedStartBeat, 0, regionLengthBeats); + const clampedEndBeat = clamp( + Math.max(clampedStartBeat + quantizedLengthStep, quantizedEndBeat), + clampedStartBeat + Math.min(quantizedLengthStep, Math.max(regionLengthBeats - clampedStartBeat, 0)), + regionLengthBeats, + ); + if (clampedEndBeat <= clampedStartBeat) { + return null; + } + + return { + startBeat: clampedStartBeat, + endBeat: clampedEndBeat, + pitch: note.pitch, + velocity: clamp(Math.round(45 + note.heat * 82), 1, 127), + } satisfies RawMidiNote; + }) + .filter((note): note is RawMidiNote => note !== null) + .sort((left, right) => left.startBeat - right.startBeat || left.pitch - right.pitch); + + const merged: RawMidiNote[] = []; + for (const note of converted) { + const previous = merged[merged.length - 1]; + if ( + previous && + previous.pitch === note.pitch && + note.startBeat <= previous.endBeat + 1e-6 + ) { + previous.endBeat = Math.max(previous.endBeat, note.endBeat); + previous.velocity = Math.max(previous.velocity, note.velocity); + continue; + } + + merged.push({ ...note }); + } + + return merged; +} diff --git a/src/util/audioToMidiCore.ts b/src/util/audioToMidiCore.ts new file mode 100644 index 0000000..124cc1d --- /dev/null +++ b/src/util/audioToMidiCore.ts @@ -0,0 +1,358 @@ +import FFT from 'fft.js'; +import { + SPECTROGRAM_FULL_MAX_MIDI_PITCH, + SPECTROGRAM_FULL_MIN_MIDI_PITCH, + getSpectrogramAnalysisResolution, + getSpectrogramPitchBinCount, +} from './spectrogramUtil'; + +const FFT_SIZE = 16384; +const HOP_SIZE = 512; +const MAX_HEIGHT_RESOLUTION = 5; +const MIN_ANALYSIS_FREQUENCY = 20; +const MAX_ANALYSIS_FREQUENCY = 5000; + +export interface AudioToMidiDetectionRequest { + pcm: Float32Array; + sampleRate: number; + startSeconds: number; + endSeconds: number; + floorDb: number; + pitchRangeStart: number; + pitchRangeEnd: number; + groupAdjacentPitchesToHighest: boolean; +} + +export interface AudioToMidiDetectedNote { + startOffsetSeconds: number; + endOffsetSeconds: number; + pitch: number; + heat: number; +} + +const HANN_WINDOW = (() => { + const window = new Float32Array(FFT_SIZE); + for (let i = 0; i < FFT_SIZE; i++) { + window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (FFT_SIZE - 1))); + } + return window; +})(); + +function midiToFrequency(midiPitch: number): number { + return 440 * Math.pow(2, (midiPitch - 69) / 12); +} + +function frequencyToMidiPitch(frequency: number): number | null { + if (!Number.isFinite(frequency) || frequency <= 0) { + return null; + } + return 69 + 12 * Math.log2(frequency / 440); +} + +function mapMidiPitchToPosition(midiPitch: number, resolution: number): number | null { + const pitchOffset = midiPitch - SPECTROGRAM_FULL_MIN_MIDI_PITCH; + if (pitchOffset < 0 || pitchOffset > SPECTROGRAM_FULL_MAX_MIDI_PITCH) { + return null; + } + + const scaled = pitchOffset * resolution + (resolution - 1) / 2; + const maxBin = getSpectrogramPitchBinCount(resolution) - 1; + return Math.max(0, Math.min(maxBin, scaled)); +} + +function estimateParabolicPeakOffset(leftMagnitude: number, centerMagnitude: number, rightMagnitude: number): number { + const denominator = leftMagnitude - 2 * centerMagnitude + rightMagnitude; + if (!Number.isFinite(denominator) || Math.abs(denominator) < Number.EPSILON) { + return 0; + } + + const offset = 0.5 * (leftMagnitude - rightMagnitude) / denominator; + return Math.max(-0.5, Math.min(0.5, offset)); +} + +function estimatePeakFrequency(bin: number, hzPerBin: number, magnitudes: Float32Array): number { + const centerFrequency = bin * hzPerBin; + if (bin <= 1 || bin >= magnitudes.length - 1) { + return centerFrequency; + } + + const leftMagnitude = magnitudes[bin - 1]; + const centerMagnitude = magnitudes[bin]; + const rightMagnitude = magnitudes[bin + 1]; + + if (centerMagnitude <= leftMagnitude || centerMagnitude <= rightMagnitude) { + return centerFrequency; + } + + return (bin + estimateParabolicPeakOffset(leftMagnitude, centerMagnitude, rightMagnitude)) * hzPerBin; +} + +function getFrequencyBandEdges(centerFrequency: number, hzPerBin: number): { lowerFrequency: number; upperFrequency: number } { + const halfBandwidth = hzPerBin / 2; + return { + lowerFrequency: Math.max(Number.EPSILON, centerFrequency - halfBandwidth), + upperFrequency: centerFrequency + halfBandwidth, + }; +} + +function paintMagnitudeAcrossPitchSpan( + target: Float32Array, + pitchBins: number, + startPosition: number, + endPosition: number, + centerPosition: number, + magnitude: number, +): void { + const clampedStart = Math.max(0, Math.min(startPosition, pitchBins - 1)); + const clampedEnd = Math.max(0, Math.min(endPosition, pitchBins - 1)); + const startBin = Math.max(0, Math.floor(clampedStart)); + const endBin = Math.min(pitchBins - 1, Math.ceil(clampedEnd)); + const spanWidth = Math.max(clampedEnd - clampedStart, 1); + + for (let targetBin = startBin; targetBin <= endBin; targetBin++) { + const cellStart = targetBin - 0.5; + const cellEnd = targetBin + 0.5; + const overlap = Math.max(0, Math.min(clampedEnd, cellEnd) - Math.max(clampedStart, cellStart)); + if (overlap <= 0) { + continue; + } + + const overlapWeight = Math.min(1, overlap); + const centerDistance = Math.abs(targetBin - centerPosition); + const centerWeight = Math.max(0, 1 - centerDistance / (spanWidth + 1)); + const weight = Math.max(overlapWeight * (0.6 + 0.4 * centerWeight), overlapWeight * 0.35); + const weightedMagnitude = magnitude * weight; + + if (weightedMagnitude > target[targetBin]) { + target[targetBin] = weightedMagnitude; + } + } +} + +function collapseAdjacentPitchClusters(activeBins: number[], frameBins: Float32Array): number[] { + if (activeBins.length === 0) { + return []; + } + + const grouped: number[] = []; + let clusterStart = 0; + + while (clusterStart < activeBins.length) { + let clusterEnd = clusterStart; + let strongestBin = activeBins[clusterStart]; + let strongestValue = frameBins[strongestBin]; + + while ( + clusterEnd + 1 < activeBins.length && + activeBins[clusterEnd + 1] <= activeBins[clusterEnd] + 1 + ) { + clusterEnd += 1; + const candidateBin = activeBins[clusterEnd]; + const candidateValue = frameBins[candidateBin]; + if (candidateValue > strongestValue) { + strongestBin = candidateBin; + strongestValue = candidateValue; + } + } + + grouped.push(strongestBin); + clusterStart = clusterEnd + 1; + } + + return grouped; +} + +function normalizePitchBinToMidiPitch(bin: number, resolution: number): number { + return Math.round((bin - ((resolution - 1) / 2)) / resolution); +} + +export function detectMonophonicNotesFromAudio(request: AudioToMidiDetectionRequest): AudioToMidiDetectedNote[] { + const { + pcm, + sampleRate, + startSeconds, + endSeconds, + floorDb, + pitchRangeStart, + pitchRangeEnd, + groupAdjacentPitchesToHighest, + } = request; + + const clampedStartSeconds = Math.max(0, startSeconds); + const clampedEndSeconds = Math.max(clampedStartSeconds, endSeconds); + const startSample = Math.max(0, Math.floor(clampedStartSeconds * sampleRate)); + const endSample = Math.min(pcm.length, Math.ceil(clampedEndSeconds * sampleRate)); + const sampleCount = Math.max(0, endSample - startSample); + if (sampleCount <= 0) { + return []; + } + + const analysisResolution = getSpectrogramAnalysisResolution(MAX_HEIGHT_RESOLUTION); + const pitchBins = getSpectrogramPitchBinCount(analysisResolution); + const fft = new FFT(FFT_SIZE); + const complexOut = fft.createComplexArray() as number[]; + const inputPadded = new Float32Array(FFT_SIZE); + const hzPerBin = sampleRate / FFT_SIZE; + const totalHops = Math.max(1, Math.ceil((sampleCount - FFT_SIZE) / HOP_SIZE) + 1); + const frameBinValues: Float32Array[] = []; + let globalMax = 0; + + const minFrequency = Math.max(MIN_ANALYSIS_FREQUENCY, midiToFrequency(Math.max(0, pitchRangeStart - 2))); + const maxFrequency = Math.min(MAX_ANALYSIS_FREQUENCY, midiToFrequency(Math.min(127, pitchRangeEnd + 2))); + + for (let hop = 0; hop < totalHops; hop++) { + const frameBins = new Float32Array(pitchBins); + const offset = hop * HOP_SIZE; + inputPadded.fill(0); + const available = Math.min(FFT_SIZE, sampleCount - offset); + + for (let i = 0; i < available; i++) { + inputPadded[i] = pcm[startSample + offset + i] * HANN_WINDOW[i]; + } + + fft.realTransform(complexOut, inputPadded as unknown as number[]); + fft.completeSpectrum(complexOut); + + const numBins = FFT_SIZE / 2; + const magnitudes = new Float32Array(numBins); + for (let bin = 1; bin < numBins; bin++) { + const re = complexOut[2 * bin]; + const im = complexOut[2 * bin + 1]; + magnitudes[bin] = Math.sqrt(re * re + im * im); + } + + for (let bin = 1; bin < numBins; bin++) { + const centerFrequency = estimatePeakFrequency(bin, hzPerBin, magnitudes); + const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, hzPerBin); + if (upperFrequency < minFrequency || lowerFrequency > maxFrequency) { + continue; + } + + const magnitude = magnitudes[bin]; + if (magnitude <= 0) { + continue; + } + + const lowerMidi = frequencyToMidiPitch(Math.max(lowerFrequency, minFrequency)); + const upperMidi = frequencyToMidiPitch(Math.min(upperFrequency, maxFrequency)); + const centerMidi = frequencyToMidiPitch(centerFrequency); + if (lowerMidi === null || upperMidi === null || centerMidi === null) { + continue; + } + + const startPosition = mapMidiPitchToPosition(lowerMidi, analysisResolution); + const endPosition = mapMidiPitchToPosition(upperMidi, analysisResolution); + const centerPosition = mapMidiPitchToPosition(centerMidi, analysisResolution); + if (startPosition === null || endPosition === null || centerPosition === null) { + continue; + } + + paintMagnitudeAcrossPitchSpan( + frameBins, + pitchBins, + Math.min(startPosition, endPosition), + Math.max(startPosition, endPosition), + centerPosition, + magnitude, + ); + } + + for (let i = 0; i < frameBins.length; i++) { + if (frameBins[i] > globalMax) { + globalMax = frameBins[i]; + } + } + + frameBinValues.push(frameBins); + } + + if (globalMax <= 0) { + return []; + } + + const linearThreshold = Math.pow(10, floorDb / 20); + const hopDurationSeconds = HOP_SIZE / sampleRate; + const notes: AudioToMidiDetectedNote[] = []; + let current: AudioToMidiDetectedNote | null = null; + let currentHeatSum = 0; + let currentFrameCount = 0; + + for (let frameIndex = 0; frameIndex < frameBinValues.length; frameIndex++) { + const normalizedBins = frameBinValues[frameIndex].slice(); + const activeBins: number[] = []; + + for (let i = 0; i < normalizedBins.length; i++) { + normalizedBins[i] = normalizedBins[i] / globalMax; + if (normalizedBins[i] >= linearThreshold) { + activeBins.push(i); + } + } + + const candidateBins = groupAdjacentPitchesToHighest + ? collapseAdjacentPitchClusters(activeBins, normalizedBins) + : activeBins; + + let winningPitch: number | null = null; + let winningHeat = 0; + + for (const bin of candidateBins) { + const heat = normalizedBins[bin]; + if (heat <= winningHeat) { + continue; + } + + const midiPitch = normalizePitchBinToMidiPitch(bin, analysisResolution); + if (midiPitch < pitchRangeStart || midiPitch > pitchRangeEnd) { + continue; + } + + winningPitch = midiPitch; + winningHeat = heat; + } + + const frameStartSeconds = frameIndex * hopDurationSeconds; + const frameEndSeconds = Math.min((frameIndex + 1) * hopDurationSeconds, clampedEndSeconds - clampedStartSeconds); + + if (winningPitch === null) { + if (current) { + current.heat = currentHeatSum / Math.max(1, currentFrameCount); + notes.push(current); + current = null; + currentHeatSum = 0; + currentFrameCount = 0; + } + continue; + } + + if (current && current.pitch === winningPitch) { + current.endOffsetSeconds = frameEndSeconds; + currentHeatSum += winningHeat; + currentFrameCount += 1; + continue; + } + + if (current) { + current.heat = currentHeatSum / Math.max(1, currentFrameCount); + notes.push(current); + } + + // Future polyphonic support should preserve the strongest bin inside an + // adjacent cluster (for example C4 over weaker B3/D4 neighbors) before + // selecting additional simultaneous note candidates. + current = { + startOffsetSeconds: frameStartSeconds, + endOffsetSeconds: frameEndSeconds, + pitch: winningPitch, + heat: winningHeat, + }; + currentHeatSum = winningHeat; + currentFrameCount = 1; + } + + if (current) { + current.heat = currentHeatSum / Math.max(1, currentFrameCount); + notes.push(current); + } + + return notes; +} diff --git a/src/util/dialogUtil.ts b/src/util/dialogUtil.ts index 2d18ced..3a9ad54 100644 --- a/src/util/dialogUtil.ts +++ b/src/util/dialogUtil.ts @@ -37,6 +37,19 @@ export interface TempoApplyResult { autoAlignRegionToBeat: boolean; } +export interface AudioToMidiOptionsResult { + monophonic: boolean; + useCurrentFloorDb: boolean; + manualFloorDb: number; + pitchRangeStart: number; + pitchRangeEnd: number; + quantizeNoteStart: string; + quantizeNoteLength: string; + convertLoopRangeOnly: boolean; + groupAdjacentPitchesToHighest: boolean; + targetTrackId: string; +} + export interface ChoiceOption { label: string; value: string; @@ -51,6 +64,12 @@ let _showChordDetectionOptionsFn: ((message: string, defaultValue?: ChordDetecti let _showMidiChordDetectionOptionsFn: ((message: string, defaultValue?: MidiChordDetectionOptionsResult) => Promise) | null = null; let _showTempoDetectionOptionsFn: ((message: string, defaultValue?: TempoDetectionOptionsResult) => Promise) | null = null; let _showTempoApplyFn: ((message: string, choices: ChoiceOption[]) => Promise) | null = null; +let _showAudioToMidiOptionsFn: (( + message: string, + targetTracks: ChoiceOption[], + loopModeEnabled: boolean, + defaultValue?: AudioToMidiOptionsResult, +) => Promise) | null = null; export function registerDialogFns( alertFn: (message: string) => Promise, @@ -62,6 +81,12 @@ export function registerDialogFns( midiChordDetectionOptionsFn?: (message: string, defaultValue?: MidiChordDetectionOptionsResult) => Promise, tempoDetectionOptionsFn?: (message: string, defaultValue?: TempoDetectionOptionsResult) => Promise, tempoApplyFn?: (message: string, choices: ChoiceOption[]) => Promise, + audioToMidiOptionsFn?: ( + message: string, + targetTracks: ChoiceOption[], + loopModeEnabled: boolean, + defaultValue?: AudioToMidiOptionsResult, + ) => Promise, ) { _showAlertFn = alertFn; _showConfirmFn = confirmFn; @@ -72,6 +97,7 @@ export function registerDialogFns( if (midiChordDetectionOptionsFn) _showMidiChordDetectionOptionsFn = midiChordDetectionOptionsFn; if (tempoDetectionOptionsFn) _showTempoDetectionOptionsFn = tempoDetectionOptionsFn; if (tempoApplyFn) _showTempoApplyFn = tempoApplyFn; + if (audioToMidiOptionsFn) _showAudioToMidiOptionsFn = audioToMidiOptionsFn; } export function showAlert(message: string): Promise { @@ -166,3 +192,26 @@ export function showTempoApply(message: string, choices: ChoiceOption[]): Promis } return _showTempoApplyFn(message, choices); } + +export function showAudioToMidiOptions( + message: string, + targetTracks: ChoiceOption[], + loopModeEnabled: boolean, + defaultValue?: AudioToMidiOptionsResult, +): Promise { + if (!_showAudioToMidiOptionsFn) { + return Promise.resolve(defaultValue ?? { + monophonic: true, + useCurrentFloorDb: true, + manualFloorDb: -25, + pitchRangeStart: 12, + pitchRangeEnd: 107, + quantizeNoteStart: '1/16', + quantizeNoteLength: '1/16', + convertLoopRangeOnly: true, + groupAdjacentPitchesToHighest: true, + targetTrackId: targetTracks[0]?.value ?? '', + }); + } + return _showAudioToMidiOptionsFn(message, targetTracks, loopModeEnabled, defaultValue); +} diff --git a/src/util/spectrogramUtil.test.ts b/src/util/spectrogramUtil.test.ts index 0bcdd9b..27ab7b3 100644 --- a/src/util/spectrogramUtil.test.ts +++ b/src/util/spectrogramUtil.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from 'vitest'; import { + frequencyToMidiPitch, + getSpectrogramAnalysisResolution, getSpectrogramPitchBinCount, getSpectrogramVisibleBinRange, + mapFrequencyToSpectrogramPosition, mapMidiPitchToSpectrogramPosition, normalizeSpectrogramHeightResolution, } from './spectrogramUtil'; @@ -21,6 +24,12 @@ describe('spectrogramUtil', () => { expect(getSpectrogramPitchBinCount(5)).toBe(640); }); + it('doubles the internal analysis resolution for each user-facing option', () => { + expect(getSpectrogramAnalysisResolution(1)).toBe(2); + expect(getSpectrogramAnalysisResolution(3)).toBe(6); + expect(getSpectrogramAnalysisResolution(5)).toBe(10); + }); + it('maps MIDI pitches into full-range spectrogram positions', () => { expect(mapMidiPitchToSpectrogramPosition(0, 1)).toBe(0); expect(mapMidiPitchToSpectrogramPosition(12.5, 3)).toBe(38.5); @@ -29,6 +38,12 @@ describe('spectrogramUtil', () => { expect(mapMidiPitchToSpectrogramPosition(128, 3)).toBeNull(); }); + it('converts frequencies into MIDI pitch space and spectrogram positions', () => { + expect(frequencyToMidiPitch(440)).toBeCloseTo(69, 6); + expect(mapFrequencyToSpectrogramPosition(440, 5)).toBeCloseTo(347, 6); + expect(frequencyToMidiPitch(0)).toBeNull(); + }); + it('derives the visible C0-B7 subrange inside the full-resolution buffer', () => { expect(getSpectrogramVisibleBinRange(1)).toEqual({ start: 12, end: 108 }); expect(getSpectrogramVisibleBinRange(3)).toEqual({ start: 36, end: 324 }); diff --git a/src/util/spectrogramUtil.ts b/src/util/spectrogramUtil.ts index d219db2..a2e8eb7 100644 --- a/src/util/spectrogramUtil.ts +++ b/src/util/spectrogramUtil.ts @@ -6,6 +6,7 @@ export const SPECTROGRAM_FULL_SEMITONES = SPECTROGRAM_FULL_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1; export const SPECTROGRAM_VISIBLE_SEMITONES = SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_MIN_MIDI_PITCH + 1; +export const SPECTROGRAM_ANALYSIS_RESOLUTION_MULTIPLIER = 2; export type SpectrogramHeightResolution = 1 | 3 | 5; @@ -19,13 +20,13 @@ export function normalizeSpectrogramHeightResolution(value: unknown): Spectrogra } export function getSpectrogramPitchBinCount( - resolution: SpectrogramHeightResolution, + resolution: number, ): number { return SPECTROGRAM_FULL_SEMITONES * resolution; } export function getSpectrogramVisibleBinRange( - resolution: SpectrogramHeightResolution, + resolution: number, ): { start: number; end: number } { return { start: (SPECTROGRAM_MIN_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH) * resolution, @@ -33,9 +34,15 @@ export function getSpectrogramVisibleBinRange( }; } +export function getSpectrogramAnalysisResolution( + resolution: SpectrogramHeightResolution, +): number { + return resolution * SPECTROGRAM_ANALYSIS_RESOLUTION_MULTIPLIER; +} + export function mapMidiPitchToSpectrogramPosition( midiPitch: number, - resolution: SpectrogramHeightResolution, + resolution: number, ): number | null { const pitchOffset = midiPitch - SPECTROGRAM_FULL_MIN_MIDI_PITCH; if (pitchOffset < 0 || pitchOffset > SPECTROGRAM_FULL_MAX_MIDI_PITCH) { @@ -48,3 +55,21 @@ export function mapMidiPitchToSpectrogramPosition( const maxBin = getSpectrogramPitchBinCount(resolution) - 1; return Math.max(0, Math.min(maxBin, scaled)); } + +export function frequencyToMidiPitch(frequency: number): number | null { + if (!Number.isFinite(frequency) || frequency <= 0) { + return null; + } + return 69 + 12 * Math.log2(frequency / 440); +} + +export function mapFrequencyToSpectrogramPosition( + frequency: number, + resolution: number, +): number | null { + const midiPitch = frequencyToMidiPitch(frequency); + if (midiPitch === null) { + return null; + } + return mapMidiPitchToSpectrogramPosition(midiPitch, resolution); +} diff --git a/src/workers/audioToMidiWorker.ts b/src/workers/audioToMidiWorker.ts new file mode 100644 index 0000000..7cc41f0 --- /dev/null +++ b/src/workers/audioToMidiWorker.ts @@ -0,0 +1,22 @@ +import { + detectMonophonicNotesFromAudio, + type AudioToMidiDetectedNote, + type AudioToMidiDetectionRequest, +} from '../util/audioToMidiCore'; + +export interface AudioToMidiWorkerResult { + type: 'result'; + notes: AudioToMidiDetectedNote[]; +} + +type WorkerScopeLike = typeof globalThis & { + onmessage: ((event: MessageEvent) => void) | null; + postMessage: (message: AudioToMidiWorkerResult) => void; +}; + +const workerScope = self as WorkerScopeLike; + +workerScope.onmessage = (event: MessageEvent) => { + const notes = detectMonophonicNotesFromAudio(event.data); + workerScope.postMessage({ type: 'result', notes }); +}; diff --git a/src/workers/spectrogramWorker.test.ts b/src/workers/spectrogramWorker.test.ts new file mode 100644 index 0000000..874c7bd --- /dev/null +++ b/src/workers/spectrogramWorker.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest'; +import { getSpectrogramPitchBinCount, mapFrequencyToSpectrogramPosition } from '../util/spectrogramUtil'; +import { + estimatePeakFrequency, + getFrequencyBandEdges, + paintMagnitudeAcrossPitchSpan, +} from './spectrogramWorker'; + +const FFT_SIZE = 16384; +const SAMPLE_RATE = 44100; +const HZ_PER_BIN = SAMPLE_RATE / FFT_SIZE; + +function paintSpanForFrequency( + centerFrequency: number, + resolution: 1 | 3 | 5, + pitchBins = getSpectrogramPitchBinCount(resolution), +): Float32Array { + const row = new Float32Array(pitchBins); + const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, HZ_PER_BIN); + const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, resolution); + const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, resolution); + const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, resolution); + + if (startPosition === null || endPosition === null || centerPosition === null) { + throw new Error('Expected mapped pitch positions for test frequency span'); + } + + paintMagnitudeAcrossPitchSpan( + row, + 0, + pitchBins, + Math.min(startPosition, endPosition), + Math.max(startPosition, endPosition), + centerPosition, + 1, + ); + return row; +} + +describe('spectrogramWorker helpers', () => { + it('keeps adjacent low-register FFT bins continuous at 5x resolution', () => { + const pitchBins = getSpectrogramPitchBinCount(5); + const row = new Float32Array(pitchBins); + const binsAroundA1 = [10, 11]; + + for (const bin of binsAroundA1) { + const centerFrequency = bin * HZ_PER_BIN; + const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, HZ_PER_BIN); + const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, 5); + const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, 5); + const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, 5); + + expect(startPosition).not.toBeNull(); + expect(endPosition).not.toBeNull(); + expect(centerPosition).not.toBeNull(); + + paintMagnitudeAcrossPitchSpan( + row, + 0, + pitchBins, + Math.min(startPosition!, endPosition!), + Math.max(startPosition!, endPosition!), + centerPosition!, + 1, + ); + } + + const nonZeroBins = [...row.entries()].filter(([, value]) => value > 0).map(([index]) => index); + expect(nonZeroBins.length).toBeGreaterThan(0); + + for (let index = nonZeroBins[0]; index <= nonZeroBins[nonZeroBins.length - 1]; index++) { + expect(row[index]).toBeGreaterThan(0); + } + }); + + it('uses sub-bin interpolation to move the ridge closer to the true peak frequency', () => { + const magnitudes = new Float32Array([0, 0.3, 1.0, 0.82, 0.1, 0]); + const rawCenterFrequency = 2 * HZ_PER_BIN; + const interpolatedFrequency = estimatePeakFrequency(2, HZ_PER_BIN, magnitudes); + const expectedFrequency = (2.35 * HZ_PER_BIN); + + expect(Math.abs(interpolatedFrequency - expectedFrequency)).toBeLessThan( + Math.abs(rawCenterFrequency - expectedFrequency), + ); + }); + + it('leaves non-peak bins at their raw FFT center while still painting a span', () => { + const magnitudes = new Float32Array([0, 0.2, 0.4, 1.0, 0.8, 0.7, 0.2, 0]); + const centerFrequency = estimatePeakFrequency(4, HZ_PER_BIN, magnitudes); + expect(centerFrequency).toBeCloseTo(4 * HZ_PER_BIN, 6); + + const row = paintSpanForFrequency(centerFrequency, 3); + expect(row.some(value => value > 0)).toBe(true); + }); + + it('maps edge-band spans into the valid spectrogram range without spilling outside the buffer', () => { + const lowRow = paintSpanForFrequency(20.5, 5); + const lowNonZeroBins = [...lowRow.entries()].filter(([, value]) => value > 0).map(([index]) => index); + expect(lowNonZeroBins.length).toBeGreaterThan(0); + expect(lowNonZeroBins[0]).toBeGreaterThanOrEqual(0); + + const highRow = paintSpanForFrequency(12530, 5); + const highNonZeroBins = [...highRow.entries()].filter(([, value]) => value > 0).map(([index]) => index); + expect(highNonZeroBins.length).toBeGreaterThan(0); + expect(highNonZeroBins[highNonZeroBins.length - 1]).toBeLessThan(highRow.length); + }); +}); diff --git a/src/workers/spectrogramWorker.ts b/src/workers/spectrogramWorker.ts index cd02a8b..3d421e0 100644 --- a/src/workers/spectrogramWorker.ts +++ b/src/workers/spectrogramWorker.ts @@ -1,7 +1,8 @@ import FFT from 'fft.js'; import { + getSpectrogramAnalysisResolution, getSpectrogramPitchBinCount, - mapMidiPitchToSpectrogramPosition, + mapFrequencyToSpectrogramPosition, normalizeSpectrogramHeightResolution, type SpectrogramHeightResolution, } from '../util/spectrogramUtil'; @@ -11,7 +12,7 @@ type WorkerScopeLike = typeof globalThis & { postMessage: (message: SpectrogramResult, transfer: Transferable[]) => void; }; -const workerScope = self as WorkerScopeLike; +const workerScope = typeof self === 'undefined' ? null : self as WorkerScopeLike; export interface SpectrogramRequest { pcm: Float32Array; @@ -28,8 +29,10 @@ export interface SpectrogramResult { pitchBins: number; } -const FFT_SIZE = 8192; +const FFT_SIZE = 16384; const HOP_SIZE = 1024; +const MIN_SPECTROGRAM_FREQUENCY = 20; +const MAX_SPECTROGRAM_FREQUENCY = 20000; function hannWindow(size: number): Float32Array { const w = new Float32Array(size); @@ -39,24 +42,98 @@ function hannWindow(size: number): Float32Array { return w; } -function spreadMagnitudeAcrossPitchBins( +export function getFrequencyBandEdges( + centerFrequency: number, + hzPerBin: number, +): { lowerFrequency: number; upperFrequency: number } { + const halfBandwidth = hzPerBin / 2; + return { + lowerFrequency: Math.max(Number.EPSILON, centerFrequency - halfBandwidth), + upperFrequency: centerFrequency + halfBandwidth, + }; +} + +export function estimateParabolicPeakOffset( + leftMagnitude: number, + centerMagnitude: number, + rightMagnitude: number, +): number { + const denominator = leftMagnitude - 2 * centerMagnitude + rightMagnitude; + if (!Number.isFinite(denominator) || Math.abs(denominator) < Number.EPSILON) { + return 0; + } + + const offset = 0.5 * (leftMagnitude - rightMagnitude) / denominator; + return Math.max(-0.5, Math.min(0.5, offset)); +} + +export function estimatePeakFrequency( + bin: number, + hzPerBin: number, + magnitudes: Float32Array, +): number { + const centerFrequency = bin * hzPerBin; + if (bin <= 1 || bin >= magnitudes.length - 1) { + return centerFrequency; + } + + const leftMagnitude = magnitudes[bin - 1]; + const centerMagnitude = magnitudes[bin]; + const rightMagnitude = magnitudes[bin + 1]; + + if (centerMagnitude <= leftMagnitude || centerMagnitude <= rightMagnitude) { + return centerFrequency; + } + + return (bin + estimateParabolicPeakOffset(leftMagnitude, centerMagnitude, rightMagnitude)) * hzPerBin; +} + +function getPitchSpanBounds( + lowerFrequency: number, + upperFrequency: number, + centerFrequency: number, + analysisResolution: number, +): { startPosition: number; endPosition: number; centerPosition: number } | null { + const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, analysisResolution); + const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, analysisResolution); + const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, analysisResolution); + + if (startPosition === null || endPosition === null || centerPosition === null) { + return null; + } + + return { + startPosition: Math.min(startPosition, endPosition), + endPosition: Math.max(startPosition, endPosition), + centerPosition, + }; +} + +export function paintMagnitudeAcrossPitchSpan( result: Float32Array, pitchRow: number, pitchBins: number, - pitchPosition: number, + startPosition: number, + endPosition: number, + centerPosition: number, magnitude: number, - heightResolution: SpectrogramHeightResolution, ): void { - const spreadRadius = Math.max(0, heightResolution - 1); - const centerBin = Math.round(pitchPosition); - const startBin = Math.max(0, centerBin - spreadRadius); - const endBin = Math.min(pitchBins - 1, centerBin + spreadRadius); - const spreadWidth = spreadRadius + 1; + const clampedStart = Math.max(0, Math.min(startPosition, pitchBins - 1)); + const clampedEnd = Math.max(0, Math.min(endPosition, pitchBins - 1)); + const startBin = Math.max(0, Math.floor(clampedStart)); + const endBin = Math.min(pitchBins - 1, Math.ceil(clampedEnd)); + const spanWidth = Math.max(clampedEnd - clampedStart, 1); for (let targetBin = startBin; targetBin <= endBin; targetBin++) { - const distance = Math.abs(targetBin - pitchPosition); - const weight = Math.max(0, 1 - distance / spreadWidth); - if (weight <= 0) continue; + const cellStart = targetBin - 0.5; + const cellEnd = targetBin + 0.5; + const overlap = Math.max(0, Math.min(clampedEnd, cellEnd) - Math.max(clampedStart, cellStart)); + if (overlap <= 0) continue; + + const overlapWeight = Math.min(1, overlap); + const centerDistance = Math.abs(targetBin - centerPosition); + const centerWeight = Math.max(0, 1 - centerDistance / (spanWidth + 1)); + const weight = Math.max(overlapWeight * (0.6 + 0.4 * centerWeight), overlapWeight * 0.35); const weightedMagnitude = magnitude * weight; const resultIndex = pitchRow + targetBin; @@ -66,7 +143,7 @@ function spreadMagnitudeAcrossPitchBins( } } -workerScope.onmessage = (e: MessageEvent) => { +function handleSpectrogramRequest(e: MessageEvent, scope: WorkerScopeLike): void { const { pcm, sampleRate, @@ -75,7 +152,8 @@ workerScope.onmessage = (e: MessageEvent) => { heightResolution, } = e.data; const normalizedResolution = normalizeSpectrogramHeightResolution(heightResolution); - const pitchBins = getSpectrogramPitchBinCount(normalizedResolution); + const analysisResolution = getSpectrogramAnalysisResolution(normalizedResolution); + const pitchBins = getSpectrogramPitchBinCount(analysisResolution); const startSample = Math.floor(clipStartOffsetSeconds * sampleRate); const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate)); @@ -85,6 +163,7 @@ workerScope.onmessage = (e: MessageEvent) => { const hann = hannWindow(FFT_SIZE); const complexOut = fft.createComplexArray() as number[]; const inputPadded = new Float32Array(FFT_SIZE); + const hzPerBin = sampleRate / FFT_SIZE; const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1); const result = new Float32Array(totalHops * pitchBins); @@ -108,25 +187,44 @@ workerScope.onmessage = (e: MessageEvent) => { // rather than summing. Summing inflates every bin by how many FFT bins land there. const pitchRow = hop * pitchBins; const numBins = FFT_SIZE / 2; + const magnitudes = new Float32Array(numBins); for (let bin = 1; bin < numBins; bin++) { - const freq = (bin * sampleRate) / FFT_SIZE; - if (freq < 20 || freq > 20000) continue; - - const midiPitch = 69 + 12 * Math.log2(freq / 440); - const pitchPosition = mapMidiPitchToSpectrogramPosition(midiPitch, normalizedResolution); - if (pitchPosition === null) continue; - const re = complexOut[2 * bin]; const im = complexOut[2 * bin + 1]; - const magnitude = Math.sqrt(re * re + im * im); - spreadMagnitudeAcrossPitchBins( + magnitudes[bin] = Math.sqrt(re * re + im * im); + } + + for (let bin = 1; bin < numBins; bin++) { + const centerFrequency = estimatePeakFrequency(bin, hzPerBin, magnitudes); + const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, hzPerBin); + if (upperFrequency < MIN_SPECTROGRAM_FREQUENCY || lowerFrequency > MAX_SPECTROGRAM_FREQUENCY) continue; + + const clampedLowerFrequency = Math.max(lowerFrequency, MIN_SPECTROGRAM_FREQUENCY); + const clampedUpperFrequency = Math.min(upperFrequency, MAX_SPECTROGRAM_FREQUENCY); + const clampedCenterFrequency = Math.min( + clampedUpperFrequency, + Math.max(clampedLowerFrequency, centerFrequency), + ); + const pitchSpan = getPitchSpanBounds( + clampedLowerFrequency, + clampedUpperFrequency, + clampedCenterFrequency, + analysisResolution, + ); + if (!pitchSpan) continue; + + const magnitude = magnitudes[bin]; + if (magnitude <= 0) continue; + + paintMagnitudeAcrossPitchSpan( result, pitchRow, pitchBins, - pitchPosition, + pitchSpan.startPosition, + pitchSpan.endPosition, + pitchSpan.centerPosition, magnitude, - normalizedResolution, ); } @@ -145,5 +243,11 @@ workerScope.onmessage = (e: MessageEvent) => { } const response: SpectrogramResult = { data: result, timeSteps: totalHops, pitchBins }; - workerScope.postMessage(response, [result.buffer]); -}; + scope.postMessage(response, [result.buffer]); +} + +if (workerScope) { + workerScope.onmessage = (e: MessageEvent) => { + handleSpectrogramRequest(e, workerScope); + }; +}