diff --git a/public/config.json b/public/config.json index 71c018f..6302f28 100644 --- a/public/config.json +++ b/public/config.json @@ -66,7 +66,8 @@ } }, "editor": { - "playhead_update_frequency": 10 + "playhead_update_frequency": 10, + "spectrogram_height_resolution": 3 }, "chatbox": { "default_open": true @@ -83,4 +84,4 @@ "chord_guide": { "chord_definition": "" } -} \ No newline at end of file +} diff --git a/src/components/piano-roll/PianoGrid.tsx b/src/components/piano-roll/PianoGrid.tsx index a406497..bd6932e 100644 --- a/src/components/piano-roll/PianoGrid.tsx +++ b/src/components/piano-roll/PianoGrid.tsx @@ -8,6 +8,7 @@ import type { KeySignature } from '../../core/KGProject'; import { KGPianoRollState } from '../../core/state/KGPianoRollState'; import SpectrogramCanvas from './SpectrogramCanvas'; import type { KGAudioRegion } from '../../core/region/KGAudioRegion'; +import type { SpectrogramHeightResolution } from '../../util/spectrogramUtil'; interface PianoGridProps { gridRef: MutableRefObject; @@ -32,6 +33,7 @@ interface PianoGridProps { bpm?: number; spectrogramThresholdDb?: number; spectrogramPower?: number; + spectrogramHeightResolution?: SpectrogramHeightResolution; pianoRollZoom?: number; mode?: 'midi-edit' | 'spectrogram' | 'hybrid'; onSpectrogramLoadingChange?: (loading: boolean) => void; @@ -62,6 +64,7 @@ const PianoGrid: React.FC = ({ bpm = 120, spectrogramThresholdDb = -25, spectrogramPower = 0.5, + spectrogramHeightResolution = 3, pianoRollZoom = 1, onSpectrogramLoadingChange, }) => { @@ -245,6 +248,7 @@ const PianoGrid: React.FC = ({ thresholdDb={spectrogramThresholdDb} power={spectrogramPower} zoom={pianoRollZoom} + heightResolution={spectrogramHeightResolution} onLoadingChange={onSpectrogramLoadingChange} /> )} @@ -305,4 +309,4 @@ const PianoGrid: React.FC = ({ ); }; -export default PianoGrid; \ No newline at end of file +export default PianoGrid; diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index 0f49c5b..eaedc6e 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -18,6 +18,10 @@ import { beatsToBar } from '../../util/midiUtil'; import { UpdateRegionCommand } from '../../core/commands'; import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil'; import { showAlert, showPrompt } from '../../util/dialogUtil'; +import { + normalizeSpectrogramHeightResolution, + type SpectrogramHeightResolution, +} from '../../util/spectrogramUtil'; interface PianoRollProps { onClose: () => void; @@ -50,6 +54,8 @@ const PianoRoll: React.FC = ({ // Spectrogram controls (only used in spectrogram mode) const [spectrogramThresholdDb, setSpectrogramThresholdDb] = useState(-25); const [spectrogramPower, setSpectrogramPower] = useState(0.5); + const [spectrogramHeightResolution, setSpectrogramHeightResolution] = + useState(3); // Piano roll zoom (1x–8x); updates --region-grid-beat-width CSS variable const [pianoRollZoom, setPianoRollZoom] = useState(1); @@ -201,6 +207,41 @@ const PianoRoll: React.FC = ({ } }, []); // Empty dependency array means this runs once on mount + useEffect(() => { + let unsubscribe: (() => void) | undefined; + + const initializeSpectrogramHeightResolution = async () => { + const configManager = ConfigManager.instance(); + if (!configManager.getIsInitialized()) { + await configManager.initialize(); + } + + const applyResolutionFromConfig = () => { + setSpectrogramHeightResolution( + normalizeSpectrogramHeightResolution( + configManager.get('editor.spectrogram_height_resolution') + ) + ); + }; + + applyResolutionFromConfig(); + unsubscribe = configManager.addChangeListener((changedKeys) => { + if ( + changedKeys.includes('__all__') || + changedKeys.includes('editor.spectrogram_height_resolution') + ) { + applyResolutionFromConfig(); + } + }); + }; + + void initializeSpectrogramHeightResolution(); + + return () => { + unsubscribe?.(); + }; + }, []); + // Add keyboard event listener for Escape useEffect(() => { const handleKeyDown = (event: KeyboardEvent) => { @@ -1048,6 +1089,7 @@ const PianoRoll: React.FC = ({ bpm={bpm} spectrogramThresholdDb={spectrogramThresholdDb} spectrogramPower={spectrogramPower} + spectrogramHeightResolution={spectrogramHeightResolution} pianoRollZoom={pianoRollZoom} /> diff --git a/src/components/piano-roll/PianoRollContent.tsx b/src/components/piano-roll/PianoRollContent.tsx index f33fe9a..63fb060 100644 --- a/src/components/piano-roll/PianoRollContent.tsx +++ b/src/components/piano-roll/PianoRollContent.tsx @@ -14,6 +14,7 @@ import { useNoteSelection } from '../../hooks/useNoteSelection'; import type { KeySignature } from '../../core/KGProject'; import type { KGAudioRegion } from '../../core/region/KGAudioRegion'; import { velocityToColor } from '../../util/velocityColor'; +import type { SpectrogramHeightResolution } from '../../util/spectrogramUtil'; interface PianoRollContentProps { contentRef: React.MutableRefObject; @@ -35,6 +36,7 @@ interface PianoRollContentProps { bpm?: number; spectrogramThresholdDb?: number; spectrogramPower?: number; + spectrogramHeightResolution?: SpectrogramHeightResolution; pianoRollZoom?: number; } @@ -58,6 +60,7 @@ const PianoRollContent: React.FC = ({ bpm = 120, spectrogramThresholdDb = -25, spectrogramPower = 0.5, + spectrogramHeightResolution = 3, pianoRollZoom = 1, }) => { const isSpectrogram = mode === 'spectrogram'; @@ -295,6 +298,7 @@ const PianoRollContent: React.FC = ({ bpm={bpm} spectrogramThresholdDb={spectrogramThresholdDb} spectrogramPower={spectrogramPower} + spectrogramHeightResolution={spectrogramHeightResolution} pianoRollZoom={pianoRollZoom} onSpectrogramLoadingChange={handleSpectrogramLoadingChange} > diff --git a/src/components/piano-roll/SpectrogramCanvas.tsx b/src/components/piano-roll/SpectrogramCanvas.tsx index 84fbf3f..bdcee5b 100644 --- a/src/components/piano-roll/SpectrogramCanvas.tsx +++ b/src/components/piano-roll/SpectrogramCanvas.tsx @@ -4,6 +4,12 @@ import { KGAudioRegion } from '../../core/region/KGAudioRegion'; import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface'; import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage'; import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker'; +import { + getSpectrogramVisibleBinRange, + normalizeSpectrogramHeightResolution, + SPECTROGRAM_VISIBLE_SEMITONES, + type SpectrogramHeightResolution, +} from '../../util/spectrogramUtil'; interface SpectrogramCanvasProps { audioRegion: KGAudioRegion; @@ -13,12 +19,10 @@ interface SpectrogramCanvasProps { thresholdDb: number; power: number; zoom: number; + heightResolution: SpectrogramHeightResolution; onLoadingChange?: (loading: boolean) => void; } -const PITCH_BINS = 128; -const HOP_SIZE = 1024; - // Piecewise-linear RGB colormap: black → dark blue → blue → purple → red → orange → yellow // Hue rotates 240°→300°→0°→60°, bypassing green entirely. const COLORMAP_STOPS: Array<[number, [number, number, number]]> = [ @@ -56,6 +60,7 @@ const SpectrogramCanvas: React.FC = ({ thresholdDb, power, zoom, + heightResolution, onLoadingChange, }) => { const canvasRef = useRef(null); @@ -80,6 +85,7 @@ const SpectrogramCanvas: React.FC = ({ regionDurationSeconds: number, thresholdDb: number, power: number, + heightResolution: SpectrogramHeightResolution, ) => { const canvas = canvasRef.current; if (!canvas) return; @@ -92,31 +98,31 @@ const SpectrogramCanvas: React.FC = ({ const totalBeats = (regionDurationSeconds * bpm) / 60; // Always draw at 1x resolution; zoom is applied as CSS width stretch const canvasWidth = Math.ceil(totalBeats * 40); - const canvasHeight = PITCH_BINS * noteHeight; + const canvasHeight = SPECTROGRAM_VISIBLE_SEMITONES * noteHeight; // Convert dB threshold to linear: values below this → black const linearThreshold = Math.pow(10, thresholdDb / 20); + const visibleRange = getSpectrogramVisibleBinRange(heightResolution); + const visiblePitchBins = visibleRange.end - visibleRange.start; - // 1. Paint at natural STFT resolution onto an offscreen canvas (timeSteps × 128). - // Row i = pitchIndex i = pitch (107 − i). Apply threshold then power curve. + // 1. Paint at natural spectrogram resolution onto an offscreen canvas. + // Result data is low-to-high pitch; draw only the visible C0-B7 window, reversed for display. const offscreen = document.createElement('canvas'); offscreen.width = result.timeSteps; - offscreen.height = PITCH_BINS; + offscreen.height = visiblePitchBins; const offCtx = offscreen.getContext('2d'); if (!offCtx) return; - const imgData = offCtx.createImageData(result.timeSteps, PITCH_BINS); + const imgData = offCtx.createImageData(result.timeSteps, visiblePitchBins); const pixels = imgData.data; - for (let row = 0; row < PITCH_BINS; row++) { - const pitch = 107 - row; + for (let row = 0; row < visiblePitchBins; row++) { + const sourceRow = visibleRange.end - 1 - row; for (let col = 0; col < result.timeSteps; col++) { const idx = (row * result.timeSteps + col) * 4; pixels[idx + 3] = 255; // always opaque - if (pitch < 12 || pitch > 107) continue; // outside range → black - - const raw = result.data[col * PITCH_BINS + pitch]; + const raw = result.data[col * result.pitchBins + sourceRow]; // Hard threshold: values below noise floor → 0 (black) // Re-scale surviving range to [0,1] then apply power curve @@ -157,9 +163,10 @@ const SpectrogramCanvas: React.FC = ({ regionDurationRef.current, thresholdDb, power, + normalizeSpectrogramHeightResolution(heightResolution), ); } - }, [thresholdDb, power, renderSpectrogram]); + }, [thresholdDb, power, renderSpectrogram, heightResolution]); // Zoom changes: stretch width only, pin height to canvas pixel height useEffect(() => { @@ -211,7 +218,14 @@ const SpectrogramCanvas: React.FC = ({ rawResultRef.current = e.data; sampleRateRef.current = sampleRate; regionDurationRef.current = regionDurationSeconds; - renderSpectrogram(e.data, sampleRate, regionDurationSeconds, thresholdDb, power); + renderSpectrogram( + e.data, + sampleRate, + regionDurationSeconds, + thresholdDb, + power, + normalizeSpectrogramHeightResolution(heightResolution), + ); setLoading(false); worker.terminate(); workerRef.current = null; @@ -223,6 +237,7 @@ const SpectrogramCanvas: React.FC = ({ clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(), regionDurationSeconds, bpm, + heightResolution: normalizeSpectrogramHeightResolution(heightResolution), }; worker.postMessage(request, [request.pcm.buffer]); } catch (err) { @@ -241,7 +256,7 @@ const SpectrogramCanvas: React.FC = ({ workerRef.current = null; }; // eslint-disable-next-line react-hooks/exhaustive-deps - }, [audioRegion, trackId, projectName, bpm]); + }, [audioRegion, trackId, projectName, bpm, heightResolution]); return ( { const [playheadUpdateFrequency, setPlayheadUpdateFrequency] = useState(10); + const [spectrogramHeightResolution, setSpectrogramHeightResolution] = useState(3); const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState(true); const [audioLookaheadTime, setAudioLookaheadTime] = useState('50'); const [playbackDelay, setPlaybackDelay] = useState('200'); @@ -23,6 +28,9 @@ const BehaviorSettings: React.FC = () => { } setPlayheadUpdateFrequency((configManager.get('editor.playhead_update_frequency') as number) ?? 10); + setSpectrogramHeightResolution( + normalizeSpectrogramHeightResolution(configManager.get('editor.spectrogram_height_resolution')) + ); setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true); const lookaheadTimeSeconds = (configManager.get('audio.lookahead_time') as number) ?? 0.05; setAudioLookaheadTime(((lookaheadTimeSeconds * 1000).toFixed(0))); @@ -50,6 +58,12 @@ const BehaviorSettings: React.FC = () => { await configManager.set('chatbox.default_open', boolValue); }; + const handleSpectrogramHeightResolutionChange = async (value: string) => { + const nextValue = normalizeSpectrogramHeightResolution(parseInt(value, 10)); + setSpectrogramHeightResolution(nextValue); + await configManager.set('editor.spectrogram_height_resolution', nextValue); + }; + const handleAudioLookaheadTimeChange = async (value: string) => { // Allow empty string, treat as 0 ms const numValueMs = value === '' ? 0 : parseFloat(value); @@ -161,6 +175,24 @@ const BehaviorSettings: React.FC = () => { Update frequency for the playhead animation during playback. Higher values (60 fps) provide smoother animation but use more CPU. Lower values (10 fps) are more efficient. Changes apply immediately without restart. + +
+ + +
+ Controls vertical spectrogram detail across the visible pitch range. Higher values sharpen pitch contours but use more CPU and memory while computing the spectrogram. +
+
@@ -289,4 +321,4 @@ const BehaviorSettings: React.FC = () => { ); }; -export default BehaviorSettings; \ No newline at end of file +export default BehaviorSettings; diff --git a/src/core/audio-interface/KGOfflineRenderer.test.ts b/src/core/audio-interface/KGOfflineRenderer.test.ts index fa8b39d..1428052 100644 --- a/src/core/audio-interface/KGOfflineRenderer.test.ts +++ b/src/core/audio-interface/KGOfflineRenderer.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { encodeWav } from './KGOfflineRenderer'; +import { encodeWav, getOfflineTrackGain, getOfflineTrackVolumeDb } from './KGOfflineRenderer'; /** * Create a minimal AudioBuffer-like object for testing. @@ -145,3 +145,22 @@ describe('encodeWav', () => { expect(view.getUint32(40, true)).toBe(0); // data size = 0 }); }); + +describe('offline track volume conversion', () => { + it('treats 0 dB as unity gain instead of silence', () => { + expect(getOfflineTrackVolumeDb(0, false)).toBe(0); + expect(getOfflineTrackGain(0, false)).toBe(1); + }); + + it('converts negative dB values to linear gain', () => { + expect(getOfflineTrackVolumeDb(-6, false)).toBe(-6); + expect(getOfflineTrackGain(-6, false)).toBeCloseTo(Math.pow(10, -6 / 20), 6); + }); + + it('silences muted tracks and floor-level volumes', () => { + expect(getOfflineTrackVolumeDb(0, true)).toBe(-Infinity); + expect(getOfflineTrackGain(0, true)).toBe(0); + expect(getOfflineTrackVolumeDb(-60, false)).toBe(-Infinity); + expect(getOfflineTrackGain(-60, false)).toBe(0); + }); +}); diff --git a/src/core/audio-interface/KGOfflineRenderer.ts b/src/core/audio-interface/KGOfflineRenderer.ts index 6f5af03..209fa32 100644 --- a/src/core/audio-interface/KGOfflineRenderer.ts +++ b/src/core/audio-interface/KGOfflineRenderer.ts @@ -3,6 +3,7 @@ import type { KGProject } from '../KGProject'; import type { KGMidiNote } from '../midi/KGMidiNote'; import type { KGAudioRegion } from '../region/KGAudioRegion'; import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants'; +import { AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants'; import { pitchToNoteNameString } from '../../util/midiUtil'; import { KGToneBuffersPool } from './KGToneBuffersPool'; import { KGToneSamplerFactory } from './KGToneSamplerFactory'; @@ -254,9 +255,8 @@ export class KGOfflineRenderer { }); }); - // Apply volume - const volumeDb = trackInfo.volume > 0 ? 20 * Math.log10(trackInfo.volume) : -Infinity; - sampler.volume.value = volumeDb; + // Track volumes are stored in dB across the app, with 0 meaning unity gain. + sampler.volume.value = getOfflineTrackVolumeDb(trackInfo.volume, trackInfo.muted); sampler.connect(masterGain); // Schedule all notes for this track @@ -288,7 +288,7 @@ export class KGOfflineRenderer { for (const trackInfo of audioTrackData) { if (!shouldPlay(trackInfo, hasSoloedTracks)) continue; - const trackGain = new Tone.Gain(trackInfo.volume); + const trackGain = new Tone.Gain(getOfflineTrackGain(trackInfo.volume, trackInfo.muted)); trackGain.connect(masterGain); for (const regionInfo of trackInfo.regions) { @@ -418,6 +418,16 @@ function shouldPlay(trackInfo: { muted: boolean; solo: boolean }, hasSoloedTrack return true; } +export function getOfflineTrackVolumeDb(volumeDb: number, muted: boolean): number { + const isSilent = muted || volumeDb <= AUDIO_INTERFACE_CONSTANTS.MIN_TRACK_VOLUME_DB; + return isSilent ? -Infinity : volumeDb; +} + +export function getOfflineTrackGain(volumeDb: number, muted: boolean): number { + const effectiveVolumeDb = getOfflineTrackVolumeDb(volumeDb, muted); + return Number.isFinite(effectiveVolumeDb) ? Math.pow(10, effectiveVolumeDb / 20) : 0; +} + // ===== WAV ENCODER ===== /** diff --git a/src/core/config/ConfigManager.ts b/src/core/config/ConfigManager.ts index 76b79c1..181574a 100644 --- a/src/core/config/ConfigManager.ts +++ b/src/core/config/ConfigManager.ts @@ -71,6 +71,7 @@ interface AppConfig { }; editor: { playhead_update_frequency: number; + spectrogram_height_resolution: 1 | 3 | 5; }; chatbox: { default_open: boolean; @@ -243,7 +244,8 @@ export class ConfigManager { }, }, editor: { - playhead_update_frequency: 10 + playhead_update_frequency: 10, + spectrogram_height_resolution: 3 }, chatbox: { default_open: true @@ -607,4 +609,4 @@ export class ConfigManager { console.error('Error disposing ConfigManager:', error); } } -} \ No newline at end of file +} diff --git a/src/util/spectrogramUtil.test.ts b/src/util/spectrogramUtil.test.ts new file mode 100644 index 0000000..0bcdd9b --- /dev/null +++ b/src/util/spectrogramUtil.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest'; +import { + getSpectrogramPitchBinCount, + getSpectrogramVisibleBinRange, + mapMidiPitchToSpectrogramPosition, + normalizeSpectrogramHeightResolution, +} from './spectrogramUtil'; + +describe('spectrogramUtil', () => { + it('normalizes unsupported resolutions to the default 3x', () => { + expect(normalizeSpectrogramHeightResolution(1)).toBe(1); + expect(normalizeSpectrogramHeightResolution(3)).toBe(3); + expect(normalizeSpectrogramHeightResolution(5)).toBe(5); + expect(normalizeSpectrogramHeightResolution(2)).toBe(3); + expect(normalizeSpectrogramHeightResolution(undefined)).toBe(3); + }); + + it('derives pitch bin counts from the full MIDI range', () => { + expect(getSpectrogramPitchBinCount(1)).toBe(128); + expect(getSpectrogramPitchBinCount(3)).toBe(384); + expect(getSpectrogramPitchBinCount(5)).toBe(640); + }); + + it('maps MIDI pitches into full-range spectrogram positions', () => { + expect(mapMidiPitchToSpectrogramPosition(0, 1)).toBe(0); + expect(mapMidiPitchToSpectrogramPosition(12.5, 3)).toBe(38.5); + expect(mapMidiPitchToSpectrogramPosition(127, 5)).toBe(637); + expect(mapMidiPitchToSpectrogramPosition(-1, 3)).toBeNull(); + expect(mapMidiPitchToSpectrogramPosition(128, 3)).toBeNull(); + }); + + it('derives the visible C0-B7 subrange inside the full-resolution buffer', () => { + expect(getSpectrogramVisibleBinRange(1)).toEqual({ start: 12, end: 108 }); + expect(getSpectrogramVisibleBinRange(3)).toEqual({ start: 36, end: 324 }); + }); +}); diff --git a/src/util/spectrogramUtil.ts b/src/util/spectrogramUtil.ts new file mode 100644 index 0000000..d219db2 --- /dev/null +++ b/src/util/spectrogramUtil.ts @@ -0,0 +1,50 @@ +export const SPECTROGRAM_FULL_MIN_MIDI_PITCH = 0; +export const SPECTROGRAM_FULL_MAX_MIDI_PITCH = 127; +export const SPECTROGRAM_MIN_MIDI_PITCH = 12; // C0 +export const SPECTROGRAM_MAX_MIDI_PITCH = 107; // B7 +export const SPECTROGRAM_FULL_SEMITONES = + SPECTROGRAM_FULL_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1; +export const SPECTROGRAM_VISIBLE_SEMITONES = + SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_MIN_MIDI_PITCH + 1; + +export type SpectrogramHeightResolution = 1 | 3 | 5; + +export const SPECTROGRAM_HEIGHT_RESOLUTION_OPTIONS: SpectrogramHeightResolution[] = [1, 3, 5]; + +export function normalizeSpectrogramHeightResolution(value: unknown): SpectrogramHeightResolution { + if (value === 1 || value === 3 || value === 5) { + return value; + } + return 3; +} + +export function getSpectrogramPitchBinCount( + resolution: SpectrogramHeightResolution, +): number { + return SPECTROGRAM_FULL_SEMITONES * resolution; +} + +export function getSpectrogramVisibleBinRange( + resolution: SpectrogramHeightResolution, +): { start: number; end: number } { + return { + start: (SPECTROGRAM_MIN_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH) * resolution, + end: (SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1) * resolution, + }; +} + +export function mapMidiPitchToSpectrogramPosition( + midiPitch: number, + resolution: SpectrogramHeightResolution, +): number | null { + const pitchOffset = midiPitch - SPECTROGRAM_FULL_MIN_MIDI_PITCH; + if (pitchOffset < 0 || pitchOffset > SPECTROGRAM_FULL_MAX_MIDI_PITCH) { + return null; + } + + // Anchor each MIDI pitch to the center of its semitone band so the + // rendered ridge lines up with the piano-roll key row rather than a boundary. + const scaled = pitchOffset * resolution + (resolution - 1) / 2; + const maxBin = getSpectrogramPitchBinCount(resolution) - 1; + return Math.max(0, Math.min(maxBin, scaled)); +} diff --git a/src/workers/spectrogramWorker.ts b/src/workers/spectrogramWorker.ts index a3a914f..cd02a8b 100644 --- a/src/workers/spectrogramWorker.ts +++ b/src/workers/spectrogramWorker.ts @@ -1,4 +1,10 @@ import FFT from 'fft.js'; +import { + getSpectrogramPitchBinCount, + mapMidiPitchToSpectrogramPosition, + normalizeSpectrogramHeightResolution, + type SpectrogramHeightResolution, +} from '../util/spectrogramUtil'; type WorkerScopeLike = typeof globalThis & { onmessage: ((event: MessageEvent) => void) | null; @@ -13,16 +19,17 @@ export interface SpectrogramRequest { clipStartOffsetSeconds: number; regionDurationSeconds: number; bpm: number; + heightResolution: SpectrogramHeightResolution; } export interface SpectrogramResult { - data: Float32Array; // [timeSteps × 128] row-major, row = time step, col = pitch 0-127 + data: Float32Array; // [timeSteps × pitchBins] row-major, low-to-high pitch bins timeSteps: number; + pitchBins: number; } const FFT_SIZE = 8192; const HOP_SIZE = 1024; -const PITCH_BINS = 128; function hannWindow(size: number): Float32Array { const w = new Float32Array(size); @@ -32,8 +39,43 @@ function hannWindow(size: number): Float32Array { return w; } +function spreadMagnitudeAcrossPitchBins( + result: Float32Array, + pitchRow: number, + pitchBins: number, + pitchPosition: number, + magnitude: number, + heightResolution: SpectrogramHeightResolution, +): void { + const spreadRadius = Math.max(0, heightResolution - 1); + const centerBin = Math.round(pitchPosition); + const startBin = Math.max(0, centerBin - spreadRadius); + const endBin = Math.min(pitchBins - 1, centerBin + spreadRadius); + const spreadWidth = spreadRadius + 1; + + for (let targetBin = startBin; targetBin <= endBin; targetBin++) { + const distance = Math.abs(targetBin - pitchPosition); + const weight = Math.max(0, 1 - distance / spreadWidth); + if (weight <= 0) continue; + + const weightedMagnitude = magnitude * weight; + const resultIndex = pitchRow + targetBin; + if (weightedMagnitude > result[resultIndex]) { + result[resultIndex] = weightedMagnitude; + } + } +} + workerScope.onmessage = (e: MessageEvent) => { - const { pcm, sampleRate, clipStartOffsetSeconds, regionDurationSeconds } = e.data; + const { + pcm, + sampleRate, + clipStartOffsetSeconds, + regionDurationSeconds, + heightResolution, + } = e.data; + const normalizedResolution = normalizeSpectrogramHeightResolution(heightResolution); + const pitchBins = getSpectrogramPitchBinCount(normalizedResolution); const startSample = Math.floor(clipStartOffsetSeconds * sampleRate); const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate)); @@ -45,7 +87,7 @@ workerScope.onmessage = (e: MessageEvent) => { const inputPadded = new Float32Array(FFT_SIZE); const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1); - const result = new Float32Array(totalHops * PITCH_BINS); + const result = new Float32Array(totalHops * pitchBins); let maxVal = 0; @@ -64,28 +106,32 @@ workerScope.onmessage = (e: MessageEvent) => { // Max-pool magnitude into pitch bins: keep the loudest FFT bin per semitone, // rather than summing. Summing inflates every bin by how many FFT bins land there. - const pitchRow = hop * PITCH_BINS; + const pitchRow = hop * pitchBins; const numBins = FFT_SIZE / 2; for (let bin = 1; bin < numBins; bin++) { const freq = (bin * sampleRate) / FFT_SIZE; if (freq < 20 || freq > 20000) continue; - // Convert frequency to MIDI pitch; clamp to piano roll range C0–B7 (MIDI 12–107) - const pitch = Math.round(69 + 12 * Math.log2(freq / 440)); - if (pitch < 12 || pitch > 107) continue; + const midiPitch = 69 + 12 * Math.log2(freq / 440); + const pitchPosition = mapMidiPitchToSpectrogramPosition(midiPitch, normalizedResolution); + if (pitchPosition === null) continue; const re = complexOut[2 * bin]; const im = complexOut[2 * bin + 1]; const magnitude = Math.sqrt(re * re + im * im); - - if (magnitude > result[pitchRow + pitch]) { - result[pitchRow + pitch] = magnitude; - } + spreadMagnitudeAcrossPitchBins( + result, + pitchRow, + pitchBins, + pitchPosition, + magnitude, + normalizedResolution, + ); } // Track global max for normalization - for (let p = 0; p < PITCH_BINS; p++) { + for (let p = 0; p < pitchBins; p++) { if (result[pitchRow + p] > maxVal) maxVal = result[pitchRow + p]; } } @@ -98,6 +144,6 @@ workerScope.onmessage = (e: MessageEvent) => { } } - const response: SpectrogramResult = { data: result, timeSteps: totalHops }; + const response: SpectrogramResult = { data: result, timeSteps: totalHops, pitchBins }; workerScope.postMessage(response, [result.buffer]); };