diff --git a/src/components/piano-roll/SpectrogramCanvas.tsx b/src/components/piano-roll/SpectrogramCanvas.tsx index bdcee5b..93c7058 100644 --- a/src/components/piano-roll/SpectrogramCanvas.tsx +++ b/src/components/piano-roll/SpectrogramCanvas.tsx @@ -5,6 +5,7 @@ import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface'; import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage'; import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker'; import { + SPECTROGRAM_FULL_SEMITONES, getSpectrogramVisibleBinRange, normalizeSpectrogramHeightResolution, SPECTROGRAM_VISIBLE_SEMITONES, @@ -52,6 +53,25 @@ function hotColormap(v: number): [number, number, number] { return COLORMAP_STOPS[COLORMAP_STOPS.length - 1][1]; } +function smoothSpectrogramVertically( + source: Float32Array, + timeSteps: number, + pitchBins: number, +): Float32Array { + const smoothed = new Float32Array(source.length); + + for (let col = 0; col < timeSteps; col++) { + for (let row = 0; row < pitchBins; row++) { + const center = source[col * pitchBins + row]; + const above = row > 0 ? source[col * pitchBins + row - 1] : center; + const below = row < pitchBins - 1 ? source[col * pitchBins + row + 1] : center; + smoothed[col * pitchBins + row] = above * 0.2 + center * 0.6 + below * 0.2; + } + } + + return smoothed; +} + const SpectrogramCanvas: React.FC = ({ audioRegion, trackId, @@ -102,18 +122,21 @@ const SpectrogramCanvas: React.FC = ({ // Convert dB threshold to linear: values below this → black const linearThreshold = Math.pow(10, thresholdDb / 20); - const visibleRange = getSpectrogramVisibleBinRange(heightResolution); + const analysisResolution = result.pitchBins / SPECTROGRAM_FULL_SEMITONES; + const visibleRange = getSpectrogramVisibleBinRange(analysisResolution); const visiblePitchBins = visibleRange.end - visibleRange.start; // 1. Paint at natural spectrogram resolution onto an offscreen canvas. // Result data is low-to-high pitch; draw only the visible C0-B7 window, reversed for display. - const offscreen = document.createElement('canvas'); - offscreen.width = result.timeSteps; - offscreen.height = visiblePitchBins; - const offCtx = offscreen.getContext('2d'); - if (!offCtx) return; + const smoothedData = smoothSpectrogramVertically(result.data, result.timeSteps, result.pitchBins); - const imgData = offCtx.createImageData(result.timeSteps, visiblePitchBins); + const sourceCanvas = document.createElement('canvas'); + sourceCanvas.width = result.timeSteps; + sourceCanvas.height = visiblePitchBins; + const sourceCtx = sourceCanvas.getContext('2d'); + if (!sourceCtx) return; + + const imgData = sourceCtx.createImageData(result.timeSteps, visiblePitchBins); const pixels = imgData.data; for (let row = 0; row < visiblePitchBins; row++) { @@ -122,7 +145,7 @@ const SpectrogramCanvas: React.FC = ({ const idx = (row * result.timeSteps + col) * 4; pixels[idx + 3] = 255; // always opaque - const raw = result.data[col * result.pitchBins + sourceRow]; + const raw = smoothedData[col * result.pitchBins + sourceRow]; // Hard threshold: values below noise floor → 0 (black) // Re-scale surviving range to [0,1] then apply power curve @@ -139,14 +162,24 @@ const SpectrogramCanvas: React.FC = ({ } } - offCtx.putImageData(imgData, 0, 0); + sourceCtx.putImageData(imgData, 0, 0); - // 2. Stretch onto the full canvas — browser bilinear filter smooths between bins. + const verticallyScaledCanvas = document.createElement('canvas'); + verticallyScaledCanvas.width = result.timeSteps; + verticallyScaledCanvas.height = canvasHeight; + const verticallyScaledCtx = verticallyScaledCanvas.getContext('2d'); + if (!verticallyScaledCtx) return; + + // Scale only along the pitch axis so note starts/stops stay crisp in time. + verticallyScaledCtx.imageSmoothingEnabled = true; + verticallyScaledCtx.imageSmoothingQuality = 'high'; + verticallyScaledCtx.drawImage(sourceCanvas, 0, 0, result.timeSteps, canvasHeight); + + // 2. Stretch onto the full canvas horizontally without temporal blur. canvas.width = canvasWidth; canvas.height = canvasHeight; - ctx.imageSmoothingEnabled = true; - ctx.imageSmoothingQuality = 'high'; - ctx.drawImage(offscreen, 0, 0, canvasWidth, canvasHeight); + ctx.imageSmoothingEnabled = false; + ctx.drawImage(verticallyScaledCanvas, 0, 0, canvasWidth, canvasHeight); // Store natural width and apply current zoom as CSS stretch (no pixel recompute on zoom) naturalWidthRef.current = canvasWidth; diff --git a/src/util/spectrogramUtil.test.ts b/src/util/spectrogramUtil.test.ts index 0bcdd9b..27ab7b3 100644 --- a/src/util/spectrogramUtil.test.ts +++ b/src/util/spectrogramUtil.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from 'vitest'; import { + frequencyToMidiPitch, + getSpectrogramAnalysisResolution, getSpectrogramPitchBinCount, getSpectrogramVisibleBinRange, + mapFrequencyToSpectrogramPosition, mapMidiPitchToSpectrogramPosition, normalizeSpectrogramHeightResolution, } from './spectrogramUtil'; @@ -21,6 +24,12 @@ describe('spectrogramUtil', () => { expect(getSpectrogramPitchBinCount(5)).toBe(640); }); + it('doubles the internal analysis resolution for each user-facing option', () => { + expect(getSpectrogramAnalysisResolution(1)).toBe(2); + expect(getSpectrogramAnalysisResolution(3)).toBe(6); + expect(getSpectrogramAnalysisResolution(5)).toBe(10); + }); + it('maps MIDI pitches into full-range spectrogram positions', () => { expect(mapMidiPitchToSpectrogramPosition(0, 1)).toBe(0); expect(mapMidiPitchToSpectrogramPosition(12.5, 3)).toBe(38.5); @@ -29,6 +38,12 @@ describe('spectrogramUtil', () => { expect(mapMidiPitchToSpectrogramPosition(128, 3)).toBeNull(); }); + it('converts frequencies into MIDI pitch space and spectrogram positions', () => { + expect(frequencyToMidiPitch(440)).toBeCloseTo(69, 6); + expect(mapFrequencyToSpectrogramPosition(440, 5)).toBeCloseTo(347, 6); + expect(frequencyToMidiPitch(0)).toBeNull(); + }); + it('derives the visible C0-B7 subrange inside the full-resolution buffer', () => { expect(getSpectrogramVisibleBinRange(1)).toEqual({ start: 12, end: 108 }); expect(getSpectrogramVisibleBinRange(3)).toEqual({ start: 36, end: 324 }); diff --git a/src/util/spectrogramUtil.ts b/src/util/spectrogramUtil.ts index d219db2..a2e8eb7 100644 --- a/src/util/spectrogramUtil.ts +++ b/src/util/spectrogramUtil.ts @@ -6,6 +6,7 @@ export const SPECTROGRAM_FULL_SEMITONES = SPECTROGRAM_FULL_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1; export const SPECTROGRAM_VISIBLE_SEMITONES = SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_MIN_MIDI_PITCH + 1; +export const SPECTROGRAM_ANALYSIS_RESOLUTION_MULTIPLIER = 2; export type SpectrogramHeightResolution = 1 | 3 | 5; @@ -19,13 +20,13 @@ export function normalizeSpectrogramHeightResolution(value: unknown): Spectrogra } export function getSpectrogramPitchBinCount( - resolution: SpectrogramHeightResolution, + resolution: number, ): number { return SPECTROGRAM_FULL_SEMITONES * resolution; } export function getSpectrogramVisibleBinRange( - resolution: SpectrogramHeightResolution, + resolution: number, ): { start: number; end: number } { return { start: (SPECTROGRAM_MIN_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH) * resolution, @@ -33,9 +34,15 @@ export function getSpectrogramVisibleBinRange( }; } +export function getSpectrogramAnalysisResolution( + resolution: SpectrogramHeightResolution, +): number { + return resolution * SPECTROGRAM_ANALYSIS_RESOLUTION_MULTIPLIER; +} + export function mapMidiPitchToSpectrogramPosition( midiPitch: number, - resolution: SpectrogramHeightResolution, + resolution: number, ): number | null { const pitchOffset = midiPitch - SPECTROGRAM_FULL_MIN_MIDI_PITCH; if (pitchOffset < 0 || pitchOffset > SPECTROGRAM_FULL_MAX_MIDI_PITCH) { @@ -48,3 +55,21 @@ export function mapMidiPitchToSpectrogramPosition( const maxBin = getSpectrogramPitchBinCount(resolution) - 1; return Math.max(0, Math.min(maxBin, scaled)); } + +export function frequencyToMidiPitch(frequency: number): number | null { + if (!Number.isFinite(frequency) || frequency <= 0) { + return null; + } + return 69 + 12 * Math.log2(frequency / 440); +} + +export function mapFrequencyToSpectrogramPosition( + frequency: number, + resolution: number, +): number | null { + const midiPitch = frequencyToMidiPitch(frequency); + if (midiPitch === null) { + return null; + } + return mapMidiPitchToSpectrogramPosition(midiPitch, resolution); +} diff --git a/src/workers/spectrogramWorker.test.ts b/src/workers/spectrogramWorker.test.ts new file mode 100644 index 0000000..874c7bd --- /dev/null +++ b/src/workers/spectrogramWorker.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest'; +import { getSpectrogramPitchBinCount, mapFrequencyToSpectrogramPosition } from '../util/spectrogramUtil'; +import { + estimatePeakFrequency, + getFrequencyBandEdges, + paintMagnitudeAcrossPitchSpan, +} from './spectrogramWorker'; + +const FFT_SIZE = 16384; +const SAMPLE_RATE = 44100; +const HZ_PER_BIN = SAMPLE_RATE / FFT_SIZE; + +function paintSpanForFrequency( + centerFrequency: number, + resolution: 1 | 3 | 5, + pitchBins = getSpectrogramPitchBinCount(resolution), +): Float32Array { + const row = new Float32Array(pitchBins); + const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, HZ_PER_BIN); + const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, resolution); + const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, resolution); + const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, resolution); + + if (startPosition === null || endPosition === null || centerPosition === null) { + throw new Error('Expected mapped pitch positions for test frequency span'); + } + + paintMagnitudeAcrossPitchSpan( + row, + 0, + pitchBins, + Math.min(startPosition, endPosition), + Math.max(startPosition, endPosition), + centerPosition, + 1, + ); + return row; +} + +describe('spectrogramWorker helpers', () => { + it('keeps adjacent low-register FFT bins continuous at 5x resolution', () => { + const pitchBins = getSpectrogramPitchBinCount(5); + const row = new Float32Array(pitchBins); + const binsAroundA1 = [10, 11]; + + for (const bin of binsAroundA1) { + const centerFrequency = bin * HZ_PER_BIN; + const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, HZ_PER_BIN); + const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, 5); + const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, 5); + const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, 5); + + expect(startPosition).not.toBeNull(); + expect(endPosition).not.toBeNull(); + expect(centerPosition).not.toBeNull(); + + paintMagnitudeAcrossPitchSpan( + row, + 0, + pitchBins, + Math.min(startPosition!, endPosition!), + Math.max(startPosition!, endPosition!), + centerPosition!, + 1, + ); + } + + const nonZeroBins = [...row.entries()].filter(([, value]) => value > 0).map(([index]) => index); + expect(nonZeroBins.length).toBeGreaterThan(0); + + for (let index = nonZeroBins[0]; index <= nonZeroBins[nonZeroBins.length - 1]; index++) { + expect(row[index]).toBeGreaterThan(0); + } + }); + + it('uses sub-bin interpolation to move the ridge closer to the true peak frequency', () => { + const magnitudes = new Float32Array([0, 0.3, 1.0, 0.82, 0.1, 0]); + const rawCenterFrequency = 2 * HZ_PER_BIN; + const interpolatedFrequency = estimatePeakFrequency(2, HZ_PER_BIN, magnitudes); + const expectedFrequency = (2.35 * HZ_PER_BIN); + + expect(Math.abs(interpolatedFrequency - expectedFrequency)).toBeLessThan( + Math.abs(rawCenterFrequency - expectedFrequency), + ); + }); + + it('leaves non-peak bins at their raw FFT center while still painting a span', () => { + const magnitudes = new Float32Array([0, 0.2, 0.4, 1.0, 0.8, 0.7, 0.2, 0]); + const centerFrequency = estimatePeakFrequency(4, HZ_PER_BIN, magnitudes); + expect(centerFrequency).toBeCloseTo(4 * HZ_PER_BIN, 6); + + const row = paintSpanForFrequency(centerFrequency, 3); + expect(row.some(value => value > 0)).toBe(true); + }); + + it('maps edge-band spans into the valid spectrogram range without spilling outside the buffer', () => { + const lowRow = paintSpanForFrequency(20.5, 5); + const lowNonZeroBins = [...lowRow.entries()].filter(([, value]) => value > 0).map(([index]) => index); + expect(lowNonZeroBins.length).toBeGreaterThan(0); + expect(lowNonZeroBins[0]).toBeGreaterThanOrEqual(0); + + const highRow = paintSpanForFrequency(12530, 5); + const highNonZeroBins = [...highRow.entries()].filter(([, value]) => value > 0).map(([index]) => index); + expect(highNonZeroBins.length).toBeGreaterThan(0); + expect(highNonZeroBins[highNonZeroBins.length - 1]).toBeLessThan(highRow.length); + }); +}); diff --git a/src/workers/spectrogramWorker.ts b/src/workers/spectrogramWorker.ts index cd02a8b..3d421e0 100644 --- a/src/workers/spectrogramWorker.ts +++ b/src/workers/spectrogramWorker.ts @@ -1,7 +1,8 @@ import FFT from 'fft.js'; import { + getSpectrogramAnalysisResolution, getSpectrogramPitchBinCount, - mapMidiPitchToSpectrogramPosition, + mapFrequencyToSpectrogramPosition, normalizeSpectrogramHeightResolution, type SpectrogramHeightResolution, } from '../util/spectrogramUtil'; @@ -11,7 +12,7 @@ type WorkerScopeLike = typeof globalThis & { postMessage: (message: SpectrogramResult, transfer: Transferable[]) => void; }; -const workerScope = self as WorkerScopeLike; +const workerScope = typeof self === 'undefined' ? null : self as WorkerScopeLike; export interface SpectrogramRequest { pcm: Float32Array; @@ -28,8 +29,10 @@ export interface SpectrogramResult { pitchBins: number; } -const FFT_SIZE = 8192; +const FFT_SIZE = 16384; const HOP_SIZE = 1024; +const MIN_SPECTROGRAM_FREQUENCY = 20; +const MAX_SPECTROGRAM_FREQUENCY = 20000; function hannWindow(size: number): Float32Array { const w = new Float32Array(size); @@ -39,24 +42,98 @@ function hannWindow(size: number): Float32Array { return w; } -function spreadMagnitudeAcrossPitchBins( +export function getFrequencyBandEdges( + centerFrequency: number, + hzPerBin: number, +): { lowerFrequency: number; upperFrequency: number } { + const halfBandwidth = hzPerBin / 2; + return { + lowerFrequency: Math.max(Number.EPSILON, centerFrequency - halfBandwidth), + upperFrequency: centerFrequency + halfBandwidth, + }; +} + +export function estimateParabolicPeakOffset( + leftMagnitude: number, + centerMagnitude: number, + rightMagnitude: number, +): number { + const denominator = leftMagnitude - 2 * centerMagnitude + rightMagnitude; + if (!Number.isFinite(denominator) || Math.abs(denominator) < Number.EPSILON) { + return 0; + } + + const offset = 0.5 * (leftMagnitude - rightMagnitude) / denominator; + return Math.max(-0.5, Math.min(0.5, offset)); +} + +export function estimatePeakFrequency( + bin: number, + hzPerBin: number, + magnitudes: Float32Array, +): number { + const centerFrequency = bin * hzPerBin; + if (bin <= 1 || bin >= magnitudes.length - 1) { + return centerFrequency; + } + + const leftMagnitude = magnitudes[bin - 1]; + const centerMagnitude = magnitudes[bin]; + const rightMagnitude = magnitudes[bin + 1]; + + if (centerMagnitude <= leftMagnitude || centerMagnitude <= rightMagnitude) { + return centerFrequency; + } + + return (bin + estimateParabolicPeakOffset(leftMagnitude, centerMagnitude, rightMagnitude)) * hzPerBin; +} + +function getPitchSpanBounds( + lowerFrequency: number, + upperFrequency: number, + centerFrequency: number, + analysisResolution: number, +): { startPosition: number; endPosition: number; centerPosition: number } | null { + const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, analysisResolution); + const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, analysisResolution); + const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, analysisResolution); + + if (startPosition === null || endPosition === null || centerPosition === null) { + return null; + } + + return { + startPosition: Math.min(startPosition, endPosition), + endPosition: Math.max(startPosition, endPosition), + centerPosition, + }; +} + +export function paintMagnitudeAcrossPitchSpan( result: Float32Array, pitchRow: number, pitchBins: number, - pitchPosition: number, + startPosition: number, + endPosition: number, + centerPosition: number, magnitude: number, - heightResolution: SpectrogramHeightResolution, ): void { - const spreadRadius = Math.max(0, heightResolution - 1); - const centerBin = Math.round(pitchPosition); - const startBin = Math.max(0, centerBin - spreadRadius); - const endBin = Math.min(pitchBins - 1, centerBin + spreadRadius); - const spreadWidth = spreadRadius + 1; + const clampedStart = Math.max(0, Math.min(startPosition, pitchBins - 1)); + const clampedEnd = Math.max(0, Math.min(endPosition, pitchBins - 1)); + const startBin = Math.max(0, Math.floor(clampedStart)); + const endBin = Math.min(pitchBins - 1, Math.ceil(clampedEnd)); + const spanWidth = Math.max(clampedEnd - clampedStart, 1); for (let targetBin = startBin; targetBin <= endBin; targetBin++) { - const distance = Math.abs(targetBin - pitchPosition); - const weight = Math.max(0, 1 - distance / spreadWidth); - if (weight <= 0) continue; + const cellStart = targetBin - 0.5; + const cellEnd = targetBin + 0.5; + const overlap = Math.max(0, Math.min(clampedEnd, cellEnd) - Math.max(clampedStart, cellStart)); + if (overlap <= 0) continue; + + const overlapWeight = Math.min(1, overlap); + const centerDistance = Math.abs(targetBin - centerPosition); + const centerWeight = Math.max(0, 1 - centerDistance / (spanWidth + 1)); + const weight = Math.max(overlapWeight * (0.6 + 0.4 * centerWeight), overlapWeight * 0.35); const weightedMagnitude = magnitude * weight; const resultIndex = pitchRow + targetBin; @@ -66,7 +143,7 @@ function spreadMagnitudeAcrossPitchBins( } } -workerScope.onmessage = (e: MessageEvent) => { +function handleSpectrogramRequest(e: MessageEvent, scope: WorkerScopeLike): void { const { pcm, sampleRate, @@ -75,7 +152,8 @@ workerScope.onmessage = (e: MessageEvent) => { heightResolution, } = e.data; const normalizedResolution = normalizeSpectrogramHeightResolution(heightResolution); - const pitchBins = getSpectrogramPitchBinCount(normalizedResolution); + const analysisResolution = getSpectrogramAnalysisResolution(normalizedResolution); + const pitchBins = getSpectrogramPitchBinCount(analysisResolution); const startSample = Math.floor(clipStartOffsetSeconds * sampleRate); const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate)); @@ -85,6 +163,7 @@ workerScope.onmessage = (e: MessageEvent) => { const hann = hannWindow(FFT_SIZE); const complexOut = fft.createComplexArray() as number[]; const inputPadded = new Float32Array(FFT_SIZE); + const hzPerBin = sampleRate / FFT_SIZE; const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1); const result = new Float32Array(totalHops * pitchBins); @@ -108,25 +187,44 @@ workerScope.onmessage = (e: MessageEvent) => { // rather than summing. Summing inflates every bin by how many FFT bins land there. const pitchRow = hop * pitchBins; const numBins = FFT_SIZE / 2; + const magnitudes = new Float32Array(numBins); for (let bin = 1; bin < numBins; bin++) { - const freq = (bin * sampleRate) / FFT_SIZE; - if (freq < 20 || freq > 20000) continue; - - const midiPitch = 69 + 12 * Math.log2(freq / 440); - const pitchPosition = mapMidiPitchToSpectrogramPosition(midiPitch, normalizedResolution); - if (pitchPosition === null) continue; - const re = complexOut[2 * bin]; const im = complexOut[2 * bin + 1]; - const magnitude = Math.sqrt(re * re + im * im); - spreadMagnitudeAcrossPitchBins( + magnitudes[bin] = Math.sqrt(re * re + im * im); + } + + for (let bin = 1; bin < numBins; bin++) { + const centerFrequency = estimatePeakFrequency(bin, hzPerBin, magnitudes); + const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, hzPerBin); + if (upperFrequency < MIN_SPECTROGRAM_FREQUENCY || lowerFrequency > MAX_SPECTROGRAM_FREQUENCY) continue; + + const clampedLowerFrequency = Math.max(lowerFrequency, MIN_SPECTROGRAM_FREQUENCY); + const clampedUpperFrequency = Math.min(upperFrequency, MAX_SPECTROGRAM_FREQUENCY); + const clampedCenterFrequency = Math.min( + clampedUpperFrequency, + Math.max(clampedLowerFrequency, centerFrequency), + ); + const pitchSpan = getPitchSpanBounds( + clampedLowerFrequency, + clampedUpperFrequency, + clampedCenterFrequency, + analysisResolution, + ); + if (!pitchSpan) continue; + + const magnitude = magnitudes[bin]; + if (magnitude <= 0) continue; + + paintMagnitudeAcrossPitchSpan( result, pitchRow, pitchBins, - pitchPosition, + pitchSpan.startPosition, + pitchSpan.endPosition, + pitchSpan.centerPosition, magnitude, - normalizedResolution, ); } @@ -145,5 +243,11 @@ workerScope.onmessage = (e: MessageEvent) => { } const response: SpectrogramResult = { data: result, timeSteps: totalHops, pitchBins }; - workerScope.postMessage(response, [result.buffer]); -}; + scope.postMessage(response, [result.buffer]); +} + +if (workerScope) { + workerScope.onmessage = (e: MessageEvent) => { + handleSpectrogramRequest(e, workerScope); + }; +}