feat: improve spectrogram pitch mapping and vertical smoothing

This commit is contained in:
Xiaohan-Tian
2026-07-06 19:34:18 -07:00
parent 08c9cf24a6
commit f4e22e50a4
5 changed files with 329 additions and 45 deletions
+46 -13
View File
@@ -5,6 +5,7 @@ import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker';
import {
SPECTROGRAM_FULL_SEMITONES,
getSpectrogramVisibleBinRange,
normalizeSpectrogramHeightResolution,
SPECTROGRAM_VISIBLE_SEMITONES,
@@ -52,6 +53,25 @@ function hotColormap(v: number): [number, number, number] {
return COLORMAP_STOPS[COLORMAP_STOPS.length - 1][1];
}
function smoothSpectrogramVertically(
source: Float32Array,
timeSteps: number,
pitchBins: number,
): Float32Array {
const smoothed = new Float32Array(source.length);
for (let col = 0; col < timeSteps; col++) {
for (let row = 0; row < pitchBins; row++) {
const center = source[col * pitchBins + row];
const above = row > 0 ? source[col * pitchBins + row - 1] : center;
const below = row < pitchBins - 1 ? source[col * pitchBins + row + 1] : center;
smoothed[col * pitchBins + row] = above * 0.2 + center * 0.6 + below * 0.2;
}
}
return smoothed;
}
const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
audioRegion,
trackId,
@@ -102,18 +122,21 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
// Convert dB threshold to linear: values below this → black
const linearThreshold = Math.pow(10, thresholdDb / 20);
const visibleRange = getSpectrogramVisibleBinRange(heightResolution);
const analysisResolution = result.pitchBins / SPECTROGRAM_FULL_SEMITONES;
const visibleRange = getSpectrogramVisibleBinRange(analysisResolution);
const visiblePitchBins = visibleRange.end - visibleRange.start;
// 1. Paint at natural spectrogram resolution onto an offscreen canvas.
// Result data is low-to-high pitch; draw only the visible C0-B7 window, reversed for display.
const offscreen = document.createElement('canvas');
offscreen.width = result.timeSteps;
offscreen.height = visiblePitchBins;
const offCtx = offscreen.getContext('2d');
if (!offCtx) return;
const smoothedData = smoothSpectrogramVertically(result.data, result.timeSteps, result.pitchBins);
const imgData = offCtx.createImageData(result.timeSteps, visiblePitchBins);
const sourceCanvas = document.createElement('canvas');
sourceCanvas.width = result.timeSteps;
sourceCanvas.height = visiblePitchBins;
const sourceCtx = sourceCanvas.getContext('2d');
if (!sourceCtx) return;
const imgData = sourceCtx.createImageData(result.timeSteps, visiblePitchBins);
const pixels = imgData.data;
for (let row = 0; row < visiblePitchBins; row++) {
@@ -122,7 +145,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
const idx = (row * result.timeSteps + col) * 4;
pixels[idx + 3] = 255; // always opaque
const raw = result.data[col * result.pitchBins + sourceRow];
const raw = smoothedData[col * result.pitchBins + sourceRow];
// Hard threshold: values below noise floor → 0 (black)
// Re-scale surviving range to [0,1] then apply power curve
@@ -139,14 +162,24 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
}
}
offCtx.putImageData(imgData, 0, 0);
sourceCtx.putImageData(imgData, 0, 0);
// 2. Stretch onto the full canvas — browser bilinear filter smooths between bins.
const verticallyScaledCanvas = document.createElement('canvas');
verticallyScaledCanvas.width = result.timeSteps;
verticallyScaledCanvas.height = canvasHeight;
const verticallyScaledCtx = verticallyScaledCanvas.getContext('2d');
if (!verticallyScaledCtx) return;
// Scale only along the pitch axis so note starts/stops stay crisp in time.
verticallyScaledCtx.imageSmoothingEnabled = true;
verticallyScaledCtx.imageSmoothingQuality = 'high';
verticallyScaledCtx.drawImage(sourceCanvas, 0, 0, result.timeSteps, canvasHeight);
// 2. Stretch onto the full canvas horizontally without temporal blur.
canvas.width = canvasWidth;
canvas.height = canvasHeight;
ctx.imageSmoothingEnabled = true;
ctx.imageSmoothingQuality = 'high';
ctx.drawImage(offscreen, 0, 0, canvasWidth, canvasHeight);
ctx.imageSmoothingEnabled = false;
ctx.drawImage(verticallyScaledCanvas, 0, 0, canvasWidth, canvasHeight);
// Store natural width and apply current zoom as CSS stretch (no pixel recompute on zoom)
naturalWidthRef.current = canvasWidth;
+15
View File
@@ -1,7 +1,10 @@
import { describe, expect, it } from 'vitest';
import {
frequencyToMidiPitch,
getSpectrogramAnalysisResolution,
getSpectrogramPitchBinCount,
getSpectrogramVisibleBinRange,
mapFrequencyToSpectrogramPosition,
mapMidiPitchToSpectrogramPosition,
normalizeSpectrogramHeightResolution,
} from './spectrogramUtil';
@@ -21,6 +24,12 @@ describe('spectrogramUtil', () => {
expect(getSpectrogramPitchBinCount(5)).toBe(640);
});
it('doubles the internal analysis resolution for each user-facing option', () => {
expect(getSpectrogramAnalysisResolution(1)).toBe(2);
expect(getSpectrogramAnalysisResolution(3)).toBe(6);
expect(getSpectrogramAnalysisResolution(5)).toBe(10);
});
it('maps MIDI pitches into full-range spectrogram positions', () => {
expect(mapMidiPitchToSpectrogramPosition(0, 1)).toBe(0);
expect(mapMidiPitchToSpectrogramPosition(12.5, 3)).toBe(38.5);
@@ -29,6 +38,12 @@ describe('spectrogramUtil', () => {
expect(mapMidiPitchToSpectrogramPosition(128, 3)).toBeNull();
});
it('converts frequencies into MIDI pitch space and spectrogram positions', () => {
expect(frequencyToMidiPitch(440)).toBeCloseTo(69, 6);
expect(mapFrequencyToSpectrogramPosition(440, 5)).toBeCloseTo(347, 6);
expect(frequencyToMidiPitch(0)).toBeNull();
});
it('derives the visible C0-B7 subrange inside the full-resolution buffer', () => {
expect(getSpectrogramVisibleBinRange(1)).toEqual({ start: 12, end: 108 });
expect(getSpectrogramVisibleBinRange(3)).toEqual({ start: 36, end: 324 });
+28 -3
View File
@@ -6,6 +6,7 @@ export const SPECTROGRAM_FULL_SEMITONES =
SPECTROGRAM_FULL_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1;
export const SPECTROGRAM_VISIBLE_SEMITONES =
SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_MIN_MIDI_PITCH + 1;
export const SPECTROGRAM_ANALYSIS_RESOLUTION_MULTIPLIER = 2;
export type SpectrogramHeightResolution = 1 | 3 | 5;
@@ -19,13 +20,13 @@ export function normalizeSpectrogramHeightResolution(value: unknown): Spectrogra
}
export function getSpectrogramPitchBinCount(
resolution: SpectrogramHeightResolution,
resolution: number,
): number {
return SPECTROGRAM_FULL_SEMITONES * resolution;
}
export function getSpectrogramVisibleBinRange(
resolution: SpectrogramHeightResolution,
resolution: number,
): { start: number; end: number } {
return {
start: (SPECTROGRAM_MIN_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH) * resolution,
@@ -33,9 +34,15 @@ export function getSpectrogramVisibleBinRange(
};
}
export function getSpectrogramAnalysisResolution(
resolution: SpectrogramHeightResolution,
): number {
return resolution * SPECTROGRAM_ANALYSIS_RESOLUTION_MULTIPLIER;
}
export function mapMidiPitchToSpectrogramPosition(
midiPitch: number,
resolution: SpectrogramHeightResolution,
resolution: number,
): number | null {
const pitchOffset = midiPitch - SPECTROGRAM_FULL_MIN_MIDI_PITCH;
if (pitchOffset < 0 || pitchOffset > SPECTROGRAM_FULL_MAX_MIDI_PITCH) {
@@ -48,3 +55,21 @@ export function mapMidiPitchToSpectrogramPosition(
const maxBin = getSpectrogramPitchBinCount(resolution) - 1;
return Math.max(0, Math.min(maxBin, scaled));
}
export function frequencyToMidiPitch(frequency: number): number | null {
if (!Number.isFinite(frequency) || frequency <= 0) {
return null;
}
return 69 + 12 * Math.log2(frequency / 440);
}
export function mapFrequencyToSpectrogramPosition(
frequency: number,
resolution: number,
): number | null {
const midiPitch = frequencyToMidiPitch(frequency);
if (midiPitch === null) {
return null;
}
return mapMidiPitchToSpectrogramPosition(midiPitch, resolution);
}
+107
View File
@@ -0,0 +1,107 @@
import { describe, expect, it } from 'vitest';
import { getSpectrogramPitchBinCount, mapFrequencyToSpectrogramPosition } from '../util/spectrogramUtil';
import {
estimatePeakFrequency,
getFrequencyBandEdges,
paintMagnitudeAcrossPitchSpan,
} from './spectrogramWorker';
const FFT_SIZE = 16384;
const SAMPLE_RATE = 44100;
const HZ_PER_BIN = SAMPLE_RATE / FFT_SIZE;
function paintSpanForFrequency(
centerFrequency: number,
resolution: 1 | 3 | 5,
pitchBins = getSpectrogramPitchBinCount(resolution),
): Float32Array {
const row = new Float32Array(pitchBins);
const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, HZ_PER_BIN);
const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, resolution);
const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, resolution);
const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, resolution);
if (startPosition === null || endPosition === null || centerPosition === null) {
throw new Error('Expected mapped pitch positions for test frequency span');
}
paintMagnitudeAcrossPitchSpan(
row,
0,
pitchBins,
Math.min(startPosition, endPosition),
Math.max(startPosition, endPosition),
centerPosition,
1,
);
return row;
}
describe('spectrogramWorker helpers', () => {
it('keeps adjacent low-register FFT bins continuous at 5x resolution', () => {
const pitchBins = getSpectrogramPitchBinCount(5);
const row = new Float32Array(pitchBins);
const binsAroundA1 = [10, 11];
for (const bin of binsAroundA1) {
const centerFrequency = bin * HZ_PER_BIN;
const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, HZ_PER_BIN);
const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, 5);
const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, 5);
const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, 5);
expect(startPosition).not.toBeNull();
expect(endPosition).not.toBeNull();
expect(centerPosition).not.toBeNull();
paintMagnitudeAcrossPitchSpan(
row,
0,
pitchBins,
Math.min(startPosition!, endPosition!),
Math.max(startPosition!, endPosition!),
centerPosition!,
1,
);
}
const nonZeroBins = [...row.entries()].filter(([, value]) => value > 0).map(([index]) => index);
expect(nonZeroBins.length).toBeGreaterThan(0);
for (let index = nonZeroBins[0]; index <= nonZeroBins[nonZeroBins.length - 1]; index++) {
expect(row[index]).toBeGreaterThan(0);
}
});
it('uses sub-bin interpolation to move the ridge closer to the true peak frequency', () => {
const magnitudes = new Float32Array([0, 0.3, 1.0, 0.82, 0.1, 0]);
const rawCenterFrequency = 2 * HZ_PER_BIN;
const interpolatedFrequency = estimatePeakFrequency(2, HZ_PER_BIN, magnitudes);
const expectedFrequency = (2.35 * HZ_PER_BIN);
expect(Math.abs(interpolatedFrequency - expectedFrequency)).toBeLessThan(
Math.abs(rawCenterFrequency - expectedFrequency),
);
});
it('leaves non-peak bins at their raw FFT center while still painting a span', () => {
const magnitudes = new Float32Array([0, 0.2, 0.4, 1.0, 0.8, 0.7, 0.2, 0]);
const centerFrequency = estimatePeakFrequency(4, HZ_PER_BIN, magnitudes);
expect(centerFrequency).toBeCloseTo(4 * HZ_PER_BIN, 6);
const row = paintSpanForFrequency(centerFrequency, 3);
expect(row.some(value => value > 0)).toBe(true);
});
it('maps edge-band spans into the valid spectrogram range without spilling outside the buffer', () => {
const lowRow = paintSpanForFrequency(20.5, 5);
const lowNonZeroBins = [...lowRow.entries()].filter(([, value]) => value > 0).map(([index]) => index);
expect(lowNonZeroBins.length).toBeGreaterThan(0);
expect(lowNonZeroBins[0]).toBeGreaterThanOrEqual(0);
const highRow = paintSpanForFrequency(12530, 5);
const highNonZeroBins = [...highRow.entries()].filter(([, value]) => value > 0).map(([index]) => index);
expect(highNonZeroBins.length).toBeGreaterThan(0);
expect(highNonZeroBins[highNonZeroBins.length - 1]).toBeLessThan(highRow.length);
});
});
+133 -29
View File
@@ -1,7 +1,8 @@
import FFT from 'fft.js';
import {
getSpectrogramAnalysisResolution,
getSpectrogramPitchBinCount,
mapMidiPitchToSpectrogramPosition,
mapFrequencyToSpectrogramPosition,
normalizeSpectrogramHeightResolution,
type SpectrogramHeightResolution,
} from '../util/spectrogramUtil';
@@ -11,7 +12,7 @@ type WorkerScopeLike = typeof globalThis & {
postMessage: (message: SpectrogramResult, transfer: Transferable[]) => void;
};
const workerScope = self as WorkerScopeLike;
const workerScope = typeof self === 'undefined' ? null : self as WorkerScopeLike;
export interface SpectrogramRequest {
pcm: Float32Array;
@@ -28,8 +29,10 @@ export interface SpectrogramResult {
pitchBins: number;
}
const FFT_SIZE = 8192;
const FFT_SIZE = 16384;
const HOP_SIZE = 1024;
const MIN_SPECTROGRAM_FREQUENCY = 20;
const MAX_SPECTROGRAM_FREQUENCY = 20000;
function hannWindow(size: number): Float32Array {
const w = new Float32Array(size);
@@ -39,24 +42,98 @@ function hannWindow(size: number): Float32Array {
return w;
}
function spreadMagnitudeAcrossPitchBins(
export function getFrequencyBandEdges(
centerFrequency: number,
hzPerBin: number,
): { lowerFrequency: number; upperFrequency: number } {
const halfBandwidth = hzPerBin / 2;
return {
lowerFrequency: Math.max(Number.EPSILON, centerFrequency - halfBandwidth),
upperFrequency: centerFrequency + halfBandwidth,
};
}
export function estimateParabolicPeakOffset(
leftMagnitude: number,
centerMagnitude: number,
rightMagnitude: number,
): number {
const denominator = leftMagnitude - 2 * centerMagnitude + rightMagnitude;
if (!Number.isFinite(denominator) || Math.abs(denominator) < Number.EPSILON) {
return 0;
}
const offset = 0.5 * (leftMagnitude - rightMagnitude) / denominator;
return Math.max(-0.5, Math.min(0.5, offset));
}
export function estimatePeakFrequency(
bin: number,
hzPerBin: number,
magnitudes: Float32Array,
): number {
const centerFrequency = bin * hzPerBin;
if (bin <= 1 || bin >= magnitudes.length - 1) {
return centerFrequency;
}
const leftMagnitude = magnitudes[bin - 1];
const centerMagnitude = magnitudes[bin];
const rightMagnitude = magnitudes[bin + 1];
if (centerMagnitude <= leftMagnitude || centerMagnitude <= rightMagnitude) {
return centerFrequency;
}
return (bin + estimateParabolicPeakOffset(leftMagnitude, centerMagnitude, rightMagnitude)) * hzPerBin;
}
function getPitchSpanBounds(
lowerFrequency: number,
upperFrequency: number,
centerFrequency: number,
analysisResolution: number,
): { startPosition: number; endPosition: number; centerPosition: number } | null {
const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, analysisResolution);
const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, analysisResolution);
const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, analysisResolution);
if (startPosition === null || endPosition === null || centerPosition === null) {
return null;
}
return {
startPosition: Math.min(startPosition, endPosition),
endPosition: Math.max(startPosition, endPosition),
centerPosition,
};
}
export function paintMagnitudeAcrossPitchSpan(
result: Float32Array,
pitchRow: number,
pitchBins: number,
pitchPosition: number,
startPosition: number,
endPosition: number,
centerPosition: number,
magnitude: number,
heightResolution: SpectrogramHeightResolution,
): void {
const spreadRadius = Math.max(0, heightResolution - 1);
const centerBin = Math.round(pitchPosition);
const startBin = Math.max(0, centerBin - spreadRadius);
const endBin = Math.min(pitchBins - 1, centerBin + spreadRadius);
const spreadWidth = spreadRadius + 1;
const clampedStart = Math.max(0, Math.min(startPosition, pitchBins - 1));
const clampedEnd = Math.max(0, Math.min(endPosition, pitchBins - 1));
const startBin = Math.max(0, Math.floor(clampedStart));
const endBin = Math.min(pitchBins - 1, Math.ceil(clampedEnd));
const spanWidth = Math.max(clampedEnd - clampedStart, 1);
for (let targetBin = startBin; targetBin <= endBin; targetBin++) {
const distance = Math.abs(targetBin - pitchPosition);
const weight = Math.max(0, 1 - distance / spreadWidth);
if (weight <= 0) continue;
const cellStart = targetBin - 0.5;
const cellEnd = targetBin + 0.5;
const overlap = Math.max(0, Math.min(clampedEnd, cellEnd) - Math.max(clampedStart, cellStart));
if (overlap <= 0) continue;
const overlapWeight = Math.min(1, overlap);
const centerDistance = Math.abs(targetBin - centerPosition);
const centerWeight = Math.max(0, 1 - centerDistance / (spanWidth + 1));
const weight = Math.max(overlapWeight * (0.6 + 0.4 * centerWeight), overlapWeight * 0.35);
const weightedMagnitude = magnitude * weight;
const resultIndex = pitchRow + targetBin;
@@ -66,7 +143,7 @@ function spreadMagnitudeAcrossPitchBins(
}
}
workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
function handleSpectrogramRequest(e: MessageEvent<SpectrogramRequest>, scope: WorkerScopeLike): void {
const {
pcm,
sampleRate,
@@ -75,7 +152,8 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
heightResolution,
} = e.data;
const normalizedResolution = normalizeSpectrogramHeightResolution(heightResolution);
const pitchBins = getSpectrogramPitchBinCount(normalizedResolution);
const analysisResolution = getSpectrogramAnalysisResolution(normalizedResolution);
const pitchBins = getSpectrogramPitchBinCount(analysisResolution);
const startSample = Math.floor(clipStartOffsetSeconds * sampleRate);
const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate));
@@ -85,6 +163,7 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
const hann = hannWindow(FFT_SIZE);
const complexOut = fft.createComplexArray() as number[];
const inputPadded = new Float32Array(FFT_SIZE);
const hzPerBin = sampleRate / FFT_SIZE;
const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1);
const result = new Float32Array(totalHops * pitchBins);
@@ -108,25 +187,44 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
// rather than summing. Summing inflates every bin by how many FFT bins land there.
const pitchRow = hop * pitchBins;
const numBins = FFT_SIZE / 2;
const magnitudes = new Float32Array(numBins);
for (let bin = 1; bin < numBins; bin++) {
const freq = (bin * sampleRate) / FFT_SIZE;
if (freq < 20 || freq > 20000) continue;
const midiPitch = 69 + 12 * Math.log2(freq / 440);
const pitchPosition = mapMidiPitchToSpectrogramPosition(midiPitch, normalizedResolution);
if (pitchPosition === null) continue;
const re = complexOut[2 * bin];
const im = complexOut[2 * bin + 1];
const magnitude = Math.sqrt(re * re + im * im);
spreadMagnitudeAcrossPitchBins(
magnitudes[bin] = Math.sqrt(re * re + im * im);
}
for (let bin = 1; bin < numBins; bin++) {
const centerFrequency = estimatePeakFrequency(bin, hzPerBin, magnitudes);
const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, hzPerBin);
if (upperFrequency < MIN_SPECTROGRAM_FREQUENCY || lowerFrequency > MAX_SPECTROGRAM_FREQUENCY) continue;
const clampedLowerFrequency = Math.max(lowerFrequency, MIN_SPECTROGRAM_FREQUENCY);
const clampedUpperFrequency = Math.min(upperFrequency, MAX_SPECTROGRAM_FREQUENCY);
const clampedCenterFrequency = Math.min(
clampedUpperFrequency,
Math.max(clampedLowerFrequency, centerFrequency),
);
const pitchSpan = getPitchSpanBounds(
clampedLowerFrequency,
clampedUpperFrequency,
clampedCenterFrequency,
analysisResolution,
);
if (!pitchSpan) continue;
const magnitude = magnitudes[bin];
if (magnitude <= 0) continue;
paintMagnitudeAcrossPitchSpan(
result,
pitchRow,
pitchBins,
pitchPosition,
pitchSpan.startPosition,
pitchSpan.endPosition,
pitchSpan.centerPosition,
magnitude,
normalizedResolution,
);
}
@@ -145,5 +243,11 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
}
const response: SpectrogramResult = { data: result, timeSteps: totalHops, pitchBins };
workerScope.postMessage(response, [result.buffer]);
};
scope.postMessage(response, [result.buffer]);
}
if (workerScope) {
workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
handleSpectrogramRequest(e, workerScope);
};
}