feat: improve spectrogram pitch mapping and vertical smoothing
This commit is contained in:
@@ -5,6 +5,7 @@ import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
|
|||||||
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
|
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
|
||||||
import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker';
|
import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker';
|
||||||
import {
|
import {
|
||||||
|
SPECTROGRAM_FULL_SEMITONES,
|
||||||
getSpectrogramVisibleBinRange,
|
getSpectrogramVisibleBinRange,
|
||||||
normalizeSpectrogramHeightResolution,
|
normalizeSpectrogramHeightResolution,
|
||||||
SPECTROGRAM_VISIBLE_SEMITONES,
|
SPECTROGRAM_VISIBLE_SEMITONES,
|
||||||
@@ -52,6 +53,25 @@ function hotColormap(v: number): [number, number, number] {
|
|||||||
return COLORMAP_STOPS[COLORMAP_STOPS.length - 1][1];
|
return COLORMAP_STOPS[COLORMAP_STOPS.length - 1][1];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function smoothSpectrogramVertically(
|
||||||
|
source: Float32Array,
|
||||||
|
timeSteps: number,
|
||||||
|
pitchBins: number,
|
||||||
|
): Float32Array {
|
||||||
|
const smoothed = new Float32Array(source.length);
|
||||||
|
|
||||||
|
for (let col = 0; col < timeSteps; col++) {
|
||||||
|
for (let row = 0; row < pitchBins; row++) {
|
||||||
|
const center = source[col * pitchBins + row];
|
||||||
|
const above = row > 0 ? source[col * pitchBins + row - 1] : center;
|
||||||
|
const below = row < pitchBins - 1 ? source[col * pitchBins + row + 1] : center;
|
||||||
|
smoothed[col * pitchBins + row] = above * 0.2 + center * 0.6 + below * 0.2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return smoothed;
|
||||||
|
}
|
||||||
|
|
||||||
const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||||
audioRegion,
|
audioRegion,
|
||||||
trackId,
|
trackId,
|
||||||
@@ -102,18 +122,21 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
|||||||
|
|
||||||
// Convert dB threshold to linear: values below this → black
|
// Convert dB threshold to linear: values below this → black
|
||||||
const linearThreshold = Math.pow(10, thresholdDb / 20);
|
const linearThreshold = Math.pow(10, thresholdDb / 20);
|
||||||
const visibleRange = getSpectrogramVisibleBinRange(heightResolution);
|
const analysisResolution = result.pitchBins / SPECTROGRAM_FULL_SEMITONES;
|
||||||
|
const visibleRange = getSpectrogramVisibleBinRange(analysisResolution);
|
||||||
const visiblePitchBins = visibleRange.end - visibleRange.start;
|
const visiblePitchBins = visibleRange.end - visibleRange.start;
|
||||||
|
|
||||||
// 1. Paint at natural spectrogram resolution onto an offscreen canvas.
|
// 1. Paint at natural spectrogram resolution onto an offscreen canvas.
|
||||||
// Result data is low-to-high pitch; draw only the visible C0-B7 window, reversed for display.
|
// Result data is low-to-high pitch; draw only the visible C0-B7 window, reversed for display.
|
||||||
const offscreen = document.createElement('canvas');
|
const smoothedData = smoothSpectrogramVertically(result.data, result.timeSteps, result.pitchBins);
|
||||||
offscreen.width = result.timeSteps;
|
|
||||||
offscreen.height = visiblePitchBins;
|
|
||||||
const offCtx = offscreen.getContext('2d');
|
|
||||||
if (!offCtx) return;
|
|
||||||
|
|
||||||
const imgData = offCtx.createImageData(result.timeSteps, visiblePitchBins);
|
const sourceCanvas = document.createElement('canvas');
|
||||||
|
sourceCanvas.width = result.timeSteps;
|
||||||
|
sourceCanvas.height = visiblePitchBins;
|
||||||
|
const sourceCtx = sourceCanvas.getContext('2d');
|
||||||
|
if (!sourceCtx) return;
|
||||||
|
|
||||||
|
const imgData = sourceCtx.createImageData(result.timeSteps, visiblePitchBins);
|
||||||
const pixels = imgData.data;
|
const pixels = imgData.data;
|
||||||
|
|
||||||
for (let row = 0; row < visiblePitchBins; row++) {
|
for (let row = 0; row < visiblePitchBins; row++) {
|
||||||
@@ -122,7 +145,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
|||||||
const idx = (row * result.timeSteps + col) * 4;
|
const idx = (row * result.timeSteps + col) * 4;
|
||||||
pixels[idx + 3] = 255; // always opaque
|
pixels[idx + 3] = 255; // always opaque
|
||||||
|
|
||||||
const raw = result.data[col * result.pitchBins + sourceRow];
|
const raw = smoothedData[col * result.pitchBins + sourceRow];
|
||||||
|
|
||||||
// Hard threshold: values below noise floor → 0 (black)
|
// Hard threshold: values below noise floor → 0 (black)
|
||||||
// Re-scale surviving range to [0,1] then apply power curve
|
// Re-scale surviving range to [0,1] then apply power curve
|
||||||
@@ -139,14 +162,24 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
offCtx.putImageData(imgData, 0, 0);
|
sourceCtx.putImageData(imgData, 0, 0);
|
||||||
|
|
||||||
// 2. Stretch onto the full canvas — browser bilinear filter smooths between bins.
|
const verticallyScaledCanvas = document.createElement('canvas');
|
||||||
|
verticallyScaledCanvas.width = result.timeSteps;
|
||||||
|
verticallyScaledCanvas.height = canvasHeight;
|
||||||
|
const verticallyScaledCtx = verticallyScaledCanvas.getContext('2d');
|
||||||
|
if (!verticallyScaledCtx) return;
|
||||||
|
|
||||||
|
// Scale only along the pitch axis so note starts/stops stay crisp in time.
|
||||||
|
verticallyScaledCtx.imageSmoothingEnabled = true;
|
||||||
|
verticallyScaledCtx.imageSmoothingQuality = 'high';
|
||||||
|
verticallyScaledCtx.drawImage(sourceCanvas, 0, 0, result.timeSteps, canvasHeight);
|
||||||
|
|
||||||
|
// 2. Stretch onto the full canvas horizontally without temporal blur.
|
||||||
canvas.width = canvasWidth;
|
canvas.width = canvasWidth;
|
||||||
canvas.height = canvasHeight;
|
canvas.height = canvasHeight;
|
||||||
ctx.imageSmoothingEnabled = true;
|
ctx.imageSmoothingEnabled = false;
|
||||||
ctx.imageSmoothingQuality = 'high';
|
ctx.drawImage(verticallyScaledCanvas, 0, 0, canvasWidth, canvasHeight);
|
||||||
ctx.drawImage(offscreen, 0, 0, canvasWidth, canvasHeight);
|
|
||||||
|
|
||||||
// Store natural width and apply current zoom as CSS stretch (no pixel recompute on zoom)
|
// Store natural width and apply current zoom as CSS stretch (no pixel recompute on zoom)
|
||||||
naturalWidthRef.current = canvasWidth;
|
naturalWidthRef.current = canvasWidth;
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
import { describe, expect, it } from 'vitest';
|
import { describe, expect, it } from 'vitest';
|
||||||
import {
|
import {
|
||||||
|
frequencyToMidiPitch,
|
||||||
|
getSpectrogramAnalysisResolution,
|
||||||
getSpectrogramPitchBinCount,
|
getSpectrogramPitchBinCount,
|
||||||
getSpectrogramVisibleBinRange,
|
getSpectrogramVisibleBinRange,
|
||||||
|
mapFrequencyToSpectrogramPosition,
|
||||||
mapMidiPitchToSpectrogramPosition,
|
mapMidiPitchToSpectrogramPosition,
|
||||||
normalizeSpectrogramHeightResolution,
|
normalizeSpectrogramHeightResolution,
|
||||||
} from './spectrogramUtil';
|
} from './spectrogramUtil';
|
||||||
@@ -21,6 +24,12 @@ describe('spectrogramUtil', () => {
|
|||||||
expect(getSpectrogramPitchBinCount(5)).toBe(640);
|
expect(getSpectrogramPitchBinCount(5)).toBe(640);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('doubles the internal analysis resolution for each user-facing option', () => {
|
||||||
|
expect(getSpectrogramAnalysisResolution(1)).toBe(2);
|
||||||
|
expect(getSpectrogramAnalysisResolution(3)).toBe(6);
|
||||||
|
expect(getSpectrogramAnalysisResolution(5)).toBe(10);
|
||||||
|
});
|
||||||
|
|
||||||
it('maps MIDI pitches into full-range spectrogram positions', () => {
|
it('maps MIDI pitches into full-range spectrogram positions', () => {
|
||||||
expect(mapMidiPitchToSpectrogramPosition(0, 1)).toBe(0);
|
expect(mapMidiPitchToSpectrogramPosition(0, 1)).toBe(0);
|
||||||
expect(mapMidiPitchToSpectrogramPosition(12.5, 3)).toBe(38.5);
|
expect(mapMidiPitchToSpectrogramPosition(12.5, 3)).toBe(38.5);
|
||||||
@@ -29,6 +38,12 @@ describe('spectrogramUtil', () => {
|
|||||||
expect(mapMidiPitchToSpectrogramPosition(128, 3)).toBeNull();
|
expect(mapMidiPitchToSpectrogramPosition(128, 3)).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('converts frequencies into MIDI pitch space and spectrogram positions', () => {
|
||||||
|
expect(frequencyToMidiPitch(440)).toBeCloseTo(69, 6);
|
||||||
|
expect(mapFrequencyToSpectrogramPosition(440, 5)).toBeCloseTo(347, 6);
|
||||||
|
expect(frequencyToMidiPitch(0)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
it('derives the visible C0-B7 subrange inside the full-resolution buffer', () => {
|
it('derives the visible C0-B7 subrange inside the full-resolution buffer', () => {
|
||||||
expect(getSpectrogramVisibleBinRange(1)).toEqual({ start: 12, end: 108 });
|
expect(getSpectrogramVisibleBinRange(1)).toEqual({ start: 12, end: 108 });
|
||||||
expect(getSpectrogramVisibleBinRange(3)).toEqual({ start: 36, end: 324 });
|
expect(getSpectrogramVisibleBinRange(3)).toEqual({ start: 36, end: 324 });
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ export const SPECTROGRAM_FULL_SEMITONES =
|
|||||||
SPECTROGRAM_FULL_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1;
|
SPECTROGRAM_FULL_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1;
|
||||||
export const SPECTROGRAM_VISIBLE_SEMITONES =
|
export const SPECTROGRAM_VISIBLE_SEMITONES =
|
||||||
SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_MIN_MIDI_PITCH + 1;
|
SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_MIN_MIDI_PITCH + 1;
|
||||||
|
export const SPECTROGRAM_ANALYSIS_RESOLUTION_MULTIPLIER = 2;
|
||||||
|
|
||||||
export type SpectrogramHeightResolution = 1 | 3 | 5;
|
export type SpectrogramHeightResolution = 1 | 3 | 5;
|
||||||
|
|
||||||
@@ -19,13 +20,13 @@ export function normalizeSpectrogramHeightResolution(value: unknown): Spectrogra
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function getSpectrogramPitchBinCount(
|
export function getSpectrogramPitchBinCount(
|
||||||
resolution: SpectrogramHeightResolution,
|
resolution: number,
|
||||||
): number {
|
): number {
|
||||||
return SPECTROGRAM_FULL_SEMITONES * resolution;
|
return SPECTROGRAM_FULL_SEMITONES * resolution;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getSpectrogramVisibleBinRange(
|
export function getSpectrogramVisibleBinRange(
|
||||||
resolution: SpectrogramHeightResolution,
|
resolution: number,
|
||||||
): { start: number; end: number } {
|
): { start: number; end: number } {
|
||||||
return {
|
return {
|
||||||
start: (SPECTROGRAM_MIN_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH) * resolution,
|
start: (SPECTROGRAM_MIN_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH) * resolution,
|
||||||
@@ -33,9 +34,15 @@ export function getSpectrogramVisibleBinRange(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getSpectrogramAnalysisResolution(
|
||||||
|
resolution: SpectrogramHeightResolution,
|
||||||
|
): number {
|
||||||
|
return resolution * SPECTROGRAM_ANALYSIS_RESOLUTION_MULTIPLIER;
|
||||||
|
}
|
||||||
|
|
||||||
export function mapMidiPitchToSpectrogramPosition(
|
export function mapMidiPitchToSpectrogramPosition(
|
||||||
midiPitch: number,
|
midiPitch: number,
|
||||||
resolution: SpectrogramHeightResolution,
|
resolution: number,
|
||||||
): number | null {
|
): number | null {
|
||||||
const pitchOffset = midiPitch - SPECTROGRAM_FULL_MIN_MIDI_PITCH;
|
const pitchOffset = midiPitch - SPECTROGRAM_FULL_MIN_MIDI_PITCH;
|
||||||
if (pitchOffset < 0 || pitchOffset > SPECTROGRAM_FULL_MAX_MIDI_PITCH) {
|
if (pitchOffset < 0 || pitchOffset > SPECTROGRAM_FULL_MAX_MIDI_PITCH) {
|
||||||
@@ -48,3 +55,21 @@ export function mapMidiPitchToSpectrogramPosition(
|
|||||||
const maxBin = getSpectrogramPitchBinCount(resolution) - 1;
|
const maxBin = getSpectrogramPitchBinCount(resolution) - 1;
|
||||||
return Math.max(0, Math.min(maxBin, scaled));
|
return Math.max(0, Math.min(maxBin, scaled));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function frequencyToMidiPitch(frequency: number): number | null {
|
||||||
|
if (!Number.isFinite(frequency) || frequency <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return 69 + 12 * Math.log2(frequency / 440);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mapFrequencyToSpectrogramPosition(
|
||||||
|
frequency: number,
|
||||||
|
resolution: number,
|
||||||
|
): number | null {
|
||||||
|
const midiPitch = frequencyToMidiPitch(frequency);
|
||||||
|
if (midiPitch === null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return mapMidiPitchToSpectrogramPosition(midiPitch, resolution);
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { getSpectrogramPitchBinCount, mapFrequencyToSpectrogramPosition } from '../util/spectrogramUtil';
|
||||||
|
import {
|
||||||
|
estimatePeakFrequency,
|
||||||
|
getFrequencyBandEdges,
|
||||||
|
paintMagnitudeAcrossPitchSpan,
|
||||||
|
} from './spectrogramWorker';
|
||||||
|
|
||||||
|
const FFT_SIZE = 16384;
|
||||||
|
const SAMPLE_RATE = 44100;
|
||||||
|
const HZ_PER_BIN = SAMPLE_RATE / FFT_SIZE;
|
||||||
|
|
||||||
|
function paintSpanForFrequency(
|
||||||
|
centerFrequency: number,
|
||||||
|
resolution: 1 | 3 | 5,
|
||||||
|
pitchBins = getSpectrogramPitchBinCount(resolution),
|
||||||
|
): Float32Array {
|
||||||
|
const row = new Float32Array(pitchBins);
|
||||||
|
const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, HZ_PER_BIN);
|
||||||
|
const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, resolution);
|
||||||
|
const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, resolution);
|
||||||
|
const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, resolution);
|
||||||
|
|
||||||
|
if (startPosition === null || endPosition === null || centerPosition === null) {
|
||||||
|
throw new Error('Expected mapped pitch positions for test frequency span');
|
||||||
|
}
|
||||||
|
|
||||||
|
paintMagnitudeAcrossPitchSpan(
|
||||||
|
row,
|
||||||
|
0,
|
||||||
|
pitchBins,
|
||||||
|
Math.min(startPosition, endPosition),
|
||||||
|
Math.max(startPosition, endPosition),
|
||||||
|
centerPosition,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('spectrogramWorker helpers', () => {
|
||||||
|
it('keeps adjacent low-register FFT bins continuous at 5x resolution', () => {
|
||||||
|
const pitchBins = getSpectrogramPitchBinCount(5);
|
||||||
|
const row = new Float32Array(pitchBins);
|
||||||
|
const binsAroundA1 = [10, 11];
|
||||||
|
|
||||||
|
for (const bin of binsAroundA1) {
|
||||||
|
const centerFrequency = bin * HZ_PER_BIN;
|
||||||
|
const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, HZ_PER_BIN);
|
||||||
|
const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, 5);
|
||||||
|
const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, 5);
|
||||||
|
const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, 5);
|
||||||
|
|
||||||
|
expect(startPosition).not.toBeNull();
|
||||||
|
expect(endPosition).not.toBeNull();
|
||||||
|
expect(centerPosition).not.toBeNull();
|
||||||
|
|
||||||
|
paintMagnitudeAcrossPitchSpan(
|
||||||
|
row,
|
||||||
|
0,
|
||||||
|
pitchBins,
|
||||||
|
Math.min(startPosition!, endPosition!),
|
||||||
|
Math.max(startPosition!, endPosition!),
|
||||||
|
centerPosition!,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const nonZeroBins = [...row.entries()].filter(([, value]) => value > 0).map(([index]) => index);
|
||||||
|
expect(nonZeroBins.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
for (let index = nonZeroBins[0]; index <= nonZeroBins[nonZeroBins.length - 1]; index++) {
|
||||||
|
expect(row[index]).toBeGreaterThan(0);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses sub-bin interpolation to move the ridge closer to the true peak frequency', () => {
|
||||||
|
const magnitudes = new Float32Array([0, 0.3, 1.0, 0.82, 0.1, 0]);
|
||||||
|
const rawCenterFrequency = 2 * HZ_PER_BIN;
|
||||||
|
const interpolatedFrequency = estimatePeakFrequency(2, HZ_PER_BIN, magnitudes);
|
||||||
|
const expectedFrequency = (2.35 * HZ_PER_BIN);
|
||||||
|
|
||||||
|
expect(Math.abs(interpolatedFrequency - expectedFrequency)).toBeLessThan(
|
||||||
|
Math.abs(rawCenterFrequency - expectedFrequency),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('leaves non-peak bins at their raw FFT center while still painting a span', () => {
|
||||||
|
const magnitudes = new Float32Array([0, 0.2, 0.4, 1.0, 0.8, 0.7, 0.2, 0]);
|
||||||
|
const centerFrequency = estimatePeakFrequency(4, HZ_PER_BIN, magnitudes);
|
||||||
|
expect(centerFrequency).toBeCloseTo(4 * HZ_PER_BIN, 6);
|
||||||
|
|
||||||
|
const row = paintSpanForFrequency(centerFrequency, 3);
|
||||||
|
expect(row.some(value => value > 0)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('maps edge-band spans into the valid spectrogram range without spilling outside the buffer', () => {
|
||||||
|
const lowRow = paintSpanForFrequency(20.5, 5);
|
||||||
|
const lowNonZeroBins = [...lowRow.entries()].filter(([, value]) => value > 0).map(([index]) => index);
|
||||||
|
expect(lowNonZeroBins.length).toBeGreaterThan(0);
|
||||||
|
expect(lowNonZeroBins[0]).toBeGreaterThanOrEqual(0);
|
||||||
|
|
||||||
|
const highRow = paintSpanForFrequency(12530, 5);
|
||||||
|
const highNonZeroBins = [...highRow.entries()].filter(([, value]) => value > 0).map(([index]) => index);
|
||||||
|
expect(highNonZeroBins.length).toBeGreaterThan(0);
|
||||||
|
expect(highNonZeroBins[highNonZeroBins.length - 1]).toBeLessThan(highRow.length);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,7 +1,8 @@
|
|||||||
import FFT from 'fft.js';
|
import FFT from 'fft.js';
|
||||||
import {
|
import {
|
||||||
|
getSpectrogramAnalysisResolution,
|
||||||
getSpectrogramPitchBinCount,
|
getSpectrogramPitchBinCount,
|
||||||
mapMidiPitchToSpectrogramPosition,
|
mapFrequencyToSpectrogramPosition,
|
||||||
normalizeSpectrogramHeightResolution,
|
normalizeSpectrogramHeightResolution,
|
||||||
type SpectrogramHeightResolution,
|
type SpectrogramHeightResolution,
|
||||||
} from '../util/spectrogramUtil';
|
} from '../util/spectrogramUtil';
|
||||||
@@ -11,7 +12,7 @@ type WorkerScopeLike = typeof globalThis & {
|
|||||||
postMessage: (message: SpectrogramResult, transfer: Transferable[]) => void;
|
postMessage: (message: SpectrogramResult, transfer: Transferable[]) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
const workerScope = self as WorkerScopeLike;
|
const workerScope = typeof self === 'undefined' ? null : self as WorkerScopeLike;
|
||||||
|
|
||||||
export interface SpectrogramRequest {
|
export interface SpectrogramRequest {
|
||||||
pcm: Float32Array;
|
pcm: Float32Array;
|
||||||
@@ -28,8 +29,10 @@ export interface SpectrogramResult {
|
|||||||
pitchBins: number;
|
pitchBins: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
const FFT_SIZE = 8192;
|
const FFT_SIZE = 16384;
|
||||||
const HOP_SIZE = 1024;
|
const HOP_SIZE = 1024;
|
||||||
|
const MIN_SPECTROGRAM_FREQUENCY = 20;
|
||||||
|
const MAX_SPECTROGRAM_FREQUENCY = 20000;
|
||||||
|
|
||||||
function hannWindow(size: number): Float32Array {
|
function hannWindow(size: number): Float32Array {
|
||||||
const w = new Float32Array(size);
|
const w = new Float32Array(size);
|
||||||
@@ -39,24 +42,98 @@ function hannWindow(size: number): Float32Array {
|
|||||||
return w;
|
return w;
|
||||||
}
|
}
|
||||||
|
|
||||||
function spreadMagnitudeAcrossPitchBins(
|
export function getFrequencyBandEdges(
|
||||||
|
centerFrequency: number,
|
||||||
|
hzPerBin: number,
|
||||||
|
): { lowerFrequency: number; upperFrequency: number } {
|
||||||
|
const halfBandwidth = hzPerBin / 2;
|
||||||
|
return {
|
||||||
|
lowerFrequency: Math.max(Number.EPSILON, centerFrequency - halfBandwidth),
|
||||||
|
upperFrequency: centerFrequency + halfBandwidth,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function estimateParabolicPeakOffset(
|
||||||
|
leftMagnitude: number,
|
||||||
|
centerMagnitude: number,
|
||||||
|
rightMagnitude: number,
|
||||||
|
): number {
|
||||||
|
const denominator = leftMagnitude - 2 * centerMagnitude + rightMagnitude;
|
||||||
|
if (!Number.isFinite(denominator) || Math.abs(denominator) < Number.EPSILON) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const offset = 0.5 * (leftMagnitude - rightMagnitude) / denominator;
|
||||||
|
return Math.max(-0.5, Math.min(0.5, offset));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function estimatePeakFrequency(
|
||||||
|
bin: number,
|
||||||
|
hzPerBin: number,
|
||||||
|
magnitudes: Float32Array,
|
||||||
|
): number {
|
||||||
|
const centerFrequency = bin * hzPerBin;
|
||||||
|
if (bin <= 1 || bin >= magnitudes.length - 1) {
|
||||||
|
return centerFrequency;
|
||||||
|
}
|
||||||
|
|
||||||
|
const leftMagnitude = magnitudes[bin - 1];
|
||||||
|
const centerMagnitude = magnitudes[bin];
|
||||||
|
const rightMagnitude = magnitudes[bin + 1];
|
||||||
|
|
||||||
|
if (centerMagnitude <= leftMagnitude || centerMagnitude <= rightMagnitude) {
|
||||||
|
return centerFrequency;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (bin + estimateParabolicPeakOffset(leftMagnitude, centerMagnitude, rightMagnitude)) * hzPerBin;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getPitchSpanBounds(
|
||||||
|
lowerFrequency: number,
|
||||||
|
upperFrequency: number,
|
||||||
|
centerFrequency: number,
|
||||||
|
analysisResolution: number,
|
||||||
|
): { startPosition: number; endPosition: number; centerPosition: number } | null {
|
||||||
|
const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, analysisResolution);
|
||||||
|
const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, analysisResolution);
|
||||||
|
const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, analysisResolution);
|
||||||
|
|
||||||
|
if (startPosition === null || endPosition === null || centerPosition === null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
startPosition: Math.min(startPosition, endPosition),
|
||||||
|
endPosition: Math.max(startPosition, endPosition),
|
||||||
|
centerPosition,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function paintMagnitudeAcrossPitchSpan(
|
||||||
result: Float32Array,
|
result: Float32Array,
|
||||||
pitchRow: number,
|
pitchRow: number,
|
||||||
pitchBins: number,
|
pitchBins: number,
|
||||||
pitchPosition: number,
|
startPosition: number,
|
||||||
|
endPosition: number,
|
||||||
|
centerPosition: number,
|
||||||
magnitude: number,
|
magnitude: number,
|
||||||
heightResolution: SpectrogramHeightResolution,
|
|
||||||
): void {
|
): void {
|
||||||
const spreadRadius = Math.max(0, heightResolution - 1);
|
const clampedStart = Math.max(0, Math.min(startPosition, pitchBins - 1));
|
||||||
const centerBin = Math.round(pitchPosition);
|
const clampedEnd = Math.max(0, Math.min(endPosition, pitchBins - 1));
|
||||||
const startBin = Math.max(0, centerBin - spreadRadius);
|
const startBin = Math.max(0, Math.floor(clampedStart));
|
||||||
const endBin = Math.min(pitchBins - 1, centerBin + spreadRadius);
|
const endBin = Math.min(pitchBins - 1, Math.ceil(clampedEnd));
|
||||||
const spreadWidth = spreadRadius + 1;
|
const spanWidth = Math.max(clampedEnd - clampedStart, 1);
|
||||||
|
|
||||||
for (let targetBin = startBin; targetBin <= endBin; targetBin++) {
|
for (let targetBin = startBin; targetBin <= endBin; targetBin++) {
|
||||||
const distance = Math.abs(targetBin - pitchPosition);
|
const cellStart = targetBin - 0.5;
|
||||||
const weight = Math.max(0, 1 - distance / spreadWidth);
|
const cellEnd = targetBin + 0.5;
|
||||||
if (weight <= 0) continue;
|
const overlap = Math.max(0, Math.min(clampedEnd, cellEnd) - Math.max(clampedStart, cellStart));
|
||||||
|
if (overlap <= 0) continue;
|
||||||
|
|
||||||
|
const overlapWeight = Math.min(1, overlap);
|
||||||
|
const centerDistance = Math.abs(targetBin - centerPosition);
|
||||||
|
const centerWeight = Math.max(0, 1 - centerDistance / (spanWidth + 1));
|
||||||
|
const weight = Math.max(overlapWeight * (0.6 + 0.4 * centerWeight), overlapWeight * 0.35);
|
||||||
|
|
||||||
const weightedMagnitude = magnitude * weight;
|
const weightedMagnitude = magnitude * weight;
|
||||||
const resultIndex = pitchRow + targetBin;
|
const resultIndex = pitchRow + targetBin;
|
||||||
@@ -66,7 +143,7 @@ function spreadMagnitudeAcrossPitchBins(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
function handleSpectrogramRequest(e: MessageEvent<SpectrogramRequest>, scope: WorkerScopeLike): void {
|
||||||
const {
|
const {
|
||||||
pcm,
|
pcm,
|
||||||
sampleRate,
|
sampleRate,
|
||||||
@@ -75,7 +152,8 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
|||||||
heightResolution,
|
heightResolution,
|
||||||
} = e.data;
|
} = e.data;
|
||||||
const normalizedResolution = normalizeSpectrogramHeightResolution(heightResolution);
|
const normalizedResolution = normalizeSpectrogramHeightResolution(heightResolution);
|
||||||
const pitchBins = getSpectrogramPitchBinCount(normalizedResolution);
|
const analysisResolution = getSpectrogramAnalysisResolution(normalizedResolution);
|
||||||
|
const pitchBins = getSpectrogramPitchBinCount(analysisResolution);
|
||||||
|
|
||||||
const startSample = Math.floor(clipStartOffsetSeconds * sampleRate);
|
const startSample = Math.floor(clipStartOffsetSeconds * sampleRate);
|
||||||
const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate));
|
const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate));
|
||||||
@@ -85,6 +163,7 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
|||||||
const hann = hannWindow(FFT_SIZE);
|
const hann = hannWindow(FFT_SIZE);
|
||||||
const complexOut = fft.createComplexArray() as number[];
|
const complexOut = fft.createComplexArray() as number[];
|
||||||
const inputPadded = new Float32Array(FFT_SIZE);
|
const inputPadded = new Float32Array(FFT_SIZE);
|
||||||
|
const hzPerBin = sampleRate / FFT_SIZE;
|
||||||
|
|
||||||
const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1);
|
const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1);
|
||||||
const result = new Float32Array(totalHops * pitchBins);
|
const result = new Float32Array(totalHops * pitchBins);
|
||||||
@@ -108,25 +187,44 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
|||||||
// rather than summing. Summing inflates every bin by how many FFT bins land there.
|
// rather than summing. Summing inflates every bin by how many FFT bins land there.
|
||||||
const pitchRow = hop * pitchBins;
|
const pitchRow = hop * pitchBins;
|
||||||
const numBins = FFT_SIZE / 2;
|
const numBins = FFT_SIZE / 2;
|
||||||
|
const magnitudes = new Float32Array(numBins);
|
||||||
|
|
||||||
for (let bin = 1; bin < numBins; bin++) {
|
for (let bin = 1; bin < numBins; bin++) {
|
||||||
const freq = (bin * sampleRate) / FFT_SIZE;
|
|
||||||
if (freq < 20 || freq > 20000) continue;
|
|
||||||
|
|
||||||
const midiPitch = 69 + 12 * Math.log2(freq / 440);
|
|
||||||
const pitchPosition = mapMidiPitchToSpectrogramPosition(midiPitch, normalizedResolution);
|
|
||||||
if (pitchPosition === null) continue;
|
|
||||||
|
|
||||||
const re = complexOut[2 * bin];
|
const re = complexOut[2 * bin];
|
||||||
const im = complexOut[2 * bin + 1];
|
const im = complexOut[2 * bin + 1];
|
||||||
const magnitude = Math.sqrt(re * re + im * im);
|
magnitudes[bin] = Math.sqrt(re * re + im * im);
|
||||||
spreadMagnitudeAcrossPitchBins(
|
}
|
||||||
|
|
||||||
|
for (let bin = 1; bin < numBins; bin++) {
|
||||||
|
const centerFrequency = estimatePeakFrequency(bin, hzPerBin, magnitudes);
|
||||||
|
const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, hzPerBin);
|
||||||
|
if (upperFrequency < MIN_SPECTROGRAM_FREQUENCY || lowerFrequency > MAX_SPECTROGRAM_FREQUENCY) continue;
|
||||||
|
|
||||||
|
const clampedLowerFrequency = Math.max(lowerFrequency, MIN_SPECTROGRAM_FREQUENCY);
|
||||||
|
const clampedUpperFrequency = Math.min(upperFrequency, MAX_SPECTROGRAM_FREQUENCY);
|
||||||
|
const clampedCenterFrequency = Math.min(
|
||||||
|
clampedUpperFrequency,
|
||||||
|
Math.max(clampedLowerFrequency, centerFrequency),
|
||||||
|
);
|
||||||
|
const pitchSpan = getPitchSpanBounds(
|
||||||
|
clampedLowerFrequency,
|
||||||
|
clampedUpperFrequency,
|
||||||
|
clampedCenterFrequency,
|
||||||
|
analysisResolution,
|
||||||
|
);
|
||||||
|
if (!pitchSpan) continue;
|
||||||
|
|
||||||
|
const magnitude = magnitudes[bin];
|
||||||
|
if (magnitude <= 0) continue;
|
||||||
|
|
||||||
|
paintMagnitudeAcrossPitchSpan(
|
||||||
result,
|
result,
|
||||||
pitchRow,
|
pitchRow,
|
||||||
pitchBins,
|
pitchBins,
|
||||||
pitchPosition,
|
pitchSpan.startPosition,
|
||||||
|
pitchSpan.endPosition,
|
||||||
|
pitchSpan.centerPosition,
|
||||||
magnitude,
|
magnitude,
|
||||||
normalizedResolution,
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,5 +243,11 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const response: SpectrogramResult = { data: result, timeSteps: totalHops, pitchBins };
|
const response: SpectrogramResult = { data: result, timeSteps: totalHops, pitchBins };
|
||||||
workerScope.postMessage(response, [result.buffer]);
|
scope.postMessage(response, [result.buffer]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (workerScope) {
|
||||||
|
workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||||
|
handleSpectrogramRequest(e, workerScope);
|
||||||
};
|
};
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user