feat: improve spectrogram pitch mapping and vertical smoothing
This commit is contained in:
@@ -5,6 +5,7 @@ import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
|
||||
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
|
||||
import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker';
|
||||
import {
|
||||
SPECTROGRAM_FULL_SEMITONES,
|
||||
getSpectrogramVisibleBinRange,
|
||||
normalizeSpectrogramHeightResolution,
|
||||
SPECTROGRAM_VISIBLE_SEMITONES,
|
||||
@@ -52,6 +53,25 @@ function hotColormap(v: number): [number, number, number] {
|
||||
return COLORMAP_STOPS[COLORMAP_STOPS.length - 1][1];
|
||||
}
|
||||
|
||||
function smoothSpectrogramVertically(
|
||||
source: Float32Array,
|
||||
timeSteps: number,
|
||||
pitchBins: number,
|
||||
): Float32Array {
|
||||
const smoothed = new Float32Array(source.length);
|
||||
|
||||
for (let col = 0; col < timeSteps; col++) {
|
||||
for (let row = 0; row < pitchBins; row++) {
|
||||
const center = source[col * pitchBins + row];
|
||||
const above = row > 0 ? source[col * pitchBins + row - 1] : center;
|
||||
const below = row < pitchBins - 1 ? source[col * pitchBins + row + 1] : center;
|
||||
smoothed[col * pitchBins + row] = above * 0.2 + center * 0.6 + below * 0.2;
|
||||
}
|
||||
}
|
||||
|
||||
return smoothed;
|
||||
}
|
||||
|
||||
const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
audioRegion,
|
||||
trackId,
|
||||
@@ -102,18 +122,21 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
|
||||
// Convert dB threshold to linear: values below this → black
|
||||
const linearThreshold = Math.pow(10, thresholdDb / 20);
|
||||
const visibleRange = getSpectrogramVisibleBinRange(heightResolution);
|
||||
const analysisResolution = result.pitchBins / SPECTROGRAM_FULL_SEMITONES;
|
||||
const visibleRange = getSpectrogramVisibleBinRange(analysisResolution);
|
||||
const visiblePitchBins = visibleRange.end - visibleRange.start;
|
||||
|
||||
// 1. Paint at natural spectrogram resolution onto an offscreen canvas.
|
||||
// Result data is low-to-high pitch; draw only the visible C0-B7 window, reversed for display.
|
||||
const offscreen = document.createElement('canvas');
|
||||
offscreen.width = result.timeSteps;
|
||||
offscreen.height = visiblePitchBins;
|
||||
const offCtx = offscreen.getContext('2d');
|
||||
if (!offCtx) return;
|
||||
const smoothedData = smoothSpectrogramVertically(result.data, result.timeSteps, result.pitchBins);
|
||||
|
||||
const imgData = offCtx.createImageData(result.timeSteps, visiblePitchBins);
|
||||
const sourceCanvas = document.createElement('canvas');
|
||||
sourceCanvas.width = result.timeSteps;
|
||||
sourceCanvas.height = visiblePitchBins;
|
||||
const sourceCtx = sourceCanvas.getContext('2d');
|
||||
if (!sourceCtx) return;
|
||||
|
||||
const imgData = sourceCtx.createImageData(result.timeSteps, visiblePitchBins);
|
||||
const pixels = imgData.data;
|
||||
|
||||
for (let row = 0; row < visiblePitchBins; row++) {
|
||||
@@ -122,7 +145,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
const idx = (row * result.timeSteps + col) * 4;
|
||||
pixels[idx + 3] = 255; // always opaque
|
||||
|
||||
const raw = result.data[col * result.pitchBins + sourceRow];
|
||||
const raw = smoothedData[col * result.pitchBins + sourceRow];
|
||||
|
||||
// Hard threshold: values below noise floor → 0 (black)
|
||||
// Re-scale surviving range to [0,1] then apply power curve
|
||||
@@ -139,14 +162,24 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
}
|
||||
}
|
||||
|
||||
offCtx.putImageData(imgData, 0, 0);
|
||||
sourceCtx.putImageData(imgData, 0, 0);
|
||||
|
||||
// 2. Stretch onto the full canvas — browser bilinear filter smooths between bins.
|
||||
const verticallyScaledCanvas = document.createElement('canvas');
|
||||
verticallyScaledCanvas.width = result.timeSteps;
|
||||
verticallyScaledCanvas.height = canvasHeight;
|
||||
const verticallyScaledCtx = verticallyScaledCanvas.getContext('2d');
|
||||
if (!verticallyScaledCtx) return;
|
||||
|
||||
// Scale only along the pitch axis so note starts/stops stay crisp in time.
|
||||
verticallyScaledCtx.imageSmoothingEnabled = true;
|
||||
verticallyScaledCtx.imageSmoothingQuality = 'high';
|
||||
verticallyScaledCtx.drawImage(sourceCanvas, 0, 0, result.timeSteps, canvasHeight);
|
||||
|
||||
// 2. Stretch onto the full canvas horizontally without temporal blur.
|
||||
canvas.width = canvasWidth;
|
||||
canvas.height = canvasHeight;
|
||||
ctx.imageSmoothingEnabled = true;
|
||||
ctx.imageSmoothingQuality = 'high';
|
||||
ctx.drawImage(offscreen, 0, 0, canvasWidth, canvasHeight);
|
||||
ctx.imageSmoothingEnabled = false;
|
||||
ctx.drawImage(verticallyScaledCanvas, 0, 0, canvasWidth, canvasHeight);
|
||||
|
||||
// Store natural width and apply current zoom as CSS stretch (no pixel recompute on zoom)
|
||||
naturalWidthRef.current = canvasWidth;
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
frequencyToMidiPitch,
|
||||
getSpectrogramAnalysisResolution,
|
||||
getSpectrogramPitchBinCount,
|
||||
getSpectrogramVisibleBinRange,
|
||||
mapFrequencyToSpectrogramPosition,
|
||||
mapMidiPitchToSpectrogramPosition,
|
||||
normalizeSpectrogramHeightResolution,
|
||||
} from './spectrogramUtil';
|
||||
@@ -21,6 +24,12 @@ describe('spectrogramUtil', () => {
|
||||
expect(getSpectrogramPitchBinCount(5)).toBe(640);
|
||||
});
|
||||
|
||||
it('doubles the internal analysis resolution for each user-facing option', () => {
|
||||
expect(getSpectrogramAnalysisResolution(1)).toBe(2);
|
||||
expect(getSpectrogramAnalysisResolution(3)).toBe(6);
|
||||
expect(getSpectrogramAnalysisResolution(5)).toBe(10);
|
||||
});
|
||||
|
||||
it('maps MIDI pitches into full-range spectrogram positions', () => {
|
||||
expect(mapMidiPitchToSpectrogramPosition(0, 1)).toBe(0);
|
||||
expect(mapMidiPitchToSpectrogramPosition(12.5, 3)).toBe(38.5);
|
||||
@@ -29,6 +38,12 @@ describe('spectrogramUtil', () => {
|
||||
expect(mapMidiPitchToSpectrogramPosition(128, 3)).toBeNull();
|
||||
});
|
||||
|
||||
it('converts frequencies into MIDI pitch space and spectrogram positions', () => {
|
||||
expect(frequencyToMidiPitch(440)).toBeCloseTo(69, 6);
|
||||
expect(mapFrequencyToSpectrogramPosition(440, 5)).toBeCloseTo(347, 6);
|
||||
expect(frequencyToMidiPitch(0)).toBeNull();
|
||||
});
|
||||
|
||||
it('derives the visible C0-B7 subrange inside the full-resolution buffer', () => {
|
||||
expect(getSpectrogramVisibleBinRange(1)).toEqual({ start: 12, end: 108 });
|
||||
expect(getSpectrogramVisibleBinRange(3)).toEqual({ start: 36, end: 324 });
|
||||
|
||||
@@ -6,6 +6,7 @@ export const SPECTROGRAM_FULL_SEMITONES =
|
||||
SPECTROGRAM_FULL_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1;
|
||||
export const SPECTROGRAM_VISIBLE_SEMITONES =
|
||||
SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_MIN_MIDI_PITCH + 1;
|
||||
export const SPECTROGRAM_ANALYSIS_RESOLUTION_MULTIPLIER = 2;
|
||||
|
||||
export type SpectrogramHeightResolution = 1 | 3 | 5;
|
||||
|
||||
@@ -19,13 +20,13 @@ export function normalizeSpectrogramHeightResolution(value: unknown): Spectrogra
|
||||
}
|
||||
|
||||
export function getSpectrogramPitchBinCount(
|
||||
resolution: SpectrogramHeightResolution,
|
||||
resolution: number,
|
||||
): number {
|
||||
return SPECTROGRAM_FULL_SEMITONES * resolution;
|
||||
}
|
||||
|
||||
export function getSpectrogramVisibleBinRange(
|
||||
resolution: SpectrogramHeightResolution,
|
||||
resolution: number,
|
||||
): { start: number; end: number } {
|
||||
return {
|
||||
start: (SPECTROGRAM_MIN_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH) * resolution,
|
||||
@@ -33,9 +34,15 @@ export function getSpectrogramVisibleBinRange(
|
||||
};
|
||||
}
|
||||
|
||||
export function getSpectrogramAnalysisResolution(
|
||||
resolution: SpectrogramHeightResolution,
|
||||
): number {
|
||||
return resolution * SPECTROGRAM_ANALYSIS_RESOLUTION_MULTIPLIER;
|
||||
}
|
||||
|
||||
export function mapMidiPitchToSpectrogramPosition(
|
||||
midiPitch: number,
|
||||
resolution: SpectrogramHeightResolution,
|
||||
resolution: number,
|
||||
): number | null {
|
||||
const pitchOffset = midiPitch - SPECTROGRAM_FULL_MIN_MIDI_PITCH;
|
||||
if (pitchOffset < 0 || pitchOffset > SPECTROGRAM_FULL_MAX_MIDI_PITCH) {
|
||||
@@ -48,3 +55,21 @@ export function mapMidiPitchToSpectrogramPosition(
|
||||
const maxBin = getSpectrogramPitchBinCount(resolution) - 1;
|
||||
return Math.max(0, Math.min(maxBin, scaled));
|
||||
}
|
||||
|
||||
export function frequencyToMidiPitch(frequency: number): number | null {
|
||||
if (!Number.isFinite(frequency) || frequency <= 0) {
|
||||
return null;
|
||||
}
|
||||
return 69 + 12 * Math.log2(frequency / 440);
|
||||
}
|
||||
|
||||
export function mapFrequencyToSpectrogramPosition(
|
||||
frequency: number,
|
||||
resolution: number,
|
||||
): number | null {
|
||||
const midiPitch = frequencyToMidiPitch(frequency);
|
||||
if (midiPitch === null) {
|
||||
return null;
|
||||
}
|
||||
return mapMidiPitchToSpectrogramPosition(midiPitch, resolution);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { getSpectrogramPitchBinCount, mapFrequencyToSpectrogramPosition } from '../util/spectrogramUtil';
|
||||
import {
|
||||
estimatePeakFrequency,
|
||||
getFrequencyBandEdges,
|
||||
paintMagnitudeAcrossPitchSpan,
|
||||
} from './spectrogramWorker';
|
||||
|
||||
const FFT_SIZE = 16384;
|
||||
const SAMPLE_RATE = 44100;
|
||||
const HZ_PER_BIN = SAMPLE_RATE / FFT_SIZE;
|
||||
|
||||
function paintSpanForFrequency(
|
||||
centerFrequency: number,
|
||||
resolution: 1 | 3 | 5,
|
||||
pitchBins = getSpectrogramPitchBinCount(resolution),
|
||||
): Float32Array {
|
||||
const row = new Float32Array(pitchBins);
|
||||
const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, HZ_PER_BIN);
|
||||
const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, resolution);
|
||||
const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, resolution);
|
||||
const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, resolution);
|
||||
|
||||
if (startPosition === null || endPosition === null || centerPosition === null) {
|
||||
throw new Error('Expected mapped pitch positions for test frequency span');
|
||||
}
|
||||
|
||||
paintMagnitudeAcrossPitchSpan(
|
||||
row,
|
||||
0,
|
||||
pitchBins,
|
||||
Math.min(startPosition, endPosition),
|
||||
Math.max(startPosition, endPosition),
|
||||
centerPosition,
|
||||
1,
|
||||
);
|
||||
return row;
|
||||
}
|
||||
|
||||
describe('spectrogramWorker helpers', () => {
|
||||
it('keeps adjacent low-register FFT bins continuous at 5x resolution', () => {
|
||||
const pitchBins = getSpectrogramPitchBinCount(5);
|
||||
const row = new Float32Array(pitchBins);
|
||||
const binsAroundA1 = [10, 11];
|
||||
|
||||
for (const bin of binsAroundA1) {
|
||||
const centerFrequency = bin * HZ_PER_BIN;
|
||||
const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, HZ_PER_BIN);
|
||||
const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, 5);
|
||||
const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, 5);
|
||||
const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, 5);
|
||||
|
||||
expect(startPosition).not.toBeNull();
|
||||
expect(endPosition).not.toBeNull();
|
||||
expect(centerPosition).not.toBeNull();
|
||||
|
||||
paintMagnitudeAcrossPitchSpan(
|
||||
row,
|
||||
0,
|
||||
pitchBins,
|
||||
Math.min(startPosition!, endPosition!),
|
||||
Math.max(startPosition!, endPosition!),
|
||||
centerPosition!,
|
||||
1,
|
||||
);
|
||||
}
|
||||
|
||||
const nonZeroBins = [...row.entries()].filter(([, value]) => value > 0).map(([index]) => index);
|
||||
expect(nonZeroBins.length).toBeGreaterThan(0);
|
||||
|
||||
for (let index = nonZeroBins[0]; index <= nonZeroBins[nonZeroBins.length - 1]; index++) {
|
||||
expect(row[index]).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('uses sub-bin interpolation to move the ridge closer to the true peak frequency', () => {
|
||||
const magnitudes = new Float32Array([0, 0.3, 1.0, 0.82, 0.1, 0]);
|
||||
const rawCenterFrequency = 2 * HZ_PER_BIN;
|
||||
const interpolatedFrequency = estimatePeakFrequency(2, HZ_PER_BIN, magnitudes);
|
||||
const expectedFrequency = (2.35 * HZ_PER_BIN);
|
||||
|
||||
expect(Math.abs(interpolatedFrequency - expectedFrequency)).toBeLessThan(
|
||||
Math.abs(rawCenterFrequency - expectedFrequency),
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves non-peak bins at their raw FFT center while still painting a span', () => {
|
||||
const magnitudes = new Float32Array([0, 0.2, 0.4, 1.0, 0.8, 0.7, 0.2, 0]);
|
||||
const centerFrequency = estimatePeakFrequency(4, HZ_PER_BIN, magnitudes);
|
||||
expect(centerFrequency).toBeCloseTo(4 * HZ_PER_BIN, 6);
|
||||
|
||||
const row = paintSpanForFrequency(centerFrequency, 3);
|
||||
expect(row.some(value => value > 0)).toBe(true);
|
||||
});
|
||||
|
||||
it('maps edge-band spans into the valid spectrogram range without spilling outside the buffer', () => {
|
||||
const lowRow = paintSpanForFrequency(20.5, 5);
|
||||
const lowNonZeroBins = [...lowRow.entries()].filter(([, value]) => value > 0).map(([index]) => index);
|
||||
expect(lowNonZeroBins.length).toBeGreaterThan(0);
|
||||
expect(lowNonZeroBins[0]).toBeGreaterThanOrEqual(0);
|
||||
|
||||
const highRow = paintSpanForFrequency(12530, 5);
|
||||
const highNonZeroBins = [...highRow.entries()].filter(([, value]) => value > 0).map(([index]) => index);
|
||||
expect(highNonZeroBins.length).toBeGreaterThan(0);
|
||||
expect(highNonZeroBins[highNonZeroBins.length - 1]).toBeLessThan(highRow.length);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,8 @@
|
||||
import FFT from 'fft.js';
|
||||
import {
|
||||
getSpectrogramAnalysisResolution,
|
||||
getSpectrogramPitchBinCount,
|
||||
mapMidiPitchToSpectrogramPosition,
|
||||
mapFrequencyToSpectrogramPosition,
|
||||
normalizeSpectrogramHeightResolution,
|
||||
type SpectrogramHeightResolution,
|
||||
} from '../util/spectrogramUtil';
|
||||
@@ -11,7 +12,7 @@ type WorkerScopeLike = typeof globalThis & {
|
||||
postMessage: (message: SpectrogramResult, transfer: Transferable[]) => void;
|
||||
};
|
||||
|
||||
const workerScope = self as WorkerScopeLike;
|
||||
const workerScope = typeof self === 'undefined' ? null : self as WorkerScopeLike;
|
||||
|
||||
export interface SpectrogramRequest {
|
||||
pcm: Float32Array;
|
||||
@@ -28,8 +29,10 @@ export interface SpectrogramResult {
|
||||
pitchBins: number;
|
||||
}
|
||||
|
||||
const FFT_SIZE = 8192;
|
||||
const FFT_SIZE = 16384;
|
||||
const HOP_SIZE = 1024;
|
||||
const MIN_SPECTROGRAM_FREQUENCY = 20;
|
||||
const MAX_SPECTROGRAM_FREQUENCY = 20000;
|
||||
|
||||
function hannWindow(size: number): Float32Array {
|
||||
const w = new Float32Array(size);
|
||||
@@ -39,24 +42,98 @@ function hannWindow(size: number): Float32Array {
|
||||
return w;
|
||||
}
|
||||
|
||||
function spreadMagnitudeAcrossPitchBins(
|
||||
export function getFrequencyBandEdges(
|
||||
centerFrequency: number,
|
||||
hzPerBin: number,
|
||||
): { lowerFrequency: number; upperFrequency: number } {
|
||||
const halfBandwidth = hzPerBin / 2;
|
||||
return {
|
||||
lowerFrequency: Math.max(Number.EPSILON, centerFrequency - halfBandwidth),
|
||||
upperFrequency: centerFrequency + halfBandwidth,
|
||||
};
|
||||
}
|
||||
|
||||
export function estimateParabolicPeakOffset(
|
||||
leftMagnitude: number,
|
||||
centerMagnitude: number,
|
||||
rightMagnitude: number,
|
||||
): number {
|
||||
const denominator = leftMagnitude - 2 * centerMagnitude + rightMagnitude;
|
||||
if (!Number.isFinite(denominator) || Math.abs(denominator) < Number.EPSILON) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const offset = 0.5 * (leftMagnitude - rightMagnitude) / denominator;
|
||||
return Math.max(-0.5, Math.min(0.5, offset));
|
||||
}
|
||||
|
||||
export function estimatePeakFrequency(
|
||||
bin: number,
|
||||
hzPerBin: number,
|
||||
magnitudes: Float32Array,
|
||||
): number {
|
||||
const centerFrequency = bin * hzPerBin;
|
||||
if (bin <= 1 || bin >= magnitudes.length - 1) {
|
||||
return centerFrequency;
|
||||
}
|
||||
|
||||
const leftMagnitude = magnitudes[bin - 1];
|
||||
const centerMagnitude = magnitudes[bin];
|
||||
const rightMagnitude = magnitudes[bin + 1];
|
||||
|
||||
if (centerMagnitude <= leftMagnitude || centerMagnitude <= rightMagnitude) {
|
||||
return centerFrequency;
|
||||
}
|
||||
|
||||
return (bin + estimateParabolicPeakOffset(leftMagnitude, centerMagnitude, rightMagnitude)) * hzPerBin;
|
||||
}
|
||||
|
||||
function getPitchSpanBounds(
|
||||
lowerFrequency: number,
|
||||
upperFrequency: number,
|
||||
centerFrequency: number,
|
||||
analysisResolution: number,
|
||||
): { startPosition: number; endPosition: number; centerPosition: number } | null {
|
||||
const startPosition = mapFrequencyToSpectrogramPosition(lowerFrequency, analysisResolution);
|
||||
const endPosition = mapFrequencyToSpectrogramPosition(upperFrequency, analysisResolution);
|
||||
const centerPosition = mapFrequencyToSpectrogramPosition(centerFrequency, analysisResolution);
|
||||
|
||||
if (startPosition === null || endPosition === null || centerPosition === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
startPosition: Math.min(startPosition, endPosition),
|
||||
endPosition: Math.max(startPosition, endPosition),
|
||||
centerPosition,
|
||||
};
|
||||
}
|
||||
|
||||
export function paintMagnitudeAcrossPitchSpan(
|
||||
result: Float32Array,
|
||||
pitchRow: number,
|
||||
pitchBins: number,
|
||||
pitchPosition: number,
|
||||
startPosition: number,
|
||||
endPosition: number,
|
||||
centerPosition: number,
|
||||
magnitude: number,
|
||||
heightResolution: SpectrogramHeightResolution,
|
||||
): void {
|
||||
const spreadRadius = Math.max(0, heightResolution - 1);
|
||||
const centerBin = Math.round(pitchPosition);
|
||||
const startBin = Math.max(0, centerBin - spreadRadius);
|
||||
const endBin = Math.min(pitchBins - 1, centerBin + spreadRadius);
|
||||
const spreadWidth = spreadRadius + 1;
|
||||
const clampedStart = Math.max(0, Math.min(startPosition, pitchBins - 1));
|
||||
const clampedEnd = Math.max(0, Math.min(endPosition, pitchBins - 1));
|
||||
const startBin = Math.max(0, Math.floor(clampedStart));
|
||||
const endBin = Math.min(pitchBins - 1, Math.ceil(clampedEnd));
|
||||
const spanWidth = Math.max(clampedEnd - clampedStart, 1);
|
||||
|
||||
for (let targetBin = startBin; targetBin <= endBin; targetBin++) {
|
||||
const distance = Math.abs(targetBin - pitchPosition);
|
||||
const weight = Math.max(0, 1 - distance / spreadWidth);
|
||||
if (weight <= 0) continue;
|
||||
const cellStart = targetBin - 0.5;
|
||||
const cellEnd = targetBin + 0.5;
|
||||
const overlap = Math.max(0, Math.min(clampedEnd, cellEnd) - Math.max(clampedStart, cellStart));
|
||||
if (overlap <= 0) continue;
|
||||
|
||||
const overlapWeight = Math.min(1, overlap);
|
||||
const centerDistance = Math.abs(targetBin - centerPosition);
|
||||
const centerWeight = Math.max(0, 1 - centerDistance / (spanWidth + 1));
|
||||
const weight = Math.max(overlapWeight * (0.6 + 0.4 * centerWeight), overlapWeight * 0.35);
|
||||
|
||||
const weightedMagnitude = magnitude * weight;
|
||||
const resultIndex = pitchRow + targetBin;
|
||||
@@ -66,7 +143,7 @@ function spreadMagnitudeAcrossPitchBins(
|
||||
}
|
||||
}
|
||||
|
||||
workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||
function handleSpectrogramRequest(e: MessageEvent<SpectrogramRequest>, scope: WorkerScopeLike): void {
|
||||
const {
|
||||
pcm,
|
||||
sampleRate,
|
||||
@@ -75,7 +152,8 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||
heightResolution,
|
||||
} = e.data;
|
||||
const normalizedResolution = normalizeSpectrogramHeightResolution(heightResolution);
|
||||
const pitchBins = getSpectrogramPitchBinCount(normalizedResolution);
|
||||
const analysisResolution = getSpectrogramAnalysisResolution(normalizedResolution);
|
||||
const pitchBins = getSpectrogramPitchBinCount(analysisResolution);
|
||||
|
||||
const startSample = Math.floor(clipStartOffsetSeconds * sampleRate);
|
||||
const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate));
|
||||
@@ -85,6 +163,7 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||
const hann = hannWindow(FFT_SIZE);
|
||||
const complexOut = fft.createComplexArray() as number[];
|
||||
const inputPadded = new Float32Array(FFT_SIZE);
|
||||
const hzPerBin = sampleRate / FFT_SIZE;
|
||||
|
||||
const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1);
|
||||
const result = new Float32Array(totalHops * pitchBins);
|
||||
@@ -108,25 +187,44 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||
// rather than summing. Summing inflates every bin by how many FFT bins land there.
|
||||
const pitchRow = hop * pitchBins;
|
||||
const numBins = FFT_SIZE / 2;
|
||||
const magnitudes = new Float32Array(numBins);
|
||||
|
||||
for (let bin = 1; bin < numBins; bin++) {
|
||||
const freq = (bin * sampleRate) / FFT_SIZE;
|
||||
if (freq < 20 || freq > 20000) continue;
|
||||
|
||||
const midiPitch = 69 + 12 * Math.log2(freq / 440);
|
||||
const pitchPosition = mapMidiPitchToSpectrogramPosition(midiPitch, normalizedResolution);
|
||||
if (pitchPosition === null) continue;
|
||||
|
||||
const re = complexOut[2 * bin];
|
||||
const im = complexOut[2 * bin + 1];
|
||||
const magnitude = Math.sqrt(re * re + im * im);
|
||||
spreadMagnitudeAcrossPitchBins(
|
||||
magnitudes[bin] = Math.sqrt(re * re + im * im);
|
||||
}
|
||||
|
||||
for (let bin = 1; bin < numBins; bin++) {
|
||||
const centerFrequency = estimatePeakFrequency(bin, hzPerBin, magnitudes);
|
||||
const { lowerFrequency, upperFrequency } = getFrequencyBandEdges(centerFrequency, hzPerBin);
|
||||
if (upperFrequency < MIN_SPECTROGRAM_FREQUENCY || lowerFrequency > MAX_SPECTROGRAM_FREQUENCY) continue;
|
||||
|
||||
const clampedLowerFrequency = Math.max(lowerFrequency, MIN_SPECTROGRAM_FREQUENCY);
|
||||
const clampedUpperFrequency = Math.min(upperFrequency, MAX_SPECTROGRAM_FREQUENCY);
|
||||
const clampedCenterFrequency = Math.min(
|
||||
clampedUpperFrequency,
|
||||
Math.max(clampedLowerFrequency, centerFrequency),
|
||||
);
|
||||
const pitchSpan = getPitchSpanBounds(
|
||||
clampedLowerFrequency,
|
||||
clampedUpperFrequency,
|
||||
clampedCenterFrequency,
|
||||
analysisResolution,
|
||||
);
|
||||
if (!pitchSpan) continue;
|
||||
|
||||
const magnitude = magnitudes[bin];
|
||||
if (magnitude <= 0) continue;
|
||||
|
||||
paintMagnitudeAcrossPitchSpan(
|
||||
result,
|
||||
pitchRow,
|
||||
pitchBins,
|
||||
pitchPosition,
|
||||
pitchSpan.startPosition,
|
||||
pitchSpan.endPosition,
|
||||
pitchSpan.centerPosition,
|
||||
magnitude,
|
||||
normalizedResolution,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -145,5 +243,11 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||
}
|
||||
|
||||
const response: SpectrogramResult = { data: result, timeSteps: totalHops, pitchBins };
|
||||
workerScope.postMessage(response, [result.buffer]);
|
||||
};
|
||||
scope.postMessage(response, [result.buffer]);
|
||||
}
|
||||
|
||||
if (workerScope) {
|
||||
workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||
handleSpectrogramRequest(e, workerScope);
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user