feat: added an option to let user choose the vertical resolution of the spectrogram, also fixed a bug that peak of pitch in the spectrogram has been mapped to the start of the note bin instead of the center
This commit is contained in:
+2
-1
@@ -66,7 +66,8 @@
|
||||
}
|
||||
},
|
||||
"editor": {
|
||||
"playhead_update_frequency": 10
|
||||
"playhead_update_frequency": 10,
|
||||
"spectrogram_height_resolution": 3
|
||||
},
|
||||
"chatbox": {
|
||||
"default_open": true
|
||||
|
||||
@@ -8,6 +8,7 @@ import type { KeySignature } from '../../core/KGProject';
|
||||
import { KGPianoRollState } from '../../core/state/KGPianoRollState';
|
||||
import SpectrogramCanvas from './SpectrogramCanvas';
|
||||
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||
import type { SpectrogramHeightResolution } from '../../util/spectrogramUtil';
|
||||
|
||||
interface PianoGridProps {
|
||||
gridRef: MutableRefObject<HTMLDivElement | null>;
|
||||
@@ -32,6 +33,7 @@ interface PianoGridProps {
|
||||
bpm?: number;
|
||||
spectrogramThresholdDb?: number;
|
||||
spectrogramPower?: number;
|
||||
spectrogramHeightResolution?: SpectrogramHeightResolution;
|
||||
pianoRollZoom?: number;
|
||||
mode?: 'midi-edit' | 'spectrogram' | 'hybrid';
|
||||
onSpectrogramLoadingChange?: (loading: boolean) => void;
|
||||
@@ -62,6 +64,7 @@ const PianoGrid: React.FC<PianoGridProps> = ({
|
||||
bpm = 120,
|
||||
spectrogramThresholdDb = -25,
|
||||
spectrogramPower = 0.5,
|
||||
spectrogramHeightResolution = 3,
|
||||
pianoRollZoom = 1,
|
||||
onSpectrogramLoadingChange,
|
||||
}) => {
|
||||
@@ -245,6 +248,7 @@ const PianoGrid: React.FC<PianoGridProps> = ({
|
||||
thresholdDb={spectrogramThresholdDb}
|
||||
power={spectrogramPower}
|
||||
zoom={pianoRollZoom}
|
||||
heightResolution={spectrogramHeightResolution}
|
||||
onLoadingChange={onSpectrogramLoadingChange}
|
||||
/>
|
||||
)}
|
||||
|
||||
@@ -18,6 +18,10 @@ import { beatsToBar } from '../../util/midiUtil';
|
||||
import { UpdateRegionCommand } from '../../core/commands';
|
||||
import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil';
|
||||
import { showAlert, showPrompt } from '../../util/dialogUtil';
|
||||
import {
|
||||
normalizeSpectrogramHeightResolution,
|
||||
type SpectrogramHeightResolution,
|
||||
} from '../../util/spectrogramUtil';
|
||||
|
||||
interface PianoRollProps {
|
||||
onClose: () => void;
|
||||
@@ -50,6 +54,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
// Spectrogram controls (only used in spectrogram mode)
|
||||
const [spectrogramThresholdDb, setSpectrogramThresholdDb] = useState<number>(-25);
|
||||
const [spectrogramPower, setSpectrogramPower] = useState<number>(0.5);
|
||||
const [spectrogramHeightResolution, setSpectrogramHeightResolution] =
|
||||
useState<SpectrogramHeightResolution>(3);
|
||||
|
||||
// Piano roll zoom (1x–8x); updates --region-grid-beat-width CSS variable
|
||||
const [pianoRollZoom, setPianoRollZoom] = useState<number>(1);
|
||||
@@ -201,6 +207,41 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
}
|
||||
}, []); // Empty dependency array means this runs once on mount
|
||||
|
||||
useEffect(() => {
|
||||
let unsubscribe: (() => void) | undefined;
|
||||
|
||||
const initializeSpectrogramHeightResolution = async () => {
|
||||
const configManager = ConfigManager.instance();
|
||||
if (!configManager.getIsInitialized()) {
|
||||
await configManager.initialize();
|
||||
}
|
||||
|
||||
const applyResolutionFromConfig = () => {
|
||||
setSpectrogramHeightResolution(
|
||||
normalizeSpectrogramHeightResolution(
|
||||
configManager.get('editor.spectrogram_height_resolution')
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
applyResolutionFromConfig();
|
||||
unsubscribe = configManager.addChangeListener((changedKeys) => {
|
||||
if (
|
||||
changedKeys.includes('__all__') ||
|
||||
changedKeys.includes('editor.spectrogram_height_resolution')
|
||||
) {
|
||||
applyResolutionFromConfig();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
void initializeSpectrogramHeightResolution();
|
||||
|
||||
return () => {
|
||||
unsubscribe?.();
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Add keyboard event listener for Escape
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (event: KeyboardEvent) => {
|
||||
@@ -1048,6 +1089,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
bpm={bpm}
|
||||
spectrogramThresholdDb={spectrogramThresholdDb}
|
||||
spectrogramPower={spectrogramPower}
|
||||
spectrogramHeightResolution={spectrogramHeightResolution}
|
||||
pianoRollZoom={pianoRollZoom}
|
||||
/>
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import { useNoteSelection } from '../../hooks/useNoteSelection';
|
||||
import type { KeySignature } from '../../core/KGProject';
|
||||
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||
import { velocityToColor } from '../../util/velocityColor';
|
||||
import type { SpectrogramHeightResolution } from '../../util/spectrogramUtil';
|
||||
|
||||
interface PianoRollContentProps {
|
||||
contentRef: React.MutableRefObject<HTMLDivElement | null>;
|
||||
@@ -35,6 +36,7 @@ interface PianoRollContentProps {
|
||||
bpm?: number;
|
||||
spectrogramThresholdDb?: number;
|
||||
spectrogramPower?: number;
|
||||
spectrogramHeightResolution?: SpectrogramHeightResolution;
|
||||
pianoRollZoom?: number;
|
||||
}
|
||||
|
||||
@@ -58,6 +60,7 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
bpm = 120,
|
||||
spectrogramThresholdDb = -25,
|
||||
spectrogramPower = 0.5,
|
||||
spectrogramHeightResolution = 3,
|
||||
pianoRollZoom = 1,
|
||||
}) => {
|
||||
const isSpectrogram = mode === 'spectrogram';
|
||||
@@ -295,6 +298,7 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
bpm={bpm}
|
||||
spectrogramThresholdDb={spectrogramThresholdDb}
|
||||
spectrogramPower={spectrogramPower}
|
||||
spectrogramHeightResolution={spectrogramHeightResolution}
|
||||
pianoRollZoom={pianoRollZoom}
|
||||
onSpectrogramLoadingChange={handleSpectrogramLoadingChange}
|
||||
>
|
||||
|
||||
@@ -4,6 +4,12 @@ import { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
|
||||
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
|
||||
import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker';
|
||||
import {
|
||||
getSpectrogramVisibleBinRange,
|
||||
normalizeSpectrogramHeightResolution,
|
||||
SPECTROGRAM_VISIBLE_SEMITONES,
|
||||
type SpectrogramHeightResolution,
|
||||
} from '../../util/spectrogramUtil';
|
||||
|
||||
interface SpectrogramCanvasProps {
|
||||
audioRegion: KGAudioRegion;
|
||||
@@ -13,12 +19,10 @@ interface SpectrogramCanvasProps {
|
||||
thresholdDb: number;
|
||||
power: number;
|
||||
zoom: number;
|
||||
heightResolution: SpectrogramHeightResolution;
|
||||
onLoadingChange?: (loading: boolean) => void;
|
||||
}
|
||||
|
||||
const PITCH_BINS = 128;
|
||||
const HOP_SIZE = 1024;
|
||||
|
||||
// Piecewise-linear RGB colormap: black → dark blue → blue → purple → red → orange → yellow
|
||||
// Hue rotates 240°→300°→0°→60°, bypassing green entirely.
|
||||
const COLORMAP_STOPS: Array<[number, [number, number, number]]> = [
|
||||
@@ -56,6 +60,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
thresholdDb,
|
||||
power,
|
||||
zoom,
|
||||
heightResolution,
|
||||
onLoadingChange,
|
||||
}) => {
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
@@ -80,6 +85,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
regionDurationSeconds: number,
|
||||
thresholdDb: number,
|
||||
power: number,
|
||||
heightResolution: SpectrogramHeightResolution,
|
||||
) => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
@@ -92,31 +98,31 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
const totalBeats = (regionDurationSeconds * bpm) / 60;
|
||||
// Always draw at 1x resolution; zoom is applied as CSS width stretch
|
||||
const canvasWidth = Math.ceil(totalBeats * 40);
|
||||
const canvasHeight = PITCH_BINS * noteHeight;
|
||||
const canvasHeight = SPECTROGRAM_VISIBLE_SEMITONES * noteHeight;
|
||||
|
||||
// Convert dB threshold to linear: values below this → black
|
||||
const linearThreshold = Math.pow(10, thresholdDb / 20);
|
||||
const visibleRange = getSpectrogramVisibleBinRange(heightResolution);
|
||||
const visiblePitchBins = visibleRange.end - visibleRange.start;
|
||||
|
||||
// 1. Paint at natural STFT resolution onto an offscreen canvas (timeSteps × 128).
|
||||
// Row i = pitchIndex i = pitch (107 − i). Apply threshold then power curve.
|
||||
// 1. Paint at natural spectrogram resolution onto an offscreen canvas.
|
||||
// Result data is low-to-high pitch; draw only the visible C0-B7 window, reversed for display.
|
||||
const offscreen = document.createElement('canvas');
|
||||
offscreen.width = result.timeSteps;
|
||||
offscreen.height = PITCH_BINS;
|
||||
offscreen.height = visiblePitchBins;
|
||||
const offCtx = offscreen.getContext('2d');
|
||||
if (!offCtx) return;
|
||||
|
||||
const imgData = offCtx.createImageData(result.timeSteps, PITCH_BINS);
|
||||
const imgData = offCtx.createImageData(result.timeSteps, visiblePitchBins);
|
||||
const pixels = imgData.data;
|
||||
|
||||
for (let row = 0; row < PITCH_BINS; row++) {
|
||||
const pitch = 107 - row;
|
||||
for (let row = 0; row < visiblePitchBins; row++) {
|
||||
const sourceRow = visibleRange.end - 1 - row;
|
||||
for (let col = 0; col < result.timeSteps; col++) {
|
||||
const idx = (row * result.timeSteps + col) * 4;
|
||||
pixels[idx + 3] = 255; // always opaque
|
||||
|
||||
if (pitch < 12 || pitch > 107) continue; // outside range → black
|
||||
|
||||
const raw = result.data[col * PITCH_BINS + pitch];
|
||||
const raw = result.data[col * result.pitchBins + sourceRow];
|
||||
|
||||
// Hard threshold: values below noise floor → 0 (black)
|
||||
// Re-scale surviving range to [0,1] then apply power curve
|
||||
@@ -157,9 +163,10 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
regionDurationRef.current,
|
||||
thresholdDb,
|
||||
power,
|
||||
normalizeSpectrogramHeightResolution(heightResolution),
|
||||
);
|
||||
}
|
||||
}, [thresholdDb, power, renderSpectrogram]);
|
||||
}, [thresholdDb, power, renderSpectrogram, heightResolution]);
|
||||
|
||||
// Zoom changes: stretch width only, pin height to canvas pixel height
|
||||
useEffect(() => {
|
||||
@@ -211,7 +218,14 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
rawResultRef.current = e.data;
|
||||
sampleRateRef.current = sampleRate;
|
||||
regionDurationRef.current = regionDurationSeconds;
|
||||
renderSpectrogram(e.data, sampleRate, regionDurationSeconds, thresholdDb, power);
|
||||
renderSpectrogram(
|
||||
e.data,
|
||||
sampleRate,
|
||||
regionDurationSeconds,
|
||||
thresholdDb,
|
||||
power,
|
||||
normalizeSpectrogramHeightResolution(heightResolution),
|
||||
);
|
||||
setLoading(false);
|
||||
worker.terminate();
|
||||
workerRef.current = null;
|
||||
@@ -223,6 +237,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
|
||||
regionDurationSeconds,
|
||||
bpm,
|
||||
heightResolution: normalizeSpectrogramHeightResolution(heightResolution),
|
||||
};
|
||||
worker.postMessage(request, [request.pcm.buffer]);
|
||||
} catch (err) {
|
||||
@@ -241,7 +256,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
workerRef.current = null;
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [audioRegion, trackId, projectName, bpm]);
|
||||
}, [audioRegion, trackId, projectName, bpm, heightResolution]);
|
||||
|
||||
return (
|
||||
<canvas
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { ConfigManager } from '../../../core/config/ConfigManager';
|
||||
import { KGAudioInterface } from '../../../core/audio-interface/KGAudioInterface';
|
||||
import {
|
||||
normalizeSpectrogramHeightResolution,
|
||||
type SpectrogramHeightResolution,
|
||||
} from '../../../util/spectrogramUtil';
|
||||
|
||||
const BehaviorSettings: React.FC = () => {
|
||||
const [playheadUpdateFrequency, setPlayheadUpdateFrequency] = useState<number>(10);
|
||||
const [spectrogramHeightResolution, setSpectrogramHeightResolution] = useState<SpectrogramHeightResolution>(3);
|
||||
const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState<boolean>(true);
|
||||
const [audioLookaheadTime, setAudioLookaheadTime] = useState<string>('50');
|
||||
const [playbackDelay, setPlaybackDelay] = useState<string>('200');
|
||||
@@ -23,6 +28,9 @@ const BehaviorSettings: React.FC = () => {
|
||||
}
|
||||
|
||||
setPlayheadUpdateFrequency((configManager.get('editor.playhead_update_frequency') as number) ?? 10);
|
||||
setSpectrogramHeightResolution(
|
||||
normalizeSpectrogramHeightResolution(configManager.get('editor.spectrogram_height_resolution'))
|
||||
);
|
||||
setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true);
|
||||
const lookaheadTimeSeconds = (configManager.get('audio.lookahead_time') as number) ?? 0.05;
|
||||
setAudioLookaheadTime(((lookaheadTimeSeconds * 1000).toFixed(0)));
|
||||
@@ -50,6 +58,12 @@ const BehaviorSettings: React.FC = () => {
|
||||
await configManager.set('chatbox.default_open', boolValue);
|
||||
};
|
||||
|
||||
const handleSpectrogramHeightResolutionChange = async (value: string) => {
|
||||
const nextValue = normalizeSpectrogramHeightResolution(parseInt(value, 10));
|
||||
setSpectrogramHeightResolution(nextValue);
|
||||
await configManager.set('editor.spectrogram_height_resolution', nextValue);
|
||||
};
|
||||
|
||||
const handleAudioLookaheadTimeChange = async (value: string) => {
|
||||
// Allow empty string, treat as 0 ms
|
||||
const numValueMs = value === '' ? 0 : parseFloat(value);
|
||||
@@ -161,6 +175,24 @@ const BehaviorSettings: React.FC = () => {
|
||||
Update frequency for the playhead animation during playback. Higher values (60 fps) provide smoother animation but use more CPU. Lower values (10 fps) are more efficient. Changes apply immediately without restart.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="settings-item">
|
||||
<label className="settings-label">
|
||||
Spectrogram Height Resolution
|
||||
</label>
|
||||
<select
|
||||
className="settings-select"
|
||||
value={spectrogramHeightResolution}
|
||||
onChange={(e) => handleSpectrogramHeightResolutionChange(e.target.value)}
|
||||
>
|
||||
<option value="1">1x</option>
|
||||
<option value="3">3x</option>
|
||||
<option value="5">5x</option>
|
||||
</select>
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
|
||||
Controls vertical spectrogram detail across the visible pitch range. Higher values sharpen pitch contours but use more CPU and memory while computing the spectrogram.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="settings-group">
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { encodeWav } from './KGOfflineRenderer';
|
||||
import { encodeWav, getOfflineTrackGain, getOfflineTrackVolumeDb } from './KGOfflineRenderer';
|
||||
|
||||
/**
|
||||
* Create a minimal AudioBuffer-like object for testing.
|
||||
@@ -145,3 +145,22 @@ describe('encodeWav', () => {
|
||||
expect(view.getUint32(40, true)).toBe(0); // data size = 0
|
||||
});
|
||||
});
|
||||
|
||||
describe('offline track volume conversion', () => {
|
||||
it('treats 0 dB as unity gain instead of silence', () => {
|
||||
expect(getOfflineTrackVolumeDb(0, false)).toBe(0);
|
||||
expect(getOfflineTrackGain(0, false)).toBe(1);
|
||||
});
|
||||
|
||||
it('converts negative dB values to linear gain', () => {
|
||||
expect(getOfflineTrackVolumeDb(-6, false)).toBe(-6);
|
||||
expect(getOfflineTrackGain(-6, false)).toBeCloseTo(Math.pow(10, -6 / 20), 6);
|
||||
});
|
||||
|
||||
it('silences muted tracks and floor-level volumes', () => {
|
||||
expect(getOfflineTrackVolumeDb(0, true)).toBe(-Infinity);
|
||||
expect(getOfflineTrackGain(0, true)).toBe(0);
|
||||
expect(getOfflineTrackVolumeDb(-60, false)).toBe(-Infinity);
|
||||
expect(getOfflineTrackGain(-60, false)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { KGProject } from '../KGProject';
|
||||
import type { KGMidiNote } from '../midi/KGMidiNote';
|
||||
import type { KGAudioRegion } from '../region/KGAudioRegion';
|
||||
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
|
||||
import { AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants';
|
||||
import { pitchToNoteNameString } from '../../util/midiUtil';
|
||||
import { KGToneBuffersPool } from './KGToneBuffersPool';
|
||||
import { KGToneSamplerFactory } from './KGToneSamplerFactory';
|
||||
@@ -254,9 +255,8 @@ export class KGOfflineRenderer {
|
||||
});
|
||||
});
|
||||
|
||||
// Apply volume
|
||||
const volumeDb = trackInfo.volume > 0 ? 20 * Math.log10(trackInfo.volume) : -Infinity;
|
||||
sampler.volume.value = volumeDb;
|
||||
// Track volumes are stored in dB across the app, with 0 meaning unity gain.
|
||||
sampler.volume.value = getOfflineTrackVolumeDb(trackInfo.volume, trackInfo.muted);
|
||||
sampler.connect(masterGain);
|
||||
|
||||
// Schedule all notes for this track
|
||||
@@ -288,7 +288,7 @@ export class KGOfflineRenderer {
|
||||
for (const trackInfo of audioTrackData) {
|
||||
if (!shouldPlay(trackInfo, hasSoloedTracks)) continue;
|
||||
|
||||
const trackGain = new Tone.Gain(trackInfo.volume);
|
||||
const trackGain = new Tone.Gain(getOfflineTrackGain(trackInfo.volume, trackInfo.muted));
|
||||
trackGain.connect(masterGain);
|
||||
|
||||
for (const regionInfo of trackInfo.regions) {
|
||||
@@ -418,6 +418,16 @@ function shouldPlay(trackInfo: { muted: boolean; solo: boolean }, hasSoloedTrack
|
||||
return true;
|
||||
}
|
||||
|
||||
export function getOfflineTrackVolumeDb(volumeDb: number, muted: boolean): number {
|
||||
const isSilent = muted || volumeDb <= AUDIO_INTERFACE_CONSTANTS.MIN_TRACK_VOLUME_DB;
|
||||
return isSilent ? -Infinity : volumeDb;
|
||||
}
|
||||
|
||||
export function getOfflineTrackGain(volumeDb: number, muted: boolean): number {
|
||||
const effectiveVolumeDb = getOfflineTrackVolumeDb(volumeDb, muted);
|
||||
return Number.isFinite(effectiveVolumeDb) ? Math.pow(10, effectiveVolumeDb / 20) : 0;
|
||||
}
|
||||
|
||||
// ===== WAV ENCODER =====
|
||||
|
||||
/**
|
||||
|
||||
@@ -71,6 +71,7 @@ interface AppConfig {
|
||||
};
|
||||
editor: {
|
||||
playhead_update_frequency: number;
|
||||
spectrogram_height_resolution: 1 | 3 | 5;
|
||||
};
|
||||
chatbox: {
|
||||
default_open: boolean;
|
||||
@@ -243,7 +244,8 @@ export class ConfigManager {
|
||||
},
|
||||
},
|
||||
editor: {
|
||||
playhead_update_frequency: 10
|
||||
playhead_update_frequency: 10,
|
||||
spectrogram_height_resolution: 3
|
||||
},
|
||||
chatbox: {
|
||||
default_open: true
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
getSpectrogramPitchBinCount,
|
||||
getSpectrogramVisibleBinRange,
|
||||
mapMidiPitchToSpectrogramPosition,
|
||||
normalizeSpectrogramHeightResolution,
|
||||
} from './spectrogramUtil';
|
||||
|
||||
describe('spectrogramUtil', () => {
|
||||
it('normalizes unsupported resolutions to the default 3x', () => {
|
||||
expect(normalizeSpectrogramHeightResolution(1)).toBe(1);
|
||||
expect(normalizeSpectrogramHeightResolution(3)).toBe(3);
|
||||
expect(normalizeSpectrogramHeightResolution(5)).toBe(5);
|
||||
expect(normalizeSpectrogramHeightResolution(2)).toBe(3);
|
||||
expect(normalizeSpectrogramHeightResolution(undefined)).toBe(3);
|
||||
});
|
||||
|
||||
it('derives pitch bin counts from the full MIDI range', () => {
|
||||
expect(getSpectrogramPitchBinCount(1)).toBe(128);
|
||||
expect(getSpectrogramPitchBinCount(3)).toBe(384);
|
||||
expect(getSpectrogramPitchBinCount(5)).toBe(640);
|
||||
});
|
||||
|
||||
it('maps MIDI pitches into full-range spectrogram positions', () => {
|
||||
expect(mapMidiPitchToSpectrogramPosition(0, 1)).toBe(0);
|
||||
expect(mapMidiPitchToSpectrogramPosition(12.5, 3)).toBe(38.5);
|
||||
expect(mapMidiPitchToSpectrogramPosition(127, 5)).toBe(637);
|
||||
expect(mapMidiPitchToSpectrogramPosition(-1, 3)).toBeNull();
|
||||
expect(mapMidiPitchToSpectrogramPosition(128, 3)).toBeNull();
|
||||
});
|
||||
|
||||
it('derives the visible C0-B7 subrange inside the full-resolution buffer', () => {
|
||||
expect(getSpectrogramVisibleBinRange(1)).toEqual({ start: 12, end: 108 });
|
||||
expect(getSpectrogramVisibleBinRange(3)).toEqual({ start: 36, end: 324 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
export const SPECTROGRAM_FULL_MIN_MIDI_PITCH = 0;
|
||||
export const SPECTROGRAM_FULL_MAX_MIDI_PITCH = 127;
|
||||
export const SPECTROGRAM_MIN_MIDI_PITCH = 12; // C0
|
||||
export const SPECTROGRAM_MAX_MIDI_PITCH = 107; // B7
|
||||
export const SPECTROGRAM_FULL_SEMITONES =
|
||||
SPECTROGRAM_FULL_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1;
|
||||
export const SPECTROGRAM_VISIBLE_SEMITONES =
|
||||
SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_MIN_MIDI_PITCH + 1;
|
||||
|
||||
export type SpectrogramHeightResolution = 1 | 3 | 5;
|
||||
|
||||
export const SPECTROGRAM_HEIGHT_RESOLUTION_OPTIONS: SpectrogramHeightResolution[] = [1, 3, 5];
|
||||
|
||||
export function normalizeSpectrogramHeightResolution(value: unknown): SpectrogramHeightResolution {
|
||||
if (value === 1 || value === 3 || value === 5) {
|
||||
return value;
|
||||
}
|
||||
return 3;
|
||||
}
|
||||
|
||||
export function getSpectrogramPitchBinCount(
|
||||
resolution: SpectrogramHeightResolution,
|
||||
): number {
|
||||
return SPECTROGRAM_FULL_SEMITONES * resolution;
|
||||
}
|
||||
|
||||
export function getSpectrogramVisibleBinRange(
|
||||
resolution: SpectrogramHeightResolution,
|
||||
): { start: number; end: number } {
|
||||
return {
|
||||
start: (SPECTROGRAM_MIN_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH) * resolution,
|
||||
end: (SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1) * resolution,
|
||||
};
|
||||
}
|
||||
|
||||
export function mapMidiPitchToSpectrogramPosition(
|
||||
midiPitch: number,
|
||||
resolution: SpectrogramHeightResolution,
|
||||
): number | null {
|
||||
const pitchOffset = midiPitch - SPECTROGRAM_FULL_MIN_MIDI_PITCH;
|
||||
if (pitchOffset < 0 || pitchOffset > SPECTROGRAM_FULL_MAX_MIDI_PITCH) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Anchor each MIDI pitch to the center of its semitone band so the
|
||||
// rendered ridge lines up with the piano-roll key row rather than a boundary.
|
||||
const scaled = pitchOffset * resolution + (resolution - 1) / 2;
|
||||
const maxBin = getSpectrogramPitchBinCount(resolution) - 1;
|
||||
return Math.max(0, Math.min(maxBin, scaled));
|
||||
}
|
||||
@@ -1,4 +1,10 @@
|
||||
import FFT from 'fft.js';
|
||||
import {
|
||||
getSpectrogramPitchBinCount,
|
||||
mapMidiPitchToSpectrogramPosition,
|
||||
normalizeSpectrogramHeightResolution,
|
||||
type SpectrogramHeightResolution,
|
||||
} from '../util/spectrogramUtil';
|
||||
|
||||
type WorkerScopeLike = typeof globalThis & {
|
||||
onmessage: ((event: MessageEvent<SpectrogramRequest>) => void) | null;
|
||||
@@ -13,16 +19,17 @@ export interface SpectrogramRequest {
|
||||
clipStartOffsetSeconds: number;
|
||||
regionDurationSeconds: number;
|
||||
bpm: number;
|
||||
heightResolution: SpectrogramHeightResolution;
|
||||
}
|
||||
|
||||
export interface SpectrogramResult {
|
||||
data: Float32Array; // [timeSteps × 128] row-major, row = time step, col = pitch 0-127
|
||||
data: Float32Array; // [timeSteps × pitchBins] row-major, low-to-high pitch bins
|
||||
timeSteps: number;
|
||||
pitchBins: number;
|
||||
}
|
||||
|
||||
const FFT_SIZE = 8192;
|
||||
const HOP_SIZE = 1024;
|
||||
const PITCH_BINS = 128;
|
||||
|
||||
function hannWindow(size: number): Float32Array {
|
||||
const w = new Float32Array(size);
|
||||
@@ -32,8 +39,43 @@ function hannWindow(size: number): Float32Array {
|
||||
return w;
|
||||
}
|
||||
|
||||
function spreadMagnitudeAcrossPitchBins(
|
||||
result: Float32Array,
|
||||
pitchRow: number,
|
||||
pitchBins: number,
|
||||
pitchPosition: number,
|
||||
magnitude: number,
|
||||
heightResolution: SpectrogramHeightResolution,
|
||||
): void {
|
||||
const spreadRadius = Math.max(0, heightResolution - 1);
|
||||
const centerBin = Math.round(pitchPosition);
|
||||
const startBin = Math.max(0, centerBin - spreadRadius);
|
||||
const endBin = Math.min(pitchBins - 1, centerBin + spreadRadius);
|
||||
const spreadWidth = spreadRadius + 1;
|
||||
|
||||
for (let targetBin = startBin; targetBin <= endBin; targetBin++) {
|
||||
const distance = Math.abs(targetBin - pitchPosition);
|
||||
const weight = Math.max(0, 1 - distance / spreadWidth);
|
||||
if (weight <= 0) continue;
|
||||
|
||||
const weightedMagnitude = magnitude * weight;
|
||||
const resultIndex = pitchRow + targetBin;
|
||||
if (weightedMagnitude > result[resultIndex]) {
|
||||
result[resultIndex] = weightedMagnitude;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||
const { pcm, sampleRate, clipStartOffsetSeconds, regionDurationSeconds } = e.data;
|
||||
const {
|
||||
pcm,
|
||||
sampleRate,
|
||||
clipStartOffsetSeconds,
|
||||
regionDurationSeconds,
|
||||
heightResolution,
|
||||
} = e.data;
|
||||
const normalizedResolution = normalizeSpectrogramHeightResolution(heightResolution);
|
||||
const pitchBins = getSpectrogramPitchBinCount(normalizedResolution);
|
||||
|
||||
const startSample = Math.floor(clipStartOffsetSeconds * sampleRate);
|
||||
const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate));
|
||||
@@ -45,7 +87,7 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||
const inputPadded = new Float32Array(FFT_SIZE);
|
||||
|
||||
const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1);
|
||||
const result = new Float32Array(totalHops * PITCH_BINS);
|
||||
const result = new Float32Array(totalHops * pitchBins);
|
||||
|
||||
let maxVal = 0;
|
||||
|
||||
@@ -64,28 +106,32 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||
|
||||
// Max-pool magnitude into pitch bins: keep the loudest FFT bin per semitone,
|
||||
// rather than summing. Summing inflates every bin by how many FFT bins land there.
|
||||
const pitchRow = hop * PITCH_BINS;
|
||||
const pitchRow = hop * pitchBins;
|
||||
const numBins = FFT_SIZE / 2;
|
||||
|
||||
for (let bin = 1; bin < numBins; bin++) {
|
||||
const freq = (bin * sampleRate) / FFT_SIZE;
|
||||
if (freq < 20 || freq > 20000) continue;
|
||||
|
||||
// Convert frequency to MIDI pitch; clamp to piano roll range C0–B7 (MIDI 12–107)
|
||||
const pitch = Math.round(69 + 12 * Math.log2(freq / 440));
|
||||
if (pitch < 12 || pitch > 107) continue;
|
||||
const midiPitch = 69 + 12 * Math.log2(freq / 440);
|
||||
const pitchPosition = mapMidiPitchToSpectrogramPosition(midiPitch, normalizedResolution);
|
||||
if (pitchPosition === null) continue;
|
||||
|
||||
const re = complexOut[2 * bin];
|
||||
const im = complexOut[2 * bin + 1];
|
||||
const magnitude = Math.sqrt(re * re + im * im);
|
||||
|
||||
if (magnitude > result[pitchRow + pitch]) {
|
||||
result[pitchRow + pitch] = magnitude;
|
||||
}
|
||||
spreadMagnitudeAcrossPitchBins(
|
||||
result,
|
||||
pitchRow,
|
||||
pitchBins,
|
||||
pitchPosition,
|
||||
magnitude,
|
||||
normalizedResolution,
|
||||
);
|
||||
}
|
||||
|
||||
// Track global max for normalization
|
||||
for (let p = 0; p < PITCH_BINS; p++) {
|
||||
for (let p = 0; p < pitchBins; p++) {
|
||||
if (result[pitchRow + p] > maxVal) maxVal = result[pitchRow + p];
|
||||
}
|
||||
}
|
||||
@@ -98,6 +144,6 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||
}
|
||||
}
|
||||
|
||||
const response: SpectrogramResult = { data: result, timeSteps: totalHops };
|
||||
const response: SpectrogramResult = { data: result, timeSteps: totalHops, pitchBins };
|
||||
workerScope.postMessage(response, [result.buffer]);
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user