feat: added an option to let user choose the vertical resolution of the spectrogram, also fixed a bug that peak of pitch in the spectrogram has been mapped to the start of the note bin instead of the center
This commit is contained in:
+2
-1
@@ -66,7 +66,8 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"editor": {
|
"editor": {
|
||||||
"playhead_update_frequency": 10
|
"playhead_update_frequency": 10,
|
||||||
|
"spectrogram_height_resolution": 3
|
||||||
},
|
},
|
||||||
"chatbox": {
|
"chatbox": {
|
||||||
"default_open": true
|
"default_open": true
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import type { KeySignature } from '../../core/KGProject';
|
|||||||
import { KGPianoRollState } from '../../core/state/KGPianoRollState';
|
import { KGPianoRollState } from '../../core/state/KGPianoRollState';
|
||||||
import SpectrogramCanvas from './SpectrogramCanvas';
|
import SpectrogramCanvas from './SpectrogramCanvas';
|
||||||
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||||
|
import type { SpectrogramHeightResolution } from '../../util/spectrogramUtil';
|
||||||
|
|
||||||
interface PianoGridProps {
|
interface PianoGridProps {
|
||||||
gridRef: MutableRefObject<HTMLDivElement | null>;
|
gridRef: MutableRefObject<HTMLDivElement | null>;
|
||||||
@@ -32,6 +33,7 @@ interface PianoGridProps {
|
|||||||
bpm?: number;
|
bpm?: number;
|
||||||
spectrogramThresholdDb?: number;
|
spectrogramThresholdDb?: number;
|
||||||
spectrogramPower?: number;
|
spectrogramPower?: number;
|
||||||
|
spectrogramHeightResolution?: SpectrogramHeightResolution;
|
||||||
pianoRollZoom?: number;
|
pianoRollZoom?: number;
|
||||||
mode?: 'midi-edit' | 'spectrogram' | 'hybrid';
|
mode?: 'midi-edit' | 'spectrogram' | 'hybrid';
|
||||||
onSpectrogramLoadingChange?: (loading: boolean) => void;
|
onSpectrogramLoadingChange?: (loading: boolean) => void;
|
||||||
@@ -62,6 +64,7 @@ const PianoGrid: React.FC<PianoGridProps> = ({
|
|||||||
bpm = 120,
|
bpm = 120,
|
||||||
spectrogramThresholdDb = -25,
|
spectrogramThresholdDb = -25,
|
||||||
spectrogramPower = 0.5,
|
spectrogramPower = 0.5,
|
||||||
|
spectrogramHeightResolution = 3,
|
||||||
pianoRollZoom = 1,
|
pianoRollZoom = 1,
|
||||||
onSpectrogramLoadingChange,
|
onSpectrogramLoadingChange,
|
||||||
}) => {
|
}) => {
|
||||||
@@ -245,6 +248,7 @@ const PianoGrid: React.FC<PianoGridProps> = ({
|
|||||||
thresholdDb={spectrogramThresholdDb}
|
thresholdDb={spectrogramThresholdDb}
|
||||||
power={spectrogramPower}
|
power={spectrogramPower}
|
||||||
zoom={pianoRollZoom}
|
zoom={pianoRollZoom}
|
||||||
|
heightResolution={spectrogramHeightResolution}
|
||||||
onLoadingChange={onSpectrogramLoadingChange}
|
onLoadingChange={onSpectrogramLoadingChange}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -18,6 +18,10 @@ import { beatsToBar } from '../../util/midiUtil';
|
|||||||
import { UpdateRegionCommand } from '../../core/commands';
|
import { UpdateRegionCommand } from '../../core/commands';
|
||||||
import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil';
|
import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil';
|
||||||
import { showAlert, showPrompt } from '../../util/dialogUtil';
|
import { showAlert, showPrompt } from '../../util/dialogUtil';
|
||||||
|
import {
|
||||||
|
normalizeSpectrogramHeightResolution,
|
||||||
|
type SpectrogramHeightResolution,
|
||||||
|
} from '../../util/spectrogramUtil';
|
||||||
|
|
||||||
interface PianoRollProps {
|
interface PianoRollProps {
|
||||||
onClose: () => void;
|
onClose: () => void;
|
||||||
@@ -50,6 +54,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
|||||||
// Spectrogram controls (only used in spectrogram mode)
|
// Spectrogram controls (only used in spectrogram mode)
|
||||||
const [spectrogramThresholdDb, setSpectrogramThresholdDb] = useState<number>(-25);
|
const [spectrogramThresholdDb, setSpectrogramThresholdDb] = useState<number>(-25);
|
||||||
const [spectrogramPower, setSpectrogramPower] = useState<number>(0.5);
|
const [spectrogramPower, setSpectrogramPower] = useState<number>(0.5);
|
||||||
|
const [spectrogramHeightResolution, setSpectrogramHeightResolution] =
|
||||||
|
useState<SpectrogramHeightResolution>(3);
|
||||||
|
|
||||||
// Piano roll zoom (1x–8x); updates --region-grid-beat-width CSS variable
|
// Piano roll zoom (1x–8x); updates --region-grid-beat-width CSS variable
|
||||||
const [pianoRollZoom, setPianoRollZoom] = useState<number>(1);
|
const [pianoRollZoom, setPianoRollZoom] = useState<number>(1);
|
||||||
@@ -201,6 +207,41 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
|||||||
}
|
}
|
||||||
}, []); // Empty dependency array means this runs once on mount
|
}, []); // Empty dependency array means this runs once on mount
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let unsubscribe: (() => void) | undefined;
|
||||||
|
|
||||||
|
const initializeSpectrogramHeightResolution = async () => {
|
||||||
|
const configManager = ConfigManager.instance();
|
||||||
|
if (!configManager.getIsInitialized()) {
|
||||||
|
await configManager.initialize();
|
||||||
|
}
|
||||||
|
|
||||||
|
const applyResolutionFromConfig = () => {
|
||||||
|
setSpectrogramHeightResolution(
|
||||||
|
normalizeSpectrogramHeightResolution(
|
||||||
|
configManager.get('editor.spectrogram_height_resolution')
|
||||||
|
)
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
applyResolutionFromConfig();
|
||||||
|
unsubscribe = configManager.addChangeListener((changedKeys) => {
|
||||||
|
if (
|
||||||
|
changedKeys.includes('__all__') ||
|
||||||
|
changedKeys.includes('editor.spectrogram_height_resolution')
|
||||||
|
) {
|
||||||
|
applyResolutionFromConfig();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
void initializeSpectrogramHeightResolution();
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
unsubscribe?.();
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
// Add keyboard event listener for Escape
|
// Add keyboard event listener for Escape
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const handleKeyDown = (event: KeyboardEvent) => {
|
const handleKeyDown = (event: KeyboardEvent) => {
|
||||||
@@ -1048,6 +1089,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
|||||||
bpm={bpm}
|
bpm={bpm}
|
||||||
spectrogramThresholdDb={spectrogramThresholdDb}
|
spectrogramThresholdDb={spectrogramThresholdDb}
|
||||||
spectrogramPower={spectrogramPower}
|
spectrogramPower={spectrogramPower}
|
||||||
|
spectrogramHeightResolution={spectrogramHeightResolution}
|
||||||
pianoRollZoom={pianoRollZoom}
|
pianoRollZoom={pianoRollZoom}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import { useNoteSelection } from '../../hooks/useNoteSelection';
|
|||||||
import type { KeySignature } from '../../core/KGProject';
|
import type { KeySignature } from '../../core/KGProject';
|
||||||
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||||
import { velocityToColor } from '../../util/velocityColor';
|
import { velocityToColor } from '../../util/velocityColor';
|
||||||
|
import type { SpectrogramHeightResolution } from '../../util/spectrogramUtil';
|
||||||
|
|
||||||
interface PianoRollContentProps {
|
interface PianoRollContentProps {
|
||||||
contentRef: React.MutableRefObject<HTMLDivElement | null>;
|
contentRef: React.MutableRefObject<HTMLDivElement | null>;
|
||||||
@@ -35,6 +36,7 @@ interface PianoRollContentProps {
|
|||||||
bpm?: number;
|
bpm?: number;
|
||||||
spectrogramThresholdDb?: number;
|
spectrogramThresholdDb?: number;
|
||||||
spectrogramPower?: number;
|
spectrogramPower?: number;
|
||||||
|
spectrogramHeightResolution?: SpectrogramHeightResolution;
|
||||||
pianoRollZoom?: number;
|
pianoRollZoom?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,6 +60,7 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
|||||||
bpm = 120,
|
bpm = 120,
|
||||||
spectrogramThresholdDb = -25,
|
spectrogramThresholdDb = -25,
|
||||||
spectrogramPower = 0.5,
|
spectrogramPower = 0.5,
|
||||||
|
spectrogramHeightResolution = 3,
|
||||||
pianoRollZoom = 1,
|
pianoRollZoom = 1,
|
||||||
}) => {
|
}) => {
|
||||||
const isSpectrogram = mode === 'spectrogram';
|
const isSpectrogram = mode === 'spectrogram';
|
||||||
@@ -295,6 +298,7 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
|||||||
bpm={bpm}
|
bpm={bpm}
|
||||||
spectrogramThresholdDb={spectrogramThresholdDb}
|
spectrogramThresholdDb={spectrogramThresholdDb}
|
||||||
spectrogramPower={spectrogramPower}
|
spectrogramPower={spectrogramPower}
|
||||||
|
spectrogramHeightResolution={spectrogramHeightResolution}
|
||||||
pianoRollZoom={pianoRollZoom}
|
pianoRollZoom={pianoRollZoom}
|
||||||
onSpectrogramLoadingChange={handleSpectrogramLoadingChange}
|
onSpectrogramLoadingChange={handleSpectrogramLoadingChange}
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -4,6 +4,12 @@ import { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
|||||||
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
|
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
|
||||||
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
|
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
|
||||||
import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker';
|
import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker';
|
||||||
|
import {
|
||||||
|
getSpectrogramVisibleBinRange,
|
||||||
|
normalizeSpectrogramHeightResolution,
|
||||||
|
SPECTROGRAM_VISIBLE_SEMITONES,
|
||||||
|
type SpectrogramHeightResolution,
|
||||||
|
} from '../../util/spectrogramUtil';
|
||||||
|
|
||||||
interface SpectrogramCanvasProps {
|
interface SpectrogramCanvasProps {
|
||||||
audioRegion: KGAudioRegion;
|
audioRegion: KGAudioRegion;
|
||||||
@@ -13,12 +19,10 @@ interface SpectrogramCanvasProps {
|
|||||||
thresholdDb: number;
|
thresholdDb: number;
|
||||||
power: number;
|
power: number;
|
||||||
zoom: number;
|
zoom: number;
|
||||||
|
heightResolution: SpectrogramHeightResolution;
|
||||||
onLoadingChange?: (loading: boolean) => void;
|
onLoadingChange?: (loading: boolean) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
const PITCH_BINS = 128;
|
|
||||||
const HOP_SIZE = 1024;
|
|
||||||
|
|
||||||
// Piecewise-linear RGB colormap: black → dark blue → blue → purple → red → orange → yellow
|
// Piecewise-linear RGB colormap: black → dark blue → blue → purple → red → orange → yellow
|
||||||
// Hue rotates 240°→300°→0°→60°, bypassing green entirely.
|
// Hue rotates 240°→300°→0°→60°, bypassing green entirely.
|
||||||
const COLORMAP_STOPS: Array<[number, [number, number, number]]> = [
|
const COLORMAP_STOPS: Array<[number, [number, number, number]]> = [
|
||||||
@@ -56,6 +60,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
|||||||
thresholdDb,
|
thresholdDb,
|
||||||
power,
|
power,
|
||||||
zoom,
|
zoom,
|
||||||
|
heightResolution,
|
||||||
onLoadingChange,
|
onLoadingChange,
|
||||||
}) => {
|
}) => {
|
||||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||||
@@ -80,6 +85,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
|||||||
regionDurationSeconds: number,
|
regionDurationSeconds: number,
|
||||||
thresholdDb: number,
|
thresholdDb: number,
|
||||||
power: number,
|
power: number,
|
||||||
|
heightResolution: SpectrogramHeightResolution,
|
||||||
) => {
|
) => {
|
||||||
const canvas = canvasRef.current;
|
const canvas = canvasRef.current;
|
||||||
if (!canvas) return;
|
if (!canvas) return;
|
||||||
@@ -92,31 +98,31 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
|||||||
const totalBeats = (regionDurationSeconds * bpm) / 60;
|
const totalBeats = (regionDurationSeconds * bpm) / 60;
|
||||||
// Always draw at 1x resolution; zoom is applied as CSS width stretch
|
// Always draw at 1x resolution; zoom is applied as CSS width stretch
|
||||||
const canvasWidth = Math.ceil(totalBeats * 40);
|
const canvasWidth = Math.ceil(totalBeats * 40);
|
||||||
const canvasHeight = PITCH_BINS * noteHeight;
|
const canvasHeight = SPECTROGRAM_VISIBLE_SEMITONES * noteHeight;
|
||||||
|
|
||||||
// Convert dB threshold to linear: values below this → black
|
// Convert dB threshold to linear: values below this → black
|
||||||
const linearThreshold = Math.pow(10, thresholdDb / 20);
|
const linearThreshold = Math.pow(10, thresholdDb / 20);
|
||||||
|
const visibleRange = getSpectrogramVisibleBinRange(heightResolution);
|
||||||
|
const visiblePitchBins = visibleRange.end - visibleRange.start;
|
||||||
|
|
||||||
// 1. Paint at natural STFT resolution onto an offscreen canvas (timeSteps × 128).
|
// 1. Paint at natural spectrogram resolution onto an offscreen canvas.
|
||||||
// Row i = pitchIndex i = pitch (107 − i). Apply threshold then power curve.
|
// Result data is low-to-high pitch; draw only the visible C0-B7 window, reversed for display.
|
||||||
const offscreen = document.createElement('canvas');
|
const offscreen = document.createElement('canvas');
|
||||||
offscreen.width = result.timeSteps;
|
offscreen.width = result.timeSteps;
|
||||||
offscreen.height = PITCH_BINS;
|
offscreen.height = visiblePitchBins;
|
||||||
const offCtx = offscreen.getContext('2d');
|
const offCtx = offscreen.getContext('2d');
|
||||||
if (!offCtx) return;
|
if (!offCtx) return;
|
||||||
|
|
||||||
const imgData = offCtx.createImageData(result.timeSteps, PITCH_BINS);
|
const imgData = offCtx.createImageData(result.timeSteps, visiblePitchBins);
|
||||||
const pixels = imgData.data;
|
const pixels = imgData.data;
|
||||||
|
|
||||||
for (let row = 0; row < PITCH_BINS; row++) {
|
for (let row = 0; row < visiblePitchBins; row++) {
|
||||||
const pitch = 107 - row;
|
const sourceRow = visibleRange.end - 1 - row;
|
||||||
for (let col = 0; col < result.timeSteps; col++) {
|
for (let col = 0; col < result.timeSteps; col++) {
|
||||||
const idx = (row * result.timeSteps + col) * 4;
|
const idx = (row * result.timeSteps + col) * 4;
|
||||||
pixels[idx + 3] = 255; // always opaque
|
pixels[idx + 3] = 255; // always opaque
|
||||||
|
|
||||||
if (pitch < 12 || pitch > 107) continue; // outside range → black
|
const raw = result.data[col * result.pitchBins + sourceRow];
|
||||||
|
|
||||||
const raw = result.data[col * PITCH_BINS + pitch];
|
|
||||||
|
|
||||||
// Hard threshold: values below noise floor → 0 (black)
|
// Hard threshold: values below noise floor → 0 (black)
|
||||||
// Re-scale surviving range to [0,1] then apply power curve
|
// Re-scale surviving range to [0,1] then apply power curve
|
||||||
@@ -157,9 +163,10 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
|||||||
regionDurationRef.current,
|
regionDurationRef.current,
|
||||||
thresholdDb,
|
thresholdDb,
|
||||||
power,
|
power,
|
||||||
|
normalizeSpectrogramHeightResolution(heightResolution),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}, [thresholdDb, power, renderSpectrogram]);
|
}, [thresholdDb, power, renderSpectrogram, heightResolution]);
|
||||||
|
|
||||||
// Zoom changes: stretch width only, pin height to canvas pixel height
|
// Zoom changes: stretch width only, pin height to canvas pixel height
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -211,7 +218,14 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
|||||||
rawResultRef.current = e.data;
|
rawResultRef.current = e.data;
|
||||||
sampleRateRef.current = sampleRate;
|
sampleRateRef.current = sampleRate;
|
||||||
regionDurationRef.current = regionDurationSeconds;
|
regionDurationRef.current = regionDurationSeconds;
|
||||||
renderSpectrogram(e.data, sampleRate, regionDurationSeconds, thresholdDb, power);
|
renderSpectrogram(
|
||||||
|
e.data,
|
||||||
|
sampleRate,
|
||||||
|
regionDurationSeconds,
|
||||||
|
thresholdDb,
|
||||||
|
power,
|
||||||
|
normalizeSpectrogramHeightResolution(heightResolution),
|
||||||
|
);
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
worker.terminate();
|
worker.terminate();
|
||||||
workerRef.current = null;
|
workerRef.current = null;
|
||||||
@@ -223,6 +237,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
|||||||
clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
|
clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
|
||||||
regionDurationSeconds,
|
regionDurationSeconds,
|
||||||
bpm,
|
bpm,
|
||||||
|
heightResolution: normalizeSpectrogramHeightResolution(heightResolution),
|
||||||
};
|
};
|
||||||
worker.postMessage(request, [request.pcm.buffer]);
|
worker.postMessage(request, [request.pcm.buffer]);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -241,7 +256,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
|||||||
workerRef.current = null;
|
workerRef.current = null;
|
||||||
};
|
};
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
}, [audioRegion, trackId, projectName, bpm]);
|
}, [audioRegion, trackId, projectName, bpm, heightResolution]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<canvas
|
<canvas
|
||||||
|
|||||||
@@ -1,9 +1,14 @@
|
|||||||
import React, { useState, useEffect } from 'react';
|
import React, { useState, useEffect } from 'react';
|
||||||
import { ConfigManager } from '../../../core/config/ConfigManager';
|
import { ConfigManager } from '../../../core/config/ConfigManager';
|
||||||
import { KGAudioInterface } from '../../../core/audio-interface/KGAudioInterface';
|
import { KGAudioInterface } from '../../../core/audio-interface/KGAudioInterface';
|
||||||
|
import {
|
||||||
|
normalizeSpectrogramHeightResolution,
|
||||||
|
type SpectrogramHeightResolution,
|
||||||
|
} from '../../../util/spectrogramUtil';
|
||||||
|
|
||||||
const BehaviorSettings: React.FC = () => {
|
const BehaviorSettings: React.FC = () => {
|
||||||
const [playheadUpdateFrequency, setPlayheadUpdateFrequency] = useState<number>(10);
|
const [playheadUpdateFrequency, setPlayheadUpdateFrequency] = useState<number>(10);
|
||||||
|
const [spectrogramHeightResolution, setSpectrogramHeightResolution] = useState<SpectrogramHeightResolution>(3);
|
||||||
const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState<boolean>(true);
|
const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState<boolean>(true);
|
||||||
const [audioLookaheadTime, setAudioLookaheadTime] = useState<string>('50');
|
const [audioLookaheadTime, setAudioLookaheadTime] = useState<string>('50');
|
||||||
const [playbackDelay, setPlaybackDelay] = useState<string>('200');
|
const [playbackDelay, setPlaybackDelay] = useState<string>('200');
|
||||||
@@ -23,6 +28,9 @@ const BehaviorSettings: React.FC = () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
setPlayheadUpdateFrequency((configManager.get('editor.playhead_update_frequency') as number) ?? 10);
|
setPlayheadUpdateFrequency((configManager.get('editor.playhead_update_frequency') as number) ?? 10);
|
||||||
|
setSpectrogramHeightResolution(
|
||||||
|
normalizeSpectrogramHeightResolution(configManager.get('editor.spectrogram_height_resolution'))
|
||||||
|
);
|
||||||
setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true);
|
setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true);
|
||||||
const lookaheadTimeSeconds = (configManager.get('audio.lookahead_time') as number) ?? 0.05;
|
const lookaheadTimeSeconds = (configManager.get('audio.lookahead_time') as number) ?? 0.05;
|
||||||
setAudioLookaheadTime(((lookaheadTimeSeconds * 1000).toFixed(0)));
|
setAudioLookaheadTime(((lookaheadTimeSeconds * 1000).toFixed(0)));
|
||||||
@@ -50,6 +58,12 @@ const BehaviorSettings: React.FC = () => {
|
|||||||
await configManager.set('chatbox.default_open', boolValue);
|
await configManager.set('chatbox.default_open', boolValue);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const handleSpectrogramHeightResolutionChange = async (value: string) => {
|
||||||
|
const nextValue = normalizeSpectrogramHeightResolution(parseInt(value, 10));
|
||||||
|
setSpectrogramHeightResolution(nextValue);
|
||||||
|
await configManager.set('editor.spectrogram_height_resolution', nextValue);
|
||||||
|
};
|
||||||
|
|
||||||
const handleAudioLookaheadTimeChange = async (value: string) => {
|
const handleAudioLookaheadTimeChange = async (value: string) => {
|
||||||
// Allow empty string, treat as 0 ms
|
// Allow empty string, treat as 0 ms
|
||||||
const numValueMs = value === '' ? 0 : parseFloat(value);
|
const numValueMs = value === '' ? 0 : parseFloat(value);
|
||||||
@@ -161,6 +175,24 @@ const BehaviorSettings: React.FC = () => {
|
|||||||
Update frequency for the playhead animation during playback. Higher values (60 fps) provide smoother animation but use more CPU. Lower values (10 fps) are more efficient. Changes apply immediately without restart.
|
Update frequency for the playhead animation during playback. Higher values (60 fps) provide smoother animation but use more CPU. Lower values (10 fps) are more efficient. Changes apply immediately without restart.
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div className="settings-item">
|
||||||
|
<label className="settings-label">
|
||||||
|
Spectrogram Height Resolution
|
||||||
|
</label>
|
||||||
|
<select
|
||||||
|
className="settings-select"
|
||||||
|
value={spectrogramHeightResolution}
|
||||||
|
onChange={(e) => handleSpectrogramHeightResolutionChange(e.target.value)}
|
||||||
|
>
|
||||||
|
<option value="1">1x</option>
|
||||||
|
<option value="3">3x</option>
|
||||||
|
<option value="5">5x</option>
|
||||||
|
</select>
|
||||||
|
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
|
||||||
|
Controls vertical spectrogram detail across the visible pitch range. Higher values sharpen pitch contours but use more CPU and memory while computing the spectrogram.
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="settings-group">
|
<div className="settings-group">
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, it, expect } from 'vitest';
|
import { describe, it, expect } from 'vitest';
|
||||||
import { encodeWav } from './KGOfflineRenderer';
|
import { encodeWav, getOfflineTrackGain, getOfflineTrackVolumeDb } from './KGOfflineRenderer';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a minimal AudioBuffer-like object for testing.
|
* Create a minimal AudioBuffer-like object for testing.
|
||||||
@@ -145,3 +145,22 @@ describe('encodeWav', () => {
|
|||||||
expect(view.getUint32(40, true)).toBe(0); // data size = 0
|
expect(view.getUint32(40, true)).toBe(0); // data size = 0
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('offline track volume conversion', () => {
|
||||||
|
it('treats 0 dB as unity gain instead of silence', () => {
|
||||||
|
expect(getOfflineTrackVolumeDb(0, false)).toBe(0);
|
||||||
|
expect(getOfflineTrackGain(0, false)).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('converts negative dB values to linear gain', () => {
|
||||||
|
expect(getOfflineTrackVolumeDb(-6, false)).toBe(-6);
|
||||||
|
expect(getOfflineTrackGain(-6, false)).toBeCloseTo(Math.pow(10, -6 / 20), 6);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('silences muted tracks and floor-level volumes', () => {
|
||||||
|
expect(getOfflineTrackVolumeDb(0, true)).toBe(-Infinity);
|
||||||
|
expect(getOfflineTrackGain(0, true)).toBe(0);
|
||||||
|
expect(getOfflineTrackVolumeDb(-60, false)).toBe(-Infinity);
|
||||||
|
expect(getOfflineTrackGain(-60, false)).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import type { KGProject } from '../KGProject';
|
|||||||
import type { KGMidiNote } from '../midi/KGMidiNote';
|
import type { KGMidiNote } from '../midi/KGMidiNote';
|
||||||
import type { KGAudioRegion } from '../region/KGAudioRegion';
|
import type { KGAudioRegion } from '../region/KGAudioRegion';
|
||||||
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
|
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
|
||||||
|
import { AUDIO_INTERFACE_CONSTANTS } from '../../constants/coreConstants';
|
||||||
import { pitchToNoteNameString } from '../../util/midiUtil';
|
import { pitchToNoteNameString } from '../../util/midiUtil';
|
||||||
import { KGToneBuffersPool } from './KGToneBuffersPool';
|
import { KGToneBuffersPool } from './KGToneBuffersPool';
|
||||||
import { KGToneSamplerFactory } from './KGToneSamplerFactory';
|
import { KGToneSamplerFactory } from './KGToneSamplerFactory';
|
||||||
@@ -254,9 +255,8 @@ export class KGOfflineRenderer {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// Apply volume
|
// Track volumes are stored in dB across the app, with 0 meaning unity gain.
|
||||||
const volumeDb = trackInfo.volume > 0 ? 20 * Math.log10(trackInfo.volume) : -Infinity;
|
sampler.volume.value = getOfflineTrackVolumeDb(trackInfo.volume, trackInfo.muted);
|
||||||
sampler.volume.value = volumeDb;
|
|
||||||
sampler.connect(masterGain);
|
sampler.connect(masterGain);
|
||||||
|
|
||||||
// Schedule all notes for this track
|
// Schedule all notes for this track
|
||||||
@@ -288,7 +288,7 @@ export class KGOfflineRenderer {
|
|||||||
for (const trackInfo of audioTrackData) {
|
for (const trackInfo of audioTrackData) {
|
||||||
if (!shouldPlay(trackInfo, hasSoloedTracks)) continue;
|
if (!shouldPlay(trackInfo, hasSoloedTracks)) continue;
|
||||||
|
|
||||||
const trackGain = new Tone.Gain(trackInfo.volume);
|
const trackGain = new Tone.Gain(getOfflineTrackGain(trackInfo.volume, trackInfo.muted));
|
||||||
trackGain.connect(masterGain);
|
trackGain.connect(masterGain);
|
||||||
|
|
||||||
for (const regionInfo of trackInfo.regions) {
|
for (const regionInfo of trackInfo.regions) {
|
||||||
@@ -418,6 +418,16 @@ function shouldPlay(trackInfo: { muted: boolean; solo: boolean }, hasSoloedTrack
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getOfflineTrackVolumeDb(volumeDb: number, muted: boolean): number {
|
||||||
|
const isSilent = muted || volumeDb <= AUDIO_INTERFACE_CONSTANTS.MIN_TRACK_VOLUME_DB;
|
||||||
|
return isSilent ? -Infinity : volumeDb;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getOfflineTrackGain(volumeDb: number, muted: boolean): number {
|
||||||
|
const effectiveVolumeDb = getOfflineTrackVolumeDb(volumeDb, muted);
|
||||||
|
return Number.isFinite(effectiveVolumeDb) ? Math.pow(10, effectiveVolumeDb / 20) : 0;
|
||||||
|
}
|
||||||
|
|
||||||
// ===== WAV ENCODER =====
|
// ===== WAV ENCODER =====
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -71,6 +71,7 @@ interface AppConfig {
|
|||||||
};
|
};
|
||||||
editor: {
|
editor: {
|
||||||
playhead_update_frequency: number;
|
playhead_update_frequency: number;
|
||||||
|
spectrogram_height_resolution: 1 | 3 | 5;
|
||||||
};
|
};
|
||||||
chatbox: {
|
chatbox: {
|
||||||
default_open: boolean;
|
default_open: boolean;
|
||||||
@@ -243,7 +244,8 @@ export class ConfigManager {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
editor: {
|
editor: {
|
||||||
playhead_update_frequency: 10
|
playhead_update_frequency: 10,
|
||||||
|
spectrogram_height_resolution: 3
|
||||||
},
|
},
|
||||||
chatbox: {
|
chatbox: {
|
||||||
default_open: true
|
default_open: true
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import {
|
||||||
|
getSpectrogramPitchBinCount,
|
||||||
|
getSpectrogramVisibleBinRange,
|
||||||
|
mapMidiPitchToSpectrogramPosition,
|
||||||
|
normalizeSpectrogramHeightResolution,
|
||||||
|
} from './spectrogramUtil';
|
||||||
|
|
||||||
|
describe('spectrogramUtil', () => {
|
||||||
|
it('normalizes unsupported resolutions to the default 3x', () => {
|
||||||
|
expect(normalizeSpectrogramHeightResolution(1)).toBe(1);
|
||||||
|
expect(normalizeSpectrogramHeightResolution(3)).toBe(3);
|
||||||
|
expect(normalizeSpectrogramHeightResolution(5)).toBe(5);
|
||||||
|
expect(normalizeSpectrogramHeightResolution(2)).toBe(3);
|
||||||
|
expect(normalizeSpectrogramHeightResolution(undefined)).toBe(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('derives pitch bin counts from the full MIDI range', () => {
|
||||||
|
expect(getSpectrogramPitchBinCount(1)).toBe(128);
|
||||||
|
expect(getSpectrogramPitchBinCount(3)).toBe(384);
|
||||||
|
expect(getSpectrogramPitchBinCount(5)).toBe(640);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('maps MIDI pitches into full-range spectrogram positions', () => {
|
||||||
|
expect(mapMidiPitchToSpectrogramPosition(0, 1)).toBe(0);
|
||||||
|
expect(mapMidiPitchToSpectrogramPosition(12.5, 3)).toBe(38.5);
|
||||||
|
expect(mapMidiPitchToSpectrogramPosition(127, 5)).toBe(637);
|
||||||
|
expect(mapMidiPitchToSpectrogramPosition(-1, 3)).toBeNull();
|
||||||
|
expect(mapMidiPitchToSpectrogramPosition(128, 3)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('derives the visible C0-B7 subrange inside the full-resolution buffer', () => {
|
||||||
|
expect(getSpectrogramVisibleBinRange(1)).toEqual({ start: 12, end: 108 });
|
||||||
|
expect(getSpectrogramVisibleBinRange(3)).toEqual({ start: 36, end: 324 });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
export const SPECTROGRAM_FULL_MIN_MIDI_PITCH = 0;
|
||||||
|
export const SPECTROGRAM_FULL_MAX_MIDI_PITCH = 127;
|
||||||
|
export const SPECTROGRAM_MIN_MIDI_PITCH = 12; // C0
|
||||||
|
export const SPECTROGRAM_MAX_MIDI_PITCH = 107; // B7
|
||||||
|
export const SPECTROGRAM_FULL_SEMITONES =
|
||||||
|
SPECTROGRAM_FULL_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1;
|
||||||
|
export const SPECTROGRAM_VISIBLE_SEMITONES =
|
||||||
|
SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_MIN_MIDI_PITCH + 1;
|
||||||
|
|
||||||
|
export type SpectrogramHeightResolution = 1 | 3 | 5;
|
||||||
|
|
||||||
|
export const SPECTROGRAM_HEIGHT_RESOLUTION_OPTIONS: SpectrogramHeightResolution[] = [1, 3, 5];
|
||||||
|
|
||||||
|
export function normalizeSpectrogramHeightResolution(value: unknown): SpectrogramHeightResolution {
|
||||||
|
if (value === 1 || value === 3 || value === 5) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
return 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getSpectrogramPitchBinCount(
|
||||||
|
resolution: SpectrogramHeightResolution,
|
||||||
|
): number {
|
||||||
|
return SPECTROGRAM_FULL_SEMITONES * resolution;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getSpectrogramVisibleBinRange(
|
||||||
|
resolution: SpectrogramHeightResolution,
|
||||||
|
): { start: number; end: number } {
|
||||||
|
return {
|
||||||
|
start: (SPECTROGRAM_MIN_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH) * resolution,
|
||||||
|
end: (SPECTROGRAM_MAX_MIDI_PITCH - SPECTROGRAM_FULL_MIN_MIDI_PITCH + 1) * resolution,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mapMidiPitchToSpectrogramPosition(
|
||||||
|
midiPitch: number,
|
||||||
|
resolution: SpectrogramHeightResolution,
|
||||||
|
): number | null {
|
||||||
|
const pitchOffset = midiPitch - SPECTROGRAM_FULL_MIN_MIDI_PITCH;
|
||||||
|
if (pitchOffset < 0 || pitchOffset > SPECTROGRAM_FULL_MAX_MIDI_PITCH) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Anchor each MIDI pitch to the center of its semitone band so the
|
||||||
|
// rendered ridge lines up with the piano-roll key row rather than a boundary.
|
||||||
|
const scaled = pitchOffset * resolution + (resolution - 1) / 2;
|
||||||
|
const maxBin = getSpectrogramPitchBinCount(resolution) - 1;
|
||||||
|
return Math.max(0, Math.min(maxBin, scaled));
|
||||||
|
}
|
||||||
@@ -1,4 +1,10 @@
|
|||||||
import FFT from 'fft.js';
|
import FFT from 'fft.js';
|
||||||
|
import {
|
||||||
|
getSpectrogramPitchBinCount,
|
||||||
|
mapMidiPitchToSpectrogramPosition,
|
||||||
|
normalizeSpectrogramHeightResolution,
|
||||||
|
type SpectrogramHeightResolution,
|
||||||
|
} from '../util/spectrogramUtil';
|
||||||
|
|
||||||
type WorkerScopeLike = typeof globalThis & {
|
type WorkerScopeLike = typeof globalThis & {
|
||||||
onmessage: ((event: MessageEvent<SpectrogramRequest>) => void) | null;
|
onmessage: ((event: MessageEvent<SpectrogramRequest>) => void) | null;
|
||||||
@@ -13,16 +19,17 @@ export interface SpectrogramRequest {
|
|||||||
clipStartOffsetSeconds: number;
|
clipStartOffsetSeconds: number;
|
||||||
regionDurationSeconds: number;
|
regionDurationSeconds: number;
|
||||||
bpm: number;
|
bpm: number;
|
||||||
|
heightResolution: SpectrogramHeightResolution;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SpectrogramResult {
|
export interface SpectrogramResult {
|
||||||
data: Float32Array; // [timeSteps × 128] row-major, row = time step, col = pitch 0-127
|
data: Float32Array; // [timeSteps × pitchBins] row-major, low-to-high pitch bins
|
||||||
timeSteps: number;
|
timeSteps: number;
|
||||||
|
pitchBins: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
const FFT_SIZE = 8192;
|
const FFT_SIZE = 8192;
|
||||||
const HOP_SIZE = 1024;
|
const HOP_SIZE = 1024;
|
||||||
const PITCH_BINS = 128;
|
|
||||||
|
|
||||||
function hannWindow(size: number): Float32Array {
|
function hannWindow(size: number): Float32Array {
|
||||||
const w = new Float32Array(size);
|
const w = new Float32Array(size);
|
||||||
@@ -32,8 +39,43 @@ function hannWindow(size: number): Float32Array {
|
|||||||
return w;
|
return w;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function spreadMagnitudeAcrossPitchBins(
|
||||||
|
result: Float32Array,
|
||||||
|
pitchRow: number,
|
||||||
|
pitchBins: number,
|
||||||
|
pitchPosition: number,
|
||||||
|
magnitude: number,
|
||||||
|
heightResolution: SpectrogramHeightResolution,
|
||||||
|
): void {
|
||||||
|
const spreadRadius = Math.max(0, heightResolution - 1);
|
||||||
|
const centerBin = Math.round(pitchPosition);
|
||||||
|
const startBin = Math.max(0, centerBin - spreadRadius);
|
||||||
|
const endBin = Math.min(pitchBins - 1, centerBin + spreadRadius);
|
||||||
|
const spreadWidth = spreadRadius + 1;
|
||||||
|
|
||||||
|
for (let targetBin = startBin; targetBin <= endBin; targetBin++) {
|
||||||
|
const distance = Math.abs(targetBin - pitchPosition);
|
||||||
|
const weight = Math.max(0, 1 - distance / spreadWidth);
|
||||||
|
if (weight <= 0) continue;
|
||||||
|
|
||||||
|
const weightedMagnitude = magnitude * weight;
|
||||||
|
const resultIndex = pitchRow + targetBin;
|
||||||
|
if (weightedMagnitude > result[resultIndex]) {
|
||||||
|
result[resultIndex] = weightedMagnitude;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||||
const { pcm, sampleRate, clipStartOffsetSeconds, regionDurationSeconds } = e.data;
|
const {
|
||||||
|
pcm,
|
||||||
|
sampleRate,
|
||||||
|
clipStartOffsetSeconds,
|
||||||
|
regionDurationSeconds,
|
||||||
|
heightResolution,
|
||||||
|
} = e.data;
|
||||||
|
const normalizedResolution = normalizeSpectrogramHeightResolution(heightResolution);
|
||||||
|
const pitchBins = getSpectrogramPitchBinCount(normalizedResolution);
|
||||||
|
|
||||||
const startSample = Math.floor(clipStartOffsetSeconds * sampleRate);
|
const startSample = Math.floor(clipStartOffsetSeconds * sampleRate);
|
||||||
const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate));
|
const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate));
|
||||||
@@ -45,7 +87,7 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
|||||||
const inputPadded = new Float32Array(FFT_SIZE);
|
const inputPadded = new Float32Array(FFT_SIZE);
|
||||||
|
|
||||||
const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1);
|
const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1);
|
||||||
const result = new Float32Array(totalHops * PITCH_BINS);
|
const result = new Float32Array(totalHops * pitchBins);
|
||||||
|
|
||||||
let maxVal = 0;
|
let maxVal = 0;
|
||||||
|
|
||||||
@@ -64,28 +106,32 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
|||||||
|
|
||||||
// Max-pool magnitude into pitch bins: keep the loudest FFT bin per semitone,
|
// Max-pool magnitude into pitch bins: keep the loudest FFT bin per semitone,
|
||||||
// rather than summing. Summing inflates every bin by how many FFT bins land there.
|
// rather than summing. Summing inflates every bin by how many FFT bins land there.
|
||||||
const pitchRow = hop * PITCH_BINS;
|
const pitchRow = hop * pitchBins;
|
||||||
const numBins = FFT_SIZE / 2;
|
const numBins = FFT_SIZE / 2;
|
||||||
|
|
||||||
for (let bin = 1; bin < numBins; bin++) {
|
for (let bin = 1; bin < numBins; bin++) {
|
||||||
const freq = (bin * sampleRate) / FFT_SIZE;
|
const freq = (bin * sampleRate) / FFT_SIZE;
|
||||||
if (freq < 20 || freq > 20000) continue;
|
if (freq < 20 || freq > 20000) continue;
|
||||||
|
|
||||||
// Convert frequency to MIDI pitch; clamp to piano roll range C0–B7 (MIDI 12–107)
|
const midiPitch = 69 + 12 * Math.log2(freq / 440);
|
||||||
const pitch = Math.round(69 + 12 * Math.log2(freq / 440));
|
const pitchPosition = mapMidiPitchToSpectrogramPosition(midiPitch, normalizedResolution);
|
||||||
if (pitch < 12 || pitch > 107) continue;
|
if (pitchPosition === null) continue;
|
||||||
|
|
||||||
const re = complexOut[2 * bin];
|
const re = complexOut[2 * bin];
|
||||||
const im = complexOut[2 * bin + 1];
|
const im = complexOut[2 * bin + 1];
|
||||||
const magnitude = Math.sqrt(re * re + im * im);
|
const magnitude = Math.sqrt(re * re + im * im);
|
||||||
|
spreadMagnitudeAcrossPitchBins(
|
||||||
if (magnitude > result[pitchRow + pitch]) {
|
result,
|
||||||
result[pitchRow + pitch] = magnitude;
|
pitchRow,
|
||||||
}
|
pitchBins,
|
||||||
|
pitchPosition,
|
||||||
|
magnitude,
|
||||||
|
normalizedResolution,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Track global max for normalization
|
// Track global max for normalization
|
||||||
for (let p = 0; p < PITCH_BINS; p++) {
|
for (let p = 0; p < pitchBins; p++) {
|
||||||
if (result[pitchRow + p] > maxVal) maxVal = result[pitchRow + p];
|
if (result[pitchRow + p] > maxVal) maxVal = result[pitchRow + p];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -98,6 +144,6 @@ workerScope.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const response: SpectrogramResult = { data: result, timeSteps: totalHops };
|
const response: SpectrogramResult = { data: result, timeSteps: totalHops, pitchBins };
|
||||||
workerScope.postMessage(response, [result.buffer]);
|
workerScope.postMessage(response, [result.buffer]);
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user