3dtours/KGStudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: added an option to let user choose the vertical resolution of the spectrogram, also fixed a bug that peak of pitch in the spectrogram has been mapped to the start of the note bin instead of the center

Browse Source
This commit is contained in:
Xiaohan-Tian
2026-05-04 20:14:04 -07:00
parent d089456676
commit 8a6e904ab4
12 changed files with 302 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/piano-roll/PianoGrid.tsx
+5 -1
View File
@@ -8,6 +8,7 @@ import type { KeySignature } from '../../core/KGProject';
import { KGPianoRollState } from '../../core/state/KGPianoRollState';
import SpectrogramCanvas from './SpectrogramCanvas';
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
import type { SpectrogramHeightResolution } from '../../util/spectrogramUtil';
interface PianoGridProps {
gridRef: MutableRefObject<HTMLDivElement | null>;
@@ -32,6 +33,7 @@ interface PianoGridProps {
bpm?: number;
spectrogramThresholdDb?: number;
spectrogramPower?: number;
spectrogramHeightResolution?: SpectrogramHeightResolution;
pianoRollZoom?: number;
mode?: 'midi-edit' | 'spectrogram' | 'hybrid';
onSpectrogramLoadingChange?: (loading: boolean) => void;
@@ -62,6 +64,7 @@ const PianoGrid: React.FC<PianoGridProps> = ({
bpm = 120,
spectrogramThresholdDb = -25,
spectrogramPower = 0.5,
spectrogramHeightResolution = 3,
pianoRollZoom = 1,
onSpectrogramLoadingChange,
}) => {
@@ -245,6 +248,7 @@ const PianoGrid: React.FC<PianoGridProps> = ({
thresholdDb={spectrogramThresholdDb}
power={spectrogramPower}
zoom={pianoRollZoom}
heightResolution={spectrogramHeightResolution}
onLoadingChange={onSpectrogramLoadingChange}
/>
)}
@@ -305,4 +309,4 @@ const PianoGrid: React.FC<PianoGridProps> = ({
);
};
export default PianoGrid;
export default PianoGrid;
src/components/piano-roll/PianoRoll.tsx
+42
View File
@@ -18,6 +18,10 @@ import { beatsToBar } from '../../util/midiUtil';
import { UpdateRegionCommand } from '../../core/commands';
import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil';
import { showAlert, showPrompt } from '../../util/dialogUtil';
import {
normalizeSpectrogramHeightResolution,
type SpectrogramHeightResolution,
} from '../../util/spectrogramUtil';
interface PianoRollProps {
onClose: () => void;
@@ -50,6 +54,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
// Spectrogram controls (only used in spectrogram mode)
const [spectrogramThresholdDb, setSpectrogramThresholdDb] = useState<number>(-25);
const [spectrogramPower, setSpectrogramPower] = useState<number>(0.5);
const [spectrogramHeightResolution, setSpectrogramHeightResolution] =
useState<SpectrogramHeightResolution>(3);
// Piano roll zoom (1x–8x); updates --region-grid-beat-width CSS variable
const [pianoRollZoom, setPianoRollZoom] = useState<number>(1);
@@ -201,6 +207,41 @@ const PianoRoll: React.FC<PianoRollProps> = ({
}
}, []); // Empty dependency array means this runs once on mount
useEffect(() => {
let unsubscribe: (() => void) | undefined;
const initializeSpectrogramHeightResolution = async () => {
const configManager = ConfigManager.instance();
if (!configManager.getIsInitialized()) {
await configManager.initialize();
}
const applyResolutionFromConfig = () => {
setSpectrogramHeightResolution(
normalizeSpectrogramHeightResolution(
configManager.get('editor.spectrogram_height_resolution')
)
);
};
applyResolutionFromConfig();
unsubscribe = configManager.addChangeListener((changedKeys) => {
if (
changedKeys.includes('__all__') ||
changedKeys.includes('editor.spectrogram_height_resolution')
) {
applyResolutionFromConfig();
}
});
};
void initializeSpectrogramHeightResolution();
return () => {
unsubscribe?.();
};
}, []);
// Add keyboard event listener for Escape
useEffect(() => {
const handleKeyDown = (event: KeyboardEvent) => {
@@ -1048,6 +1089,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
bpm={bpm}
spectrogramThresholdDb={spectrogramThresholdDb}
spectrogramPower={spectrogramPower}
spectrogramHeightResolution={spectrogramHeightResolution}
pianoRollZoom={pianoRollZoom}
/>
src/components/piano-roll/PianoRollContent.tsx
+4
View File
@@ -14,6 +14,7 @@ import { useNoteSelection } from '../../hooks/useNoteSelection';
import type { KeySignature } from '../../core/KGProject';
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
import { velocityToColor } from '../../util/velocityColor';
import type { SpectrogramHeightResolution } from '../../util/spectrogramUtil';
interface PianoRollContentProps {
contentRef: React.MutableRefObject<HTMLDivElement | null>;
@@ -35,6 +36,7 @@ interface PianoRollContentProps {
bpm?: number;
spectrogramThresholdDb?: number;
spectrogramPower?: number;
spectrogramHeightResolution?: SpectrogramHeightResolution;
pianoRollZoom?: number;
}
@@ -58,6 +60,7 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
bpm = 120,
spectrogramThresholdDb = -25,
spectrogramPower = 0.5,
spectrogramHeightResolution = 3,
pianoRollZoom = 1,
}) => {
const isSpectrogram = mode === 'spectrogram';
@@ -295,6 +298,7 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
bpm={bpm}
spectrogramThresholdDb={spectrogramThresholdDb}
spectrogramPower={spectrogramPower}
spectrogramHeightResolution={spectrogramHeightResolution}
pianoRollZoom={pianoRollZoom}
onSpectrogramLoadingChange={handleSpectrogramLoadingChange}
>
src/components/piano-roll/SpectrogramCanvas.tsx
+31 -16
View File
@@ -4,6 +4,12 @@ import { KGAudioRegion } from '../../core/region/KGAudioRegion';
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker';
import {
getSpectrogramVisibleBinRange,
normalizeSpectrogramHeightResolution,
SPECTROGRAM_VISIBLE_SEMITONES,
type SpectrogramHeightResolution,
} from '../../util/spectrogramUtil';
interface SpectrogramCanvasProps {
audioRegion: KGAudioRegion;
@@ -13,12 +19,10 @@ interface SpectrogramCanvasProps {
thresholdDb: number;
power: number;
zoom: number;
heightResolution: SpectrogramHeightResolution;
onLoadingChange?: (loading: boolean) => void;
}
const PITCH_BINS = 128;
const HOP_SIZE = 1024;
// Piecewise-linear RGB colormap: black → dark blue → blue → purple → red → orange → yellow
// Hue rotates 240°→300°→0°→60°, bypassing green entirely.
const COLORMAP_STOPS: Array<[number, [number, number, number]]> = [
@@ -56,6 +60,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
thresholdDb,
power,
zoom,
heightResolution,
onLoadingChange,
}) => {
const canvasRef = useRef<HTMLCanvasElement | null>(null);
@@ -80,6 +85,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
regionDurationSeconds: number,
thresholdDb: number,
power: number,
heightResolution: SpectrogramHeightResolution,
) => {
const canvas = canvasRef.current;
if (!canvas) return;
@@ -92,31 +98,31 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
const totalBeats = (regionDurationSeconds * bpm) / 60;
// Always draw at 1x resolution; zoom is applied as CSS width stretch
const canvasWidth = Math.ceil(totalBeats * 40);
const canvasHeight = PITCH_BINS * noteHeight;
const canvasHeight = SPECTROGRAM_VISIBLE_SEMITONES * noteHeight;
// Convert dB threshold to linear: values below this → black
const linearThreshold = Math.pow(10, thresholdDb / 20);
const visibleRange = getSpectrogramVisibleBinRange(heightResolution);
const visiblePitchBins = visibleRange.end - visibleRange.start;
// 1. Paint at natural STFT resolution onto an offscreen canvas (timeSteps × 128).
// Row i = pitchIndex i = pitch (107 − i). Apply threshold then power curve.
// 1. Paint at natural spectrogram resolution onto an offscreen canvas.
// Result data is low-to-high pitch; draw only the visible C0-B7 window, reversed for display.
const offscreen = document.createElement('canvas');
offscreen.width = result.timeSteps;
offscreen.height = PITCH_BINS;
offscreen.height = visiblePitchBins;
const offCtx = offscreen.getContext('2d');
if (!offCtx) return;
const imgData = offCtx.createImageData(result.timeSteps, PITCH_BINS);
const imgData = offCtx.createImageData(result.timeSteps, visiblePitchBins);
const pixels = imgData.data;
for (let row = 0; row < PITCH_BINS; row++) {
const pitch = 107 - row;
for (let row = 0; row < visiblePitchBins; row++) {
const sourceRow = visibleRange.end - 1 - row;
for (let col = 0; col < result.timeSteps; col++) {
const idx = (row * result.timeSteps + col) * 4;
pixels[idx + 3] = 255; // always opaque
if (pitch < 12 || pitch > 107) continue; // outside range → black
const raw = result.data[col * PITCH_BINS + pitch];
const raw = result.data[col * result.pitchBins + sourceRow];
// Hard threshold: values below noise floor → 0 (black)
// Re-scale surviving range to [0,1] then apply power curve
@@ -157,9 +163,10 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
regionDurationRef.current,
thresholdDb,
power,
normalizeSpectrogramHeightResolution(heightResolution),
);
}
}, [thresholdDb, power, renderSpectrogram]);
}, [thresholdDb, power, renderSpectrogram, heightResolution]);
// Zoom changes: stretch width only, pin height to canvas pixel height
useEffect(() => {
@@ -211,7 +218,14 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
rawResultRef.current = e.data;
sampleRateRef.current = sampleRate;
regionDurationRef.current = regionDurationSeconds;
renderSpectrogram(e.data, sampleRate, regionDurationSeconds, thresholdDb, power);
renderSpectrogram(
e.data,
sampleRate,
regionDurationSeconds,
thresholdDb,
power,
normalizeSpectrogramHeightResolution(heightResolution),
);
setLoading(false);
worker.terminate();
workerRef.current = null;
@@ -223,6 +237,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
regionDurationSeconds,
bpm,
heightResolution: normalizeSpectrogramHeightResolution(heightResolution),
};
worker.postMessage(request, [request.pcm.buffer]);
} catch (err) {
@@ -241,7 +256,7 @@ const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
workerRef.current = null;
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [audioRegion, trackId, projectName, bpm]);
}, [audioRegion, trackId, projectName, bpm, heightResolution]);
return (
<canvas
src/components/settings/sections/BehaviorSettings.tsx
+33 -1
View File
@@ -1,9 +1,14 @@
import React, { useState, useEffect } from 'react';
import { ConfigManager } from '../../../core/config/ConfigManager';
import { KGAudioInterface } from '../../../core/audio-interface/KGAudioInterface';
import {
normalizeSpectrogramHeightResolution,
type SpectrogramHeightResolution,
} from '../../../util/spectrogramUtil';
const BehaviorSettings: React.FC = () => {
const [playheadUpdateFrequency, setPlayheadUpdateFrequency] = useState<number>(10);
const [spectrogramHeightResolution, setSpectrogramHeightResolution] = useState<SpectrogramHeightResolution>(3);
const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState<boolean>(true);
const [audioLookaheadTime, setAudioLookaheadTime] = useState<string>('50');
const [playbackDelay, setPlaybackDelay] = useState<string>('200');
@@ -23,6 +28,9 @@ const BehaviorSettings: React.FC = () => {
}
setPlayheadUpdateFrequency((configManager.get('editor.playhead_update_frequency') as number) ?? 10);
setSpectrogramHeightResolution(
normalizeSpectrogramHeightResolution(configManager.get('editor.spectrogram_height_resolution'))
);
setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true);
const lookaheadTimeSeconds = (configManager.get('audio.lookahead_time') as number) ?? 0.05;
setAudioLookaheadTime(((lookaheadTimeSeconds * 1000).toFixed(0)));
@@ -50,6 +58,12 @@ const BehaviorSettings: React.FC = () => {
await configManager.set('chatbox.default_open', boolValue);
};
const handleSpectrogramHeightResolutionChange = async (value: string) => {
const nextValue = normalizeSpectrogramHeightResolution(parseInt(value, 10));
setSpectrogramHeightResolution(nextValue);
await configManager.set('editor.spectrogram_height_resolution', nextValue);
};
const handleAudioLookaheadTimeChange = async (value: string) => {
// Allow empty string, treat as 0 ms
const numValueMs = value === '' ? 0 : parseFloat(value);
@@ -161,6 +175,24 @@ const BehaviorSettings: React.FC = () => {
Update frequency for the playhead animation during playback. Higher values (60 fps) provide smoother animation but use more CPU. Lower values (10 fps) are more efficient. Changes apply immediately without restart.
</div>
</div>
<div className="settings-item">
<label className="settings-label">
Spectrogram Height Resolution
</label>
<select
className="settings-select"
value={spectrogramHeightResolution}
onChange={(e) => handleSpectrogramHeightResolutionChange(e.target.value)}
>
<option value="1">1x</option>
<option value="3">3x</option>
<option value="5">5x</option>
</select>
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
Controls vertical spectrogram detail across the visible pitch range. Higher values sharpen pitch contours but use more CPU and memory while computing the spectrogram.
</div>
</div>
</div>
<div className="settings-group">
@@ -289,4 +321,4 @@ const BehaviorSettings: React.FC = () => {
);
};
export default BehaviorSettings;
export default BehaviorSettings;