3dtours/KGStudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implemented piano roll hybrid mode to allow user edit MIDI notes while observing spectrogram as reference

Browse Source
This commit is contained in:
Xiaohan-Tian
2026-05-01 18:28:45 -07:00
parent ba0e27e50a
commit 83034455cc
11 changed files with 162 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/MainContent.tsx
+45 -20
View File
@@ -44,6 +44,8 @@ const MainContent: React.FC<MainContentProps> = ({
pianoRollMode,
openMidiPianoRoll,
openSpectrogramViewer,
openHybridMode,
hybridAudioRegionId,
addTrack,
addAudioTrack,
projectName,
@@ -552,6 +554,19 @@ const MainContent: React.FC<MainContentProps> = ({
openSpectrogramViewer(regionId);
};
// Handle hybrid mode open (+ button clicked on opposite-type region)
const handleOpenHybrid = (regionId: string) => {
if (pianoRollMode === 'midi-edit' && activeRegionId) {
openHybridMode(activeRegionId, regionId);
} else if (pianoRollMode === 'spectrogram' && activeRegionId) {
openHybridMode(regionId, activeRegionId);
}
};
// + button is visible only when piano roll is open and mode is not hybrid
const showHybridButtonForAudio = showPianoRoll && pianoRollMode === 'midi-edit';
const showHybridButtonForMidi = showPianoRoll && pianoRollMode === 'spectrogram';
// Handle piano roll close
const handlePianoRollClose = () => {
setShowPianoRoll(false);
@@ -844,6 +859,9 @@ const MainContent: React.FC<MainContentProps> = ({
onRegionClick={handleRegionClick}
onOpenPianoRoll={handleOpenPianoRoll}
onOpenSpectrogram={handleOpenSpectrogram}
showHybridButtonForAudio={showHybridButtonForAudio}
showHybridButtonForMidi={showHybridButtonForMidi}
onOpenHybrid={handleOpenHybrid}
onExternalDropComplete={handleExternalDropComplete}
/>
</div>
@@ -855,26 +873,33 @@ const MainContent: React.FC<MainContentProps> = ({
onClose={handlePianoRollClose}
regionId={activeRegionId}
mode={pianoRollMode}
audioRegion={pianoRollMode === 'spectrogram' && activeRegionId
? (() => {
for (const track of tracks) {
const region = track.getRegions().find(r => r.getId() === activeRegionId);
if (region && region.getCurrentType() === 'KGAudioRegion') {
return region as unknown as KGAudioRegion;
}
}
return undefined;
})()
: undefined}
trackId={pianoRollMode === 'spectrogram' && activeRegionId
? (() => {
for (const track of tracks) {
const region = track.getRegions().find(r => r.getId() === activeRegionId);
if (region) return track.getId().toString();
}
return undefined;
})()
: undefined}
audioRegion={(() => {
// spectrogram mode: audio region IS the activeRegionId
// hybrid mode: audio region is hybridAudioRegionId
const audioId = pianoRollMode === 'spectrogram' ? activeRegionId
: pianoRollMode === 'hybrid' ? hybridAudioRegionId
: null;
if (!audioId) return undefined;
for (const track of tracks) {
const region = track.getRegions().find(r => r.getId() === audioId);
if (region && region.getCurrentType() === 'KGAudioRegion') {
return region as unknown as KGAudioRegion;
}
}
return undefined;
})()}
trackId={(() => {
const audioId = pianoRollMode === 'spectrogram' ? activeRegionId
: pianoRollMode === 'hybrid' ? hybridAudioRegionId
: null;
if (!audioId) return undefined;
for (const track of tracks) {
if (track.getRegions().some(r => r.getId() === audioId)) {
return track.getId().toString();
}
}
return undefined;
})()}
projectName={savedProjectName}
/>,
document.body
src/components/piano-roll/PianoGrid.tsx
+1
View File
@@ -32,6 +32,7 @@ interface PianoGridProps {
bpm?: number;
spectrogramThresholdDb?: number;
spectrogramPower?: number;
mode?: 'midi-edit' | 'spectrogram' | 'hybrid';
}
interface CursorPosition {
src/components/piano-roll/PianoRoll.css
+10 -2
View File
@@ -77,10 +77,17 @@
/* Ensure toolbar and its dropdowns appear above piano roll content */
}
.piano-roll-toolbar .tool-button {
width: 20px;
height: 20px;
}
/* Override pointer-events for piano roll toolbar sections */
.piano-roll-toolbar .toolbar-left,
.piano-roll-toolbar .toolbar-right {
pointer-events: auto;
/* max-width: 70%; */
width: auto;
}
.piano-roll-toolbar .quant-button {
@@ -101,6 +108,7 @@
display: flex;
align-items: center;
gap: 6px;
margin-left: 5px;
}
.spectrogram-control-label {
@@ -121,7 +129,7 @@
.spectrogram-threshold-value {
font-size: 10px;
color: #e0e0e0;
min-width: 44px;
min-width: 30px;
text-align: right;
white-space: nowrap;
}
@@ -131,7 +139,7 @@
text-align: center;
font-size: 14px;
color: #e0e0e0;
text-transform: uppercase;
/* text-transform: uppercase; */
cursor: pointer;
padding: 5px;
border-radius: 3px;
src/components/piano-roll/PianoRoll.tsx
+7 -1
View File
@@ -23,7 +23,7 @@ interface PianoRollProps {
regionId: string | null;
initialPosition?: { x: number; y: number };
initialSize?: { width: number; height: number };
mode?: 'midi-edit' | 'spectrogram';
mode?: 'midi-edit' | 'spectrogram' | 'hybrid';
audioRegion?: KGAudioRegion;
trackId?: string;
projectName?: string;
@@ -40,6 +40,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
projectName,
}) => {
const isSpectrogram = mode === 'spectrogram';
const isHybrid = mode === 'hybrid';
const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm } = useProjectStore();
// Tool state for piano roll
@@ -870,6 +871,11 @@ const PianoRoll: React.FC<PianoRollProps> = ({
// Get the title for the piano roll based on the active region
const getPianoRollTitle = () => {
if (isSpectrogram) return audioRegion ? `SPECTROGRAM — ${audioRegion.getName()}` : 'SPECTROGRAM';
if (isHybrid) {
const midiName = activeRegion?.getName() ?? 'MIDI';
const audioName = audioRegion?.getName() ?? 'Audio';
return `${midiName} + ${audioName}`;
}
if (!activeRegion) return "EDIT NOTE CLIP";
// Calculate the bar and beat position of the region
src/components/piano-roll/PianoRollContent.tsx
+1 -1
View File
@@ -27,7 +27,7 @@ interface PianoRollContentProps {
selectedMode: string;
keySignature: KeySignature;
chordGuide: string;
mode?: 'midi-edit' | 'spectrogram';
mode?: 'midi-edit' | 'spectrogram' | 'hybrid';
audioRegion?: KGAudioRegion;
trackId?: string;
projectName?: string;
src/components/piano-roll/PianoRollToolbar.tsx
+20 -23
View File
@@ -24,7 +24,7 @@ interface PianoRollToolbarProps {
chordGuide: string;
onChordGuideChange: (value: string) => void;
blinkButton?: string | null;
mode?: 'midi-edit' | 'spectrogram';
mode?: 'midi-edit' | 'spectrogram' | 'hybrid';
thresholdDb?: number;
onThresholdChange?: (db: number) => void;
power?: number;
@@ -51,11 +51,27 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
onPowerChange,
}) => {
const isSpectrogram = mode === 'spectrogram';
const showMidiControls = mode !== 'spectrogram'; // midi-edit and hybrid
const showSpecControls = mode === 'spectrogram' || mode === 'hybrid';
return (
<div className="piano-roll-toolbar">
{!isSpectrogram && (
{showMidiControls && (
<div className="toolbar-left">
<button
className={`tool-button ${activeTool === 'pointer' ? 'active' : ''}`}
onClick={() => onToolSelect('pointer')}
title="Pointer Tool"
>
<FaMousePointer />
</button>
<button
className={`tool-button ${activeTool === 'pencil' ? 'active' : ''}`}
onClick={() => onToolSelect('pencil')}
title="Pencil Tool"
>
<FaPencilAlt />
</button>
<KGDropdown
options={Object.entries(KGCore.FUNCTIONAL_CHORDS_DATA).map(([id, data]) => ({ label: data.name, value: id }))}
value={selectedMode}
@@ -80,27 +96,8 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
</div>
)}
{!isSpectrogram && (
<div className="toolbar-center">
<button
className={`tool-button ${activeTool === 'pointer' ? 'active' : ''}`}
onClick={() => onToolSelect('pointer')}
title="Pointer Tool"
>
<FaMousePointer />
</button>
<button
className={`tool-button ${activeTool === 'pencil' ? 'active' : ''}`}
onClick={() => onToolSelect('pencil')}
title="Pencil Tool"
>
<FaPencilAlt />
</button>
</div>
)}
<div className="toolbar-right">
{!isSpectrogram && (
{showMidiControls && (
<>
<KGDropdown
options={KGPianoRollState.SNAP_OPTIONS}
@@ -127,7 +124,7 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
</>
)}
{isSpectrogram && (
{showSpecControls && (
<div className="spectrogram-toolbar-controls">
<span className="spectrogram-control-label">Floor</span>
<input
src/components/track/Region.css
+21
View File
@@ -140,6 +140,27 @@
background: rgba(0, 0, 0, 0.35);
}
.region-hybrid-btn {
position: absolute;
top: 4px;
left: 24px;
background: rgba(0, 0, 0, 0.25);
color: #fff;
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 3px;
padding: 2px;
margin: 0;
cursor: pointer;
z-index: 2;
display: inline-flex;
align-items: center;
justify-content: center;
}
.region-hybrid-btn:hover {
background: rgba(0, 0, 0, 0.35);
}
/* Instrument dropdown specific styles */
.instrument-dropdown .quant-dropdown {
min-width: 80px;
src/components/track/RegionItem.tsx
+27 -2
View File
@@ -1,6 +1,6 @@
import React, { useState, useRef, useEffect } from 'react';
import './Region.css';
import { FaPencilAlt } from 'react-icons/fa';
import { FaPencilAlt, FaPlus } from 'react-icons/fa';
import { MdGraphicEq } from 'react-icons/md';
import type { ResizeAction } from '../interfaces';
import { REGION_CONSTANTS, DEBUG_MODE } from '../../constants';
@@ -30,6 +30,9 @@ interface RegionItemProps {
onOpenPianoRoll?: (regionId: string) => void;
// Open spectrogram viewer for audio regions
onOpenSpectrogram?: (regionId: string) => void;
// Enter hybrid mode (show + when piano roll is open with the opposite region type selected)
showHybridButton?: boolean;
onOpenHybrid?: (regionId: string) => void;
// MIDI region data for rendering notes
midiRegion?: KGMidiRegion;
// Audio region data for rendering waveform
@@ -53,6 +56,8 @@ const RegionItem: React.FC<RegionItemProps> = ({
onClick,
onOpenPianoRoll,
onOpenSpectrogram,
showHybridButton,
onOpenHybrid,
midiRegion,
audioRegion,
audioBuffer
@@ -64,7 +69,7 @@ const RegionItem: React.FC<RegionItemProps> = ({
const [resizeEdge, setResizeEdge] = useState<ResizeAction>('none');
const [isResizing, setIsResizing] = useState(false);
const [isDragging, setIsDragging] = useState(false);
const initialMousePosRef = useRef<{x: number, y: number}>({x: 0, y: 0});
const initialMousePosRef = useRef<{ x: number, y: number }>({ x: 0, y: 0 });
// Use refs to track states for immediate access
const isResizingRef = useRef<boolean>(false);
const isDraggingRef = useRef<boolean>(false);
@@ -600,6 +605,26 @@ const RegionItem: React.FC<RegionItemProps> = ({
<MdGraphicEq size={10} />
</button>
)}
{showHybridButton && (
<button
className="region-hybrid-btn"
title="Open in hybrid mode"
onMouseDown={(e) => {
e.preventDefault();
e.stopPropagation();
}}
onClick={(e) => {
e.preventDefault();
e.stopPropagation();
if (onOpenHybrid) {
onOpenHybrid(id);
}
}}
aria-label="Open hybrid mode"
>
<FaPlus size={10} />
</button>
)}
<canvas ref={canvasRef} />
</div>
</div>
src/components/track/TrackGridItem.tsx
+8
View File
@@ -27,6 +27,9 @@ interface TrackGridItemProps {
onRegionClick?: (regionId: string) => void;
onOpenPianoRoll?: (regionId: string) => void;
onOpenSpectrogram?: (regionId: string) => void;
showHybridButtonForAudio?: boolean;
showHybridButtonForMidi?: boolean;
onOpenHybrid?: (regionId: string) => void;
allTracks?: KGTrack[]; // Added to access all tracks for drag operations
onKGOneClipDrop?: (e: React.DragEvent<HTMLDivElement>, trackIndex: number) => void;
}
@@ -49,6 +52,9 @@ const TrackGridItem: React.FC<TrackGridItemProps> = ({
onRegionClick,
onOpenPianoRoll,
onOpenSpectrogram,
showHybridButtonForAudio,
showHybridButtonForMidi,
onOpenHybrid,
allTracks,
onKGOneClipDrop,
}) => {
@@ -571,6 +577,8 @@ const TrackGridItem: React.FC<TrackGridItemProps> = ({
onOpenSpectrogram={audioRegion ? (regionId) => {
onOpenSpectrogram?.(regionId);
} : undefined}
showHybridButton={audioRegion ? showHybridButtonForAudio : showHybridButtonForMidi}
onOpenHybrid={onOpenHybrid}
midiRegion={midiRegion}
audioRegion={audioRegion}
audioBuffer={audioBuffer}
src/components/track/TrackGridPanel.tsx
+9
View File
@@ -31,6 +31,9 @@ interface TrackGridPanelProps {
onRegionClick?: (regionId: string) => void;
onOpenPianoRoll?: (regionId: string) => void;
onOpenSpectrogram?: (regionId: string) => void;
showHybridButtonForAudio?: boolean;
showHybridButtonForMidi?: boolean;
onOpenHybrid?: (regionId: string) => void;
onExternalDropComplete?: (trackIndex: number, regionUI: RegionUI) => void;
}
@@ -48,6 +51,9 @@ const TrackGridPanel: React.FC<TrackGridPanelProps> = ({
onRegionClick,
onOpenPianoRoll,
onOpenSpectrogram,
showHybridButtonForAudio,
showHybridButtonForMidi,
onOpenHybrid,
onExternalDropComplete,
}) => {
const gridContainerRef = useRef<HTMLDivElement>(null);
@@ -654,6 +660,9 @@ const TrackGridPanel: React.FC<TrackGridPanelProps> = ({
onRegionClick={handleRegionClick}
onOpenPianoRoll={onOpenPianoRoll}
onOpenSpectrogram={onOpenSpectrogram}
showHybridButtonForAudio={showHybridButtonForAudio}
showHybridButtonForMidi={showHybridButtonForMidi}
onOpenHybrid={onOpenHybrid}
allTracks={tracks}
onKGOneClipDrop={handleExternalDrop}
/>
src/stores/projectStore.ts
+13 -6
View File
@@ -78,7 +78,8 @@ interface ProjectState {
// Piano roll state
showPianoRoll: boolean;
activeRegionId: string | null;
pianoRollMode: 'midi-edit' | 'spectrogram';
pianoRollMode: 'midi-edit' | 'spectrogram' | 'hybrid';
hybridAudioRegionId: string | null;
// ChatBox state
showChatBox: boolean;
@@ -149,6 +150,7 @@ interface ProjectState {
setActiveRegionId: (regionId: string | null) => void;
openMidiPianoRoll: (regionId: string) => void;
openSpectrogramViewer: (regionId: string) => void;
openHybridMode: (midiRegionId: string, audioRegionId: string) => void;
// Project state cleanup
cleanupProjectState: () => void;
@@ -308,6 +310,7 @@ export const useProjectStore = create<ProjectState>((set, get) => {
showPianoRoll: false,
activeRegionId: null,
pianoRollMode: 'midi-edit' as const,
hybridAudioRegionId: null,
// Initial ChatBox state
showChatBox: initialChatBoxState,
@@ -1038,20 +1041,24 @@ export const useProjectStore = create<ProjectState>((set, get) => {
},
openMidiPianoRoll: (regionId: string) => {
set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'midi-edit' });
set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'midi-edit', hybridAudioRegionId: null });
},
openSpectrogramViewer: (regionId: string) => {
set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'spectrogram' });
set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'spectrogram', hybridAudioRegionId: null });
},
openHybridMode: (midiRegionId: string, audioRegionId: string) => {
set({ showPianoRoll: true, activeRegionId: midiRegionId, hybridAudioRegionId: audioRegionId, pianoRollMode: 'hybrid' });
},
// Project state cleanup - used when starting new/loading projects
cleanupProjectState: () => {
// Close piano roll if it's visible
set({ showPianoRoll: false });
// Clear active region
set({ activeRegionId: null });
// Clear active region and hybrid state
set({ activeRegionId: null, hybridAudioRegionId: null, pianoRollMode: 'midi-edit' });
// Clear any selected items
KGCore.instance().clearSelectedItems();