3dtours/KGStudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: added recording MIDI device feature.

Browse Source
This commit is contained in:
Xiaohan-Tian
2026-05-01 10:59:36 -07:00
parent bd578cb01a
commit 77885c8e2e
8 changed files with 277 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
public/config.json
+2 -1
View File
@@ -73,7 +73,8 @@
"audio": {
"enable_audio_capture_for_screen_sharing": false,
"lookahead_time": 0.05,
"playback_delay": 0.2
"playback_delay": 0.2,
"recording_offset": 0
},
"templates": {
"custom_instructions": ""
src/components/Toolbar.tsx
+57 -1
View File
@@ -12,9 +12,11 @@ import {
FaUndo, FaRedo, FaMousePointer, FaStepBackward,
FaPlay, FaPause, FaComments, FaSync,
FaFolderOpen, FaSave, FaDownload, FaUpload, FaPlus,
FaCog, FaMagnet, FaCut
FaCog, FaMagnet, FaCut, FaCircle
} from 'react-icons/fa';
import { KGProject, type KeySignature } from '../core/KGProject';
import { KGMidiInput } from '../core/midi-input/KGMidiInput';
import { KGMidiRegion } from '../core/region/KGMidiRegion';
import { plainToInstance } from 'class-transformer';
import { FaPencil, FaCopy, FaPaste, FaTrash, FaWandMagicSparkles } from 'react-icons/fa6';
import { KGMainContentState } from '../core/state/KGMainContentState';
@@ -45,6 +47,7 @@ const Toolbar: React.FC = () => {
isLooping, toggleLoop,
canUndo, canRedo, undoDescription, redoDescription, undo, redo,
toggleChatBox, toggleSettings, toggleKGOnePanel, showKGOnePanel, cleanupProjectState, toggleMetronome, isMetronomeEnabled,
isRecording, startRecording, stopRecording,
// Piano roll state/actions
showPianoRoll, setShowPianoRoll, activeRegionId, setActiveRegionId,
// Selection state
@@ -515,6 +518,11 @@ const Toolbar: React.FC = () => {
console.log("Pause button clicked");
}
try {
if (isRecording) {
await stopRecording();
setStatus("Recording stopped — notes committed");
return;
}
await stopPlaying();
} catch (error) {
console.error("Failed to stop playback:", error);
@@ -860,6 +868,46 @@ const Toolbar: React.FC = () => {
}
};
const handleRecordClick = async () => {
if (isRecording) {
await stopRecording();
setStatus("Recording stopped — notes committed");
return;
}
// Require an active or selected MIDI region
const candidateId = activeRegionId ?? (selectedRegionIds[0] ?? null);
if (!candidateId) {
await showAlert("Please open a MIDI region in the Piano Roll before starting recording.");
return;
}
const tracks = KGCore.instance().getCurrentProject().getTracks();
let isMidi = false;
for (const track of tracks) {
const region = track.getRegions().find(r => r.getId() === candidateId);
if (region) { isMidi = region instanceof KGMidiRegion; break; }
}
if (!isMidi) {
await showAlert("Please select a MIDI region before starting recording.");
return;
}
// Require at least one connected MIDI device
if (KGMidiInput.instance().getConnectedInputCount() === 0) {
await showAlert("No MIDI device detected. Please connect a MIDI keyboard and try again.");
return;
}
// Ensure the piano roll is open showing the target region
if (!activeRegionId) {
setActiveRegionId(candidateId);
setShowPianoRoll(true);
}
await startRecording();
setStatus("Recording started...");
};
return (
<>
<div className="toolbar">
@@ -943,6 +991,14 @@ const Toolbar: React.FC = () => {
) : (
<button title="Pause" className="button-pause" onClick={handlePauseClick}><FaPause /></button>
)}
<button
title={isRecording ? "Stop Recording" : "Record"}
className={`tool-button ${isRecording ? 'active' : ''}`}
style={isRecording ? { color: 'red' } : undefined}
onClick={handleRecordClick}
>
<FaCircle />
</button>
<button
title="Loop"
className={`tool-button ${isLooping ? 'active' : ''}`}
src/components/piano-roll/PianoRoll.css
+8
View File
@@ -255,6 +255,14 @@
transition: opacity 0.1s ease;
}
.piano-grid-recording-note {
position: absolute;
background-color: rgba(255, 60, 60, 0.35);
border: 1px solid rgba(255, 60, 60, 0.85);
pointer-events: none;
z-index: 3;
}
.resize-handle {
position: absolute;
right: 5px;
src/components/piano-roll/PianoRollContent.tsx
+24
View File
@@ -4,6 +4,7 @@ import { KGMidiRegion } from '../../core/region/KGMidiRegion';
import { KGMidiNote } from '../../core/midi/KGMidiNote';
import { KGTrack } from '../../core/track/KGTrack';
import { KGCore } from '../../core/KGCore';
import { useProjectStore } from '../../stores/projectStore';
import PianoNote from './PianoNote';
import PianoKeys from './PianoKeys';
import PianoGridHeader from './PianoGridHeader';
@@ -44,6 +45,10 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
// Get KGCore instance
const core = KGCore.instance();
// Recording state
const isRecording = useProjectStore(s => s.isRecording);
const recordingNotes = useProjectStore(s => s.recordingNotes);
// Use the note operations hook for resize and drag functionality
const {
resizingNoteId,
@@ -216,6 +221,24 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
});
}, [activeRegion, noteUpdateCounter, resizingNoteId, draggingNoteId, tempNoteStyles, selectedNoteIds, selectionBoxRender, tracks]);
const recordingNoteOverlays = useMemo(() => {
if (!isRecording || !activeRegion || recordingNotes.length === 0) return null;
const beatWidth = parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-grid-beat-width')) || 40;
const noteHeight = parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-height')) || 20;
return recordingNotes.map((note, index) => (
<div
key={`recording-note-${index}`}
className="piano-grid-recording-note"
style={{
left: note.startBeat * beatWidth,
top: (107 - note.pitch) * noteHeight,
width: Math.max((note.endBeat - note.startBeat) * beatWidth, 4),
height: noteHeight,
}}
/>
));
}, [isRecording, recordingNotes, activeRegion]);
return (
<div
className="piano-roll-content"
@@ -239,6 +262,7 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
chordGuide={chordGuide}
>
{memoizedNotes}
{recordingNoteOverlays}
</PianoGrid>
</div>
</div>
src/components/settings/sections/BehaviorSettings.tsx
+53
View File
@@ -7,9 +7,11 @@ const BehaviorSettings: React.FC = () => {
const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState<boolean>(true);
const [audioLookaheadTime, setAudioLookaheadTime] = useState<string>('50');
const [playbackDelay, setPlaybackDelay] = useState<string>('200');
const [recordingOffset, setRecordingOffset] = useState<string>('0');
const [enableAudioCapture, setEnableAudioCapture] = useState<boolean>(false);
const [lookaheadValidationErrors, setLookaheadValidationErrors] = useState<string[]>([]);
const [playbackDelayValidationErrors, setPlaybackDelayValidationErrors] = useState<string[]>([]);
const [recordingOffsetValidationErrors, setRecordingOffsetValidationErrors] = useState<string[]>([]);
const configManager = ConfigManager.instance();
@@ -26,6 +28,8 @@ const BehaviorSettings: React.FC = () => {
setAudioLookaheadTime(((lookaheadTimeSeconds * 1000).toFixed(0)));
const playbackDelaySeconds = (configManager.get('audio.playback_delay') as number) ?? 0.2;
setPlaybackDelay(((playbackDelaySeconds * 1000).toFixed(0)));
const recordingOffsetSeconds = (configManager.get('audio.recording_offset') as number) ?? 0;
setRecordingOffset(((recordingOffsetSeconds * 1000).toFixed(0)));
setEnableAudioCapture((configManager.get('audio.enable_audio_capture_for_screen_sharing') as boolean) ?? false);
};
@@ -102,6 +106,28 @@ const BehaviorSettings: React.FC = () => {
}
};
const handleRecordingOffsetChange = async (value: string) => {
const numValueMs = value === '' ? 0 : parseFloat(value);
const numValueSeconds = numValueMs / 1000;
const errors: string[] = [];
if (isNaN(numValueMs)) {
errors.push('MIDI input latency must be a valid number');
} else if (numValueSeconds < 0) {
errors.push('MIDI input latency must be between 0 and 0.5 seconds (0-500ms)');
} else if (numValueSeconds > 0.5) {
errors.push('MIDI input latency must be between 0 and 0.5 seconds (0-500ms)');
}
setRecordingOffsetValidationErrors(errors);
if (errors.length === 0) {
setRecordingOffset(value);
await configManager.set('audio.recording_offset', numValueSeconds);
console.log(`MIDI input latency changed to: ${numValueSeconds}s (${numValueMs}ms)`);
}
};
const handleEnableAudioCaptureChange = async (value: string) => {
const boolValue = value === 'yes';
setEnableAudioCapture(boolValue);
@@ -212,6 +238,33 @@ const BehaviorSettings: React.FC = () => {
</div>
</div>
<div className="settings-item">
<label className="settings-label">
MIDI Input Latency (ms)
</label>
<input
type="number"
className="settings-select"
value={recordingOffset}
onChange={(e) => handleRecordingOffsetChange(e.target.value)}
min="0"
max="500"
step="1"
/>
{recordingOffsetValidationErrors.length > 0 && (
<div className="settings-validation-errors">
{recordingOffsetValidationErrors.map((error, index) => (
<div key={index} className="settings-validation-error">
{error}
</div>
))}
</div>
)}
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
Timing correction for MIDI recording (0-500ms). If recorded notes appear slightly late compared to where you intended to play them, increase this value to match your MIDI device's input latency. Each note's position is shifted back by this amount when committed.
</div>
</div>
<div className="settings-item">
<label className="settings-label">
Capture Audio for Screen Sharing
src/core/config/ConfigManager.ts
+3 -1
View File
@@ -79,6 +79,7 @@ interface AppConfig {
enable_audio_capture_for_screen_sharing: boolean;
lookahead_time: number;
playback_delay: number;
recording_offset: number;
};
templates: {
custom_instructions: string;
@@ -250,7 +251,8 @@ export class ConfigManager {
audio: {
enable_audio_capture_for_screen_sharing: false,
lookahead_time: 0.05,
playback_delay: 0.2
playback_delay: 0.2,
recording_offset: 0
},
templates: {
custom_instructions: ''
src/core/midi-input/KGMidiInput.ts
+16
View File
@@ -15,6 +15,10 @@ export class KGMidiInput {
private isInitialized: boolean = false;
private connectedInputs: Map<string, MIDIInput> = new Map();
// Recording callbacks
private onRecordNoteOn: ((pitch: number) => void) | null = null;
private onRecordNoteOff: ((pitch: number) => void) | null = null;
// Private constructor to prevent direct instantiation
private constructor() {
console.log("KGMidiInput initialized");
@@ -161,11 +165,13 @@ export class KGMidiInput {
if (command === 0x90 && velocity > 0) {
console.log(`MIDI Note On: pitch=${pitch}, velocity=${velocity}, channel=${channel}`);
this.triggerNoteOn(pitch, velocity);
this.onRecordNoteOn?.(pitch);
}
// Note Off: command = 0x80 (128) or Note On with velocity 0
else if (command === 0x80 || (command === 0x90 && velocity === 0)) {
console.log(`MIDI Note Off: pitch=${pitch}, channel=${channel}`);
this.triggerNoteOff(pitch);
this.onRecordNoteOff?.(pitch);
}
// Control Change: command = 0xB0 (176)
else if (command === 0xb0) {
@@ -264,6 +270,16 @@ export class KGMidiInput {
}
}
// ===== RECORDING =====
public setRecordingCallbacks(
onNoteOn: ((pitch: number) => void) | null,
onNoteOff: ((pitch: number) => void) | null
): void {
this.onRecordNoteOn = onNoteOn;
this.onRecordNoteOff = onNoteOff;
}
// ===== GETTERS =====
public getIsInitialized(): boolean {
src/stores/projectStore.ts
+110
View File
@@ -18,6 +18,10 @@ import { upgradeProjectToLatest } from '../core/project-upgrader/KGProjectUpgrad
import { toggleLoop } from '../util/loopUtil';
import { TOOLBAR_CONSTANTS } from '../constants/uiConstants';
import * as Tone from 'tone';
import { KGMidiInput } from '../core/midi-input/KGMidiInput';
import { KGMidiRegion } from '../core/region/KGMidiRegion';
import { CreateNotesCommand } from '../core/commands/note/CreateNotesCommand';
import type { NoteCreationData } from '../core/commands/note/CreateNotesCommand';
/**
* Update CSS custom property for time signature numerator
@@ -92,6 +96,12 @@ interface ProjectState {
// Settings state
showSettings: boolean;
// Recording state
isRecording: boolean;
recordingTargetRegionId: string | null;
recordingNotes: Array<{ pitch: number; startBeat: number; endBeat: number }>;
recordingOriginalPlayhead: number;
// Undo/redo state
canUndo: boolean;
canRedo: boolean;
@@ -168,10 +178,18 @@ interface ProjectState {
// Project state refresh actions
refreshProjectState: () => void;
// Recording actions
startRecording: () => Promise<void>;
stopRecording: () => Promise<void>;
// Initialization actions
initializeFromConfig: () => Promise<void>;
}
// Module-level recording state (not reactive — only used for timing during active recording)
let _recordingActiveNotes: Map<number, number> = new Map(); // pitch → region-relative startBeat
let _recordingRegionStartBeat: number = 0;
// Create the store
export const useProjectStore = create<ProjectState>((set, get) => {
const currentProject = KGCore.instance().getCurrentProject();
@@ -309,6 +327,12 @@ export const useProjectStore = create<ProjectState>((set, get) => {
undoDescription: null,
redoDescription: null,
// Initial recording state
isRecording: false,
recordingTargetRegionId: null,
recordingNotes: [],
recordingOriginalPlayhead: 0,
// Actions
setProjectName: (name: string) => {
try {
@@ -772,6 +796,92 @@ export const useProjectStore = create<ProjectState>((set, get) => {
set({ isPlaying: false });
},
startRecording: async () => {
const { activeRegionId, timeSignature, playheadPosition, startPlaying, setPlayheadPosition } = get();
const project = KGCore.instance().getCurrentProject();
let targetRegion: KGMidiRegion | null = null;
for (const track of project.getTracks()) {
const found = track.getRegions().find(r => r.getId() === activeRegionId);
if (found instanceof KGMidiRegion) { targetRegion = found; break; }
}
if (!targetRegion) return;
_recordingRegionStartBeat = targetRegion.getStartFromBeat();
_recordingActiveNotes = new Map();
set({
isRecording: true,
recordingNotes: [],
recordingTargetRegionId: activeRegionId,
recordingOriginalPlayhead: playheadPosition,
});
const buildCorrectedBeat = (): number => {
const bpm = get().bpm;
const playbackDelaySec = (ConfigManager.instance().get('audio.playback_delay') as number) ?? 0.2;
const recordingOffsetSec = (ConfigManager.instance().get('audio.recording_offset') as number) ?? 0;
const correctionBeats = (playbackDelaySec + recordingOffsetSec) * (bpm / 60);
return KGAudioInterface.instance().getTransportPosition() - correctionBeats - _recordingRegionStartBeat;
};
KGMidiInput.instance().setRecordingCallbacks(
(pitch: number) => {
const beat = buildCorrectedBeat();
_recordingActiveNotes.set(pitch, beat);
},
(pitch: number) => {
const endBeat = buildCorrectedBeat();
const startBeat = _recordingActiveNotes.get(pitch);
if (startBeat !== undefined) {
_recordingActiveNotes.delete(pitch);
set(state => ({
recordingNotes: [...state.recordingNotes, { pitch, startBeat, endBeat }],
}));
}
}
);
setPlayheadPosition(playheadPosition - timeSignature.numerator);
await startPlaying();
},
stopRecording: async () => {
const { recordingNotes, recordingTargetRegionId, recordingOriginalPlayhead, stopPlaying, setPlayheadPosition, refreshProjectState } = get();
// Finalize any held keys
const finalNotes = [...recordingNotes];
const bpm = get().bpm;
const playbackDelaySec = (ConfigManager.instance().get('audio.playback_delay') as number) ?? 0.2;
const recordingOffsetSec = (ConfigManager.instance().get('audio.recording_offset') as number) ?? 0;
const correctionBeats = (playbackDelaySec + recordingOffsetSec) * (bpm / 60);
const endBeatForHeld = KGAudioInterface.instance().getTransportPosition() - correctionBeats - _recordingRegionStartBeat;
_recordingActiveNotes.forEach((startBeat, pitch) => {
finalNotes.push({ pitch, startBeat, endBeat: endBeatForHeld });
});
_recordingActiveNotes.clear();
KGMidiInput.instance().setRecordingCallbacks(null, null);
if (finalNotes.length > 0 && recordingTargetRegionId) {
const noteData: NoteCreationData[] = finalNotes.map(n => ({
regionId: recordingTargetRegionId,
startBeat: n.startBeat,
endBeat: n.endBeat,
pitch: n.pitch,
velocity: 127,
}));
const command = new CreateNotesCommand(noteData);
KGCore.instance().executeCommand(command);
refreshProjectState();
}
await stopPlaying();
setPlayheadPosition(recordingOriginalPlayhead);
set({ isRecording: false, recordingNotes: [], recordingTargetRegionId: null });
},
toggleLoop: () => {
const { isLooping, loopingRange, maxBars, isPlaying, stopPlaying } = get();