3dtours/KGStudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add audio track support with WAV/MP3 playback and looping

Browse Source 
Introduce KGAudioTrack and KGAudioRegion as new track/region types
alongside existing MIDI tracks. Users can import WAV, MP3, OGG, FLAC,
and AAC files into audio tracks with full playback, looping, and
solo/mute/volume support.

New core models:
- KGAudioTrack (extends KGTrack with TrackType.Wave)
- KGAudioRegion (extends KGRegion with audio file reference)
- KGAudioPlayerBus (Tone.Gain + ToneBufferSource-based playback)
- KGAudioFileStorage (OPFS media/ directory for binary audio files)

New commands:
- AddAudioTrackCommand (create audio track with player bus)
- ImportAudioCommand (import audio file, create region at playhead,
auto-expand maxBars)

Playback features:
- Audio regions scheduled via Tone.Transport alongside MIDI events
- Loop support with duration capping at loop boundaries
- Resume safety offset to prevent ToneBufferSource timing misses
- Cross-track buffer copy when moving audio regions between tracks
- Proper buffer cleanup on track deletion and project switch

UI changes:
- Separate "+ MIDI" and "+ Audio" buttons in top-left spacer
- Audio track shows upload button instead of instrument selector
- FileImportModal reused for audio file drag-and-drop upload
- Real waveform rendering on audio region canvas
- Green color scheme for audio regions (vs blue for MIDI)
- Pencil icon and resize handles hidden for audio regions
- Cross-type region moves blocked (MIDI ↔ Audio)
- Piano roll blocked for audio regions
- Audio region deletion support

Infrastructure:
- Project structure version bumped to 4 (no-op migration)
- class-transformer discriminators updated for new subtypes
- RemoveTrackCommand updated to clean up audio player buses
This commit is contained in:
Xiaohan-Tian
2026-04-09 21:54:54 -07:00
parent de2979178e
commit fc84edfc59
23 changed files with 1406 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/MainContent.css
+2
View File
@@ -25,6 +25,8 @@
border-bottom: 1px solid #3a3a3a;
border-right: 1px solid #3a3a3a;
z-index: 1002; /* Higher than other elements to ensure it's always visible */
display: flex;
flex-direction: row;
}
/* Offset spacer when instrument selection panel is visible on the left */
src/components/MainContent.tsx
+15 -3
View File
@@ -5,6 +5,7 @@ import { useProjectStore } from '../stores/projectStore';
import { KGCore } from '../core/KGCore';
import { KGTrack } from '../core/track/KGTrack';
import { KGMidiRegion } from '../core/region/KGMidiRegion';
import { KGAudioRegion } from '../core/region/KGAudioRegion';
import TrackInfoPanel from './track/TrackInfoPanel';
import TrackGridPanel from './track/TrackGridPanel';
import PianoRoll from './piano-roll/PianoRoll';
@@ -35,7 +36,8 @@ const MainContent: React.FC<MainContentProps> = ({
activeRegionId,
setShowPianoRoll,
setActiveRegionId,
addTrack
addTrack,
addAudioTrack
} = useProjectStore();
// State to store regions
@@ -130,7 +132,7 @@ const MainContent: React.FC<MainContentProps> = ({
// Iterate through all regions in the track
track.getRegions().forEach(region => {
if (region instanceof KGMidiRegion) {
if (region instanceof KGMidiRegion || region instanceof KGAudioRegion) {
// Calculate bar number and length from beats
const beatsPerBar = timeSignature.numerator;
const barNumber = Math.floor(region.getStartFromBeat() / beatsPerBar) + 1;
@@ -432,6 +434,15 @@ const MainContent: React.FC<MainContentProps> = ({
// Handle explicit pencil action: select region and open piano roll
const handleOpenPianoRoll = (regionId: string) => {
// Don't open piano roll for audio regions
const project = KGCore.instance().getCurrentProject();
for (const track of project.getTracks()) {
const region = track.getRegions().find(r => r.getId() === regionId);
if (region && region.getCurrentType() === 'KGAudioRegion') {
return;
}
}
if (DEBUG_MODE.MAIN_CONTENT) {
console.log(`Open piano roll via pencil for region: ${regionId}`);
}
@@ -691,7 +702,8 @@ const MainContent: React.FC<MainContentProps> = ({
<div className="main-content-wrapper">
{/* Top-left spacer */}
<div className="top-left-spacer">
<button className="add-track-btn" onClick={() => addTrack()}>+ Add track</button>
<button className="add-track-btn" onClick={() => addTrack()}>+ MIDI</button>
<button className="add-track-btn" onClick={() => addAudioTrack()}>+ Audio</button>
</div>
{/* Bar numbers at the top */}
src/components/track/Region.css
+22
View File
@@ -71,6 +71,28 @@
position: relative; /* Allow overlayed controls */
}
.region-content.audio-region-content {
background-color: #90EE90; /* Light green for audio regions */
}
/* Audio region overrides */
.track-region.audio-region {
background-color: #3a6b4a;
border-color: #4a8b5a;
}
.track-region.audio-region:hover {
box-shadow: 0 0 0 1px #6aab7a;
}
.track-region.audio-region .region-header {
background-color: #4a8b5a;
}
.track-region.audio-region:hover .region-header {
background-color: #5a9b6a;
}
/* Region pencil trigger inside content */
.region-pencil-btn {
position: absolute;
src/components/track/RegionItem.tsx
+106 -31
View File
@@ -4,6 +4,7 @@ import { FaPencilAlt } from 'react-icons/fa';
import type { ResizeAction } from '../interfaces';
import { REGION_CONSTANTS, DEBUG_MODE } from '../../constants';
import { KGMidiRegion } from '../../core/region/KGMidiRegion';
import { KGAudioRegion } from '../../core/region/KGAudioRegion';
import { useProjectStore } from '../../stores/projectStore';
import { KGMainContentState } from '../../core/state/KGMainContentState';
@@ -28,6 +29,9 @@ interface RegionItemProps {
onOpenPianoRoll?: (regionId: string) => void;
// MIDI region data for rendering notes
midiRegion?: KGMidiRegion;
// Audio region data for rendering waveform
audioRegion?: KGAudioRegion;
audioBuffer?: AudioBuffer;
}
const RegionItem: React.FC<RegionItemProps> = ({
@@ -45,7 +49,9 @@ const RegionItem: React.FC<RegionItemProps> = ({
onDragEnd,
onClick,
onOpenPianoRoll,
midiRegion
midiRegion,
audioRegion,
audioBuffer
}) => {
// Get selection state and time signature from store
const { selectedRegionIds, timeSignature } = useProjectStore();
@@ -204,6 +210,58 @@ const RegionItem: React.FC<RegionItemProps> = ({
}
};
// Function to render audio waveform on canvas
const renderWaveformOnCanvas = () => {
if (!canvasRef.current || !regionContentRef.current || !audioBuffer) return;
const canvas = canvasRef.current;
const ctx = canvas.getContext('2d');
if (!ctx) return;
const contentRect = regionContentRef.current.getBoundingClientRect();
const width = contentRect.width;
const height = contentRect.height;
canvas.width = width;
canvas.height = height;
ctx.clearRect(0, 0, width, height);
// Get channel data (use first channel)
const channelData = audioBuffer.getChannelData(0);
const samples = channelData.length;
// Downsample to canvas width
const samplesPerPixel = Math.max(1, Math.floor(samples / width));
const centerY = height / 2;
ctx.strokeStyle = 'rgba(255, 255, 255, 0.7)';
ctx.lineWidth = 1;
ctx.beginPath();
for (let x = 0; x < width; x++) {
const startSample = Math.floor(x * samplesPerPixel);
const endSample = Math.min(startSample + samplesPerPixel, samples);
let min = 0;
let max = 0;
for (let i = startSample; i < endSample; i++) {
const val = channelData[i];
if (val < min) min = val;
if (val > max) max = val;
}
// Draw vertical line from min to max amplitude
const yMin = centerY - max * centerY;
const yMax = centerY - min * centerY;
ctx.moveTo(x, yMin);
ctx.lineTo(x, yMax);
}
ctx.stroke();
};
// Create a stable reference to track note changes
const notesRef = useRef<string>('');
const [noteUpdateTrigger, setNoteUpdateTrigger] = useState(0);
@@ -226,15 +284,23 @@ const RegionItem: React.FC<RegionItemProps> = ({
// Set up canvas when component mounts or updates
useEffect(() => {
renderNotesOnCanvas();
}, [midiRegion, timeSignature, id, noteUpdateTrigger]);
if (audioRegion && audioBuffer) {
renderWaveformOnCanvas();
} else {
renderNotesOnCanvas();
}
}, [midiRegion, audioRegion, audioBuffer, timeSignature, id, noteUpdateTrigger]);
// Re-render canvas when region content size changes
useEffect(() => {
if (!regionContentRef.current) return;
const resizeObserver = new ResizeObserver(() => {
renderNotesOnCanvas();
if (audioRegion && audioBuffer) {
renderWaveformOnCanvas();
} else {
renderNotesOnCanvas();
}
});
resizeObserver.observe(regionContentRef.current);
@@ -244,7 +310,7 @@ const RegionItem: React.FC<RegionItemProps> = ({
resizeObserver.unobserve(regionContentRef.current);
}
};
}, [midiRegion, timeSignature]);
}, [midiRegion, audioRegion, audioBuffer, timeSignature]);
// Handle mouse movement to detect edge proximity
const handleMouseMove = (e: React.MouseEvent<HTMLDivElement>) => {
@@ -259,6 +325,13 @@ const RegionItem: React.FC<RegionItemProps> = ({
return;
}
// Audio regions: move only, no resize
if (audioRegion) {
setCursor('grab');
setResizeEdge('none');
return;
}
const regionElement = e.currentTarget;
const rect = regionElement.getBoundingClientRect();
@@ -453,7 +526,7 @@ const RegionItem: React.FC<RegionItemProps> = ({
return (
<div
key={id}
className={`track-region ${isDragging ? 'dragging' : ''} ${isSelected ? 'selected' : ''}`}
className={`track-region ${isDragging ? 'dragging' : ''} ${isSelected ? 'selected' : ''} ${audioRegion ? 'audio-region' : ''}`}
style={{ ...style, cursor }}
onMouseMove={handleMouseMove}
onMouseLeave={handleMouseLeave}
@@ -464,32 +537,34 @@ const RegionItem: React.FC<RegionItemProps> = ({
data-is-dragging={isDragging}
>
<div className="region-header">
{name}
{audioRegion ? audioRegion.getAudioFileName() : name}
</div>
<div className="region-content" ref={regionContentRef}>
<button
className="region-pencil-btn"
title="Edit notes"
onMouseDown={(e) => {
e.preventDefault();
e.stopPropagation();
}}
onClick={(e) => {
e.preventDefault();
e.stopPropagation();
if (DEBUG_MODE.REGION_ITEM) {
console.log(`Pencil clicked: open piano roll for region ${id}`);
}
if (onOpenPianoRoll) {
onOpenPianoRoll(id);
} else if (onClick) {
onClick(id);
}
}}
aria-label="Open piano roll"
>
<FaPencilAlt size={10} />
</button>
<div className={`region-content${audioRegion ? ' audio-region-content' : ''}`} ref={regionContentRef}>
{!audioRegion && (
<button
className="region-pencil-btn"
title="Edit notes"
onMouseDown={(e) => {
e.preventDefault();
e.stopPropagation();
}}
onClick={(e) => {
e.preventDefault();
e.stopPropagation();
if (DEBUG_MODE.REGION_ITEM) {
console.log(`Pencil clicked: open piano roll for region ${id}`);
}
if (onOpenPianoRoll) {
onOpenPianoRoll(id);
} else if (onClick) {
onClick(id);
}
}}
aria-label="Open piano roll"
>
<FaPencilAlt size={10} />
</button>
)}
<canvas ref={canvasRef} />
</div>
</div>
src/components/track/Track.css
+1 -1
View File
@@ -26,7 +26,7 @@
font-size: 12px;
padding: 0;
height: 100%;
width: 100%;
flex: 1;
display: flex;
align-items: center;
justify-content: center;
src/components/track/TrackGridItem.tsx
+21 -6
View File
@@ -1,6 +1,8 @@
import React, { useEffect, useState, useRef } from 'react';
import { KGTrack } from '../../core/track/KGTrack';
import { KGMidiRegion } from '../../core/region/KGMidiRegion';
import { KGAudioRegion } from '../../core/region/KGAudioRegion';
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
import RegionItem from './RegionItem';
import type { RegionUI, ResizeAction } from '../interfaces';
import { REGION_CONSTANTS, DEBUG_MODE } from '../../constants';
@@ -506,11 +508,22 @@ const TrackGridItem: React.FC<TrackGridItemProps> = ({
>
{/* Render regions for this track */}
{trackRegions.map(region => {
// Find the corresponding KGMidiRegion in the track
const midiRegion = track.getRegions().find(r => r.getId() === region.id) as KGMidiRegion | undefined;
// Find the corresponding region in the track
const coreRegion = track.getRegions().find(r => r.getId() === region.id);
const midiRegion = coreRegion?.getCurrentType() === 'KGMidiRegion' ? coreRegion as unknown as KGMidiRegion : undefined;
const audioRegion = coreRegion?.getCurrentType() === 'KGAudioRegion' ? coreRegion as unknown as KGAudioRegion : undefined;
// Get audio buffer for waveform rendering
let audioBuffer: AudioBuffer | undefined;
if (audioRegion) {
audioBuffer = KGAudioInterface.instance().getAudioBuffer(
track.getId().toString(),
audioRegion.getAudioFileId()
);
}
return (
<RegionItem
<RegionItem
key={region.id}
id={region.id}
name={region.name}
@@ -526,8 +539,8 @@ const TrackGridItem: React.FC<TrackGridItemProps> = ({
onDragEnd={handleRegionDragEnd}
// Keep onClick for selection-only logic if needed by parent
onClick={handleRegionClick}
// New explicit pencil action
onOpenPianoRoll={(regionId) => {
// New explicit pencil action — disabled for audio regions
onOpenPianoRoll={audioRegion ? undefined : (regionId) => {
if (onOpenPianoRoll) {
onOpenPianoRoll(regionId);
} else if (onRegionClick) {
@@ -536,6 +549,8 @@ const TrackGridItem: React.FC<TrackGridItemProps> = ({
}
}}
midiRegion={midiRegion}
audioRegion={audioRegion}
audioBuffer={audioBuffer}
/>
);
})}
src/components/track/TrackGridPanel.tsx
+36 -4
View File
@@ -1,5 +1,5 @@
import React, { useRef } from 'react';
import { KGTrack } from '../../core/track/KGTrack';
import { KGTrack, TrackType } from '../../core/track/KGTrack';
import { KGMidiRegion } from '../../core/region/KGMidiRegion';
import TrackGridItem from './TrackGridItem';
import { Playhead } from '../common';
@@ -9,6 +9,8 @@ import { KGMainContentState } from '../../core/state/KGMainContentState';
import { isModifierKeyPressed } from '../../util/osUtil';
import { CreateRegionCommand, ResizeRegionCommand, MoveRegionCommand } from '../../core/commands';
import { KGCore } from '../../core/KGCore';
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
import { KGAudioRegion } from '../../core/region/KGAudioRegion';
import { generateNewRegionName } from '../../util/miscUtil';
interface TrackGridPanelProps {
@@ -64,7 +66,12 @@ const TrackGridPanel: React.FC<TrackGridPanelProps> = ({
// Get the track and its ID
const track = tracks[trackIndex];
const trackId = track.getId().toString();
// Don't allow manual region creation on audio tracks
if (track.getType() === TrackType.Wave) {
return;
}
if (DEBUG_MODE.TRACK_GRID_PANEL) {
console.log(`Creating region on track ${trackIndex + 1}, bar ${barNumber}`);
}
@@ -238,7 +245,19 @@ const TrackGridPanel: React.FC<TrackGridPanelProps> = ({
// Get the target track
const targetTrack = tracks[finalTrackIndex];
if (!targetTrack) return;
// Block cross-type region moves (MIDI <-> Audio)
const sourceTrack = tracks.find(t => {
return t.getRegions().some(r => r.getId() === regionId);
});
if (sourceTrack && sourceTrack.getType() !== targetTrack.getType()) {
// Snap back — don't execute the move
if (DEBUG_MODE.TRACK_GRID_PANEL) {
console.log(`Blocked cross-type move: ${sourceTrack.getType()} region cannot move to ${targetTrack.getType()} track`);
}
return;
}
// Use command pattern to move the region
try {
const command = MoveRegionCommand.fromBarCoordinates(
@@ -251,9 +270,22 @@ const TrackGridPanel: React.FC<TrackGridPanelProps> = ({
KGCore.instance().executeCommand(command);
// Copy audio buffer to target track if this is a cross-track audio region move
if (sourceTrack && targetTrack && sourceTrack.getId() !== targetTrack.getId()) {
const coreRegion = targetTrack.getRegions().find(r => r.getId() === regionId);
if (coreRegion?.getCurrentType() === 'KGAudioRegion') {
const audioRegion = coreRegion as unknown as KGAudioRegion;
KGAudioInterface.instance().copyAudioBufferBetweenTracks(
sourceTrack.getId().toString(),
targetTrack.getId().toString(),
audioRegion.getAudioFileId()
);
}
}
if (DEBUG_MODE.TRACK_GRID_PANEL) {
console.log(`Executed MoveRegionCommand: region ${regionId} moved using command pattern`);
// Verify the command worked
const movedRegion = command.getTargetRegion();
console.log(`Verified region: ${movedRegion ? 'found' : 'not found'}, startBeat=${movedRegion?.getStartFromBeat()}, trackId=${movedRegion?.getTrackId()}`);
src/components/track/TrackInfoItem.tsx
+52 -11
View File
@@ -1,10 +1,13 @@
import React, { useState, useRef, useEffect } from 'react';
import { KGTrack } from '../../core/track/KGTrack';
import { KGMidiTrack } from '../../core/track/KGMidiTrack';
import { KGAudioTrack } from '../../core/track/KGAudioTrack';
import { useProjectStore } from '../../stores/projectStore';
import { TbPiano } from 'react-icons/tb';
import { TbSettings } from 'react-icons/tb';
import { FaFileAudio } from 'react-icons/fa';
import KGDropdown from '../common/KGDropdown';
import FileImportModal from '../common/FileImportModal';
import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
import { DEBUG_MODE } from '../../constants/uiConstants';
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
@@ -34,7 +37,7 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
onDrop,
onDragEnd
}) => {
const { selectedTrackId, setSelectedTrack, removeTrack, toggleInstrumentSelectionForTrack, tracks: allTracks } = useProjectStore();
const { selectedTrackId, setSelectedTrack, removeTrack, toggleInstrumentSelectionForTrack, importAudioToTrack, tracks: allTracks } = useProjectStore();
const isSelected = selectedTrackId === track.getId().toString();
// Inline instrument dropdown removed; use InstrumentSelection panel instead
@@ -48,6 +51,7 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
const [currentInstrument, setCurrentInstrument] = useState(getTrackInstrument());
const [showSettingsDropdown, setShowSettingsDropdown] = useState(false);
const [showAudioImportModal, setShowAudioImportModal] = useState(false);
const settingsDropdownRef = useRef<HTMLDivElement>(null);
const suppressDragRef = useRef(false);
const [volume, setVolume] = useState(track.getVolume());
@@ -195,6 +199,8 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
// Inline instrument change removed; handled by InstrumentSelection panel
const isAudioTrack = track instanceof KGAudioTrack;
// Handle piano button click
const handlePianoButtonClick = (e: React.MouseEvent) => {
e.stopPropagation();
@@ -204,6 +210,19 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
toggleInstrumentSelectionForTrack();
};
// Handle audio import button click
const handleAudioImportClick = (e: React.MouseEvent) => {
e.stopPropagation();
setSelectedTrack(track.getId().toString());
setShowAudioImportModal(true);
};
// Handle audio file import
const handleAudioFileImport = (file: File) => {
importAudioToTrack(track.getId().toString(), file);
setShowAudioImportModal(false);
};
// Handle settings button click
const handleSettingsButtonClick = (e: React.MouseEvent) => {
e.stopPropagation();
@@ -253,12 +272,18 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
<div className="track-controls">
<div className="track-name-and-volume">
<div className="instrument-image">
<img
src={`${import.meta.env.BASE_URL}resources/instruments/${String(FLUIDR3_INSTRUMENT_MAP[currentInstrument as keyof typeof FLUIDR3_INSTRUMENT_MAP]?.image || 'piano.png')}`}
alt={String(FLUIDR3_INSTRUMENT_MAP[currentInstrument as keyof typeof FLUIDR3_INSTRUMENT_MAP]?.displayName || currentInstrument)}
width="64"
height="64"
/>
{isAudioTrack ? (
<div style={{ width: 64, height: 64, display: 'flex', alignItems: 'center', justifyContent: 'center', fontSize: '28px', color: 'var(--text-secondary, #999)' }}>
<FaFileAudio />
</div>
) : (
<img
src={`${import.meta.env.BASE_URL}resources/instruments/${String(FLUIDR3_INSTRUMENT_MAP[currentInstrument as keyof typeof FLUIDR3_INSTRUMENT_MAP]?.image || 'piano.png')}`}
alt={String(FLUIDR3_INSTRUMENT_MAP[currentInstrument as keyof typeof FLUIDR3_INSTRUMENT_MAP]?.displayName || currentInstrument)}
width="64"
height="64"
/>
)}
</div>
<div className="track-name-and-controls">
<div
@@ -296,9 +321,15 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
<button className={`solo${solo ? ' active' : ''}`} onClick={handleToggleSolo}>S</button>
<button className={`mute${muted ? ' active' : ''}`} onClick={handleToggleMute}>M</button>
<div>
<button className="instrument" onClick={handlePianoButtonClick}>
<TbPiano />
</button>
{isAudioTrack ? (
<button className="instrument" onClick={handleAudioImportClick} title="Import Audio">
<FaFileAudio />
</button>
) : (
<button className="instrument" onClick={handlePianoButtonClick}>
<TbPiano />
</button>
)}
</div>
<div style={{ position: 'relative' }} ref={settingsDropdownRef}>
<button className="settings" onClick={handleSettingsButtonClick}>
@@ -319,8 +350,18 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
</div>
</div>
</div>
{isAudioTrack && (
<FileImportModal
isVisible={showAudioImportModal}
onClose={() => setShowAudioImportModal(false)}
onFileImport={handleAudioFileImport}
acceptedTypes={['.wav', '.mp3', '.ogg', '.flac', '.aac']}
title="Import Audio"
description="Drag and drop your audio file here"
/>
)}
</div>
);
};
export default TrackInfoItem;
export default TrackInfoItem;