feat: add audio track support with WAV/MP3 playback and looping

Introduce KGAudioTrack and KGAudioRegion as new track/region types alongside existing MIDI tracks. Users can import WAV, MP3, OGG, FLAC, and AAC files into audio tracks with full playback, looping, and solo/mute/volume support. New core models: - KGAudioTrack (extends KGTrack with TrackType.Wave) - KGAudioRegion (extends KGRegion with audio file reference) - KGAudioPlayerBus (Tone.Gain + ToneBufferSource-based playback) - KGAudioFileStorage (OPFS media/ directory for binary audio files) New commands: - AddAudioTrackCommand (create audio track with player bus) - ImportAudioCommand (import audio file, create region at playhead, auto-expand maxBars) Playback features: - Audio regions scheduled via Tone.Transport alongside MIDI events - Loop support with duration capping at loop boundaries - Resume safety offset to prevent ToneBufferSource timing misses - Cross-track buffer copy when moving audio regions between tracks - Proper buffer cleanup on track deletion and project switch UI changes: - Separate "+ MIDI" and "+ Audio" buttons in top-left spacer - Audio track shows upload button instead of instrument selector - FileImportModal reused for audio file drag-and-drop upload - Real waveform rendering on audio region canvas - Green color scheme for audio regions (vs blue for MIDI) - Pencil icon and resize handles hidden for audio regions - Cross-type region moves blocked (MIDI ↔ Audio) - Piano roll blocked for audio regions - Audio region deletion support Infrastructure: - Project structure version bumped to 4 (no-op migration) - class-transformer discriminators updated for new subtypes - RemoveTrackCommand updated to clean up audio player buses
2026-04-09 21:54:54 -07:00
parent de2979178e
commit fc84edfc59
23 changed files with 1406 additions and 100 deletions
@@ -71,6 +71,28 @@
  position: relative; /* Allow overlayed controls */
 }

+.region-content.audio-region-content {
+  background-color: #90EE90; /* Light green for audio regions */
+}
+
+/* Audio region overrides */
+.track-region.audio-region {
+  background-color: #3a6b4a;
+  border-color: #4a8b5a;
+}
+
+.track-region.audio-region:hover {
+  box-shadow: 0 0 0 1px #6aab7a;
+}
+
+.track-region.audio-region .region-header {
+  background-color: #4a8b5a;
+}
+
+.track-region.audio-region:hover .region-header {
+  background-color: #5a9b6a;
+}
+
 /* Region pencil trigger inside content */
 .region-pencil-btn {
  position: absolute;
@@ -4,6 +4,7 @@ import { FaPencilAlt } from 'react-icons/fa';
 import type { ResizeAction } from '../interfaces';
 import { REGION_CONSTANTS, DEBUG_MODE } from '../../constants';
 import { KGMidiRegion } from '../../core/region/KGMidiRegion';
+import { KGAudioRegion } from '../../core/region/KGAudioRegion';
 import { useProjectStore } from '../../stores/projectStore';
 import { KGMainContentState } from '../../core/state/KGMainContentState';

@@ -28,6 +29,9 @@ interface RegionItemProps {
  onOpenPianoRoll?: (regionId: string) => void;
  // MIDI region data for rendering notes
  midiRegion?: KGMidiRegion;
+  // Audio region data for rendering waveform
+  audioRegion?: KGAudioRegion;
+  audioBuffer?: AudioBuffer;
 }

 const RegionItem: React.FC<RegionItemProps> = ({
@@ -45,7 +49,9 @@ const RegionItem: React.FC<RegionItemProps> = ({
  onDragEnd,
  onClick,
  onOpenPianoRoll,
-  midiRegion
+  midiRegion,
+  audioRegion,
+  audioBuffer
 }) => {
  // Get selection state and time signature from store
  const { selectedRegionIds, timeSignature } = useProjectStore();
@@ -204,6 +210,58 @@ const RegionItem: React.FC<RegionItemProps> = ({
    }
  };

+  // Function to render audio waveform on canvas
+  const renderWaveformOnCanvas = () => {
+    if (!canvasRef.current || !regionContentRef.current || !audioBuffer) return;
+
+    const canvas = canvasRef.current;
+    const ctx = canvas.getContext('2d');
+    if (!ctx) return;
+
+    const contentRect = regionContentRef.current.getBoundingClientRect();
+    const width = contentRect.width;
+    const height = contentRect.height;
+
+    canvas.width = width;
+    canvas.height = height;
+
+    ctx.clearRect(0, 0, width, height);
+
+    // Get channel data (use first channel)
+    const channelData = audioBuffer.getChannelData(0);
+    const samples = channelData.length;
+
+    // Downsample to canvas width
+    const samplesPerPixel = Math.max(1, Math.floor(samples / width));
+    const centerY = height / 2;
+
+    ctx.strokeStyle = 'rgba(255, 255, 255, 0.7)';
+    ctx.lineWidth = 1;
+    ctx.beginPath();
+
+    for (let x = 0; x < width; x++) {
+      const startSample = Math.floor(x * samplesPerPixel);
+      const endSample = Math.min(startSample + samplesPerPixel, samples);
+
+      let min = 0;
+      let max = 0;
+      for (let i = startSample; i < endSample; i++) {
+        const val = channelData[i];
+        if (val < min) min = val;
+        if (val > max) max = val;
+      }
+
+      // Draw vertical line from min to max amplitude
+      const yMin = centerY - max * centerY;
+      const yMax = centerY - min * centerY;
+
+      ctx.moveTo(x, yMin);
+      ctx.lineTo(x, yMax);
+    }
+
+    ctx.stroke();
+  };
+
  // Create a stable reference to track note changes
  const notesRef = useRef<string>('');
  const [noteUpdateTrigger, setNoteUpdateTrigger] = useState(0);
@@ -226,15 +284,23 @@ const RegionItem: React.FC<RegionItemProps> = ({

  // Set up canvas when component mounts or updates
  useEffect(() => {
-    renderNotesOnCanvas();
-  }, [midiRegion, timeSignature, id, noteUpdateTrigger]);
+    if (audioRegion && audioBuffer) {
+      renderWaveformOnCanvas();
+    } else {
+      renderNotesOnCanvas();
+    }
+  }, [midiRegion, audioRegion, audioBuffer, timeSignature, id, noteUpdateTrigger]);

  // Re-render canvas when region content size changes
  useEffect(() => {
    if (!regionContentRef.current) return;

    const resizeObserver = new ResizeObserver(() => {
-      renderNotesOnCanvas();
+      if (audioRegion && audioBuffer) {
+        renderWaveformOnCanvas();
+      } else {
+        renderNotesOnCanvas();
+      }
    });

    resizeObserver.observe(regionContentRef.current);
@@ -244,7 +310,7 @@ const RegionItem: React.FC<RegionItemProps> = ({
        resizeObserver.unobserve(regionContentRef.current);
      }
    };
-  }, [midiRegion, timeSignature]);
+  }, [midiRegion, audioRegion, audioBuffer, timeSignature]);

  // Handle mouse movement to detect edge proximity
  const handleMouseMove = (e: React.MouseEvent<HTMLDivElement>) => {
@@ -259,6 +325,13 @@ const RegionItem: React.FC<RegionItemProps> = ({
      return;
    }

+    // Audio regions: move only, no resize
+    if (audioRegion) {
+      setCursor('grab');
+      setResizeEdge('none');
+      return;
+    }
+
    const regionElement = e.currentTarget;
    const rect = regionElement.getBoundingClientRect();

@@ -453,7 +526,7 @@ const RegionItem: React.FC<RegionItemProps> = ({
  return (
    <div
      key={id}
-      className={`track-region ${isDragging ? 'dragging' : ''} ${isSelected ? 'selected' : ''}`}
+      className={`track-region ${isDragging ? 'dragging' : ''} ${isSelected ? 'selected' : ''} ${audioRegion ? 'audio-region' : ''}`}
      style={{ ...style, cursor }}
      onMouseMove={handleMouseMove}
      onMouseLeave={handleMouseLeave}
@@ -464,32 +537,34 @@ const RegionItem: React.FC<RegionItemProps> = ({
      data-is-dragging={isDragging}
    >
      <div className="region-header">
-        {name}
+        {audioRegion ? audioRegion.getAudioFileName() : name}
      </div>
-      <div className="region-content" ref={regionContentRef}>
-        <button
-          className="region-pencil-btn"
-          title="Edit notes"
-          onMouseDown={(e) => {
-            e.preventDefault();
-            e.stopPropagation();
-          }}
-          onClick={(e) => {
-            e.preventDefault();
-            e.stopPropagation();
-            if (DEBUG_MODE.REGION_ITEM) {
-              console.log(`Pencil clicked: open piano roll for region ${id}`);
-            }
-            if (onOpenPianoRoll) {
-              onOpenPianoRoll(id);
-            } else if (onClick) {
-              onClick(id);
-            }
-          }}
-          aria-label="Open piano roll"
-        >
-          <FaPencilAlt size={10} />
-        </button>
+      <div className={`region-content${audioRegion ? ' audio-region-content' : ''}`} ref={regionContentRef}>
+        {!audioRegion && (
+          <button
+            className="region-pencil-btn"
+            title="Edit notes"
+            onMouseDown={(e) => {
+              e.preventDefault();
+              e.stopPropagation();
+            }}
+            onClick={(e) => {
+              e.preventDefault();
+              e.stopPropagation();
+              if (DEBUG_MODE.REGION_ITEM) {
+                console.log(`Pencil clicked: open piano roll for region ${id}`);
+              }
+              if (onOpenPianoRoll) {
+                onOpenPianoRoll(id);
+              } else if (onClick) {
+                onClick(id);
+              }
+            }}
+            aria-label="Open piano roll"
+          >
+            <FaPencilAlt size={10} />
+          </button>
+        )}
        <canvas ref={canvasRef} />
      </div>
    </div>
@@ -26,7 +26,7 @@
  font-size: 12px;
  padding: 0;
  height: 100%;
-  width: 100%;
+  flex: 1;
  display: flex;
  align-items: center;
  justify-content: center;
@@ -1,6 +1,8 @@
 import React, { useEffect, useState, useRef } from 'react';
 import { KGTrack } from '../../core/track/KGTrack';
 import { KGMidiRegion } from '../../core/region/KGMidiRegion';
+import { KGAudioRegion } from '../../core/region/KGAudioRegion';
+import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
 import RegionItem from './RegionItem';
 import type { RegionUI, ResizeAction } from '../interfaces';
 import { REGION_CONSTANTS, DEBUG_MODE } from '../../constants';
@@ -506,11 +508,22 @@ const TrackGridItem: React.FC<TrackGridItemProps> = ({
    >
      {/* Render regions for this track */}
      {trackRegions.map(region => {
-        // Find the corresponding KGMidiRegion in the track
-        const midiRegion = track.getRegions().find(r => r.getId() === region.id) as KGMidiRegion | undefined;
-        
+        // Find the corresponding region in the track
+        const coreRegion = track.getRegions().find(r => r.getId() === region.id);
+        const midiRegion = coreRegion?.getCurrentType() === 'KGMidiRegion' ? coreRegion as unknown as KGMidiRegion : undefined;
+        const audioRegion = coreRegion?.getCurrentType() === 'KGAudioRegion' ? coreRegion as unknown as KGAudioRegion : undefined;
+
+        // Get audio buffer for waveform rendering
+        let audioBuffer: AudioBuffer | undefined;
+        if (audioRegion) {
+          audioBuffer = KGAudioInterface.instance().getAudioBuffer(
+            track.getId().toString(),
+            audioRegion.getAudioFileId()
+          );
+        }
+
        return (
-          <RegionItem 
+          <RegionItem
            key={region.id}
            id={region.id}
            name={region.name}
@@ -526,8 +539,8 @@ const TrackGridItem: React.FC<TrackGridItemProps> = ({
            onDragEnd={handleRegionDragEnd}
            // Keep onClick for selection-only logic if needed by parent
            onClick={handleRegionClick}
-            // New explicit pencil action
-            onOpenPianoRoll={(regionId) => {
+            // New explicit pencil action — disabled for audio regions
+            onOpenPianoRoll={audioRegion ? undefined : (regionId) => {
              if (onOpenPianoRoll) {
                onOpenPianoRoll(regionId);
              } else if (onRegionClick) {
@@ -536,6 +549,8 @@ const TrackGridItem: React.FC<TrackGridItemProps> = ({
              }
            }}
            midiRegion={midiRegion}
+            audioRegion={audioRegion}
+            audioBuffer={audioBuffer}
          />
        );
      })}
@@ -1,5 +1,5 @@
 import React, { useRef } from 'react';
-import { KGTrack } from '../../core/track/KGTrack';
+import { KGTrack, TrackType } from '../../core/track/KGTrack';
 import { KGMidiRegion } from '../../core/region/KGMidiRegion';
 import TrackGridItem from './TrackGridItem';
 import { Playhead } from '../common';
@@ -9,6 +9,8 @@ import { KGMainContentState } from '../../core/state/KGMainContentState';
 import { isModifierKeyPressed } from '../../util/osUtil';
 import { CreateRegionCommand, ResizeRegionCommand, MoveRegionCommand } from '../../core/commands';
 import { KGCore } from '../../core/KGCore';
+import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
+import { KGAudioRegion } from '../../core/region/KGAudioRegion';
 import { generateNewRegionName } from '../../util/miscUtil';

 interface TrackGridPanelProps {
@@ -64,7 +66,12 @@ const TrackGridPanel: React.FC<TrackGridPanelProps> = ({
    // Get the track and its ID
    const track = tracks[trackIndex];
    const trackId = track.getId().toString();
-    
+
+    // Don't allow manual region creation on audio tracks
+    if (track.getType() === TrackType.Wave) {
+      return;
+    }
+
    if (DEBUG_MODE.TRACK_GRID_PANEL) {
      console.log(`Creating region on track ${trackIndex + 1}, bar ${barNumber}`);
    }
@@ -238,7 +245,19 @@ const TrackGridPanel: React.FC<TrackGridPanelProps> = ({
    // Get the target track
    const targetTrack = tracks[finalTrackIndex];
    if (!targetTrack) return;
-    
+
+    // Block cross-type region moves (MIDI <-> Audio)
+    const sourceTrack = tracks.find(t => {
+      return t.getRegions().some(r => r.getId() === regionId);
+    });
+    if (sourceTrack && sourceTrack.getType() !== targetTrack.getType()) {
+      // Snap back — don't execute the move
+      if (DEBUG_MODE.TRACK_GRID_PANEL) {
+        console.log(`Blocked cross-type move: ${sourceTrack.getType()} region cannot move to ${targetTrack.getType()} track`);
+      }
+      return;
+    }
+
    // Use command pattern to move the region
    try {
      const command = MoveRegionCommand.fromBarCoordinates(
@@ -251,9 +270,22 @@ const TrackGridPanel: React.FC<TrackGridPanelProps> = ({
      
      KGCore.instance().executeCommand(command);

+      // Copy audio buffer to target track if this is a cross-track audio region move
+      if (sourceTrack && targetTrack && sourceTrack.getId() !== targetTrack.getId()) {
+        const coreRegion = targetTrack.getRegions().find(r => r.getId() === regionId);
+        if (coreRegion?.getCurrentType() === 'KGAudioRegion') {
+          const audioRegion = coreRegion as unknown as KGAudioRegion;
+          KGAudioInterface.instance().copyAudioBufferBetweenTracks(
+            sourceTrack.getId().toString(),
+            targetTrack.getId().toString(),
+            audioRegion.getAudioFileId()
+          );
+        }
+      }
+
      if (DEBUG_MODE.TRACK_GRID_PANEL) {
        console.log(`Executed MoveRegionCommand: region ${regionId} moved using command pattern`);
-        
+
        // Verify the command worked
        const movedRegion = command.getTargetRegion();
        console.log(`Verified region: ${movedRegion ? 'found' : 'not found'}, startBeat=${movedRegion?.getStartFromBeat()}, trackId=${movedRegion?.getTrackId()}`);
@@ -1,10 +1,13 @@
 import React, { useState, useRef, useEffect } from 'react';
 import { KGTrack } from '../../core/track/KGTrack';
 import { KGMidiTrack } from '../../core/track/KGMidiTrack';
+import { KGAudioTrack } from '../../core/track/KGAudioTrack';
 import { useProjectStore } from '../../stores/projectStore';
 import { TbPiano } from 'react-icons/tb';
 import { TbSettings } from 'react-icons/tb';
+import { FaFileAudio } from 'react-icons/fa';
 import KGDropdown from '../common/KGDropdown';
+import FileImportModal from '../common/FileImportModal';
 import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
 import { DEBUG_MODE } from '../../constants/uiConstants';
 import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
@@ -34,7 +37,7 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
  onDrop,
  onDragEnd
 }) => {
-  const { selectedTrackId, setSelectedTrack, removeTrack, toggleInstrumentSelectionForTrack, tracks: allTracks } = useProjectStore();
+  const { selectedTrackId, setSelectedTrack, removeTrack, toggleInstrumentSelectionForTrack, importAudioToTrack, tracks: allTracks } = useProjectStore();
  const isSelected = selectedTrackId === track.getId().toString();
  // Inline instrument dropdown removed; use InstrumentSelection panel instead
  
@@ -48,6 +51,7 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
  
  const [currentInstrument, setCurrentInstrument] = useState(getTrackInstrument());
  const [showSettingsDropdown, setShowSettingsDropdown] = useState(false);
+  const [showAudioImportModal, setShowAudioImportModal] = useState(false);
  const settingsDropdownRef = useRef<HTMLDivElement>(null);
  const suppressDragRef = useRef(false);
  const [volume, setVolume] = useState(track.getVolume());
@@ -195,6 +199,8 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({

  // Inline instrument change removed; handled by InstrumentSelection panel

+  const isAudioTrack = track instanceof KGAudioTrack;
+
  // Handle piano button click
  const handlePianoButtonClick = (e: React.MouseEvent) => {
    e.stopPropagation();
@@ -204,6 +210,19 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
    toggleInstrumentSelectionForTrack();
  };

+  // Handle audio import button click
+  const handleAudioImportClick = (e: React.MouseEvent) => {
+    e.stopPropagation();
+    setSelectedTrack(track.getId().toString());
+    setShowAudioImportModal(true);
+  };
+
+  // Handle audio file import
+  const handleAudioFileImport = (file: File) => {
+    importAudioToTrack(track.getId().toString(), file);
+    setShowAudioImportModal(false);
+  };
+
  // Handle settings button click
  const handleSettingsButtonClick = (e: React.MouseEvent) => {
    e.stopPropagation();
@@ -253,12 +272,18 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
      <div className="track-controls">
        <div className="track-name-and-volume">
          <div className="instrument-image">
-            <img 
-              src={`${import.meta.env.BASE_URL}resources/instruments/${String(FLUIDR3_INSTRUMENT_MAP[currentInstrument as keyof typeof FLUIDR3_INSTRUMENT_MAP]?.image || 'piano.png')}`} 
-              alt={String(FLUIDR3_INSTRUMENT_MAP[currentInstrument as keyof typeof FLUIDR3_INSTRUMENT_MAP]?.displayName || currentInstrument)}
-              width="64"
-              height="64"
-            />
+            {isAudioTrack ? (
+              <div style={{ width: 64, height: 64, display: 'flex', alignItems: 'center', justifyContent: 'center', fontSize: '28px', color: 'var(--text-secondary, #999)' }}>
+                <FaFileAudio />
+              </div>
+            ) : (
+              <img
+                src={`${import.meta.env.BASE_URL}resources/instruments/${String(FLUIDR3_INSTRUMENT_MAP[currentInstrument as keyof typeof FLUIDR3_INSTRUMENT_MAP]?.image || 'piano.png')}`}
+                alt={String(FLUIDR3_INSTRUMENT_MAP[currentInstrument as keyof typeof FLUIDR3_INSTRUMENT_MAP]?.displayName || currentInstrument)}
+                width="64"
+                height="64"
+              />
+            )}
          </div>
          <div className="track-name-and-controls">
            <div 
@@ -296,9 +321,15 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
          <button className={`solo${solo ? ' active' : ''}`} onClick={handleToggleSolo}>S</button>
          <button className={`mute${muted ? ' active' : ''}`} onClick={handleToggleMute}>M</button>
          <div>
-            <button className="instrument" onClick={handlePianoButtonClick}>
-              <TbPiano />
-            </button>
+            {isAudioTrack ? (
+              <button className="instrument" onClick={handleAudioImportClick} title="Import Audio">
+                <FaFileAudio />
+              </button>
+            ) : (
+              <button className="instrument" onClick={handlePianoButtonClick}>
+                <TbPiano />
+              </button>
+            )}
          </div>
          <div style={{ position: 'relative' }} ref={settingsDropdownRef}>
            <button className="settings" onClick={handleSettingsButtonClick}>
@@ -319,8 +350,18 @@ const TrackInfoItem: React.FC<TrackInfoItemProps> = ({
          </div>
        </div>
      </div>
+      {isAudioTrack && (
+        <FileImportModal
+          isVisible={showAudioImportModal}
+          onClose={() => setShowAudioImportModal(false)}
+          onFileImport={handleAudioFileImport}
+          acceptedTypes={['.wav', '.mp3', '.ogg', '.flac', '.aac']}
+          title="Import Audio"
+          description="Drag and drop your audio file here"
+        />
+      )}
    </div>
  );
 };

-export default TrackInfoItem; 
+export default TrackInfoItem;