feat: add audio track support with WAV/MP3 playback and looping

Introduce KGAudioTrack and KGAudioRegion as new track/region types alongside existing MIDI tracks. Users can import WAV, MP3, OGG, FLAC, and AAC files into audio tracks with full playback, looping, and solo/mute/volume support. New core models: - KGAudioTrack (extends KGTrack with TrackType.Wave) - KGAudioRegion (extends KGRegion with audio file reference) - KGAudioPlayerBus (Tone.Gain + ToneBufferSource-based playback) - KGAudioFileStorage (OPFS media/ directory for binary audio files) New commands: - AddAudioTrackCommand (create audio track with player bus) - ImportAudioCommand (import audio file, create region at playhead, auto-expand maxBars) Playback features: - Audio regions scheduled via Tone.Transport alongside MIDI events - Loop support with duration capping at loop boundaries - Resume safety offset to prevent ToneBufferSource timing misses - Cross-track buffer copy when moving audio regions between tracks - Proper buffer cleanup on track deletion and project switch UI changes: - Separate "+ MIDI" and "+ Audio" buttons in top-left spacer - Audio track shows upload button instead of instrument selector - FileImportModal reused for audio file drag-and-drop upload - Real waveform rendering on audio region canvas - Green color scheme for audio regions (vs blue for MIDI) - Pencil icon and resize handles hidden for audio regions - Cross-type region moves blocked (MIDI ↔ Audio) - Piano roll blocked for audio regions - Audio region deletion support Infrastructure: - Project structure version bumped to 4 (no-op migration) - class-transformer discriminators updated for new subtypes - RemoveTrackCommand updated to clean up audio player buses
2026-04-09 21:54:54 -07:00
parent de2979178e
commit fc84edfc59
23 changed files with 1406 additions and 100 deletions
@@ -9,10 +9,14 @@ import { KGAudioInterface } from '../core/audio-interface/KGAudioInterface';
 import { KGPianoRollState } from '../core/state/KGPianoRollState';
 import { KGMidiNote } from '../core/midi/KGMidiNote';
 import { KGRegion } from '../core/region/KGRegion';
-import { AddTrackCommand, RemoveTrackCommand, ReorderTracksCommand, UpdateTrackCommand, type TrackUpdateProperties, PasteRegionsCommand, PasteNotesCommand, ChangeProjectPropertyCommand } from '../core/commands';
+import { AddTrackCommand, AddAudioTrackCommand, RemoveTrackCommand, ReorderTracksCommand, UpdateTrackCommand, type TrackUpdateProperties, PasteRegionsCommand, PasteNotesCommand, ChangeProjectPropertyCommand, ImportAudioCommand } from '../core/commands';
+import { KGAudioTrack } from '../core/track/KGAudioTrack';
+import { KGAudioRegion } from '../core/region/KGAudioRegion';
+import { KGAudioFileStorage } from '../core/io/KGAudioFileStorage';
 import { ConfigManager } from '../core/config/ConfigManager';
 import { upgradeProjectToLatest } from '../core/project-upgrader/KGProjectUpgrader';
 import { toggleLoop } from '../util/loopUtil';
+import * as Tone from 'tone';

 /**
 * Update CSS custom property for time signature numerator
@@ -63,9 +67,13 @@ interface ProjectState {
  showInstrumentSelection: boolean;
  // instrumentSelectionTrackId removed; panel now follows selectedTrackId
  
+  // Audio import modal state
+  showAudioImportModal: boolean;
+  audioImportTargetTrackId: string | null;
+
  // Settings state
  showSettings: boolean;
-  
+
  // Undo/redo state
  canUndo: boolean;
  canRedo: boolean;
@@ -75,6 +83,10 @@ interface ProjectState {
  // Actions
  setProjectName: (name: string) => void;
  addTrack: () => Promise<void>;
+  addAudioTrack: () => Promise<void>;
+  importAudioToTrack: (trackId: string, file: File) => Promise<void>;
+  openAudioImportModal: (trackId: string) => void;
+  closeAudioImportModal: () => void;
  removeTrack: (id: number) => Promise<void>;
  updateTrack: (track: KGTrack) => Promise<void>;
  updateTrackProperties: (trackId: number, properties: TrackUpdateProperties) => Promise<void>;
@@ -250,6 +262,10 @@ export const useProjectStore = create<ProjectState>((set, get) => {
    // Initial Instrument Selection panel state
    showInstrumentSelection: initialShowInstrumentSelection,
    
+    // Initial audio import modal state
+    showAudioImportModal: false,
+    audioImportTargetTrackId: null,
+
    // Initial Settings state
    showSettings: false,
    
@@ -300,6 +316,117 @@ export const useProjectStore = create<ProjectState>((set, get) => {
      }
    },

+    addAudioTrack: async () => {
+      try {
+        const command = new AddAudioTrackCommand();
+        KGCore.instance().executeCommand(command);
+
+        const project = KGCore.instance().getCurrentProject();
+        set({ tracks: [...project.getTracks()] as KGTrack[] });
+
+        const newTrackId = command.getTrackId().toString();
+        set({
+          selectedTrackId: newTrackId,
+          showAudioImportModal: true,
+          audioImportTargetTrackId: newTrackId,
+        });
+
+        console.log(`Added audio track ${command.getTrackId()}`);
+      } catch (error) {
+        console.error('Error adding audio track:', error);
+        get().setStatus('Failed to add audio track');
+      }
+    },
+
+    importAudioToTrack: async (trackId: string, file: File) => {
+      try {
+        get().setStatus(`Importing "${file.name}"...`);
+
+        // Decode the audio file to get duration
+        const arrayBuffer = await file.arrayBuffer();
+        const toneBuffer = new Tone.ToneAudioBuffer();
+        await new Promise<void>((resolve, reject) => {
+          toneBuffer.onload = () => resolve();
+          // Set buffer from array buffer
+          const audioContext = Tone.getContext().rawContext as AudioContext;
+          audioContext.decodeAudioData(
+            arrayBuffer.slice(0),  // slice to avoid detached buffer
+            (decoded) => {
+              toneBuffer.set(decoded);
+              resolve();
+            },
+            (err) => reject(err)
+          );
+        });
+
+        const audioDurationSeconds = toneBuffer.duration;
+        const { bpm, timeSignature, playheadPosition, maxBars } = get();
+
+        // Calculate duration in beats
+        const durationInBeats = audioDurationSeconds * (bpm / 60);
+
+        // Calculate if we need to expand maxBars
+        const beatsPerBar = timeSignature.numerator;
+        const endBeat = playheadPosition + durationInBeats;
+        const requiredBars = Math.ceil(endBeat / beatsPerBar);
+        const newMaxBars = Math.max(maxBars, requiredBars);
+
+        // Store audio file in OPFS
+        const projectName = get().projectName;
+        const audioFileId = KGAudioFileStorage.generateAudioFileId(file.name);
+        await KGAudioFileStorage.storeAudioFile(projectName, audioFileId, file);
+
+        // Load buffer into the audio player bus
+        const audioInterface = KGAudioInterface.instance();
+        audioInterface.loadAudioBufferForTrack(trackId, audioFileId, toneBuffer);
+
+        // Find the track to get trackIndex
+        const project = KGCore.instance().getCurrentProject();
+        const track = project.getTracks().find(t => t.getId().toString() === trackId);
+        if (!track) {
+          throw new Error(`Track ${trackId} not found`);
+        }
+
+        // Execute the import command
+        const command = new ImportAudioCommand(
+          track.getId(),
+          track.getTrackIndex(),
+          audioFileId,
+          file.name,
+          audioDurationSeconds,
+          playheadPosition,
+          durationInBeats,
+          maxBars,
+          newMaxBars
+        );
+        KGCore.instance().executeCommand(command);
+
+        // Update store state
+        const updatedState: Partial<ProjectState> = {
+          tracks: [...project.getTracks()] as KGTrack[],
+        };
+        if (newMaxBars > maxBars) {
+          updatedState.maxBars = newMaxBars;
+          updateMaxBarsCSS(newMaxBars);
+        }
+        set(updatedState);
+
+        get().setStatus(`Imported "${file.name}" successfully`);
+        console.log(`Imported audio "${file.name}" to track ${trackId}`);
+      } catch (error) {
+        console.error('Error importing audio:', error);
+        get().setStatus(`Failed to import audio: ${error}`);
+      }
+    },
+
+    openAudioImportModal: (trackId: string) => {
+      set({ showAudioImportModal: true, audioImportTargetTrackId: trackId });
+    },
+
+    closeAudioImportModal: () => {
+      set({ showAudioImportModal: false, audioImportTargetTrackId: null });
+    },
+
    removeTrack: async (id: number) => {
      try {
        // Get the current tracks and find the index of the track being deleted
@@ -488,23 +615,52 @@ export const useProjectStore = create<ProjectState>((set, get) => {
        // Setup audio synths for all tracks
        const audioInterface = KGAudioInterface.instance();
        
-        // Clear any existing synths first
-        tracks.forEach(track => {
-          audioInterface.removeTrackSynth(track.getId().toString());
-        });
-        
-        // Create synths for all tracks (with their stored volumes)
+        // Clear any existing synths/buses first
        tracks.forEach(track => {
          const trackId = track.getId().toString();
-          // Get instrument from track model if it's a MIDI track
-          let instrument: InstrumentType = 'acoustic_grand_piano'; // Default fallback
-          if (track.getCurrentType() === 'KGMidiTrack' && 'getInstrument' in track) {
-            instrument = (track as KGMidiTrack).getInstrument();
-          }
-          audioInterface.createTrackSynth(trackId, instrument);
-          // Volume is applied during bus creation; ensure sync if bus already existed
-          audioInterface.setTrackVolume(trackId, track.getVolume());
+          audioInterface.removeTrackSynth(trackId);
+          audioInterface.removeTrackAudioPlayerBus(trackId);
        });
+
+        // Create synths/buses for all tracks (with their stored volumes)
+        const projectName = projectToLoad.getName();
+        for (const track of tracks) {
+          const trackId = track.getId().toString();
+
+          if (track.getCurrentType() === 'KGAudioTrack') {
+            // Audio track: create player bus and load audio buffers
+            await audioInterface.createTrackAudioPlayerBus(trackId, track.getVolume());
+
+            // Load audio buffers for all regions in this audio track
+            const audioTrack = track as KGAudioTrack;
+            for (const region of audioTrack.getRegions()) {
+              if (region.getCurrentType() === 'KGAudioRegion') {
+                const audioRegion = region as KGAudioRegion;
+                const audioFileId = audioRegion.getAudioFileId();
+                if (audioFileId) {
+                  try {
+                    const arrayBuffer = await KGAudioFileStorage.loadAudioFile(projectName, audioFileId);
+                    const audioContext = Tone.getContext().rawContext as AudioContext;
+                    const decoded = await audioContext.decodeAudioData(arrayBuffer);
+                    const toneBuffer = new Tone.ToneAudioBuffer();
+                    toneBuffer.set(decoded);
+                    audioInterface.loadAudioBufferForTrack(trackId, audioFileId, toneBuffer);
+                  } catch (err) {
+                    console.error(`Failed to load audio file ${audioFileId}:`, err);
+                  }
+                }
+              }
+            }
+          } else {
+            // MIDI track: create sampler-based audio bus
+            let instrument: InstrumentType = 'acoustic_grand_piano';
+            if (track.getCurrentType() === 'KGMidiTrack' && 'getInstrument' in track) {
+              instrument = (track as KGMidiTrack).getInstrument();
+            }
+            audioInterface.createTrackSynth(trackId, instrument);
+            audioInterface.setTrackVolume(trackId, track.getVolume());
+          }
+        }
        
        // Update CSS variable for time signature numerator
        updateTimeSignatureCSS(timeSignature);