From 7b82572d5c5c86c82135fc4744b68f7a7bca7b90 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Wed, 15 Apr 2026 22:40:47 -0700 Subject: [PATCH] feat: implemented drag-n-drop to import KGOne generated audio/MIDI file feature --- src/components/KGOnePanel.css | 35 ++++ src/components/KGOnePanel.tsx | 104 +++++++++++- src/components/MainContent.tsx | 18 +- src/components/track/TrackGridItem.tsx | 18 +- src/components/track/TrackGridPanel.tsx | 154 +++++++++++++++++- src/core/commands/index.ts | 1 + .../commands/region/ImportMidiClipCommand.ts | 113 +++++++++++++ src/util/midiUtil.ts | 34 ++++ 8 files changed, 464 insertions(+), 13 deletions(-) create mode 100644 src/core/commands/region/ImportMidiClipCommand.ts diff --git a/src/components/KGOnePanel.css b/src/components/KGOnePanel.css index ec08d1e..fb22554 100644 --- a/src/components/KGOnePanel.css +++ b/src/components/KGOnePanel.css @@ -124,6 +124,11 @@ margin-top: 2px; } +.kgone-hint strong { + color: #999; + font-weight: 600; +} + /* Two-column row (e.g. note + scale) */ .kgone-row { display: flex; @@ -284,6 +289,36 @@ flex-shrink: 0; } +.kgone-player-drag-handle { + color: #555; + cursor: grab; + font-size: 12px; + padding: 2px 4px; + flex-shrink: 0; + display: flex; + align-items: center; +} + +.kgone-player-drag-handle:hover { + color: #888; +} + +.kgone-player-download-btn { + background: transparent; + border: none; + color: #888; + cursor: pointer; + font-size: 12px; + padding: 2px 4px; + flex-shrink: 0; + display: flex; + align-items: center; +} + +.kgone-player-download-btn:hover { + color: #7a9cff; +} + /* ── Generation state feedback ─────────────────────────────────────────────── */ .kgone-gen-hint { color: #888; diff --git a/src/components/KGOnePanel.tsx b/src/components/KGOnePanel.tsx index 7cd8d9e..386aa4a 100644 --- a/src/components/KGOnePanel.tsx +++ b/src/components/KGOnePanel.tsx @@ -1,7 +1,7 @@ import React, { useState, useMemo, useRef, useEffect, useCallback } from 'react'; import './KGOnePanel.css'; -import { FaPlay, FaPause } from 'react-icons/fa'; -import { FaCircleNotch } from 'react-icons/fa6'; +import { FaPlay, FaPause, FaDownload } from 'react-icons/fa'; +import { FaCircleNotch, FaGripVertical } from 'react-icons/fa6'; import { useProjectStore } from '../stores/projectStore'; import { KGCore } from '../core/KGCore'; import { KGAudioRegion } from '../core/region/KGAudioRegion'; @@ -77,9 +77,14 @@ const Expander: React.FC = ({ label, children }) => { interface AudioPlayerProps { src: string; + /** When provided, makes the player draggable (shows grip handle) and adds a download button */ + dragData?: { + midiUrl?: string; // full URL to /v1/clip/midi/{taskId}; absent = MIDI import not supported + audioFileName: string; // filename used for download and OPFS storage + }; } -const AudioPlayer: React.FC = ({ src }) => { +const AudioPlayer: React.FC = ({ src, dragData }) => { const audioRef = useRef(null); const [isPlaying, setIsPlaying] = useState(false); const [currentTime, setCurrentTime] = useState(0); @@ -104,10 +109,34 @@ const AudioPlayer: React.FC = ({ src }) => { setCurrentTime(ratio * duration); }; + const handleDragStart = (e: React.DragEvent) => { + if (!dragData) return; + e.dataTransfer.setData('application/kgone-clip', JSON.stringify({ + midiUrl: dragData.midiUrl, + audioUrl: src, + audioDurationSeconds: duration, + audioFileName: dragData.audioFileName, + })); + e.dataTransfer.effectAllowed = 'copy'; + }; + + const handleDownload = () => { + const a = document.createElement('a'); + a.href = src; + a.download = dragData?.audioFileName ?? 'kgone_clip.wav'; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + }; + const progress = duration > 0 ? (currentTime / duration) * 100 : 0; return ( -
+
); }; @@ -161,6 +200,7 @@ const ClipTab: React.FC = ({ bpm, keySignature }) => { const [audioUrl, setAudioUrl] = useState(null); const abortRef = useRef(null); + const taskIdRef = useRef(''); // Revoke blob URL on unmount useEffect(() => { @@ -248,6 +288,7 @@ const ClipTab: React.FC = ({ bpm, keySignature }) => { const genJson = (await genResp.json()) as { task_id: string }; kgoneLog('RES', `POST /v1/clip/generate → ${genResp.status}`, genJson); const { task_id } = genJson; + taskIdRef.current = task_id; // ── 3. Poll for completion ────────────────────────────────────────────── setGenStatus('polling'); @@ -431,7 +472,25 @@ const ClipTab: React.FC = ({ bpm, keySignature }) => { {/* Audio preview player — shown once generation is complete */} - {audioUrl && } + {audioUrl && ( + + )} + + {/* Drag-to-track hint — shown after a successful generation */} + {genStatus === 'done' && ( +
+ Drag the player above to a track to import the clip. + Drop onto an audio track to import as a WAV region (recommended), + or onto a MIDI track to import as a MIDI region. + Note: MIDI is transcribed from the audio and may not be perfectly accurate. +
+ )} {/* Error message */} {genStatus === 'error' && errorMsg && ( @@ -473,6 +532,7 @@ const FullSongTab: React.FC = () => { const [audioUrl, setAudioUrl] = useState(null); const abortRef = useRef(null); + const taskIdRef = useRef(''); useEffect(() => { return () => { @@ -554,6 +614,7 @@ const FullSongTab: React.FC = () => { const genJson = (await genResp.json()) as { data: { task_id: string }; code: number }; kgoneLog('RES', `POST /v1/fullsong/generate → ${genResp.status}`, genJson); const task_id = genJson.data.task_id; + taskIdRef.current = task_id; // ── 3. Poll for completion ────────────────────────────────────────────── setGenStatus('polling'); @@ -714,7 +775,21 @@ const FullSongTab: React.FC = () => {
- {audioUrl && } + {audioUrl && ( + + )} + + {genStatus === 'done' && ( +
+ Drag the player above to an audio track to import the song. + Dropping onto a MIDI track is not supported for full song generation. +
+ )} {genStatus === 'error' && errorMsg && (
{errorMsg}
@@ -753,6 +828,7 @@ const SeparatorTab: React.FC = () => { const [stemAudioUrls, setStemAudioUrls] = useState>([]); const abortRef = useRef(null); + const taskIdRef = useRef(''); // Revoke all blob URLs on unmount useEffect(() => { @@ -853,6 +929,7 @@ const SeparatorTab: React.FC = () => { const sepJson = (await sepResp.json()) as { task_id: string }; kgoneLog('RES', `POST /v1/separator/separate → ${sepResp.status}`, sepJson); const { task_id } = sepJson; + taskIdRef.current = task_id; // ── 3. Poll for completion ───────────────────────────────────────────── setGenStatus('polling'); @@ -966,12 +1043,25 @@ const SeparatorTab: React.FC = () => { {stemAudioUrls.map(stem => (
{stem.name}
- +
))} )} + {/* Drag-to-track hint — shown after successful separation */} + {genStatus === 'done' && ( +
+ Drag each stem player above to an audio track to import it. + Dropping onto a MIDI track is not supported for stem separation. +
+ )} + {/* Error message */} {genStatus === 'error' && errorMsg && (
{errorMsg}
diff --git a/src/components/MainContent.tsx b/src/components/MainContent.tsx index 8ef912d..f1df80e 100644 --- a/src/components/MainContent.tsx +++ b/src/components/MainContent.tsx @@ -38,7 +38,8 @@ const MainContent: React.FC = ({ setShowPianoRoll, setActiveRegionId, addTrack, - addAudioTrack + addAudioTrack, + projectName, } = useProjectStore(); // State to store regions @@ -239,6 +240,19 @@ const MainContent: React.FC = ({ }); }; + // Handle regions dropped from K.G.One panel (external drag-and-drop) + const handleExternalDropComplete = (trackIndex: number, regionUI: RegionUI) => { + const track = tracks[trackIndex]; + if (!track) return; + updateTrack(track); + setSelectedTrack(track.getId().toString()); + setRegions(prev => { + const updated = [...prev, regionUI]; + selectRegion(regionUI.id, updated); + return updated; + }); + }; + // Handle region updates (resize, move, etc.) const handleRegionUpdated = ( regionId: string, @@ -742,10 +756,12 @@ const MainContent: React.FC = ({ draggedTrackIndex={draggedTrackIndex} dragOverTrackIndex={dragOverTrackIndex} selectedRegionId={selectedRegionId} + projectName={projectName} onRegionCreated={handleRegionCreated} onRegionUpdated={handleRegionUpdated} onRegionClick={handleRegionClick} onOpenPianoRoll={handleOpenPianoRoll} + onExternalDropComplete={handleExternalDropComplete} /> diff --git a/src/components/track/TrackGridItem.tsx b/src/components/track/TrackGridItem.tsx index 3b518e8..08ee19b 100644 --- a/src/components/track/TrackGridItem.tsx +++ b/src/components/track/TrackGridItem.tsx @@ -27,6 +27,7 @@ interface TrackGridItemProps { onRegionClick?: (regionId: string) => void; onOpenPianoRoll?: (regionId: string) => void; allTracks?: KGTrack[]; // Added to access all tracks for drag operations + onKGOneClipDrop?: (e: React.DragEvent, trackIndex: number) => void; } const TrackGridItem: React.FC = ({ @@ -46,7 +47,8 @@ const TrackGridItem: React.FC = ({ onRegionDragEnd, onRegionClick, onOpenPianoRoll, - allTracks + allTracks, + onKGOneClipDrop, }) => { const [containerWidth, setContainerWidth] = useState(0); const [resizingRegion, setResizingRegion] = useState(null); @@ -503,12 +505,24 @@ const TrackGridItem: React.FC = ({ const trackRegions = regions.filter(region => region.trackIndex === index); return ( -
onDoubleClick(e, index)} onClick={(e) => onClick && onClick(e, index)} ref={trackElementRef} + onDragOver={(e) => { + if (Array.from(e.dataTransfer.types).includes('application/kgone-clip')) { + e.preventDefault(); + e.dataTransfer.dropEffect = 'copy'; + } + }} + onDrop={(e) => { + if (Array.from(e.dataTransfer.types).includes('application/kgone-clip')) { + e.preventDefault(); + onKGOneClipDrop?.(e, index); + } + }} > {/* Render regions for this track */} {trackRegions.map(region => { diff --git a/src/components/track/TrackGridPanel.tsx b/src/components/track/TrackGridPanel.tsx index dad1391..bb8511f 100644 --- a/src/components/track/TrackGridPanel.tsx +++ b/src/components/track/TrackGridPanel.tsx @@ -7,11 +7,14 @@ import type { RegionUI } from '../interfaces'; import { DEBUG_MODE, REGION_CONSTANTS } from '../../constants'; import { KGMainContentState } from '../../core/state/KGMainContentState'; import { isModifierKeyPressed } from '../../util/osUtil'; -import { CreateRegionCommand, ResizeRegionCommand, MoveRegionCommand } from '../../core/commands'; +import { CreateRegionCommand, ResizeRegionCommand, MoveRegionCommand, ImportAudioCommand, ImportMidiClipCommand } from '../../core/commands'; import { KGCore } from '../../core/KGCore'; import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface'; import { KGAudioRegion } from '../../core/region/KGAudioRegion'; import { generateNewRegionName } from '../../util/miscUtil'; +import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage'; +import { parseMidiFirstTrackNotes } from '../../util/midiUtil'; +import * as Tone from 'tone'; interface TrackGridPanelProps { tracks: KGTrack[]; @@ -21,10 +24,12 @@ interface TrackGridPanelProps { draggedTrackIndex: number | null; dragOverTrackIndex: number | null; selectedRegionId: string | null; + projectName: string; onRegionCreated: (trackIndex: number, region: RegionUI, midiRegion: KGMidiRegion) => void; onRegionUpdated?: (regionId: string, updates: Partial, expectedModelUpdates?: { startBeat: number, length: number }) => void; onRegionClick?: (regionId: string) => void; onOpenPianoRoll?: (regionId: string) => void; + onExternalDropComplete?: (trackIndex: number, regionUI: RegionUI) => void; } const TrackGridPanel: React.FC = ({ @@ -35,10 +40,12 @@ const TrackGridPanel: React.FC = ({ draggedTrackIndex, dragOverTrackIndex, selectedRegionId, + projectName, onRegionCreated, onRegionUpdated, onRegionClick, - onOpenPianoRoll + onOpenPianoRoll, + onExternalDropComplete, }) => { const gridContainerRef = useRef(null); @@ -396,11 +403,151 @@ const TrackGridPanel: React.FC = ({ } }; + // Handle external K.G.One clip drop onto a track row + const handleExternalDrop = async (e: React.DragEvent, trackIndex: number) => { + const raw = e.dataTransfer.getData('application/kgone-clip'); + if (!raw) return; + + let dropData: { midiUrl?: string; audioUrl: string; audioDurationSeconds: number; audioFileName: string }; + try { + dropData = JSON.parse(raw); + } catch { + console.error('[KGOne] Invalid drop data'); + return; + } + + // Calculate drop bar position + if (!gridContainerRef.current) return; + const gridRect = gridContainerRef.current.getBoundingClientRect(); + const relativeX = e.clientX - gridRect.left; + const barWidth = gridContainerRef.current.clientWidth / maxBars; + const rawBar = relativeX / barWidth + 1; + const snap = KGMainContentState.instance().isSnappingEnabled(); + const barNumber = Math.max(1, snap ? Math.round(rawBar) : Math.floor(rawBar)); + + const track = tracks[trackIndex]; + if (!track) return; + + const beatsPerBar = timeSignature.numerator; + + try { + if (track.getType() === TrackType.MIDI) { + if (!dropData.midiUrl) { + window.alert( + 'This audio clip can only be imported into an audio track.\n' + + 'Please drag it onto an audio track instead.' + ); + return; + } + // ── MIDI track: fetch MIDI file and create a KGMidiRegion ────────── + const midiResp = await fetch(dropData.midiUrl); + if (!midiResp.ok) throw new Error(`MIDI fetch failed (${midiResp.status})`); + const buf = await midiResp.arrayBuffer(); + const { notes, totalBeats } = parseMidiFirstTrackNotes(new Uint8Array(buf)); + const lengthInBars = Math.max(1, Math.ceil(totalBeats / beatsPerBar)); + + const cmd = ImportMidiClipCommand.fromBarCoordinates( + track.getId().toString(), + trackIndex, + barNumber, + lengthInBars, + beatsPerBar, + notes, + 'KGOne Clip' + ); + KGCore.instance().executeCommand(cmd); + + const created = cmd.getCreatedRegion(); + if (created && onExternalDropComplete) { + const regionUI: RegionUI = { + id: created.getId(), + trackId: track.getId().toString(), + trackIndex, + barNumber, + length: lengthInBars, + name: created.getName(), + }; + onExternalDropComplete(trackIndex, regionUI); + } + + if (DEBUG_MODE.TRACK_GRID_PANEL) { + console.log(`[KGOne] Imported MIDI clip to track ${trackIndex}, bar ${barNumber}, ${notes.length} notes`); + } + + } else if (track.getType() === TrackType.Wave) { + // ── Audio track: save blob to OPFS and create a KGAudioRegion ────── + const blob = await fetch(dropData.audioUrl).then(r => r.blob()); + const audioFile = new File([blob], dropData.audioFileName, { type: 'audio/wav' }); + const fileId = `kgone_${Date.now()}_${Math.random().toString(36).slice(2, 7)}`; + + // Decode audio to get accurate duration and load into player bus + const arrayBuffer = await audioFile.arrayBuffer(); + const toneBuffer = new Tone.ToneAudioBuffer(); + await new Promise((resolve, reject) => { + const audioContext = Tone.getContext().rawContext as AudioContext; + audioContext.decodeAudioData( + arrayBuffer.slice(0), + (decoded) => { toneBuffer.set(decoded); resolve(); }, + (err) => reject(err) + ); + }); + + const audioDurationSeconds = toneBuffer.duration; + + await KGAudioFileStorage.storeAudioFile(projectName, fileId, audioFile); + KGAudioInterface.instance().loadAudioBufferForTrack( + track.getId().toString(), + fileId, + toneBuffer + ); + + const bpm = KGCore.instance().getCurrentProject().getBpm(); + const durationInBeats = audioDurationSeconds * (bpm / 60); + const insertBeat = (barNumber - 1) * beatsPerBar; + const lengthInBars = Math.max(1, Math.ceil(durationInBeats / beatsPerBar)); + const prevMaxBars = maxBars; + const newMaxBars = Math.max(maxBars, barNumber + lengthInBars - 1); + + const cmd = new ImportAudioCommand( + track.getId() as unknown as number, + trackIndex, + fileId, + dropData.audioFileName, + audioDurationSeconds, + insertBeat, + durationInBeats, + prevMaxBars, + newMaxBars + ); + KGCore.instance().executeCommand(cmd); + + const created = cmd.getCreatedRegion(); + if (created && onExternalDropComplete) { + const regionUI: RegionUI = { + id: created.getId(), + trackId: track.getId().toString(), + trackIndex, + barNumber, + length: lengthInBars, + name: created.getName(), + }; + onExternalDropComplete(trackIndex, regionUI); + } + + if (DEBUG_MODE.TRACK_GRID_PANEL) { + console.log(`[KGOne] Imported audio clip to track ${trackIndex}, bar ${barNumber}, ${audioDurationSeconds.toFixed(2)}s`); + } + } + } catch (err) { + console.error('[KGOne] Drop import failed:', err); + } + }; + return (
{/* Playhead */} - + {/* Track grids */} {tracks.map((track, index) => ( = ({ onRegionClick={handleRegionClick} onOpenPianoRoll={onOpenPianoRoll} allTracks={tracks} + onKGOneClipDrop={handleExternalDrop} /> ))}
diff --git a/src/core/commands/index.ts b/src/core/commands/index.ts index 06b73f0..0aada8e 100644 --- a/src/core/commands/index.ts +++ b/src/core/commands/index.ts @@ -21,6 +21,7 @@ export { MoveRegionCommand } from './region/MoveRegionCommand'; export { PasteRegionsCommand } from './region/PasteRegionsCommand'; export { UpdateRegionCommand, type RegionUpdateProperties } from './region/UpdateRegionCommand'; export { ImportAudioCommand } from './region/ImportAudioCommand'; +export { ImportMidiClipCommand } from './region/ImportMidiClipCommand'; // Note commands export { CreateNoteCommand } from './note/CreateNoteCommand'; diff --git a/src/core/commands/region/ImportMidiClipCommand.ts b/src/core/commands/region/ImportMidiClipCommand.ts new file mode 100644 index 0000000..8116a8e --- /dev/null +++ b/src/core/commands/region/ImportMidiClipCommand.ts @@ -0,0 +1,113 @@ +import { KGCommand } from '../KGCommand'; +import { KGCore } from '../../KGCore'; +import { KGMidiRegion } from '../../region/KGMidiRegion'; +import { KGMidiNote } from '../../midi/KGMidiNote'; +import { generateUniqueId } from '../../../util/miscUtil'; +import type { RawMidiNote } from '../../../util/midiUtil'; + +/** + * Command to insert a MIDI clip (from K.G.One generation) into a MIDI track. + * All note data is stored so that redo recreates the full region with notes. + */ +export class ImportMidiClipCommand extends KGCommand { + private trackId: string; + private trackIndex: number; + private regionId: string; + private regionName: string; + private startBeat: number; + private lengthInBeats: number; + private rawNotes: RawMidiNote[]; + private createdRegion: KGMidiRegion | null = null; + + constructor( + trackId: string, + trackIndex: number, + startBeat: number, + lengthInBeats: number, + rawNotes: RawMidiNote[], + regionName?: string, + regionId?: string + ) { + super(); + this.trackId = trackId; + this.trackIndex = trackIndex; + this.startBeat = startBeat; + this.lengthInBeats = lengthInBeats; + this.rawNotes = rawNotes; + this.regionId = regionId || generateUniqueId('KGMidiRegion'); + this.regionName = regionName || 'KGOne Clip'; + } + + execute(): void { + const core = KGCore.instance(); + const currentProject = core.getCurrentProject(); + const track = currentProject.getTracks().find(t => t.getId().toString() === this.trackId); + if (!track) { + throw new Error(`Track ${this.trackId} not found`); + } + + // Create the region + this.createdRegion = new KGMidiRegion( + this.regionId, + this.trackId, + this.trackIndex, + this.regionName, + this.startBeat, + this.lengthInBeats + ); + + // Populate notes + for (const raw of this.rawNotes) { + const note = new KGMidiNote( + generateUniqueId('KGMidiNote'), + raw.startBeat, + raw.endBeat, + raw.pitch, + raw.velocity + ); + this.createdRegion.addNote(note); + } + + track.addRegion(this.createdRegion); + console.log(`Imported MIDI clip "${this.regionName}" (${this.rawNotes.length} notes) at beat ${this.startBeat}`); + } + + undo(): void { + const core = KGCore.instance(); + const currentProject = core.getCurrentProject(); + const track = currentProject.getTracks().find(t => t.getId().toString() === this.trackId); + if (track) { + track.removeRegion(this.regionId); + } + console.log(`Undid MIDI clip import "${this.regionName}"`); + } + + getDescription(): string { + return `Import MIDI clip "${this.regionName}"`; + } + + getCreatedRegion(): KGMidiRegion | null { + return this.createdRegion; + } + + /** + * Factory from bar-based coordinates (common UI pattern). + */ + static fromBarCoordinates( + trackId: string, + trackIndex: number, + barNumber: number, + lengthInBars: number, + beatsPerBar: number, + rawNotes: RawMidiNote[], + regionName?: string, + regionId?: string + ): ImportMidiClipCommand { + const startBeat = (barNumber - 1) * beatsPerBar; + const lengthInBeats = lengthInBars * beatsPerBar; + return new ImportMidiClipCommand( + trackId, trackIndex, startBeat, lengthInBeats, + rawNotes, regionName, regionId + ); + } +} diff --git a/src/util/midiUtil.ts b/src/util/midiUtil.ts index c3c0a3b..d79d652 100644 --- a/src/util/midiUtil.ts +++ b/src/util/midiUtil.ts @@ -465,6 +465,40 @@ function getKeySignatureBytes(keySignature: KeySignature): Uint8Array { return new Uint8Array([0, 0]); } +// ─── K.G.One clip MIDI import helpers ──────────────────────────────────────── + +export interface RawMidiNote { + startBeat: number; + endBeat: number; + pitch: number; + velocity: number; +} + +/** + * Parses a MIDI binary and returns all notes from the first track that has + * notes, with beat offsets normalised so the earliest note starts at beat 0. + * Used by the K.G.One Clip drag-to-MIDI-track feature. + */ +export function parseMidiFirstTrackNotes(data: Uint8Array): { + notes: RawMidiNote[]; + totalBeats: number; +} { + const midiFile = parseMidiFile(data); + const firstTrack = midiFile.tracks.find(t => t.notes.length > 0); + if (!firstTrack || firstTrack.notes.length === 0) { + return { notes: [], totalBeats: 0 }; + } + const minBeat = Math.min(...firstTrack.notes.map(n => n.startBeat)); + const notes: RawMidiNote[] = firstTrack.notes.map(n => ({ + startBeat: n.startBeat - minBeat, + endBeat: n.endBeat - minBeat, + pitch: n.pitch, + velocity: n.velocity, + })); + const totalBeats = Math.max(...notes.map(n => n.endBeat)); + return { notes, totalBeats }; +} + /** * Converts a MIDI binary file to a KGSP project * @param midiData - The MIDI file data as Uint8Array