From f60c7b44ec2aa08bbbc2f648598a75f1e6b073fa Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Wed, 29 Apr 2026 18:28:23 -0700 Subject: [PATCH] feat: added remix and repaint support --- src/components/KGOnePanel.tsx | 1032 ++++++++++++++++++++++++++++++++- 1 file changed, 1029 insertions(+), 3 deletions(-) diff --git a/src/components/KGOnePanel.tsx b/src/components/KGOnePanel.tsx index db30a23..78b6102 100644 --- a/src/components/KGOnePanel.tsx +++ b/src/components/KGOnePanel.tsx @@ -14,10 +14,11 @@ import { sliceAudioToWav } from '../util/audioUtil'; import type { KeySignature } from '../core/KGProject'; import { ImportStemsCommand } from '../core/commands'; import type { StemImportEntry } from '../core/commands'; +import { showAlert } from './common/DialogProvider'; // ─── Types ──────────────────────────────────────────────────────────────────── -type Tab = 'clip' | 'fullsong' | 'separator'; +type Tab = 'clip' | 'fullsong' | 'remix' | 'repaint' | 'separator'; type GenStatus = 'idle' | 'loading-model' | 'generating' | 'polling' | 'downloading' | 'done' | 'error'; @@ -813,6 +814,22 @@ function extractStemName(filename: string): string { return match ? match[1] : filename; } +/** Count existing remix tracks derived from `sourceTrackName` (for naming new ones). */ +function countRemixTracks(sourceTrackName: string): number { + const tracks = KGCore.instance().getCurrentProject().getTracks(); + const escaped = sourceTrackName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const pattern = new RegExp(`^${escaped} - remix \\(\\d+\\)$`); + return tracks.filter(t => pattern.test(t.getName())).length; +} + +/** Count existing repaint tracks derived from `sourceTrackName` (for naming new ones). */ +function countRepaintTracks(sourceTrackName: string): number { + const tracks = KGCore.instance().getCurrentProject().getTracks(); + const escaped = sourceTrackName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const pattern = new RegExp(`^${escaped} - repaint \\(\\d+\\)$`); + return tracks.filter(t => pattern.test(t.getName())).length; +} + const SeparatorTab: React.FC = () => { const { selectedRegionIds, projectName, bpm, timeSignature, maxBars, refreshProjectState } = useProjectStore(); const [model, setModel] = useState(SEPARATOR_MODELS[0].value); @@ -1202,6 +1219,1008 @@ const SeparatorTab: React.FC = () => { ); }; +// ─── Remix Tab ──────────────────────────────────────────────────────────────── + +const RemixTab: React.FC = () => { + const { selectedRegionIds, projectName, bpm, maxBars, refreshProjectState } = useProjectStore(); + + // Form state (mirrors FullSongTab) + const [caption, setCaption] = useState(''); + const [lyrics, setLyrics] = useState(''); + const [instrumental, setInstrumental] = useState(false); + const [inferenceSteps, setInferenceSteps] = useState(8); + const [guidanceScale, setGuidanceScale] = useState(7.0); + const [useRandomSeed, setUseRandomSeed] = useState(true); + const [seed, setSeed] = useState(-1); + const [thinking, setThinking] = useState(true); + + // Remix-specific params + const [audioCoverStrength, setAudioCoverStrength] = useState(0.5); + const [coverNoiseStrength, setCoverNoiseStrength] = useState(0.2); + + // Generation state + const [genStatus, setGenStatus] = useState('idle'); + const [genHint, setGenHint] = useState(''); + const [errorMsg, setErrorMsg] = useState(''); + const [audioUrl, setAudioUrl] = useState(null); + + // Import state + const [isImporting, setIsImporting] = useState(false); + const [importError, setImportError] = useState(''); + + const abortRef = useRef(null); + const taskIdRef = useRef(''); + const originalRegionRef = useRef<{ + regionName: string; + trackName: string; + startFromBeat: number; + trackIndex: number; + } | null>(null); + + // Revoke blob URL on unmount + useEffect(() => { + return () => { + abortRef.current?.abort(); + if (audioUrl) URL.revokeObjectURL(audioUrl); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + const selectedAudioRegion = useMemo(() => { + if (!selectedRegionIds.length) return null; + const project = KGCore.instance().getCurrentProject(); + for (const track of project.getTracks()) { + for (const region of track.getRegions()) { + if ( + selectedRegionIds.includes(region.getId()) && + region.getCurrentType() === 'KGAudioRegion' + ) { + return { region: region as KGAudioRegion, trackName: track.getName(), trackIndex: track.getTrackIndex() }; + } + } + } + return null; + }, [selectedRegionIds]); + + const isGenerating = genStatus !== 'idle' && genStatus !== 'done' && genStatus !== 'error'; + + const handleRemix = useCallback(async () => { + if (!selectedAudioRegion) return; + + // Capture snapshot before anything changes — selection may shift during generation + originalRegionRef.current = { + regionName: selectedAudioRegion.region.getName(), + trackName: selectedAudioRegion.trackName, + startFromBeat: selectedAudioRegion.region.getStartFromBeat(), + trackIndex: selectedAudioRegion.trackIndex, + }; + setImportError(''); + + // Abort any in-flight request + abortRef.current?.abort(); + const ctrl = new AbortController(); + abortRef.current = ctrl; + const { signal } = ctrl; + + if (audioUrl) { + URL.revokeObjectURL(audioUrl); + setAudioUrl(null); + } + setErrorMsg(''); + + try { + const baseUrl = getKGOneBaseUrl(); + + // ── 1. Load the fullsong model ────────────────────────────────────────── + setGenStatus('loading-model'); + setGenHint('Loading model — this can take 60+ seconds, please wait...'); + + const loadPayload = { model: 'fullsong' }; + kgoneLog('REQ', 'POST /v1/models/load', loadPayload); + const loadResp = await fetch(`${baseUrl}/v1/models/load`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(loadPayload), + signal, + }); + + if (!loadResp.ok) { + const body = await loadResp.text().catch(() => ''); + kgoneLog('RES', `POST /v1/models/load → ${loadResp.status}`, body); + throw new Error(`Model load failed (${loadResp.status}): ${body}`); + } + kgoneLog('RES', `POST /v1/models/load → ${loadResp.status}`, await loadResp.clone().json().catch(() => '(unparseable)')); + + if (signal.aborted) return; + + // ── 2. Load audio from OPFS and build multipart form ─────────────────── + setGenStatus('generating'); + setGenHint('Reading audio file...'); + + const audioFileId = selectedAudioRegion.region.getAudioFileId(); + const audioFileName = selectedAudioRegion.region.getAudioFileName(); + const clipStart = selectedAudioRegion.region.getClipStartOffsetSeconds(); + const fullDuration = selectedAudioRegion.region.getAudioDurationSeconds(); + const regionLengthSec = selectedAudioRegion.region.getLength() * (60 / bpm); + const effectiveDuration = Math.min(regionLengthSec, fullDuration - clipStart); + + const rawBuffer = await KGAudioFileStorage.loadAudioFile(projectName, audioFileId); + const needsSlice = clipStart > 0.01 || effectiveDuration < fullDuration - 0.01; + + let uploadBuffer: ArrayBuffer; + let uploadFileName: string; + let uploadMimeType: string; + + if (needsSlice) { + setGenHint('Trimming audio to region range...'); + uploadBuffer = await sliceAudioToWav(rawBuffer, clipStart, effectiveDuration); + uploadFileName = audioFileName.replace(/\.[^.]+$/, '.wav'); + uploadMimeType = 'audio/wav'; + } else { + uploadBuffer = rawBuffer; + uploadFileName = audioFileName; + uploadMimeType = 'audio/mpeg'; + } + + const audioFile = new File([uploadBuffer], uploadFileName, { type: uploadMimeType }); + + const formData = new FormData(); + formData.append('audio_file', audioFile, uploadFileName); + formData.append('caption', caption); + formData.append('lyrics', instrumental ? '' : lyrics); + formData.append('instrumental', String(instrumental)); + formData.append('inference_steps', String(inferenceSteps)); + formData.append('guidance_scale', String(guidanceScale)); + formData.append('use_random_seed', String(useRandomSeed)); + formData.append('seed', String(seed)); + formData.append('thinking', String(thinking)); + formData.append('batch_size', '1'); + formData.append('audio_format', 'mp3'); + formData.append('audio_cover_strength', String(audioCoverStrength)); + formData.append('cover_noise_strength', String(coverNoiseStrength)); + + if (signal.aborted) return; + + setGenHint('Submitting remix request...'); + + kgoneLog('REQ', 'POST /v1/fullsong/remix', { + file: uploadFileName, caption, instrumental, inference_steps: inferenceSteps, + guidance_scale: guidanceScale, use_random_seed: useRandomSeed, seed, thinking, + audio_cover_strength: audioCoverStrength, cover_noise_strength: coverNoiseStrength, + }); + const remixResp = await fetch(`${baseUrl}/v1/fullsong/remix`, { + method: 'POST', + body: formData, + signal, + }); + + if (!remixResp.ok) { + const body = await remixResp.text().catch(() => ''); + kgoneLog('RES', `POST /v1/fullsong/remix → ${remixResp.status}`, body); + throw new Error(`Remix request failed (${remixResp.status}): ${body}`); + } + + // Response shape: { "data": { "task_id": "...", ... }, "code": 200, ... } + const remixJson = (await remixResp.json()) as { data: { task_id: string }; code: number }; + kgoneLog('RES', `POST /v1/fullsong/remix → ${remixResp.status}`, remixJson); + + const task_id = remixJson.data.task_id; + taskIdRef.current = task_id; + + // ── 3. Poll for completion (same as FullSongTab) ─────────────────────── + setGenStatus('polling'); + setGenHint('Generating remix...'); + + type ResultItem = { progress: number; stage: string; status: number }; + type PollResponse = { data: Array<{ status: number; result: string }>; code: number }; + + // eslint-disable-next-line no-constant-condition + while (true) { + if (signal.aborted) return; + + await new Promise(r => { + const t = setTimeout(r, 5000); + signal.addEventListener('abort', () => { clearTimeout(t); r(); }, { once: true }); + }); + + if (signal.aborted) return; + + kgoneLog('REQ', `GET /v1/fullsong/result/${task_id}`, null); + const resultResp = await fetchWithRetry(`${baseUrl}/v1/fullsong/result/${task_id}`, { signal }); + + const pollJson = (await resultResp.json()) as PollResponse; + kgoneLog('RES', `GET /v1/fullsong/result/${task_id} → ${resultResp.status}`, pollJson); + + const outer = pollJson.data?.[0]; + if (!outer) continue; + + try { + const inner = JSON.parse(outer.result) as ResultItem[]; + const item = inner[0]; + if (item) { + const pct = Math.round((item.progress ?? 0) * 100); + const stage = item.stage ?? ''; + setGenHint(stage ? `Generating remix... ${pct}% — ${stage}` : `Generating remix... ${pct}%`); + } + } catch { + // result may be empty string while still queued — ignore parse errors + } + + if (outer.status === 1) break; + } + + // ── 4. Download the MP3 ───────────────────────────────────────────────── + setGenStatus('downloading'); + setGenHint('Downloading audio...'); + + kgoneLog('REQ', `GET /v1/fullsong/audio/${task_id}?index=0`, null); + const audioResp = await fetchWithRetry(`${baseUrl}/v1/fullsong/audio/${task_id}?index=0`, { signal }); + + const blob = await audioResp.blob(); + const url = URL.createObjectURL(blob); + kgoneLog('RES', `GET /v1/fullsong/audio/${task_id}?index=0 → ${audioResp.status} (binary MP3)`, url); + + setAudioUrl(url); + setGenStatus('done'); + setGenHint(''); + } catch (err) { + if (signal.aborted) return; + console.error('[KGOne] Remix error:', err); + setErrorMsg(err instanceof Error ? err.message : String(err)); + setGenStatus('error'); + setGenHint(''); + } + }, [selectedAudioRegion, projectName, bpm, caption, lyrics, instrumental, inferenceSteps, guidanceScale, useRandomSeed, seed, thinking, audioCoverStrength, coverNoiseStrength, audioUrl]); + + const handleImportAligned = useCallback(async () => { + const snap = originalRegionRef.current; + if (!snap || !audioUrl) return; + + setIsImporting(true); + setImportError(''); + + try { + const audioContext = Tone.getContext().rawContext as AudioContext; + + const blob = await fetch(audioUrl).then(r => r.blob()); + const fileName = `KGOne_Remix_${taskIdRef.current || Date.now()}.mp3`; + const fileId = `kgone_remix_${Date.now()}_${Math.random().toString(36).slice(2, 7)}`; + const audioFile = new File([blob], fileName, { type: 'audio/mpeg' }); + + const arrayBuffer = await blob.arrayBuffer(); + const toneBuffer = new Tone.ToneAudioBuffer(); + await new Promise((resolve, reject) => { + audioContext.decodeAudioData( + arrayBuffer.slice(0), + decoded => { toneBuffer.set(decoded); resolve(); }, + reject, + ); + }); + + await KGAudioFileStorage.storeAudioFile(projectName, fileId, audioFile); + + const project = KGCore.instance().getCurrentProject(); + const count = countRemixTracks(snap.trackName) + 1; + const stemEntry: StemImportEntry = { + trackName: `${snap.trackName} - remix (${count})`, + regionName: `${snap.regionName} - remix (${count})`, + audioFileId: fileId, + audioFileName: fileName, + audioDurationSeconds: toneBuffer.duration, + toneBuffer, + }; + + const cmd = new ImportStemsCommand( + project.getTracks().length, + snap.trackIndex, + snap.startFromBeat, + [stemEntry], + maxBars, + ); + KGCore.instance().executeCommand(cmd); + refreshProjectState(); + } catch (err) { + setImportError(err instanceof Error ? err.message : String(err)); + } finally { + setIsImporting(false); + } + }, [audioUrl, projectName, maxBars, refreshProjectState]); + + const btnLabel = () => { + switch (genStatus) { + case 'loading-model': return 'Loading model...'; + case 'generating': return 'Preparing upload...'; + case 'polling': return 'Processing remix...'; + case 'downloading': return 'Downloading...'; + default: return 'Generate Remix'; + } + }; + + return ( + <> + {selectedAudioRegion ? ( + <> +
+
Selected Region
+
{selectedAudioRegion.region.getName()}
+
Track
+
{selectedAudioRegion.trackName}
+
+ +
+ +