Initial commit: ACE-Step UI - Open source music generation interface
This commit is contained in:
fspecii
@@ -0,0 +1,458 @@
|
||||
/**
|
||||
* Demucs Web - Stem Extraction for SunoAce
|
||||
*/
|
||||
import * as ort from 'https://cdn.jsdelivr.net/npm/onnxruntime-web@1.21.0/dist/ort.all.mjs';
|
||||
import { DemucsProcessor, CONSTANTS } from './src/index.js';
|
||||
|
||||
const { SAMPLE_RATE, TRAINING_SAMPLES, TRACKS, DEFAULT_MODEL_URL } = CONSTANTS;
|
||||
|
||||
const LOCAL_MODEL_URL = '../models/htdemucs_embedded.onnx';
|
||||
|
||||
let processor = null;
|
||||
let audioContext = null;
|
||||
let audioBuffer = null;
|
||||
let isProcessing = false;
|
||||
|
||||
// DOM elements
|
||||
const dropZone = document.getElementById('dropZone');
|
||||
const fileInput = document.getElementById('fileInput');
|
||||
const processBtn = document.getElementById('processBtn');
|
||||
const progressFill = document.getElementById('progressFill');
|
||||
const status = document.getElementById('status');
|
||||
const results = document.getElementById('results');
|
||||
const trackList = document.getElementById('trackList');
|
||||
const backendBadge = document.getElementById('backendBadge');
|
||||
const audioFileName = document.getElementById('audioFileName');
|
||||
const statusDetail = document.getElementById('statusDetail');
|
||||
const statsRow = document.getElementById('statsRow');
|
||||
const statElapsed = document.getElementById('statElapsed');
|
||||
const statSegment = document.getElementById('statSegment');
|
||||
const statSpeed = document.getElementById('statSpeed');
|
||||
const statETA = document.getElementById('statETA');
|
||||
|
||||
let processStartTime = null;
|
||||
|
||||
function log(phase, message) {
|
||||
const now = new Date();
|
||||
const timeStr = now.toLocaleTimeString('en-US', { hour12: false });
|
||||
const logLine = document.createElement('div');
|
||||
logLine.className = 'text-zinc-400 py-1 border-b border-zinc-800/50 last:border-0';
|
||||
logLine.innerHTML = `<span class="text-emerald-400">[${timeStr}]</span> <span class="text-teal-400">[${phase}]</span> ${message}`;
|
||||
statusDetail.appendChild(logLine);
|
||||
statusDetail.scrollTop = statusDetail.scrollHeight;
|
||||
console.log(`[${phase}] ${message}`);
|
||||
}
|
||||
|
||||
function formatTime(seconds) {
|
||||
if (!isFinite(seconds) || seconds < 0) return '--:--';
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = Math.floor(seconds % 60);
|
||||
return `${mins}:${secs.toString().padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
async function init() {
|
||||
let backend = 'wasm';
|
||||
|
||||
if ('gpu' in navigator) {
|
||||
try {
|
||||
const gpuAdapter = await navigator.gpu.requestAdapter();
|
||||
if (gpuAdapter) {
|
||||
backend = 'webgpu';
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('WebGPU not available:', e);
|
||||
}
|
||||
}
|
||||
|
||||
ort.env.wasm.numThreads = navigator.hardwareConcurrency || 4;
|
||||
|
||||
if (backend === 'webgpu') {
|
||||
ort.env.webgpu = ort.env.webgpu || {};
|
||||
ort.env.webgpu.powerPreference = 'high-performance';
|
||||
backendBadge.textContent = 'WebGPU (GPU)';
|
||||
backendBadge.className = 'badge badge-gpu';
|
||||
} else {
|
||||
const threads = navigator.hardwareConcurrency || 4;
|
||||
backendBadge.textContent = `WASM (${threads} threads)`;
|
||||
backendBadge.className = 'badge badge-cpu';
|
||||
}
|
||||
|
||||
processor = new DemucsProcessor({
|
||||
ort,
|
||||
onProgress: ({ progress, currentSegment, totalSegments }) => {
|
||||
progressFill.style.width = (5 + progress * 90) + '%';
|
||||
|
||||
const elapsed = (Date.now() - processStartTime) / 1000;
|
||||
statElapsed.textContent = formatTime(elapsed);
|
||||
statSegment.textContent = `${currentSegment}/${totalSegments}`;
|
||||
|
||||
if (currentSegment > 0 && audioBuffer) {
|
||||
const processedDuration = (currentSegment / totalSegments) * audioBuffer.duration;
|
||||
const speed = processedDuration / elapsed;
|
||||
statSpeed.textContent = speed.toFixed(2) + 'x';
|
||||
|
||||
const remainingSegments = totalSegments - currentSegment;
|
||||
const avgTimePerSegment = elapsed / currentSegment;
|
||||
const eta = remainingSegments * avgTimePerSegment;
|
||||
statETA.textContent = formatTime(eta);
|
||||
}
|
||||
},
|
||||
onLog: log,
|
||||
onDownloadProgress: (loaded, total) => {
|
||||
const percent = ((loaded / total) * 100).toFixed(1);
|
||||
const loadedMB = (loaded / 1024 / 1024).toFixed(1);
|
||||
const totalMB = (total / 1024 / 1024).toFixed(1);
|
||||
status.textContent = `Downloading model... ${loadedMB}MB / ${totalMB}MB (${percent}%)`;
|
||||
progressFill.style.width = (loaded / total * 100) + '%';
|
||||
}
|
||||
});
|
||||
|
||||
status.textContent = 'Loading AI model...';
|
||||
|
||||
try {
|
||||
try {
|
||||
status.textContent = 'Downloading model (~172MB)...';
|
||||
await processor.loadModel(DEFAULT_MODEL_URL);
|
||||
} catch {
|
||||
status.textContent = 'Loading local model...';
|
||||
await processor.loadModel(LOCAL_MODEL_URL);
|
||||
}
|
||||
status.textContent = 'Ready - Select an audio file';
|
||||
progressFill.style.width = '0%';
|
||||
} catch (e) {
|
||||
status.textContent = 'Failed to load model: ' + e.message;
|
||||
console.error('Failed to load model:', e);
|
||||
}
|
||||
|
||||
audioContext = new (window.AudioContext || window.webkitAudioContext)({
|
||||
sampleRate: SAMPLE_RATE
|
||||
});
|
||||
|
||||
// Check for audio URL parameter and auto-start
|
||||
const urlParams = new URLSearchParams(window.location.search);
|
||||
const audioUrl = urlParams.get('audioUrl');
|
||||
if (audioUrl) {
|
||||
await loadAudioFromUrl(audioUrl);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadAudioFromUrl(url) {
|
||||
try {
|
||||
status.textContent = 'Loading audio...';
|
||||
const fileName = decodeURIComponent(url.split('/').pop() || 'audio.mp3');
|
||||
audioFileName.textContent = fileName;
|
||||
|
||||
// Force fresh fetch to avoid 304 Not Modified with empty body
|
||||
const response = await fetch(url, { cache: 'no-store' });
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
|
||||
|
||||
const duration = audioBuffer.duration.toFixed(1);
|
||||
status.textContent = `Loaded: ${duration}s - Starting extraction...`;
|
||||
processBtn.disabled = false;
|
||||
|
||||
// Auto-start extraction
|
||||
setTimeout(() => startProcessing(), 500);
|
||||
} catch (e) {
|
||||
status.textContent = 'Failed to load audio: ' + e.message;
|
||||
console.error('Failed to load audio from URL:', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Drag and drop handlers
|
||||
dropZone.addEventListener('click', () => fileInput.click());
|
||||
dropZone.addEventListener('dragover', (e) => {
|
||||
e.preventDefault();
|
||||
dropZone.classList.add('border-emerald-500', 'bg-emerald-500/5');
|
||||
});
|
||||
dropZone.addEventListener('dragleave', () => {
|
||||
dropZone.classList.remove('border-emerald-500', 'bg-emerald-500/5');
|
||||
});
|
||||
dropZone.addEventListener('drop', (e) => {
|
||||
e.preventDefault();
|
||||
dropZone.classList.remove('border-emerald-500', 'bg-emerald-500/5');
|
||||
const file = e.dataTransfer.files[0];
|
||||
if (file && file.type.startsWith('audio/')) {
|
||||
handleFile(file);
|
||||
}
|
||||
});
|
||||
fileInput.addEventListener('change', (e) => {
|
||||
const file = e.target.files[0];
|
||||
if (file) handleFile(file);
|
||||
});
|
||||
|
||||
async function handleFile(file) {
|
||||
audioFileName.textContent = file.name;
|
||||
status.textContent = 'Reading audio...';
|
||||
|
||||
try {
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
|
||||
const duration = audioBuffer.duration.toFixed(1);
|
||||
status.textContent = `Loaded: ${duration}s - Ready to extract`;
|
||||
processBtn.disabled = false;
|
||||
} catch (e) {
|
||||
status.textContent = 'Failed to read audio: ' + e.message;
|
||||
console.error('Failed to decode audio:', e);
|
||||
}
|
||||
}
|
||||
|
||||
processBtn.addEventListener('click', startProcessing);
|
||||
|
||||
async function startProcessing() {
|
||||
if (!audioBuffer || !processor || isProcessing) return;
|
||||
|
||||
isProcessing = true;
|
||||
processBtn.disabled = true;
|
||||
processBtn.textContent = 'Processing...';
|
||||
results.classList.remove('visible');
|
||||
processStartTime = Date.now();
|
||||
statusDetail.innerHTML = '';
|
||||
statusDetail.classList.add('visible');
|
||||
statsRow.classList.add('visible');
|
||||
|
||||
try {
|
||||
log('Init', 'Starting stem extraction...');
|
||||
status.textContent = 'Preparing audio...';
|
||||
progressFill.style.width = '2%';
|
||||
|
||||
let leftChannel = audioBuffer.getChannelData(0);
|
||||
let rightChannel = audioBuffer.numberOfChannels > 1
|
||||
? audioBuffer.getChannelData(1)
|
||||
: leftChannel;
|
||||
|
||||
if (audioBuffer.sampleRate !== SAMPLE_RATE) {
|
||||
log('Resample', `${audioBuffer.sampleRate}Hz → ${SAMPLE_RATE}Hz`);
|
||||
const ratio = SAMPLE_RATE / audioBuffer.sampleRate;
|
||||
const newLength = Math.floor(leftChannel.length * ratio);
|
||||
const newLeft = new Float32Array(newLength);
|
||||
const newRight = new Float32Array(newLength);
|
||||
|
||||
for (let i = 0; i < newLength; i++) {
|
||||
const srcIdx = i / ratio;
|
||||
const idx0 = Math.floor(srcIdx);
|
||||
const idx1 = Math.min(idx0 + 1, leftChannel.length - 1);
|
||||
const frac = srcIdx - idx0;
|
||||
newLeft[i] = leftChannel[idx0] * (1 - frac) + leftChannel[idx1] * frac;
|
||||
newRight[i] = rightChannel[idx0] * (1 - frac) + rightChannel[idx1] * frac;
|
||||
}
|
||||
|
||||
leftChannel = newLeft;
|
||||
rightChannel = newRight;
|
||||
}
|
||||
|
||||
status.textContent = 'Extracting stems...';
|
||||
const separatedTracks = await processor.separate(leftChannel, rightChannel);
|
||||
displayResults(separatedTracks);
|
||||
|
||||
const totalTime = ((Date.now() - processStartTime) / 1000).toFixed(1);
|
||||
const speedRatio = (audioBuffer.duration / parseFloat(totalTime)).toFixed(2);
|
||||
|
||||
log('Done', `Completed in ${totalTime}s (${speedRatio}x realtime)`);
|
||||
status.textContent = `Complete! Extracted 4 stems in ${totalTime}s`;
|
||||
progressFill.style.width = '100%';
|
||||
|
||||
} catch (e) {
|
||||
status.textContent = 'Processing failed: ' + e.message;
|
||||
console.error('Processing failed:', e);
|
||||
}
|
||||
|
||||
isProcessing = false;
|
||||
processBtn.disabled = false;
|
||||
processBtn.textContent = 'Extract Stems';
|
||||
}
|
||||
|
||||
// Store track URLs for download all feature
|
||||
let trackUrls = {};
|
||||
|
||||
function displayResults(tracks) {
|
||||
trackList.innerHTML = '';
|
||||
trackUrls = {};
|
||||
|
||||
const TRACK_CONFIG = {
|
||||
drums: { icon: '🥁', label: 'Drums' },
|
||||
bass: { icon: '🎸', label: 'Bass' },
|
||||
other: { icon: '🎹', label: 'Instrumental' },
|
||||
vocals: { icon: '🎤', label: 'Vocals' }
|
||||
};
|
||||
|
||||
for (const [name, track] of Object.entries(tracks)) {
|
||||
const config = TRACK_CONFIG[name] || { icon: '🎵', label: name };
|
||||
const trackBuffer = audioContext.createBuffer(2, track.left.length, SAMPLE_RATE);
|
||||
trackBuffer.getChannelData(0).set(track.left);
|
||||
trackBuffer.getChannelData(1).set(track.right);
|
||||
|
||||
const audioBlob = audioBufferToWav(trackBuffer);
|
||||
const audioUrl = URL.createObjectURL(audioBlob);
|
||||
const trackId = `track-${name}`;
|
||||
const fileName = config.label.toLowerCase();
|
||||
|
||||
// Store for download all
|
||||
trackUrls[fileName] = audioUrl;
|
||||
|
||||
const trackDiv = document.createElement('div');
|
||||
trackDiv.className = 'track';
|
||||
trackDiv.innerHTML = `
|
||||
<div class="track-row">
|
||||
<div class="track-info">
|
||||
<div class="track-icon ${name}">${config.icon}</div>
|
||||
<div>
|
||||
<div class="track-name">${config.label}</div>
|
||||
<div class="track-duration">${formatTime(trackBuffer.duration)}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="track-player">
|
||||
<button id="play-${trackId}" class="play-btn" onclick="togglePlay('${trackId}')">
|
||||
<svg fill="currentColor" viewBox="0 0 24 24"><path d="M8 5v14l11-7z"/></svg>
|
||||
</button>
|
||||
|
||||
<div id="progress-bg-${trackId}" class="track-progress" onclick="seekTrack(event, '${trackId}')">
|
||||
<div id="progress-${trackId}" class="track-progress-fill ${name}"></div>
|
||||
</div>
|
||||
|
||||
<span id="time-${trackId}" class="track-time">0:00 / ${formatTime(trackBuffer.duration)}</span>
|
||||
</div>
|
||||
|
||||
<a href="${audioUrl}" download="${fileName}.wav" class="download-btn">
|
||||
<svg fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-4l-4 4m0 0l-4-4m4 4V4"/>
|
||||
</svg>
|
||||
WAV
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<audio id="audio-${trackId}" src="${audioUrl}" preload="metadata"></audio>
|
||||
`;
|
||||
|
||||
trackList.appendChild(trackDiv);
|
||||
|
||||
const audio = document.getElementById(`audio-${trackId}`);
|
||||
audio.addEventListener('timeupdate', () => updateProgress(trackId, audio));
|
||||
audio.addEventListener('ended', () => resetPlayer(trackId));
|
||||
}
|
||||
|
||||
results.classList.add('visible');
|
||||
}
|
||||
|
||||
// Download all stems
|
||||
window.downloadAllStems = function() {
|
||||
const entries = Object.entries(trackUrls);
|
||||
let index = 0;
|
||||
|
||||
function downloadNext() {
|
||||
if (index >= entries.length) return;
|
||||
const [name, url] = entries[index];
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = `${name}.wav`;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
index++;
|
||||
setTimeout(downloadNext, 500);
|
||||
}
|
||||
|
||||
downloadNext();
|
||||
};
|
||||
|
||||
// Player functions (global scope for onclick handlers)
|
||||
window.togglePlay = function(trackId) {
|
||||
const audio = document.getElementById(`audio-${trackId}`);
|
||||
const playBtn = document.getElementById(`play-${trackId}`);
|
||||
|
||||
// Pause all other tracks
|
||||
document.querySelectorAll('audio').forEach(a => {
|
||||
if (a.id !== `audio-${trackId}` && !a.paused) {
|
||||
a.pause();
|
||||
const otherId = a.id.replace('audio-', '');
|
||||
resetPlayer(otherId);
|
||||
}
|
||||
});
|
||||
|
||||
if (audio.paused) {
|
||||
audio.play();
|
||||
playBtn.innerHTML = `<svg class="w-4 h-4" fill="currentColor" viewBox="0 0 24 24"><path d="M6 4h4v16H6zm8 0h4v16h-4z"/></svg>`;
|
||||
} else {
|
||||
audio.pause();
|
||||
playBtn.innerHTML = `<svg class="w-4 h-4 ml-0.5" fill="currentColor" viewBox="0 0 24 24"><path d="M8 5v14l11-7z"/></svg>`;
|
||||
}
|
||||
};
|
||||
|
||||
window.seekTrack = function(event, trackId) {
|
||||
const audio = document.getElementById(`audio-${trackId}`);
|
||||
const progressBg = document.getElementById(`progress-bg-${trackId}`);
|
||||
const rect = progressBg.getBoundingClientRect();
|
||||
const percent = (event.clientX - rect.left) / rect.width;
|
||||
audio.currentTime = percent * audio.duration;
|
||||
};
|
||||
|
||||
function updateProgress(trackId, audio) {
|
||||
const progress = document.getElementById(`progress-${trackId}`);
|
||||
const timeDisplay = document.getElementById(`time-${trackId}`);
|
||||
const percent = (audio.currentTime / audio.duration) * 100;
|
||||
progress.style.width = `${percent}%`;
|
||||
timeDisplay.textContent = `${formatTime(audio.currentTime)} / ${formatTime(audio.duration)}`;
|
||||
}
|
||||
|
||||
function resetPlayer(trackId) {
|
||||
const playBtn = document.getElementById(`play-${trackId}`);
|
||||
const progress = document.getElementById(`progress-${trackId}`);
|
||||
playBtn.innerHTML = `<svg class="w-4 h-4 ml-0.5" fill="currentColor" viewBox="0 0 24 24"><path d="M8 5v14l11-7z"/></svg>`;
|
||||
progress.style.width = '0%';
|
||||
}
|
||||
|
||||
function audioBufferToWav(buffer) {
|
||||
const numChannels = buffer.numberOfChannels;
|
||||
const sampleRate = buffer.sampleRate;
|
||||
const bitDepth = 16;
|
||||
const bytesPerSample = bitDepth / 8;
|
||||
const blockAlign = numChannels * bytesPerSample;
|
||||
const samples = buffer.length;
|
||||
const dataSize = samples * blockAlign;
|
||||
const bufferSize = 44 + dataSize;
|
||||
|
||||
const arrayBuffer = new ArrayBuffer(bufferSize);
|
||||
const view = new DataView(arrayBuffer);
|
||||
|
||||
const writeString = (offset, string) => {
|
||||
for (let i = 0; i < string.length; i++) {
|
||||
view.setUint8(offset + i, string.charCodeAt(i));
|
||||
}
|
||||
};
|
||||
|
||||
writeString(0, 'RIFF');
|
||||
view.setUint32(4, bufferSize - 8, true);
|
||||
writeString(8, 'WAVE');
|
||||
writeString(12, 'fmt ');
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, numChannels, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, sampleRate * blockAlign, true);
|
||||
view.setUint16(32, blockAlign, true);
|
||||
view.setUint16(34, bitDepth, true);
|
||||
writeString(36, 'data');
|
||||
view.setUint32(40, dataSize, true);
|
||||
|
||||
const channels = [];
|
||||
for (let c = 0; c < numChannels; c++) {
|
||||
channels.push(buffer.getChannelData(c));
|
||||
}
|
||||
|
||||
let offset = 44;
|
||||
for (let i = 0; i < samples; i++) {
|
||||
for (let c = 0; c < numChannels; c++) {
|
||||
const sample = Math.max(-1, Math.min(1, channels[c][i]));
|
||||
const intSample = sample < 0 ? sample * 0x8000 : sample * 0x7FFF;
|
||||
view.setInt16(offset, intSample, true);
|
||||
offset += 2;
|
||||
}
|
||||
}
|
||||
|
||||
return new Blob([arrayBuffer], { type: 'audio/wav' });
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -0,0 +1,389 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Extract Stems - ACE-Step UI</title>
|
||||
<style>
|
||||
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap');
|
||||
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
|
||||
:root {
|
||||
--suno-bg: #09090b;
|
||||
--suno-panel: #121214;
|
||||
--suno-card: #18181b;
|
||||
--suno-hover: #27272a;
|
||||
--suno-border: #27272a;
|
||||
--zinc-400: #a1a1aa;
|
||||
--zinc-500: #71717a;
|
||||
--zinc-600: #52525b;
|
||||
--zinc-700: #3f3f46;
|
||||
--zinc-800: #27272a;
|
||||
--emerald-400: #34d399;
|
||||
--emerald-500: #10b981;
|
||||
--emerald-600: #059669;
|
||||
--teal-400: #2dd4bf;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif;
|
||||
background: var(--suno-bg);
|
||||
color: #fff;
|
||||
min-height: 100vh;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar { width: 6px; height: 6px; }
|
||||
::-webkit-scrollbar-track { background: transparent; }
|
||||
::-webkit-scrollbar-thumb { background: var(--zinc-700); border-radius: 3px; }
|
||||
::-webkit-scrollbar-thumb:hover { background: var(--zinc-600); }
|
||||
|
||||
.container { max-width: 900px; margin: 0 auto; padding: 2rem 1rem; }
|
||||
|
||||
.header { text-align: center; margin-bottom: 2rem; }
|
||||
.header h1 {
|
||||
font-size: 2rem;
|
||||
font-weight: 700;
|
||||
margin-bottom: 0.5rem;
|
||||
background: linear-gradient(to right, var(--emerald-500), var(--teal-400));
|
||||
-webkit-background-clip: text;
|
||||
-webkit-text-fill-color: transparent;
|
||||
background-clip: text;
|
||||
}
|
||||
.header p { color: var(--zinc-500); font-size: 0.875rem; }
|
||||
|
||||
.badge {
|
||||
display: inline-block;
|
||||
padding: 0.25rem 0.75rem;
|
||||
border-radius: 9999px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
margin-top: 0.75rem;
|
||||
}
|
||||
.badge-detecting { background: var(--zinc-800); color: var(--zinc-400); }
|
||||
.badge-gpu { background: rgba(16, 185, 129, 0.2); color: var(--emerald-400); }
|
||||
.badge-cpu { background: rgba(245, 158, 11, 0.2); color: #fbbf24; }
|
||||
|
||||
.card {
|
||||
background: var(--suno-card);
|
||||
border: 1px solid var(--suno-border);
|
||||
border-radius: 0.75rem;
|
||||
padding: 1.5rem;
|
||||
margin-bottom: 1.5rem;
|
||||
}
|
||||
|
||||
.drop-zone {
|
||||
border: 2px dashed var(--zinc-700);
|
||||
border-radius: 0.75rem;
|
||||
padding: 3rem 2rem;
|
||||
text-align: center;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
.drop-zone:hover, .drop-zone.dragover {
|
||||
border-color: var(--emerald-500);
|
||||
background: rgba(16, 185, 129, 0.05);
|
||||
}
|
||||
.drop-zone-icon { font-size: 3rem; margin-bottom: 0.75rem; }
|
||||
.drop-zone p { color: var(--zinc-400); }
|
||||
.drop-zone .hint { color: var(--zinc-500); font-size: 0.875rem; margin-top: 0.25rem; }
|
||||
.file-name { margin-top: 1rem; color: var(--emerald-400); font-weight: 500; font-size: 0.875rem; }
|
||||
|
||||
input[type="file"] { display: none; }
|
||||
|
||||
.processing-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
.processing-header h3 { font-weight: 600; }
|
||||
|
||||
.btn {
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 0.5rem;
|
||||
font-size: 0.875rem;
|
||||
font-weight: 600;
|
||||
border: none;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
.btn-primary {
|
||||
background: var(--emerald-600);
|
||||
color: white;
|
||||
}
|
||||
.btn-primary:hover:not(:disabled) { background: var(--emerald-500); }
|
||||
.btn-primary:disabled {
|
||||
background: var(--zinc-700);
|
||||
color: var(--zinc-500);
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.progress-bar {
|
||||
height: 0.5rem;
|
||||
background: var(--zinc-800);
|
||||
border-radius: 9999px;
|
||||
overflow: hidden;
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
.progress-fill {
|
||||
height: 100%;
|
||||
background: linear-gradient(to right, var(--emerald-500), var(--teal-400));
|
||||
transition: width 0.3s;
|
||||
width: 0%;
|
||||
}
|
||||
|
||||
.status { color: var(--zinc-400); font-size: 0.875rem; }
|
||||
|
||||
.stats-row {
|
||||
display: none;
|
||||
grid-template-columns: repeat(4, 1fr);
|
||||
gap: 1rem;
|
||||
margin-top: 1rem;
|
||||
padding-top: 1rem;
|
||||
border-top: 1px solid var(--zinc-800);
|
||||
}
|
||||
.stats-row.visible { display: grid; }
|
||||
.stat-item { text-align: center; }
|
||||
.stat-value { font-size: 1.125rem; font-weight: 600; color: var(--emerald-400); }
|
||||
.stat-label { font-size: 0.75rem; color: var(--zinc-500); }
|
||||
|
||||
.log-detail {
|
||||
display: none;
|
||||
margin-top: 1rem;
|
||||
padding: 0.75rem;
|
||||
background: rgba(0, 0, 0, 0.3);
|
||||
border-radius: 0.5rem;
|
||||
max-height: 150px;
|
||||
overflow-y: auto;
|
||||
font-family: monospace;
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
.log-detail.visible { display: block; }
|
||||
.log-line { padding: 0.25rem 0; border-bottom: 1px solid rgba(255,255,255,0.05); }
|
||||
.log-line:last-child { border-bottom: none; }
|
||||
.log-time { color: var(--emerald-400); margin-right: 0.5rem; }
|
||||
.log-phase { color: var(--teal-400); margin-right: 0.5rem; }
|
||||
|
||||
.results { display: none; }
|
||||
.results.visible { display: block; }
|
||||
.results-card {
|
||||
background: var(--suno-card);
|
||||
border: 1px solid var(--suno-border);
|
||||
border-radius: 0.75rem;
|
||||
overflow: hidden;
|
||||
}
|
||||
.results-header {
|
||||
padding: 1rem 1.5rem;
|
||||
border-bottom: 1px solid var(--zinc-800);
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
.results-header-text h3 { font-weight: 600; margin-bottom: 0.25rem; }
|
||||
.results-header-text p { color: var(--zinc-500); font-size: 0.75rem; }
|
||||
|
||||
.download-all-btn {
|
||||
padding: 0.5rem 1rem;
|
||||
background: var(--emerald-600);
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 0.5rem;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
.download-all-btn:hover { background: var(--emerald-500); }
|
||||
.download-all-btn svg { width: 16px; height: 16px; }
|
||||
|
||||
.track {
|
||||
padding: 1rem 1.5rem;
|
||||
border-bottom: 1px solid var(--zinc-800);
|
||||
transition: background 0.2s;
|
||||
}
|
||||
.track:last-child { border-bottom: none; }
|
||||
.track:hover { background: rgba(255,255,255,0.03); }
|
||||
|
||||
.track-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.track-info {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
width: 140px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.track-icon {
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
border-radius: 0.5rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 1.25rem;
|
||||
}
|
||||
.track-icon.drums { background: rgba(249, 115, 22, 0.1); border: 1px solid rgba(249, 115, 22, 0.2); }
|
||||
.track-icon.bass { background: rgba(168, 85, 247, 0.1); border: 1px solid rgba(168, 85, 247, 0.2); }
|
||||
.track-icon.other { background: rgba(59, 130, 246, 0.1); border: 1px solid rgba(59, 130, 246, 0.2); }
|
||||
.track-icon.vocals { background: rgba(236, 72, 153, 0.1); border: 1px solid rgba(236, 72, 153, 0.2); }
|
||||
|
||||
.track-name { font-weight: 600; }
|
||||
.track-duration { font-size: 0.75rem; color: var(--zinc-500); }
|
||||
|
||||
.track-player {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.play-btn {
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border-radius: 50%;
|
||||
background: white;
|
||||
color: black;
|
||||
border: none;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
transition: transform 0.2s;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.play-btn:hover { transform: scale(1.05); }
|
||||
.play-btn svg { width: 14px; height: 14px; }
|
||||
|
||||
.track-progress {
|
||||
flex: 1;
|
||||
height: 6px;
|
||||
background: var(--zinc-700);
|
||||
border-radius: 3px;
|
||||
cursor: pointer;
|
||||
position: relative;
|
||||
}
|
||||
.track-progress-fill {
|
||||
height: 100%;
|
||||
border-radius: 3px;
|
||||
transition: width 0.1s;
|
||||
width: 0%;
|
||||
}
|
||||
.track-progress-fill.drums { background: linear-gradient(to right, #f97316, #f59e0b); }
|
||||
.track-progress-fill.bass { background: linear-gradient(to right, #a855f7, #8b5cf6); }
|
||||
.track-progress-fill.other { background: linear-gradient(to right, #3b82f6, #06b6d4); }
|
||||
.track-progress-fill.vocals { background: linear-gradient(to right, #ec4899, #f43f5e); }
|
||||
|
||||
.track-time {
|
||||
font-size: 0.75rem;
|
||||
color: var(--zinc-500);
|
||||
font-family: monospace;
|
||||
width: 90px;
|
||||
text-align: right;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.download-btn {
|
||||
padding: 0.375rem 0.75rem;
|
||||
background: var(--zinc-800);
|
||||
color: var(--zinc-400);
|
||||
border: none;
|
||||
border-radius: 0.375rem;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
text-decoration: none;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.375rem;
|
||||
transition: all 0.2s;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.download-btn:hover { background: var(--zinc-700); color: white; }
|
||||
.download-btn svg { width: 14px; height: 14px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<!-- Header -->
|
||||
<div class="header">
|
||||
<h1>Stem Extraction</h1>
|
||||
<p>AI-powered audio separation using Demucs</p>
|
||||
<span id="backendBadge" class="badge badge-detecting">Detecting...</span>
|
||||
</div>
|
||||
|
||||
<!-- Upload Card -->
|
||||
<div class="card">
|
||||
<div id="dropZone" class="drop-zone">
|
||||
<div class="drop-zone-icon">🎵</div>
|
||||
<p>Drop audio file here</p>
|
||||
<p class="hint">or click to select</p>
|
||||
<input type="file" id="fileInput" accept="audio/*">
|
||||
<p id="audioFileName" class="file-name"></p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Processing Card -->
|
||||
<div class="card">
|
||||
<div class="processing-header">
|
||||
<h3>Processing</h3>
|
||||
<button id="processBtn" class="btn btn-primary" disabled>Extract Stems</button>
|
||||
</div>
|
||||
<div class="progress-bar">
|
||||
<div id="progressFill" class="progress-fill"></div>
|
||||
</div>
|
||||
<p id="status" class="status">Select an audio file to begin</p>
|
||||
|
||||
<div id="statsRow" class="stats-row">
|
||||
<div class="stat-item">
|
||||
<div id="statElapsed" class="stat-value">0:00</div>
|
||||
<div class="stat-label">Elapsed</div>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<div id="statSegment" class="stat-value">0/0</div>
|
||||
<div class="stat-label">Segment</div>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<div id="statSpeed" class="stat-value">-</div>
|
||||
<div class="stat-label">Speed</div>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<div id="statETA" class="stat-value">--:--</div>
|
||||
<div class="stat-label">ETA</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="statusDetail" class="log-detail"></div>
|
||||
</div>
|
||||
|
||||
<!-- Results -->
|
||||
<div id="results" class="results">
|
||||
<div class="results-card">
|
||||
<div class="results-header">
|
||||
<div class="results-header-text">
|
||||
<h3>Separated Tracks</h3>
|
||||
<p>Click to play, download individual stems</p>
|
||||
</div>
|
||||
<button class="download-all-btn" onclick="downloadAllStems()">
|
||||
<svg fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-4l-4 4m0 0l-4-4m4 4V4"/>
|
||||
</svg>
|
||||
Download All
|
||||
</button>
|
||||
</div>
|
||||
<div id="trackList"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script type="module" src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,16 @@
|
||||
/**
|
||||
* Constants for Demucs model
|
||||
*/
|
||||
export const CONSTANTS = {
|
||||
SAMPLE_RATE: 44100,
|
||||
FFT_SIZE: 4096,
|
||||
HOP_SIZE: 1024,
|
||||
TRAINING_SAMPLES: 343980,
|
||||
MODEL_SPEC_BINS: 2048,
|
||||
MODEL_SPEC_FRAMES: 336,
|
||||
SEGMENT_OVERLAP: 0.25,
|
||||
TRACKS: ['drums', 'bass', 'other', 'vocals'],
|
||||
|
||||
// Default model URL (Hugging Face Hub)
|
||||
DEFAULT_MODEL_URL: 'https://huggingface.co/timcsy/demucs-web-onnx/resolve/main/htdemucs_embedded.onnx'
|
||||
};
|
||||
@@ -0,0 +1,216 @@
|
||||
/**
|
||||
* Fast FFT/iFFT implementation using Cooley-Tukey radix-2 algorithm
|
||||
*/
|
||||
|
||||
const fftTwiddles = new Map();
|
||||
const ifftTwiddles = new Map();
|
||||
const hannWindows = new Map();
|
||||
|
||||
function getFFTTwiddles(n) {
|
||||
if (fftTwiddles.has(n)) return fftTwiddles.get(n);
|
||||
const real = new Float32Array(n / 2);
|
||||
const imag = new Float32Array(n / 2);
|
||||
for (let k = 0; k < n / 2; k++) {
|
||||
const angle = -2 * Math.PI * k / n;
|
||||
real[k] = Math.cos(angle);
|
||||
imag[k] = Math.sin(angle);
|
||||
}
|
||||
const twiddles = { real, imag };
|
||||
fftTwiddles.set(n, twiddles);
|
||||
return twiddles;
|
||||
}
|
||||
|
||||
function getIFFTTwiddles(n) {
|
||||
if (ifftTwiddles.has(n)) return ifftTwiddles.get(n);
|
||||
const real = new Float32Array(n / 2);
|
||||
const imag = new Float32Array(n / 2);
|
||||
for (let k = 0; k < n / 2; k++) {
|
||||
const angle = 2 * Math.PI * k / n;
|
||||
real[k] = Math.cos(angle);
|
||||
imag[k] = Math.sin(angle);
|
||||
}
|
||||
const twiddles = { real, imag };
|
||||
ifftTwiddles.set(n, twiddles);
|
||||
return twiddles;
|
||||
}
|
||||
|
||||
export function getHannWindow(size) {
|
||||
if (hannWindows.has(size)) return hannWindows.get(size);
|
||||
const window = new Float32Array(size);
|
||||
for (let i = 0; i < size; i++) {
|
||||
window[i] = 0.5 * (1 - Math.cos(2 * Math.PI * i / size));
|
||||
}
|
||||
hannWindows.set(size, window);
|
||||
return window;
|
||||
}
|
||||
|
||||
function bitReverse(n, bits) {
|
||||
let result = 0;
|
||||
for (let i = 0; i < bits; i++) {
|
||||
result = (result << 1) | (n & 1);
|
||||
n >>= 1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function fft(realOut, imagOut, realIn, n) {
|
||||
const bits = Math.log2(n) | 0;
|
||||
const twiddles = getFFTTwiddles(n);
|
||||
|
||||
for (let i = 0; i < n; i++) {
|
||||
const j = bitReverse(i, bits);
|
||||
realOut[i] = realIn[j];
|
||||
imagOut[i] = 0;
|
||||
}
|
||||
|
||||
for (let size = 2; size <= n; size *= 2) {
|
||||
const halfSize = size / 2;
|
||||
const step = n / size;
|
||||
for (let i = 0; i < n; i += size) {
|
||||
for (let j = 0; j < halfSize; j++) {
|
||||
const k = j * step;
|
||||
const tReal = twiddles.real[k];
|
||||
const tImag = twiddles.imag[k];
|
||||
const idx1 = i + j;
|
||||
const idx2 = i + j + halfSize;
|
||||
const eReal = realOut[idx1];
|
||||
const eImag = imagOut[idx1];
|
||||
const oReal = realOut[idx2] * tReal - imagOut[idx2] * tImag;
|
||||
const oImag = realOut[idx2] * tImag + imagOut[idx2] * tReal;
|
||||
realOut[idx1] = eReal + oReal;
|
||||
imagOut[idx1] = eImag + oImag;
|
||||
realOut[idx2] = eReal - oReal;
|
||||
imagOut[idx2] = eImag - oImag;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function ifft(realOut, imagOut, realIn, imagIn, n) {
|
||||
const bits = Math.log2(n) | 0;
|
||||
const twiddles = getIFFTTwiddles(n);
|
||||
|
||||
for (let i = 0; i < n; i++) {
|
||||
const j = bitReverse(i, bits);
|
||||
realOut[i] = realIn[j];
|
||||
imagOut[i] = imagIn[j];
|
||||
}
|
||||
|
||||
for (let size = 2; size <= n; size *= 2) {
|
||||
const halfSize = size / 2;
|
||||
const step = n / size;
|
||||
for (let i = 0; i < n; i += size) {
|
||||
for (let j = 0; j < halfSize; j++) {
|
||||
const k = j * step;
|
||||
const tReal = twiddles.real[k];
|
||||
const tImag = twiddles.imag[k];
|
||||
const idx1 = i + j;
|
||||
const idx2 = i + j + halfSize;
|
||||
const eReal = realOut[idx1];
|
||||
const eImag = imagOut[idx1];
|
||||
const oReal = realOut[idx2] * tReal - imagOut[idx2] * tImag;
|
||||
const oImag = realOut[idx2] * tImag + imagOut[idx2] * tReal;
|
||||
realOut[idx1] = eReal + oReal;
|
||||
imagOut[idx1] = eImag + oImag;
|
||||
realOut[idx2] = eReal - oReal;
|
||||
imagOut[idx2] = eImag - oImag;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < n; i++) {
|
||||
realOut[i] /= n;
|
||||
imagOut[i] /= n;
|
||||
}
|
||||
}
|
||||
|
||||
export function stft(signal, fftSize, hopSize) {
|
||||
const numFrames = Math.floor((signal.length - fftSize) / hopSize) + 1;
|
||||
const numBins = fftSize / 2 + 1;
|
||||
const window = getHannWindow(fftSize);
|
||||
const scale = 1.0 / Math.sqrt(fftSize);
|
||||
|
||||
const specReal = new Float32Array(numFrames * numBins);
|
||||
const specImag = new Float32Array(numFrames * numBins);
|
||||
const frameReal = new Float32Array(fftSize);
|
||||
const frameImag = new Float32Array(fftSize);
|
||||
const windowedFrame = new Float32Array(fftSize);
|
||||
|
||||
for (let frame = 0; frame < numFrames; frame++) {
|
||||
const start = frame * hopSize;
|
||||
for (let i = 0; i < fftSize; i++) {
|
||||
windowedFrame[i] = signal[start + i] * window[i];
|
||||
}
|
||||
fft(frameReal, frameImag, windowedFrame, fftSize);
|
||||
const outOffset = frame * numBins;
|
||||
for (let k = 0; k < numBins; k++) {
|
||||
specReal[outOffset + k] = frameReal[k] * scale;
|
||||
specImag[outOffset + k] = frameImag[k] * scale;
|
||||
}
|
||||
}
|
||||
|
||||
return { real: specReal, imag: specImag, numFrames, numBins };
|
||||
}
|
||||
|
||||
export function istft(specReal, specImag, numFrames, numBins, fftSize, hopSize, length) {
|
||||
const outputLength = length || (numFrames - 1) * hopSize + fftSize;
|
||||
const output = new Float32Array(outputLength);
|
||||
const windowSum = new Float32Array(outputLength);
|
||||
const window = getHannWindow(fftSize);
|
||||
const scale = Math.sqrt(fftSize);
|
||||
|
||||
const fullReal = new Float32Array(fftSize);
|
||||
const fullImag = new Float32Array(fftSize);
|
||||
const outReal = new Float32Array(fftSize);
|
||||
const outImag = new Float32Array(fftSize);
|
||||
|
||||
for (let frame = 0; frame < numFrames; frame++) {
|
||||
fullReal.fill(0);
|
||||
fullImag.fill(0);
|
||||
|
||||
for (let k = 0; k < numBins; k++) {
|
||||
fullReal[k] = specReal[frame * numBins + k];
|
||||
fullImag[k] = specImag[frame * numBins + k];
|
||||
}
|
||||
|
||||
for (let k = 1; k < numBins - 1; k++) {
|
||||
fullReal[fftSize - k] = fullReal[k];
|
||||
fullImag[fftSize - k] = -fullImag[k];
|
||||
}
|
||||
|
||||
ifft(outReal, outImag, fullReal, fullImag, fftSize);
|
||||
|
||||
const start = frame * hopSize;
|
||||
for (let i = 0; i < fftSize && start + i < outputLength; i++) {
|
||||
output[start + i] += outReal[i] * window[i] * scale;
|
||||
windowSum[start + i] += window[i] * window[i];
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < outputLength; i++) {
|
||||
if (windowSum[i] > 1e-8) {
|
||||
output[i] /= windowSum[i];
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
export function reflectPad(signal, padLeft, padRight) {
|
||||
const length = signal.length;
|
||||
const output = new Float32Array(padLeft + length + padRight);
|
||||
|
||||
for (let i = 0; i < padLeft; i++) {
|
||||
const srcIdx = Math.min(padLeft - i, length - 1);
|
||||
output[i] = signal[srcIdx];
|
||||
}
|
||||
|
||||
output.set(signal, padLeft);
|
||||
|
||||
for (let i = 0; i < padRight; i++) {
|
||||
const srcIdx = Math.max(0, length - 2 - i);
|
||||
output[padLeft + length + i] = signal[srcIdx];
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
/**
|
||||
* Demucs Web - Music Source Separation using ONNX Runtime Web
|
||||
* @module demucs-web
|
||||
*/
|
||||
|
||||
export { CONSTANTS } from './constants.js';
|
||||
export { fft, ifft, stft, istft, reflectPad, getHannWindow } from './fft.js';
|
||||
export { DemucsProcessor, standaloneMask, standaloneIspec, prepareModelInput } from './processor.js';
|
||||
@@ -0,0 +1,352 @@
|
||||
/**
|
||||
* Demucs audio processor - Core separation logic
|
||||
*/
|
||||
|
||||
import { CONSTANTS } from './constants.js';
|
||||
import { stft, istft, reflectPad } from './fft.js';
|
||||
|
||||
const { SAMPLE_RATE, FFT_SIZE, HOP_SIZE, TRAINING_SAMPLES, MODEL_SPEC_BINS, MODEL_SPEC_FRAMES, SEGMENT_OVERLAP, TRACKS } = CONSTANTS;
|
||||
|
||||
/**
|
||||
* Convert model frequency output to complex spectrogram per track
|
||||
*/
|
||||
export function standaloneMask(freqOutput) {
|
||||
const numTracks = 4;
|
||||
const numChannels = 4;
|
||||
const numBins = MODEL_SPEC_BINS;
|
||||
const numFrames = MODEL_SPEC_FRAMES;
|
||||
const result = [];
|
||||
|
||||
for (let t = 0; t < numTracks; t++) {
|
||||
const trackSpec = {
|
||||
leftReal: new Float32Array(numBins * numFrames),
|
||||
leftImag: new Float32Array(numBins * numFrames),
|
||||
rightReal: new Float32Array(numBins * numFrames),
|
||||
rightImag: new Float32Array(numBins * numFrames)
|
||||
};
|
||||
|
||||
for (let f = 0; f < numFrames; f++) {
|
||||
for (let b = 0; b < numBins; b++) {
|
||||
const baseIdx = t * numChannels * numBins * numFrames;
|
||||
const outIdx = b * numFrames + f;
|
||||
trackSpec.leftReal[outIdx] = freqOutput[baseIdx + 0 * numBins * numFrames + b * numFrames + f];
|
||||
trackSpec.leftImag[outIdx] = freqOutput[baseIdx + 1 * numBins * numFrames + b * numFrames + f];
|
||||
trackSpec.rightReal[outIdx] = freqOutput[baseIdx + 2 * numBins * numFrames + b * numFrames + f];
|
||||
trackSpec.rightImag[outIdx] = freqOutput[baseIdx + 3 * numBins * numFrames + b * numFrames + f];
|
||||
}
|
||||
}
|
||||
result.push(trackSpec);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert complex spectrogram back to time domain (iSTFT with proper offsets)
|
||||
*/
|
||||
export function standaloneIspec(trackSpec, targetLength) {
|
||||
const numBins = MODEL_SPEC_BINS;
|
||||
const numFrames = MODEL_SPEC_FRAMES;
|
||||
const hopLength = HOP_SIZE;
|
||||
const paddedBins = numBins + 1;
|
||||
const paddedFrames = numFrames + 4;
|
||||
|
||||
const padChannel = (real, imag) => {
|
||||
const paddedReal = new Float32Array(paddedFrames * paddedBins);
|
||||
const paddedImag = new Float32Array(paddedFrames * paddedBins);
|
||||
|
||||
for (let f = 0; f < numFrames; f++) {
|
||||
for (let b = 0; b < numBins; b++) {
|
||||
const srcIdx = b * numFrames + f;
|
||||
const dstFrame = f + 2;
|
||||
const dstIdx = dstFrame * paddedBins + b;
|
||||
paddedReal[dstIdx] = real[srcIdx];
|
||||
paddedImag[dstIdx] = imag[srcIdx];
|
||||
}
|
||||
}
|
||||
return { real: paddedReal, imag: paddedImag };
|
||||
};
|
||||
|
||||
const leftPadded = padChannel(trackSpec.leftReal, trackSpec.leftImag);
|
||||
const rightPadded = padChannel(trackSpec.rightReal, trackSpec.rightImag);
|
||||
|
||||
const centerPad = FFT_SIZE / 2;
|
||||
const pad = Math.floor(hopLength / 2) * 3;
|
||||
const istftLength = (paddedFrames - 1) * hopLength + FFT_SIZE;
|
||||
|
||||
const leftOut = istft(leftPadded.real, leftPadded.imag, paddedFrames, paddedBins, FFT_SIZE, hopLength, istftLength);
|
||||
const rightOut = istft(rightPadded.real, rightPadded.imag, paddedFrames, paddedBins, FFT_SIZE, hopLength, istftLength);
|
||||
|
||||
const totalOffset = centerPad + pad;
|
||||
const left = leftOut.subarray(totalOffset, totalOffset + targetLength);
|
||||
const right = rightOut.subarray(totalOffset, totalOffset + targetLength);
|
||||
|
||||
return { left: new Float32Array(left), right: new Float32Array(right) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare model input from stereo audio
|
||||
*/
|
||||
export function prepareModelInput(leftChannel, rightChannel) {
|
||||
const inputLength = TRAINING_SAMPLES;
|
||||
|
||||
const paddedLeft = new Float32Array(inputLength);
|
||||
const paddedRight = new Float32Array(inputLength);
|
||||
const copyLen = Math.min(leftChannel.length, inputLength);
|
||||
paddedLeft.set(leftChannel.subarray(0, copyLen));
|
||||
paddedRight.set(rightChannel.subarray(0, copyLen));
|
||||
|
||||
const le = Math.ceil(inputLength / HOP_SIZE);
|
||||
const pad = Math.floor(HOP_SIZE / 2) * 3;
|
||||
const padRight = pad + le * HOP_SIZE - inputLength;
|
||||
|
||||
const stftInputLeft = reflectPad(paddedLeft, pad, padRight);
|
||||
const stftInputRight = reflectPad(paddedRight, pad, padRight);
|
||||
|
||||
const centerPad = FFT_SIZE / 2;
|
||||
const centeredLeft = reflectPad(stftInputLeft, centerPad, centerPad);
|
||||
const centeredRight = reflectPad(stftInputRight, centerPad, centerPad);
|
||||
|
||||
const stftLeft = stft(centeredLeft, FFT_SIZE, HOP_SIZE);
|
||||
const stftRight = stft(centeredRight, FFT_SIZE, HOP_SIZE);
|
||||
|
||||
const numBins = MODEL_SPEC_BINS;
|
||||
const numFrames = MODEL_SPEC_FRAMES;
|
||||
const frameOffset = 2;
|
||||
|
||||
const magSpec = new Float32Array(4 * numBins * numFrames);
|
||||
|
||||
for (let f = 0; f < numFrames; f++) {
|
||||
const srcFrame = f + frameOffset;
|
||||
for (let b = 0; b < numBins; b++) {
|
||||
const srcIdx = srcFrame * stftLeft.numBins + b;
|
||||
magSpec[0 * numBins * numFrames + b * numFrames + f] = stftLeft.real[srcIdx];
|
||||
magSpec[1 * numBins * numFrames + b * numFrames + f] = stftLeft.imag[srcIdx];
|
||||
magSpec[2 * numBins * numFrames + b * numFrames + f] = stftRight.real[srcIdx];
|
||||
magSpec[3 * numBins * numFrames + b * numFrames + f] = stftRight.imag[srcIdx];
|
||||
}
|
||||
}
|
||||
|
||||
const waveform = new Float32Array(2 * inputLength);
|
||||
waveform.set(paddedLeft, 0);
|
||||
waveform.set(paddedRight, inputLength);
|
||||
|
||||
return { waveform, magSpec, numBins, numFrames, originalLength: leftChannel.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Main Demucs processor class
|
||||
*/
|
||||
export class DemucsProcessor {
|
||||
constructor(options = {}) {
|
||||
this.ort = options.ort || null;
|
||||
this.session = null;
|
||||
this.modelPath = options.modelPath || './htdemucs_embedded.onnx';
|
||||
this.sessionOptions = options.sessionOptions || {};
|
||||
this.onProgress = options.onProgress || (() => {});
|
||||
this.onLog = options.onLog || (() => {});
|
||||
this.onDownloadProgress = options.onDownloadProgress || (() => {});
|
||||
}
|
||||
|
||||
async loadModel(modelPathOrBuffer) {
|
||||
if (!this.ort) {
|
||||
throw new Error('ONNX Runtime not provided. Pass ort in constructor options.');
|
||||
}
|
||||
|
||||
this.onLog('model', 'Loading model...');
|
||||
|
||||
let modelBuffer;
|
||||
if (modelPathOrBuffer instanceof ArrayBuffer) {
|
||||
modelBuffer = modelPathOrBuffer;
|
||||
} else {
|
||||
const response = await fetch(modelPathOrBuffer || this.modelPath);
|
||||
|
||||
// Check if we can track progress
|
||||
const contentLength = response.headers.get('Content-Length');
|
||||
if (contentLength && response.body) {
|
||||
const totalSize = parseInt(contentLength, 10);
|
||||
const reader = response.body.getReader();
|
||||
const chunks = [];
|
||||
let loadedSize = 0;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
loadedSize += value.length;
|
||||
this.onDownloadProgress(loadedSize, totalSize);
|
||||
}
|
||||
|
||||
// Combine chunks into single ArrayBuffer
|
||||
const combined = new Uint8Array(loadedSize);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
combined.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
modelBuffer = combined.buffer;
|
||||
} else {
|
||||
// Fallback: no progress tracking
|
||||
modelBuffer = await response.arrayBuffer();
|
||||
}
|
||||
}
|
||||
|
||||
const defaultSessionOptions = {
|
||||
executionProviders: ['webgpu', 'wasm'],
|
||||
graphOptimizationLevel: 'basic'
|
||||
};
|
||||
|
||||
this.session = await this.ort.InferenceSession.create(modelBuffer, {
|
||||
...defaultSessionOptions,
|
||||
...this.sessionOptions
|
||||
});
|
||||
|
||||
this.onLog('model', 'Model loaded successfully');
|
||||
return this.session;
|
||||
}
|
||||
|
||||
async separate(leftChannel, rightChannel) {
|
||||
if (!this.session) {
|
||||
throw new Error('Model not loaded. Call loadModel() first.');
|
||||
}
|
||||
|
||||
const totalSamples = leftChannel.length;
|
||||
const stride = Math.floor(TRAINING_SAMPLES * (1 - SEGMENT_OVERLAP));
|
||||
const numSegments = Math.ceil((totalSamples - TRAINING_SAMPLES) / stride) + 1;
|
||||
|
||||
const outputs = TRACKS.map(() => ({
|
||||
left: new Float32Array(totalSamples),
|
||||
right: new Float32Array(totalSamples)
|
||||
}));
|
||||
const weights = new Float32Array(totalSamples);
|
||||
|
||||
let segmentIdx = 0;
|
||||
|
||||
for (let start = 0; start < totalSamples; start += stride) {
|
||||
const end = Math.min(start + TRAINING_SAMPLES, totalSamples);
|
||||
const segmentLength = end - start;
|
||||
|
||||
const segLeft = new Float32Array(TRAINING_SAMPLES);
|
||||
const segRight = new Float32Array(TRAINING_SAMPLES);
|
||||
|
||||
for (let i = 0; i < segmentLength; i++) {
|
||||
segLeft[i] = leftChannel[start + i];
|
||||
segRight[i] = rightChannel[start + i];
|
||||
}
|
||||
|
||||
const input = prepareModelInput(segLeft, segRight);
|
||||
|
||||
const waveformTensor = new this.ort.Tensor('float32', input.waveform, [1, 2, TRAINING_SAMPLES]);
|
||||
const magSpecTensor = new this.ort.Tensor('float32', input.magSpec, [1, 4, MODEL_SPEC_BINS, MODEL_SPEC_FRAMES]);
|
||||
|
||||
const feeds = {};
|
||||
feeds[this.session.inputNames[0]] = waveformTensor;
|
||||
if (this.session.inputNames.length > 1) {
|
||||
feeds[this.session.inputNames[1]] = magSpecTensor;
|
||||
}
|
||||
|
||||
const inferResults = await this.session.run(feeds);
|
||||
|
||||
let timeData = null, timeShape = null;
|
||||
let freqData = null;
|
||||
|
||||
for (const name of this.session.outputNames) {
|
||||
const tensor = inferResults[name];
|
||||
if (tensor.dims.length === 4 && tensor.dims[2] === 2) {
|
||||
timeData = tensor.data;
|
||||
timeShape = tensor.dims;
|
||||
} else if (tensor.dims.length === 5 && tensor.dims[2] === 4) {
|
||||
freqData = tensor.data;
|
||||
}
|
||||
}
|
||||
|
||||
if (!timeData) {
|
||||
throw new Error('Could not find time-domain output tensor');
|
||||
}
|
||||
|
||||
let combinedOutputs = null;
|
||||
if (freqData) {
|
||||
const trackSpecs = standaloneMask(freqData);
|
||||
combinedOutputs = [];
|
||||
|
||||
for (let t = 0; t < 4; t++) {
|
||||
const freqOutput = standaloneIspec(trackSpecs[t], TRAINING_SAMPLES);
|
||||
const numChannels = timeShape[2];
|
||||
const samples = timeShape[3];
|
||||
const timeLeft = new Float32Array(samples);
|
||||
const timeRight = new Float32Array(samples);
|
||||
|
||||
for (let i = 0; i < samples; i++) {
|
||||
timeLeft[i] = timeData[t * numChannels * samples + 0 * samples + i];
|
||||
timeRight[i] = timeData[t * numChannels * samples + 1 * samples + i];
|
||||
}
|
||||
|
||||
const combined = {
|
||||
left: new Float32Array(samples),
|
||||
right: new Float32Array(samples)
|
||||
};
|
||||
for (let i = 0; i < samples; i++) {
|
||||
combined.left[i] = timeLeft[i] + (freqOutput.left[i] || 0);
|
||||
combined.right[i] = timeRight[i] + (freqOutput.right[i] || 0);
|
||||
}
|
||||
combinedOutputs.push(combined);
|
||||
}
|
||||
}
|
||||
|
||||
const numTracks = timeShape[1];
|
||||
const numChannels = timeShape[2];
|
||||
const samples = timeShape[3];
|
||||
|
||||
const overlapWindow = new Float32Array(segmentLength);
|
||||
for (let i = 0; i < segmentLength; i++) {
|
||||
const fadeIn = Math.min(i / (stride * 0.5), 1);
|
||||
const fadeOut = Math.min((segmentLength - i) / (stride * 0.5), 1);
|
||||
overlapWindow[i] = Math.min(fadeIn, fadeOut);
|
||||
}
|
||||
|
||||
for (let t = 0; t < numTracks; t++) {
|
||||
for (let i = 0; i < segmentLength && start + i < totalSamples; i++) {
|
||||
let leftVal, rightVal;
|
||||
if (combinedOutputs) {
|
||||
leftVal = combinedOutputs[t].left[i];
|
||||
rightVal = combinedOutputs[t].right[i];
|
||||
} else {
|
||||
const leftIdx = t * numChannels * samples + 0 * samples + i;
|
||||
const rightIdx = t * numChannels * samples + 1 * samples + i;
|
||||
leftVal = timeData[leftIdx];
|
||||
rightVal = timeData[rightIdx];
|
||||
}
|
||||
outputs[t].left[start + i] += leftVal * overlapWindow[i];
|
||||
outputs[t].right[start + i] += rightVal * overlapWindow[i];
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < segmentLength && start + i < totalSamples; i++) {
|
||||
weights[start + i] += overlapWindow[i];
|
||||
}
|
||||
|
||||
segmentIdx++;
|
||||
this.onProgress({
|
||||
progress: segmentIdx / numSegments,
|
||||
currentSegment: segmentIdx,
|
||||
totalSegments: numSegments
|
||||
});
|
||||
}
|
||||
|
||||
for (let t = 0; t < TRACKS.length; t++) {
|
||||
for (let i = 0; i < totalSamples; i++) {
|
||||
if (weights[i] > 0) {
|
||||
outputs[t].left[i] /= weights[i];
|
||||
outputs[t].right[i] /= weights[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
drums: outputs[0],
|
||||
bass: outputs[1],
|
||||
other: outputs[2],
|
||||
vocals: outputs[3]
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user