feat: added local separator model demucs_4s support
This commit is contained in:
@@ -328,4 +328,106 @@ SOFTWARE.
|
||||
|
||||
---
|
||||
|
||||
### 11. htdemucs_4s
|
||||
|
||||
This project uses the htdemucs_4s model from the Demucs project by Facebook Research,
|
||||
used for music source separation (stem extraction).
|
||||
|
||||
Original project: https://github.com/facebookresearch/demucs
|
||||
Model weights (original, PyTorch): https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/955717e8-8726e21a.th
|
||||
ONNX conversion (original): https://huggingface.co/timcsy/demucs-web-onnx/resolve/main/htdemucs_embedded.onnx
|
||||
ONNX model downloaded by this project (unmodified copy of the above): https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx
|
||||
|
||||
```
|
||||
MIT License
|
||||
|
||||
Copyright (c) Facebook, Inc. and its affiliates.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this model and associated documentation files, to deal in the model without
|
||||
restriction, including without limitation the rights to use, copy, modify,
|
||||
merge, publish, distribute, sublicense, and/or sell copies of the model, and
|
||||
to permit persons to whom the model is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the model.
|
||||
|
||||
THE MODEL IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE MODEL OR THE USE OR OTHER DEALINGS IN THE
|
||||
MODEL.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 12. onnxruntime-web
|
||||
|
||||
onnxruntime-web is a JavaScript library for running ONNX models in the browser,
|
||||
used for on-device neural network inference.
|
||||
|
||||
Original project: https://github.com/microsoft/onnxruntime
|
||||
|
||||
```
|
||||
MIT License
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 13. demucs-web
|
||||
|
||||
demucs-web is a browser-based wrapper for running Demucs stem separation models
|
||||
in the browser via ONNX Runtime Web.
|
||||
|
||||
Original project: https://github.com/timcsy/demucs-web
|
||||
|
||||
```
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Apache License Version 2.0, January 2004: http://www.apache.org/licenses/LICENSE-2.0
|
||||
@@ -391,4 +391,4 @@ Licensed under the Apache License, Version 2.0, with additional terms (see `LICE
|
||||
- No patent applications using this software or assets
|
||||
- Attribution required when used in public/commercial products (“Powered by K.G.Studio”)
|
||||
|
||||
Third‑party notices (FluidR3_GM SoundFont, midi‑js‑soundfonts, VexFlow, prompt structure notes, Gemma 4 E4B, UVR-MDX-NET-Inst_HQ_3, MediaPipe, Meyda, web-audio-beat-detector, and tonal) are included in `LICENSE`.
|
||||
Third‑party notices (FluidR3_GM SoundFont, midi‑js‑soundfonts, VexFlow, prompt structure notes, Gemma 4 E4B, UVR-MDX-NET-Inst_HQ_3, MediaPipe, Meyda, web-audio-beat-detector, tonal, htdemucs_4s, onnxruntime-web, and demucs-web) are included in `LICENSE`.
|
||||
|
||||
Generated
+13
@@ -10,6 +10,7 @@
|
||||
"dependencies": {
|
||||
"@breezystack/lamejs": "^1.2.7",
|
||||
"class-transformer": "^0.5.1",
|
||||
"demucs-web": "^1.0.2",
|
||||
"fft.js": "^4.0.4",
|
||||
"idb": "^8.0.3",
|
||||
"jszip": "^3.10.1",
|
||||
@@ -4440,6 +4441,18 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/demucs-web": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/demucs-web/-/demucs-web-1.0.2.tgz",
|
||||
"integrity": "sha512-G/HWbbBDvfGzaxDZQhqTp4+6i8nlCIRGEd6E1+EZJJONx5Es3bBc+jHqhfyblTlsKo/7cAdylA8HfetBpbQ98w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=16.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"onnxruntime-web": ">=1.17.0"
|
||||
}
|
||||
},
|
||||
"node_modules/deprecation": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
"dependencies": {
|
||||
"@breezystack/lamejs": "^1.2.7",
|
||||
"class-transformer": "^0.5.1",
|
||||
"demucs-web": "^1.0.2",
|
||||
"fft.js": "^4.0.4",
|
||||
"idb": "^8.0.3",
|
||||
"jszip": "^3.10.1",
|
||||
|
||||
+2
-1
@@ -31,7 +31,8 @@
|
||||
"model_url": "https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task"
|
||||
},
|
||||
"uvr5_web_runtime": {
|
||||
"mdx_net_model_url": "https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx"
|
||||
"mdx_net_model_url": "https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx",
|
||||
"htdemucs_4s_model_url": "https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx"
|
||||
},
|
||||
"soundfont": {
|
||||
"base_url": "https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/"
|
||||
|
||||
@@ -4,6 +4,7 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/react';
|
||||
import KGOnePanel from './KGOnePanel';
|
||||
import { KGAudioRegion } from '../core/region/KGAudioRegion';
|
||||
import { KGAudioTrack } from '../core/track/KGAudioTrack';
|
||||
import { LOCAL_SEPARATOR_MODEL_IDS } from '../util/local-separator/config';
|
||||
|
||||
const { mockLocalSeparatorDownload } = vi.hoisted(() => ({
|
||||
mockLocalSeparatorDownload: vi.fn(async (_url?: string, _filename?: string, _onProgress?: unknown) => undefined),
|
||||
@@ -11,7 +12,7 @@ const { mockLocalSeparatorDownload } = vi.hoisted(() => ({
|
||||
|
||||
let kgoneEnabled = false;
|
||||
let selectedRegionIds: string[] = [];
|
||||
let localModelCached = false;
|
||||
let localModelCached: Record<string, boolean> = {};
|
||||
let localSeparationResult: Array<{ name: string; blob: Blob }> = [];
|
||||
|
||||
const mockRefreshProjectState = vi.fn();
|
||||
@@ -36,6 +37,7 @@ vi.mock('../core/config/ConfigManager', () => ({
|
||||
if (key === 'general.kgone.enabled') return kgoneEnabled;
|
||||
if (key === 'general.kgone.base_url') return 'http://127.0.0.1:8000';
|
||||
if (key === 'general.uvr5_web_runtime.mdx_net_model_url') return 'https://example.com/custom-uvr5.onnx';
|
||||
if (key === 'general.uvr5_web_runtime.htdemucs_4s_model_url') return 'https://example.com/custom-htdemucs.onnx';
|
||||
return undefined;
|
||||
},
|
||||
}),
|
||||
@@ -83,13 +85,13 @@ vi.mock('../util/audioUtil', () => ({
|
||||
|
||||
vi.mock('../util/local-separator/modelCache', () => ({
|
||||
LocalSeparatorModelCache: {
|
||||
exists: vi.fn(async () => localModelCached),
|
||||
download: vi.fn(async (url: string, filename: string, onProgress: (progress: unknown) => void) => {
|
||||
localModelCached = true;
|
||||
return mockLocalSeparatorDownload(url, filename, onProgress);
|
||||
exists: vi.fn(async (modelConfig: { id: string }) => localModelCached[modelConfig.id] ?? false),
|
||||
download: vi.fn(async (modelConfig: { id: string; filename: string }, url: string, onProgress: (progress: unknown) => void) => {
|
||||
localModelCached[modelConfig.id] = true;
|
||||
return mockLocalSeparatorDownload(modelConfig.filename, url, onProgress);
|
||||
}),
|
||||
delete: vi.fn(async () => {
|
||||
localModelCached = false;
|
||||
delete: vi.fn(async (modelConfig: { id: string }) => {
|
||||
localModelCached[modelConfig.id] = false;
|
||||
}),
|
||||
getArrayBuffer: vi.fn(async () => new ArrayBuffer(16)),
|
||||
},
|
||||
@@ -125,7 +127,7 @@ describe('KGOnePanel local separator mode', () => {
|
||||
beforeEach(() => {
|
||||
kgoneEnabled = false;
|
||||
selectedRegionIds = [];
|
||||
localModelCached = false;
|
||||
localModelCached = {};
|
||||
localSeparationResult = [
|
||||
{ name: 'Instrumental', blob: new Blob(['instrumental'], { type: 'audio/wav' }) },
|
||||
{ name: 'Vocals', blob: new Blob(['vocals'], { type: 'audio/wav' }) },
|
||||
@@ -143,41 +145,42 @@ describe('KGOnePanel local separator mode', () => {
|
||||
expect(screen.getByRole('button', { name: 'Remix' })).toBeDisabled();
|
||||
expect(screen.getByRole('button', { name: 'Repaint' })).toBeDisabled();
|
||||
expect(screen.getByRole('button', { name: 'Separator' })).not.toBeDisabled();
|
||||
expect(screen.getByRole('button', { name: 'Download Model' })).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: 'Download Selected Model' })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows the single local separator model and advanced settings when the model is cached', async () => {
|
||||
localModelCached = true;
|
||||
localModelCached[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium] = true;
|
||||
selectedRegionIds = ['audio-region-1'];
|
||||
|
||||
render(<KGOnePanel isVisible={true} />);
|
||||
|
||||
await screen.findByText('Selected Region');
|
||||
const options = await screen.findAllByRole('option');
|
||||
expect(options).toHaveLength(1);
|
||||
expect(options).toHaveLength(2);
|
||||
expect(options[0]).toHaveTextContent('Vocal and Instrument (Medium Accuracy)');
|
||||
expect(options[1]).toHaveTextContent('Vocal, Drums, Bass, and Others');
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: /Advanced Settings/i }));
|
||||
expect(screen.getByLabelText('Optional audio chunk duration (seconds)')).toBeInTheDocument();
|
||||
expect(screen.getByLabelText('MDX overlap')).toBeInTheDocument();
|
||||
expect(screen.getByLabelText('Model overlap')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('uses the configured UVR5 model URL when downloading the local model', async () => {
|
||||
render(<KGOnePanel isVisible={true} />);
|
||||
|
||||
fireEvent.click(await screen.findByRole('button', { name: 'Download Model' }));
|
||||
fireEvent.click(await screen.findByRole('button', { name: 'Download Selected Model' }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockLocalSeparatorDownload).toHaveBeenCalledWith(
|
||||
'https://example.com/custom-uvr5.onnx',
|
||||
'UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||
'https://example.com/custom-uvr5.onnx',
|
||||
expect.any(Function),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('prompts for an audio region when the model is cached but nothing is selected', async () => {
|
||||
localModelCached = true;
|
||||
localModelCached[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium] = true;
|
||||
|
||||
render(<KGOnePanel isVisible={true} />);
|
||||
|
||||
@@ -185,7 +188,7 @@ describe('KGOnePanel local separator mode', () => {
|
||||
});
|
||||
|
||||
it('renders local separation outputs after processing completes', async () => {
|
||||
localModelCached = true;
|
||||
localModelCached[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium] = true;
|
||||
selectedRegionIds = ['audio-region-1'];
|
||||
|
||||
render(<KGOnePanel isVisible={true} />);
|
||||
@@ -198,4 +201,36 @@ describe('KGOnePanel local separator mode', () => {
|
||||
expect(screen.getByRole('button', { name: 'Import All Stems to Timeline' })).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it('uses Demucs defaults and renders four local stem players', async () => {
|
||||
localModelCached[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s] = true;
|
||||
localSeparationResult = [
|
||||
{ name: 'Vocals', blob: new Blob(['vocals'], { type: 'audio/wav' }) },
|
||||
{ name: 'Drums', blob: new Blob(['drums'], { type: 'audio/wav' }) },
|
||||
{ name: 'Bass', blob: new Blob(['bass'], { type: 'audio/wav' }) },
|
||||
{ name: 'Others', blob: new Blob(['others'], { type: 'audio/wav' }) },
|
||||
];
|
||||
selectedRegionIds = ['audio-region-1'];
|
||||
|
||||
render(<KGOnePanel isVisible={true} />);
|
||||
|
||||
await screen.findByText('Selected Region');
|
||||
fireEvent.change(screen.getByRole('combobox'), { target: { value: LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s } });
|
||||
fireEvent.click(screen.getByRole('button', { name: /Advanced Settings/i }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect((screen.getByLabelText('Optional audio chunk duration (seconds)') as HTMLInputElement).value).toBe('8');
|
||||
expect((screen.getByLabelText('Model overlap') as HTMLInputElement).value).toBe('0.25');
|
||||
});
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Separate Stems' }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('Vocals')).toBeInTheDocument();
|
||||
expect(screen.getByText('Drums')).toBeInTheDocument();
|
||||
expect(screen.getByText('Bass')).toBeInTheDocument();
|
||||
expect(screen.getByText('Others')).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: 'Import All Stems to Timeline' })).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -16,13 +16,15 @@ import { ImportStemsCommand } from '../core/commands';
|
||||
import type { StemImportEntry } from '../core/commands';
|
||||
import { showAlert } from '../util/dialogUtil';
|
||||
import {
|
||||
LOCAL_SEPARATOR_MODEL_CONFIG,
|
||||
LOCAL_SEPARATOR_MODEL_FILENAME,
|
||||
LOCAL_SEPARATOR_DEFAULT_MODEL_URL,
|
||||
getLocalSeparatorModelConfig,
|
||||
LOCAL_SEPARATOR_MODELS,
|
||||
LOCAL_SEPARATOR_MODEL_CONFIGS,
|
||||
LOCAL_SEPARATOR_MODEL_IDS,
|
||||
} from '../util/local-separator/config';
|
||||
import { LocalSeparatorModelCache } from '../util/local-separator/modelCache';
|
||||
import { runLocalSeparator } from '../util/local-separator/runner';
|
||||
import { LocalOrtRuntimeManager, detectLocalRuntimeSupport } from '../util/local-separator/runtime';
|
||||
import type { LocalSeparatorModelConfig, LocalSeparatorModelId } from '../util/local-separator/types';
|
||||
|
||||
// ─── Types ────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -36,9 +38,10 @@ const SERVER_SEPARATOR_MODELS = [
|
||||
{ label: 'Vocal and Instrument (High Accuracy)', value: 'MDX23C-8KFFT-InstVoc_HQ.ckpt' },
|
||||
{ label: 'Vocal, Drums, Bass, Guitar, Piano, and Others', value: 'htdemucs_6s.yaml' },
|
||||
] as const;
|
||||
const LOCAL_SEPARATOR_MODELS = [
|
||||
{ label: LOCAL_SEPARATOR_MODEL_CONFIG.displayName, value: LOCAL_SEPARATOR_MODEL_FILENAME },
|
||||
] as const;
|
||||
const LOCAL_SEPARATOR_MODEL_OPTIONS = LOCAL_SEPARATOR_MODELS.map(modelConfig => ({
|
||||
label: modelConfig.displayName,
|
||||
value: modelConfig.id,
|
||||
})) as ReadonlyArray<{ label: string; value: LocalSeparatorModelId }>;
|
||||
const KGONE_TABS = ['fullsong', 'remix', 'repaint', 'separator'] as const;
|
||||
|
||||
const CLIP_NOTES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'];
|
||||
@@ -862,8 +865,11 @@ function countRepaintTracks(sourceTrackName: string): number {
|
||||
const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
const { selectedRegionIds, projectName, bpm, timeSignature, maxBars, refreshProjectState } = useProjectStore();
|
||||
const localOnlyMode = mode === 'local-separator';
|
||||
const availableSeparatorModels = localOnlyMode ? LOCAL_SEPARATOR_MODELS : SERVER_SEPARATOR_MODELS;
|
||||
const [model, setModel] = useState<typeof SERVER_SEPARATOR_MODELS[number]['value']>(availableSeparatorModels[0].value);
|
||||
const availableSeparatorModels = localOnlyMode ? LOCAL_SEPARATOR_MODEL_OPTIONS : SERVER_SEPARATOR_MODELS;
|
||||
const [model, setModel] = useState<string>(availableSeparatorModels[0].value);
|
||||
const currentLocalModelConfig = useMemo<LocalSeparatorModelConfig>(() => {
|
||||
return getLocalSeparatorModelConfig(model);
|
||||
}, [model]);
|
||||
|
||||
// Generation state
|
||||
const [genStatus, setGenStatus] = useState<GenStatus>('idle');
|
||||
@@ -881,7 +887,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
const [localProgressPercent, setLocalProgressPercent] = useState(0);
|
||||
const [localProgressText, setLocalProgressText] = useState('');
|
||||
const [localChunkDurationSeconds, setLocalChunkDurationSeconds] = useState('');
|
||||
const [localOverlap, setLocalOverlap] = useState(String(LOCAL_SEPARATOR_MODEL_CONFIG.defaults.overlap));
|
||||
const [localOverlap, setLocalOverlap] = useState(String(currentLocalModelConfig.defaults.overlap));
|
||||
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
const taskIdRef = useRef<string>('');
|
||||
@@ -922,18 +928,28 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
setModel(availableSeparatorModels[0].value);
|
||||
}, [availableSeparatorModels]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!localOnlyMode) return;
|
||||
setLocalChunkDurationSeconds(
|
||||
currentLocalModelConfig.defaultChunkDurationSeconds == null
|
||||
? ''
|
||||
: String(currentLocalModelConfig.defaultChunkDurationSeconds),
|
||||
);
|
||||
setLocalOverlap(String(currentLocalModelConfig.defaults.overlap));
|
||||
}, [currentLocalModelConfig, localOnlyMode]);
|
||||
|
||||
const refreshLocalModelCacheState = useCallback(async () => {
|
||||
if (!localOnlyMode) return;
|
||||
setIsCheckingLocalModel(true);
|
||||
try {
|
||||
setIsLocalModelCached(await LocalSeparatorModelCache.exists());
|
||||
setIsLocalModelCached(await LocalSeparatorModelCache.exists(currentLocalModelConfig));
|
||||
} catch (err) {
|
||||
console.error('[KGOne] Local model cache check failed:', err);
|
||||
setErrorMsg(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setIsCheckingLocalModel(false);
|
||||
}
|
||||
}, [localOnlyMode]);
|
||||
}, [currentLocalModelConfig, localOnlyMode]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!localOnlyMode) return;
|
||||
@@ -960,11 +976,11 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
}, [selectedRegionIds]);
|
||||
|
||||
const getConfiguredLocalSeparatorModelUrl = useCallback(() => {
|
||||
const configured = ConfigManager.instance().get('general.uvr5_web_runtime.mdx_net_model_url');
|
||||
const configured = ConfigManager.instance().get(currentLocalModelConfig.download.configKey);
|
||||
return typeof configured === 'string' && configured.trim()
|
||||
? configured
|
||||
: LOCAL_SEPARATOR_DEFAULT_MODEL_URL;
|
||||
}, []);
|
||||
: currentLocalModelConfig.download.defaultUrl;
|
||||
}, [currentLocalModelConfig]);
|
||||
|
||||
const isGenerating = genStatus !== 'idle' && genStatus !== 'done' && genStatus !== 'error';
|
||||
|
||||
@@ -972,25 +988,25 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
setIsDownloadingLocalModel(true);
|
||||
setErrorMsg('');
|
||||
setLocalProgressPercent(0);
|
||||
setLocalProgressText('Downloading local separator model...');
|
||||
setLocalProgressText(`Downloading ${currentLocalModelConfig.displayName}...`);
|
||||
|
||||
try {
|
||||
await LocalSeparatorModelCache.download(
|
||||
currentLocalModelConfig,
|
||||
getConfiguredLocalSeparatorModelUrl(),
|
||||
LOCAL_SEPARATOR_MODEL_FILENAME,
|
||||
progress => {
|
||||
const receivedMb = (progress.receivedBytes / (1024 * 1024)).toFixed(1);
|
||||
const totalMb = progress.totalBytes ? (progress.totalBytes / (1024 * 1024)).toFixed(1) : null;
|
||||
setLocalProgressPercent(progress.totalBytes ? progress.percent : 0);
|
||||
setLocalProgressText(
|
||||
totalMb
|
||||
? `Downloading local separator model... ${receivedMb} / ${totalMb} MB`
|
||||
: `Downloading local separator model... ${receivedMb} MB`,
|
||||
? `Downloading ${currentLocalModelConfig.displayName}... ${receivedMb} / ${totalMb} MB`
|
||||
: `Downloading ${currentLocalModelConfig.displayName}... ${receivedMb} MB`,
|
||||
);
|
||||
},
|
||||
);
|
||||
setLocalProgressPercent(100);
|
||||
setLocalProgressText('Local separator model is ready.');
|
||||
setLocalProgressText(`${currentLocalModelConfig.displayName} is ready.`);
|
||||
await refreshLocalModelCacheState();
|
||||
} catch (err) {
|
||||
setLocalProgressPercent(0);
|
||||
@@ -999,13 +1015,13 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
} finally {
|
||||
setIsDownloadingLocalModel(false);
|
||||
}
|
||||
}, [getConfiguredLocalSeparatorModelUrl, refreshLocalModelCacheState]);
|
||||
}, [currentLocalModelConfig, getConfiguredLocalSeparatorModelUrl, refreshLocalModelCacheState]);
|
||||
|
||||
const handleDeleteLocalModel = useCallback(async () => {
|
||||
setIsDeletingLocalModel(true);
|
||||
setErrorMsg('');
|
||||
try {
|
||||
await LocalSeparatorModelCache.delete();
|
||||
await LocalSeparatorModelCache.delete(currentLocalModelConfig);
|
||||
localRuntimeManagerRef.current?.reset();
|
||||
setLocalProviderLabel(runtimeSupport.webgpuExposed ? 'webgpu available' : 'cpu/wasm only');
|
||||
setLocalProgressPercent(0);
|
||||
@@ -1016,7 +1032,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
} finally {
|
||||
setIsDeletingLocalModel(false);
|
||||
}
|
||||
}, [refreshLocalModelCacheState, runtimeSupport.webgpuExposed]);
|
||||
}, [currentLocalModelConfig, refreshLocalModelCacheState, runtimeSupport.webgpuExposed]);
|
||||
|
||||
const handleSeparateServer = useCallback(async () => {
|
||||
if (!selectedAudioRegion) return;
|
||||
@@ -1212,12 +1228,12 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
setLocalProviderLabel(runtimeSupport.webgpuExposed ? 'webgpu available' : 'cpu/wasm only');
|
||||
|
||||
try {
|
||||
const modelBuffer = await LocalSeparatorModelCache.getArrayBuffer();
|
||||
const modelBuffer = await LocalSeparatorModelCache.getArrayBuffer(currentLocalModelConfig);
|
||||
const runtimeManager = localRuntimeManagerRef.current ?? new LocalOrtRuntimeManager({
|
||||
onProviderChange: provider => setLocalProviderLabel(provider),
|
||||
});
|
||||
localRuntimeManagerRef.current = runtimeManager;
|
||||
const runtime = await runtimeManager.ensureRuntime(LOCAL_SEPARATOR_MODEL_CONFIG, new Uint8Array(modelBuffer));
|
||||
const runtime = await runtimeManager.ensureRuntime(currentLocalModelConfig, new Uint8Array(modelBuffer));
|
||||
|
||||
setGenStatus('generating');
|
||||
setLocalProgressPercent(3);
|
||||
@@ -1247,10 +1263,10 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
const result = await runLocalSeparator({
|
||||
session: runtime.session,
|
||||
runtimeProvider: runtime.provider,
|
||||
modelConfig: LOCAL_SEPARATOR_MODEL_CONFIG,
|
||||
modelConfig: currentLocalModelConfig,
|
||||
audioBuffer: inputBuffer,
|
||||
chunkDurationSeconds: Number.isFinite(chunkDuration) && (chunkDuration ?? 0) > 0 ? chunkDuration : null,
|
||||
overlap: Number.isFinite(overlapValue) ? overlapValue : LOCAL_SEPARATOR_MODEL_CONFIG.defaults.overlap,
|
||||
overlap: Number.isFinite(overlapValue) ? overlapValue : currentLocalModelConfig.defaults.overlap,
|
||||
onProviderChange: provider => setLocalProviderLabel(provider),
|
||||
onProgress: progress => {
|
||||
setLocalProgressPercent(progress.percent);
|
||||
@@ -1283,6 +1299,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
projectName,
|
||||
localChunkDurationSeconds,
|
||||
localOverlap,
|
||||
currentLocalModelConfig,
|
||||
]);
|
||||
|
||||
const handleSeparate = useCallback(async () => {
|
||||
@@ -1381,14 +1398,19 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
<div className="kgone-local-mode-card">
|
||||
<div className="kgone-local-mode-title">Local Separator Mode</div>
|
||||
<div className="kgone-local-mode-text">
|
||||
Only Vocal and Instrument (Medium Accuracy) is available while not integrated with K.G.One Music Studio server.
|
||||
Processing in local may take long time depending on your hardware. When fallback to CPU happens, the webpage may
|
||||
temporarily hang with little or no UI response until processing advances.{' '}
|
||||
If K.G.One Music Studio is unavailable, Local Separator Mode provides a built-in alternative for extracting stems
|
||||
directly in your browser. Two local models are available: Vocal and Instrument (Medium Accuracy), and Vocal,
|
||||
Drums, Bass, and Others. Download status below reflects the currently selected model. Vocal and Instrument
|
||||
(Medium Accuracy) usually takes longer to process than Vocal, Drums, Bass, and Others, and total processing time
|
||||
will still depend on your hardware. If processing falls back to CPU, the page may become temporarily less
|
||||
responsive while separation is running.{' '}
|
||||
<a href="https://github.com/KGAudioLab/K.G.One" target="_blank" rel="noopener noreferrer">Learn more about K.G.One Music Studio server integration.</a>
|
||||
</div>
|
||||
<div className="kgone-runtime-row">
|
||||
<div className="kgone-provider-chip">Provider: {localProviderLabel}</div>
|
||||
<div className="kgone-provider-chip">Model: {isLocalModelCached ? 'downloaded' : 'not downloaded'}</div>
|
||||
<div className="kgone-provider-chip">
|
||||
Model: {currentLocalModelConfig.displayName} ({isLocalModelCached ? 'downloaded' : 'not downloaded'})
|
||||
</div>
|
||||
</div>
|
||||
{(localProgressText || isCheckingLocalModel) && (
|
||||
<div className="kgone-progress-block">
|
||||
@@ -1414,7 +1436,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
disabled={isCheckingLocalModel || isDownloadingLocalModel || isDeletingLocalModel || isGenerating}
|
||||
onClick={() => void handleDownloadLocalModel()}
|
||||
>
|
||||
{isDownloadingLocalModel ? 'Downloading Model...' : 'Download Model'}
|
||||
{isDownloadingLocalModel ? 'Downloading Model...' : 'Download Selected Model'}
|
||||
</button>
|
||||
) : (
|
||||
<>
|
||||
@@ -1451,7 +1473,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
|
||||
<div className="kgone-field">
|
||||
<label className="kgone-label">Separation Model</label>
|
||||
<select className="kgone-select" value={model} onChange={e => setModel(e.target.value as typeof SERVER_SEPARATOR_MODELS[number]['value'])}>
|
||||
<select className="kgone-select" value={model} onChange={e => setModel(e.target.value)}>
|
||||
{availableSeparatorModels.map(m => (
|
||||
<option key={m.value} value={m.value}>{m.label}</option>
|
||||
))}
|
||||
@@ -1474,10 +1496,10 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
/>
|
||||
</div>
|
||||
<div className="kgone-field">
|
||||
<label className="kgone-label">MDX overlap</label>
|
||||
<label className="kgone-label">Model overlap</label>
|
||||
<input
|
||||
className="kgone-input"
|
||||
aria-label="MDX overlap"
|
||||
aria-label="Model overlap"
|
||||
type="number"
|
||||
min={0.001}
|
||||
max={0.999}
|
||||
@@ -1552,7 +1574,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||
) : (
|
||||
<div className="kgone-separator-hint">
|
||||
{localOnlyMode && !isLocalModelCached
|
||||
? 'Download the local separator model, then select an audio region on the timeline to extract stems from it.'
|
||||
? `Download ${currentLocalModelConfig.displayName}, then select an audio region on the timeline to extract stems from it.`
|
||||
: 'Select an audio region on the timeline to extract stems from it. Only audio regions are supported — MIDI regions cannot be separated.'}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -29,6 +29,7 @@ const configState = new Map<string, unknown>([
|
||||
['general.local_browser.context_length', 65536],
|
||||
['general.local_browser.model_url', 'https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task'],
|
||||
['general.uvr5_web_runtime.mdx_net_model_url', 'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx'],
|
||||
['general.uvr5_web_runtime.htdemucs_4s_model_url', 'https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx'],
|
||||
['general.soundfont.base_url', 'https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/'],
|
||||
['general.kgone.enabled', false],
|
||||
['general.kgone.base_url', 'http://127.0.0.1:8000'],
|
||||
@@ -140,6 +141,7 @@ describe('GeneralSettings', () => {
|
||||
|
||||
expect(await screen.findByDisplayValue('https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task')).toBeTruthy();
|
||||
expect(screen.getByDisplayValue('https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx')).toBeTruthy();
|
||||
expect(screen.getByDisplayValue('https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx')).toBeTruthy();
|
||||
|
||||
const inputs = screen.getAllByRole('textbox');
|
||||
const gemmaUrlInput = inputs.find(input =>
|
||||
@@ -148,22 +150,30 @@ describe('GeneralSettings', () => {
|
||||
const uvr5UrlInput = inputs.find(input =>
|
||||
(input as HTMLInputElement).value.includes('UVR-MDX-NET-Inst_HQ_3.onnx'),
|
||||
) as HTMLInputElement | undefined;
|
||||
const htdemucsUrlInput = inputs.find(input =>
|
||||
(input as HTMLInputElement).value.includes('htdemucs_embedded.onnx'),
|
||||
) as HTMLInputElement | undefined;
|
||||
|
||||
expect(gemmaUrlInput).toBeTruthy();
|
||||
expect(uvr5UrlInput).toBeTruthy();
|
||||
expect(htdemucsUrlInput).toBeTruthy();
|
||||
|
||||
fireEvent.change(gemmaUrlInput!, { target: { value: 'https://example.com/gemma.task' } });
|
||||
fireEvent.change(uvr5UrlInput!, { target: { value: 'https://example.com/uvr5.onnx' } });
|
||||
fireEvent.change(htdemucsUrlInput!, { target: { value: 'https://example.com/htdemucs.onnx' } });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(configManagerMock.set).toHaveBeenCalledWith('general.local_browser.model_url', 'https://example.com/gemma.task');
|
||||
expect(configManagerMock.set).toHaveBeenCalledWith('general.uvr5_web_runtime.mdx_net_model_url', 'https://example.com/uvr5.onnx');
|
||||
expect(configManagerMock.set).toHaveBeenCalledWith('general.uvr5_web_runtime.htdemucs_4s_model_url', 'https://example.com/htdemucs.onnx');
|
||||
});
|
||||
});
|
||||
|
||||
it('restores default download URLs and deletes the UVR5 model cache', async () => {
|
||||
localSeparatorModelCacheMock.exists
|
||||
.mockResolvedValueOnce(true)
|
||||
.mockResolvedValueOnce(true)
|
||||
.mockResolvedValueOnce(false)
|
||||
.mockResolvedValueOnce(false);
|
||||
|
||||
render(<GeneralSettings />);
|
||||
@@ -173,6 +183,7 @@ describe('GeneralSettings', () => {
|
||||
const restoreLinks = screen.getAllByText('Restore default');
|
||||
fireEvent.click(restoreLinks[0]);
|
||||
fireEvent.click(restoreLinks[1]);
|
||||
fireEvent.click(restoreLinks[2]);
|
||||
const uvr5DeleteButton = screen.getAllByRole('button', { name: 'Delete Cached Model' })[1];
|
||||
expect(uvr5DeleteButton).not.toBeDisabled();
|
||||
fireEvent.click(uvr5DeleteButton);
|
||||
@@ -186,6 +197,10 @@ describe('GeneralSettings', () => {
|
||||
'general.uvr5_web_runtime.mdx_net_model_url',
|
||||
'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||
);
|
||||
expect(configManagerMock.set).toHaveBeenCalledWith(
|
||||
'general.uvr5_web_runtime.htdemucs_4s_model_url',
|
||||
'https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx',
|
||||
);
|
||||
expect(localSeparatorModelCacheMock.delete).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
||||
@@ -12,7 +12,10 @@ import {
|
||||
normalizeLocalLLMContextLength,
|
||||
type LocalLLMContextLength,
|
||||
} from '../../../util/localLLMConfig';
|
||||
import { LOCAL_SEPARATOR_DEFAULT_MODEL_URL } from '../../../util/local-separator/config';
|
||||
import {
|
||||
LOCAL_SEPARATOR_MODEL_CONFIGS,
|
||||
LOCAL_SEPARATOR_MODEL_IDS,
|
||||
} from '../../../util/local-separator/config';
|
||||
|
||||
const GeneralSettings: React.FC = () => {
|
||||
const [llmProvider, setLlmProvider] = useState<string>(LOCAL_LLM_PROVIDER_KEY);
|
||||
@@ -39,9 +42,12 @@ const GeneralSettings: React.FC = () => {
|
||||
const [localModelState, setLocalModelState] = useState<LocalLLMModelState>(LocalLLMModelManager.getState());
|
||||
const [localModelUrl, setLocalModelUrl] = useState<string>('');
|
||||
const [uvr5ModelUrl, setUvr5ModelUrl] = useState<string>('');
|
||||
const [htdemucsModelUrl, setHtdemucsModelUrl] = useState<string>('');
|
||||
const [isUvr5ModelCached, setIsUvr5ModelCached] = useState<boolean>(false);
|
||||
const [isCheckingUvr5ModelCache, setIsCheckingUvr5ModelCache] = useState<boolean>(false);
|
||||
const [isDeletingUvr5Model, setIsDeletingUvr5Model] = useState<boolean>(false);
|
||||
const [isHtdemucsModelCached, setIsHtdemucsModelCached] = useState<boolean>(false);
|
||||
const [isDeletingHtdemucsModel, setIsDeletingHtdemucsModel] = useState<boolean>(false);
|
||||
|
||||
const configManager = ConfigManager.instance();
|
||||
|
||||
@@ -62,10 +68,16 @@ const GeneralSettings: React.FC = () => {
|
||||
const refreshUvr5ModelCacheState = useCallback(async () => {
|
||||
setIsCheckingUvr5ModelCache(true);
|
||||
try {
|
||||
setIsUvr5ModelCached(await LocalSeparatorModelCache.exists());
|
||||
const [mdxCached, demucsCached] = await Promise.all([
|
||||
LocalSeparatorModelCache.exists(LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium]),
|
||||
LocalSeparatorModelCache.exists(LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s]),
|
||||
]);
|
||||
setIsUvr5ModelCached(mdxCached);
|
||||
setIsHtdemucsModelCached(demucsCached);
|
||||
} catch (error) {
|
||||
console.error('Failed to check UVR5 cached model state:', error);
|
||||
setIsUvr5ModelCached(false);
|
||||
setIsHtdemucsModelCached(false);
|
||||
} finally {
|
||||
setIsCheckingUvr5ModelCache(false);
|
||||
}
|
||||
@@ -95,7 +107,14 @@ const GeneralSettings: React.FC = () => {
|
||||
setCompatibleModel((configManager.get('general.openai_compatible.model') as string) || '');
|
||||
setLocalContextLength(normalizeLocalLLMContextLength(configManager.get('general.local_browser.context_length')));
|
||||
setLocalModelUrl((configManager.get('general.local_browser.model_url') as string) || LOCAL_LLM_DEFAULT_MODEL_URL);
|
||||
setUvr5ModelUrl((configManager.get('general.uvr5_web_runtime.mdx_net_model_url') as string) || LOCAL_SEPARATOR_DEFAULT_MODEL_URL);
|
||||
setUvr5ModelUrl(
|
||||
(configManager.get('general.uvr5_web_runtime.mdx_net_model_url') as string)
|
||||
|| LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium].download.defaultUrl,
|
||||
);
|
||||
setHtdemucsModelUrl(
|
||||
(configManager.get('general.uvr5_web_runtime.htdemucs_4s_model_url') as string)
|
||||
|| LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s].download.defaultUrl,
|
||||
);
|
||||
setSoundfontBaseUrl((configManager.get('general.soundfont.base_url') as string) || '');
|
||||
setKgoneEnabled((configManager.get('general.kgone.enabled') as boolean) ?? false);
|
||||
setKgoneBaseUrl((configManager.get('general.kgone.base_url') as string) || '');
|
||||
@@ -245,6 +264,11 @@ const GeneralSettings: React.FC = () => {
|
||||
debouncedSave('general.uvr5_web_runtime.mdx_net_model_url', value);
|
||||
};
|
||||
|
||||
const handleHtdemucsModelUrlChange = (value: string) => {
|
||||
setHtdemucsModelUrl(value);
|
||||
debouncedSave('general.uvr5_web_runtime.htdemucs_4s_model_url', value);
|
||||
};
|
||||
|
||||
const handleDeleteLocalModel = async () => {
|
||||
try {
|
||||
await LocalLLMModelManager.deleteCachedModel();
|
||||
@@ -256,7 +280,7 @@ const GeneralSettings: React.FC = () => {
|
||||
const handleDeleteUvr5Model = async () => {
|
||||
setIsDeletingUvr5Model(true);
|
||||
try {
|
||||
await LocalSeparatorModelCache.delete();
|
||||
await LocalSeparatorModelCache.delete(LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium]);
|
||||
setIsUvr5ModelCached(false);
|
||||
} catch (error) {
|
||||
console.error('Failed to delete UVR5 cached model:', error);
|
||||
@@ -266,6 +290,19 @@ const GeneralSettings: React.FC = () => {
|
||||
}
|
||||
};
|
||||
|
||||
const handleDeleteHtdemucsModel = async () => {
|
||||
setIsDeletingHtdemucsModel(true);
|
||||
try {
|
||||
await LocalSeparatorModelCache.delete(LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s]);
|
||||
setIsHtdemucsModelCached(false);
|
||||
} catch (error) {
|
||||
console.error('Failed to delete HTDemucs cached model:', error);
|
||||
} finally {
|
||||
setIsDeletingHtdemucsModel(false);
|
||||
await refreshUvr5ModelCacheState();
|
||||
}
|
||||
};
|
||||
|
||||
const handleLocalContextLengthChange = async (value: string) => {
|
||||
const parsed = Number(value);
|
||||
const normalized = normalizeLocalLLMContextLength(parsed);
|
||||
@@ -448,7 +485,7 @@ const GeneralSettings: React.FC = () => {
|
||||
<input
|
||||
type="text"
|
||||
className="settings-input"
|
||||
placeholder={`e.g. ${LOCAL_SEPARATOR_DEFAULT_MODEL_URL}`}
|
||||
placeholder={`e.g. ${LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium].download.defaultUrl}`}
|
||||
value={uvr5ModelUrl}
|
||||
onChange={(e) => handleUvr5ModelUrlChange(e.target.value)}
|
||||
/>
|
||||
@@ -458,7 +495,9 @@ const GeneralSettings: React.FC = () => {
|
||||
href="#"
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
handleUvr5ModelUrlChange(LOCAL_SEPARATOR_DEFAULT_MODEL_URL);
|
||||
handleUvr5ModelUrlChange(
|
||||
LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium].download.defaultUrl,
|
||||
);
|
||||
}}
|
||||
style={{ color: '#5a9fd4', textDecoration: 'underline', cursor: 'pointer' }}
|
||||
>
|
||||
@@ -477,6 +516,45 @@ const GeneralSettings: React.FC = () => {
|
||||
{isDeletingUvr5Model ? 'Deleting...' : 'Delete Cached Model'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="settings-item">
|
||||
<label className="settings-label">
|
||||
htdemucs_4s Download URL
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
className="settings-input"
|
||||
placeholder={`e.g. ${LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s].download.defaultUrl}`}
|
||||
value={htdemucsModelUrl}
|
||||
onChange={(e) => handleHtdemucsModelUrlChange(e.target.value)}
|
||||
/>
|
||||
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
|
||||
Changing this URL may break downloads or point to an incompatible model file.{' '}
|
||||
<a
|
||||
href="#"
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
handleHtdemucsModelUrlChange(
|
||||
LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s].download.defaultUrl,
|
||||
);
|
||||
}}
|
||||
style={{ color: '#5a9fd4', textDecoration: 'underline', cursor: 'pointer' }}
|
||||
>
|
||||
Restore default
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="settings-item" style={{ marginTop: '12px' }}>
|
||||
<button
|
||||
type="button"
|
||||
className="settings-btn settings-btn-danger"
|
||||
onClick={() => void handleDeleteHtdemucsModel()}
|
||||
disabled={isCheckingUvr5ModelCache || isDeletingHtdemucsModel || !isHtdemucsModelCached}
|
||||
>
|
||||
{isDeletingHtdemucsModel ? 'Deleting...' : 'Delete Cached Model'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="settings-group">
|
||||
|
||||
@@ -14,6 +14,7 @@ interface AppConfig {
|
||||
};
|
||||
uvr5_web_runtime: {
|
||||
mdx_net_model_url: string;
|
||||
htdemucs_4s_model_url: string;
|
||||
};
|
||||
openai: {
|
||||
api_key: string;
|
||||
@@ -224,7 +225,8 @@ export class ConfigManager {
|
||||
model_url: 'https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task'
|
||||
},
|
||||
uvr5_web_runtime: {
|
||||
mdx_net_model_url: 'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx'
|
||||
mdx_net_model_url: 'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||
htdemucs_4s_model_url: 'https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx',
|
||||
},
|
||||
soundfont: {
|
||||
base_url: 'https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/'
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { LocalSeparatorModelCache } from '../../util/local-separator/modelCache';
|
||||
import {
|
||||
LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES,
|
||||
LOCAL_SEPARATOR_MODEL_FILENAME,
|
||||
LOCAL_SEPARATOR_MODEL_CONFIGS,
|
||||
LOCAL_SEPARATOR_MODEL_IDS,
|
||||
} from '../../util/local-separator/config';
|
||||
|
||||
class MockWritableFileStream {
|
||||
@@ -113,7 +113,9 @@ vi.stubGlobal('navigator', {
|
||||
});
|
||||
|
||||
describe('LocalSeparatorModelCache', () => {
|
||||
const makeModelBytes = (fill: number): Uint8Array => new Uint8Array(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES).fill(fill);
|
||||
const mdxConfig = LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium];
|
||||
const demucsConfig = LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s];
|
||||
const makeModelBytes = (size: number, fill: number): Uint8Array => new Uint8Array(size).fill(fill);
|
||||
|
||||
beforeEach(() => {
|
||||
mockRoot.clear();
|
||||
@@ -121,64 +123,64 @@ describe('LocalSeparatorModelCache', () => {
|
||||
});
|
||||
|
||||
it('downloads and stores a model in OPFS cache', async () => {
|
||||
const bytes = makeModelBytes(1);
|
||||
const bytes = makeModelBytes(mdxConfig.download.expectedSizeBytes, 1);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(bytes, {
|
||||
status: 200,
|
||||
headers: { 'Content-Length': String(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES) },
|
||||
headers: { 'Content-Length': String(mdxConfig.download.expectedSizeBytes) },
|
||||
})));
|
||||
|
||||
await LocalSeparatorModelCache.download('https://example.com/model.onnx');
|
||||
await LocalSeparatorModelCache.download(mdxConfig, 'https://example.com/model.onnx');
|
||||
|
||||
expect(await LocalSeparatorModelCache.exists()).toBe(true);
|
||||
const buffer = await LocalSeparatorModelCache.getArrayBuffer();
|
||||
expect(buffer.byteLength).toBe(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES);
|
||||
expect(await LocalSeparatorModelCache.exists(mdxConfig)).toBe(true);
|
||||
const buffer = await LocalSeparatorModelCache.getArrayBuffer(mdxConfig);
|
||||
expect(buffer.byteLength).toBe(mdxConfig.download.expectedSizeBytes);
|
||||
expect(new Uint8Array(buffer)[0]).toBe(1);
|
||||
});
|
||||
|
||||
it('replaces a broken cached file on redownload', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(1), {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(mdxConfig.download.expectedSizeBytes, 1), {
|
||||
status: 200,
|
||||
headers: { 'Content-Length': String(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES) },
|
||||
headers: { 'Content-Length': String(mdxConfig.download.expectedSizeBytes) },
|
||||
})));
|
||||
await LocalSeparatorModelCache.download('https://example.com/model.onnx');
|
||||
await LocalSeparatorModelCache.download(mdxConfig, 'https://example.com/model.onnx');
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(9), {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(mdxConfig.download.expectedSizeBytes, 9), {
|
||||
status: 200,
|
||||
headers: { 'Content-Length': String(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES) },
|
||||
headers: { 'Content-Length': String(mdxConfig.download.expectedSizeBytes) },
|
||||
})));
|
||||
await LocalSeparatorModelCache.download('https://example.com/model.onnx');
|
||||
await LocalSeparatorModelCache.download(mdxConfig, 'https://example.com/model.onnx');
|
||||
|
||||
const buffer = await LocalSeparatorModelCache.getArrayBuffer();
|
||||
expect(buffer.byteLength).toBe(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES);
|
||||
const buffer = await LocalSeparatorModelCache.getArrayBuffer(mdxConfig);
|
||||
expect(buffer.byteLength).toBe(mdxConfig.download.expectedSizeBytes);
|
||||
expect(new Uint8Array(buffer)[0]).toBe(9);
|
||||
});
|
||||
|
||||
it('deletes the cached model file', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(2), {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(mdxConfig.download.expectedSizeBytes, 2), {
|
||||
status: 200,
|
||||
headers: { 'Content-Length': String(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES) },
|
||||
headers: { 'Content-Length': String(mdxConfig.download.expectedSizeBytes) },
|
||||
})));
|
||||
await LocalSeparatorModelCache.download('https://example.com/model.onnx');
|
||||
await LocalSeparatorModelCache.download(mdxConfig, 'https://example.com/model.onnx');
|
||||
|
||||
await LocalSeparatorModelCache.delete();
|
||||
await LocalSeparatorModelCache.delete(mdxConfig);
|
||||
|
||||
expect(await LocalSeparatorModelCache.exists()).toBe(false);
|
||||
expect(await LocalSeparatorModelCache.exists(mdxConfig)).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects and deletes a cached file when the size is wrong', async () => {
|
||||
const dir = await navigator.storage.getDirectory();
|
||||
const modelsDir = await dir.getDirectoryHandle('models', { create: true });
|
||||
const fileHandle = await modelsDir.getFileHandle(LOCAL_SEPARATOR_MODEL_FILENAME, { create: true });
|
||||
const fileHandle = await modelsDir.getFileHandle(mdxConfig.filename, { create: true });
|
||||
const fileWritable = await fileHandle.createWritable();
|
||||
await fileWritable.write(new Uint8Array([1, 2, 3]));
|
||||
await fileWritable.close();
|
||||
|
||||
const sizeHandle = await modelsDir.getFileHandle(`${LOCAL_SEPARATOR_MODEL_FILENAME}.size`, { create: true });
|
||||
const sizeHandle = await modelsDir.getFileHandle(`${mdxConfig.filename}.size`, { create: true });
|
||||
const sizeWritable = await sizeHandle.createWritable();
|
||||
await sizeWritable.write(String(3));
|
||||
await sizeWritable.close();
|
||||
|
||||
expect(await LocalSeparatorModelCache.exists()).toBe(false);
|
||||
expect(await LocalSeparatorModelCache.exists(mdxConfig)).toBe(false);
|
||||
});
|
||||
|
||||
it('fails a download when the final size does not match the expected model size', async () => {
|
||||
@@ -187,7 +189,19 @@ describe('LocalSeparatorModelCache', () => {
|
||||
headers: { 'Content-Length': '3' },
|
||||
})));
|
||||
|
||||
await expect(LocalSeparatorModelCache.download('https://example.com/model.onnx')).rejects.toThrow(/size mismatch/i);
|
||||
expect(await LocalSeparatorModelCache.exists()).toBe(false);
|
||||
await expect(LocalSeparatorModelCache.download(mdxConfig, 'https://example.com/model.onnx')).rejects.toThrow(/size mismatch/i);
|
||||
expect(await LocalSeparatorModelCache.exists(mdxConfig)).toBe(false);
|
||||
});
|
||||
|
||||
it('tracks cached files independently per model', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(demucsConfig.download.expectedSizeBytes, 7), {
|
||||
status: 200,
|
||||
headers: { 'Content-Length': String(demucsConfig.download.expectedSizeBytes) },
|
||||
})));
|
||||
|
||||
await LocalSeparatorModelCache.download(demucsConfig, 'https://example.com/htdemucs.onnx');
|
||||
|
||||
expect(await LocalSeparatorModelCache.exists(demucsConfig)).toBe(true);
|
||||
expect(await LocalSeparatorModelCache.exists(mdxConfig)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
Vendored
+35
@@ -0,0 +1,35 @@
|
||||
declare module 'demucs-web' {
|
||||
export interface DemucsTrackOutput {
|
||||
left: Float32Array;
|
||||
right: Float32Array;
|
||||
}
|
||||
|
||||
export interface DemucsSeparationResult {
|
||||
drums: DemucsTrackOutput;
|
||||
bass: DemucsTrackOutput;
|
||||
other: DemucsTrackOutput;
|
||||
vocals: DemucsTrackOutput;
|
||||
}
|
||||
|
||||
export interface DemucsProgressInfo {
|
||||
progress: number;
|
||||
currentSegment: number;
|
||||
totalSegments: number;
|
||||
}
|
||||
|
||||
export interface DemucsProcessorOptions {
|
||||
ort: typeof import('onnxruntime-web/webgpu');
|
||||
modelPath?: string;
|
||||
sessionOptions?: import('onnxruntime-web/webgpu').InferenceSession.SessionOptions;
|
||||
onProgress?: (info: DemucsProgressInfo) => void;
|
||||
onLog?: (phase: string, message: string) => void;
|
||||
onDownloadProgress?: (loaded: number, total: number) => void;
|
||||
}
|
||||
|
||||
export class DemucsProcessor {
|
||||
constructor(options?: DemucsProcessorOptions);
|
||||
session: import('onnxruntime-web/webgpu').InferenceSession | null;
|
||||
loadModel(pathOrBuffer?: string | ArrayBuffer): Promise<import('onnxruntime-web/webgpu').InferenceSession>;
|
||||
separate(left: Float32Array, right: Float32Array): Promise<DemucsSeparationResult>;
|
||||
}
|
||||
}
|
||||
@@ -1,32 +1,81 @@
|
||||
import type { LocalSeparatorModelConfig } from './types';
|
||||
import type { LocalSeparatorModelConfig, LocalSeparatorModelId } from './types';
|
||||
|
||||
export const LOCAL_SEPARATOR_DEFAULT_MODEL_URL =
|
||||
'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx';
|
||||
export const LOCAL_SEPARATOR_MODEL_IDS = {
|
||||
mdxMedium: 'UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||
htdemucs4s: 'htdemucs_4s.onnx',
|
||||
} as const satisfies Record<string, LocalSeparatorModelId>;
|
||||
|
||||
export const LOCAL_SEPARATOR_MODEL_FILENAME = 'UVR-MDX-NET-Inst_HQ_3.onnx';
|
||||
export const LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES = 66759214;
|
||||
export const LOCAL_SEPARATOR_DEFAULT_MODEL_ID = LOCAL_SEPARATOR_MODEL_IDS.mdxMedium;
|
||||
|
||||
export const LOCAL_SEPARATOR_MODEL_CONFIG: LocalSeparatorModelConfig = {
|
||||
filename: LOCAL_SEPARATOR_MODEL_FILENAME,
|
||||
displayName: 'Vocal and Instrument (Medium Accuracy)',
|
||||
status: 'ready',
|
||||
defaults: {
|
||||
sampleRate: 44100,
|
||||
hopLength: 1024,
|
||||
segmentSize: 256,
|
||||
overlap: 0.25,
|
||||
batchSize: 1,
|
||||
enableDenoise: false,
|
||||
invertUsingSpec: false,
|
||||
normalizationThreshold: 0.9,
|
||||
amplificationThreshold: 0,
|
||||
matchMixOverlap: 0.02,
|
||||
export const LOCAL_SEPARATOR_MODEL_CONFIGS: Record<LocalSeparatorModelId, LocalSeparatorModelConfig> = {
|
||||
[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium]: {
|
||||
id: LOCAL_SEPARATOR_MODEL_IDS.mdxMedium,
|
||||
filename: 'UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||
kind: 'mdx',
|
||||
displayName: 'Vocal and Instrument (Medium Accuracy)',
|
||||
status: 'ready',
|
||||
outputStemNames: ['Instrumental', 'Vocals'],
|
||||
defaultChunkDurationSeconds: null,
|
||||
defaults: {
|
||||
sampleRate: 44100,
|
||||
hopLength: 1024,
|
||||
segmentSize: 256,
|
||||
overlap: 0.25,
|
||||
batchSize: 1,
|
||||
enableDenoise: false,
|
||||
invertUsingSpec: false,
|
||||
normalizationThreshold: 0.9,
|
||||
amplificationThreshold: 0,
|
||||
matchMixOverlap: 0.02,
|
||||
},
|
||||
metadata: {
|
||||
compensate: 1.021,
|
||||
mdx_dim_f_set: 3072,
|
||||
mdx_dim_t_set: 8,
|
||||
mdx_n_fft_scale_set: 7680,
|
||||
primary_stem: 'Instrumental',
|
||||
},
|
||||
download: {
|
||||
configKey: 'general.uvr5_web_runtime.mdx_net_model_url',
|
||||
defaultUrl: 'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||
expectedSizeBytes: 66759214,
|
||||
},
|
||||
},
|
||||
metadata: {
|
||||
compensate: 1.021,
|
||||
mdx_dim_f_set: 3072,
|
||||
mdx_dim_t_set: 8,
|
||||
mdx_n_fft_scale_set: 7680,
|
||||
primary_stem: 'Instrumental',
|
||||
[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s]: {
|
||||
id: LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s,
|
||||
filename: 'htdemucs_4s.onnx',
|
||||
kind: 'demucs',
|
||||
displayName: 'Vocal, Drums, Bass, and Others',
|
||||
status: 'ready',
|
||||
outputStemNames: ['Vocals', 'Drums', 'Bass', 'Others'],
|
||||
defaultChunkDurationSeconds: 8,
|
||||
defaults: {
|
||||
sampleRate: 44100,
|
||||
hopLength: 1024,
|
||||
segmentSize: 256,
|
||||
overlap: 0.25,
|
||||
batchSize: 1,
|
||||
enableDenoise: false,
|
||||
invertUsingSpec: false,
|
||||
normalizationThreshold: 0.9,
|
||||
amplificationThreshold: 0,
|
||||
matchMixOverlap: 0.02,
|
||||
},
|
||||
metadata: null,
|
||||
download: {
|
||||
configKey: 'general.uvr5_web_runtime.htdemucs_4s_model_url',
|
||||
defaultUrl: 'https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx',
|
||||
expectedSizeBytes: 180534758,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export const LOCAL_SEPARATOR_MODELS = Object.values(LOCAL_SEPARATOR_MODEL_CONFIGS);
|
||||
|
||||
export function getLocalSeparatorModelConfig(modelId: string): LocalSeparatorModelConfig {
|
||||
const config = LOCAL_SEPARATOR_MODEL_CONFIGS[modelId as LocalSeparatorModelId];
|
||||
if (!config) {
|
||||
return LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_DEFAULT_MODEL_ID];
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { LocalSeparatorModelConfig, StereoChannels } from './types';
|
||||
import type { LocalSeparatorMdxModelConfig, StereoChannels } from './types';
|
||||
import { FFT, createWindowCache, getHannPeriodic, index4d, reflectPad } from './shared';
|
||||
|
||||
interface SpectrogramPayload {
|
||||
@@ -18,7 +18,7 @@ export class LocalSeparatorCpuDsp {
|
||||
private readonly numFreqBins: number;
|
||||
private readonly inverseFft: FFT;
|
||||
|
||||
constructor(config: LocalSeparatorModelConfig) {
|
||||
constructor(config: LocalSeparatorMdxModelConfig) {
|
||||
this.nFft = config.metadata.mdx_n_fft_scale_set;
|
||||
this.hopLength = config.defaults.hopLength;
|
||||
this.dimF = config.metadata.mdx_dim_f_set;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { LocalSeparatorModelConfig } from './types';
|
||||
import type { LocalSeparatorMdxModelConfig } from './types';
|
||||
import { LocalSeparatorCpuDsp } from './cpuDsp';
|
||||
import { reflectPad } from './shared';
|
||||
|
||||
@@ -140,7 +140,7 @@ export class LocalSeparatorGpuDsp {
|
||||
private paramBuffer: GPUBufferLike | null = null;
|
||||
private readonly pipeline: GPUComputePipelineLike;
|
||||
|
||||
public static async create(config: LocalSeparatorModelConfig): Promise<LocalSeparatorGpuDsp> {
|
||||
public static async create(config: LocalSeparatorMdxModelConfig): Promise<LocalSeparatorGpuDsp> {
|
||||
if (!('gpu' in navigator)) {
|
||||
throw new Error('WebGPU is not available for GPU DSP.');
|
||||
}
|
||||
@@ -164,7 +164,7 @@ export class LocalSeparatorGpuDsp {
|
||||
return new LocalSeparatorGpuDsp(config, device);
|
||||
}
|
||||
|
||||
private constructor(config: LocalSeparatorModelConfig, device: GPUDeviceLike) {
|
||||
private constructor(config: LocalSeparatorMdxModelConfig, device: GPUDeviceLike) {
|
||||
this.device = device;
|
||||
this.cpuDsp = new LocalSeparatorCpuDsp(config);
|
||||
this.nFft = config.metadata.mdx_n_fft_scale_set;
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
import {
|
||||
LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES,
|
||||
LOCAL_SEPARATOR_MODEL_FILENAME,
|
||||
} from './config';
|
||||
import type { LocalSeparatorModelConfig } from './types';
|
||||
import { OpfsModelCache, type ModelDownloadProgress } from '../opfsModelCache';
|
||||
|
||||
const cache = new OpfsModelCache({ directoryName: 'models' });
|
||||
@@ -9,34 +6,34 @@ const cache = new OpfsModelCache({ directoryName: 'models' });
|
||||
export { type ModelDownloadProgress };
|
||||
|
||||
export class LocalSeparatorModelCache {
|
||||
public static async exists(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<boolean> {
|
||||
return cache.exists(filename, {
|
||||
expectedSizeBytes: LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES,
|
||||
public static async exists(modelConfig: LocalSeparatorModelConfig): Promise<boolean> {
|
||||
return cache.exists(modelConfig.filename, {
|
||||
expectedSizeBytes: modelConfig.download.expectedSizeBytes,
|
||||
});
|
||||
}
|
||||
|
||||
public static async getFile(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<File> {
|
||||
return cache.getFile(filename);
|
||||
public static async getFile(modelConfig: LocalSeparatorModelConfig): Promise<File> {
|
||||
return cache.getFile(modelConfig.filename);
|
||||
}
|
||||
|
||||
public static async getArrayBuffer(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<ArrayBuffer> {
|
||||
return cache.getArrayBuffer(filename);
|
||||
public static async getArrayBuffer(modelConfig: LocalSeparatorModelConfig): Promise<ArrayBuffer> {
|
||||
return cache.getArrayBuffer(modelConfig.filename);
|
||||
}
|
||||
|
||||
public static async delete(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<void> {
|
||||
await cache.delete(filename);
|
||||
public static async delete(modelConfig: LocalSeparatorModelConfig): Promise<void> {
|
||||
await cache.delete(modelConfig.filename);
|
||||
}
|
||||
|
||||
public static async download(
|
||||
modelConfig: LocalSeparatorModelConfig,
|
||||
sourceUrl: string,
|
||||
filename: string = LOCAL_SEPARATOR_MODEL_FILENAME,
|
||||
onProgress?: (progress: ModelDownloadProgress) => void,
|
||||
): Promise<void> {
|
||||
await cache.download(
|
||||
sourceUrl,
|
||||
filename,
|
||||
modelConfig.filename,
|
||||
{
|
||||
expectedSizeBytes: LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES,
|
||||
expectedSizeBytes: modelConfig.download.expectedSizeBytes,
|
||||
},
|
||||
onProgress,
|
||||
);
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import * as ort from 'onnxruntime-web/webgpu';
|
||||
import { DemucsProcessor } from 'demucs-web';
|
||||
import { LocalSeparatorCpuDsp } from './cpuDsp';
|
||||
import { LocalSeparatorGpuDsp } from './gpuDsp';
|
||||
import { LocalSeparatorTimingCollector } from './timing';
|
||||
import type {
|
||||
LocalRuntimeProvider,
|
||||
LocalSeparatorDemucsModelConfig,
|
||||
LocalSeparatorModelConfig,
|
||||
LocalSeparatorMdxModelConfig,
|
||||
LocalSeparatorProgress,
|
||||
StereoChannels,
|
||||
} from './types';
|
||||
@@ -76,8 +79,8 @@ function unpackBatchOutput(outputData: Float32Array, batchInfo: { dims: number[]
|
||||
class BrowserMdxSeparator {
|
||||
private readonly session: ort.InferenceSession;
|
||||
private readonly runtimeProvider: LocalRuntimeProvider;
|
||||
private readonly defaults: LocalSeparatorModelConfig['defaults'];
|
||||
private readonly metadata: LocalSeparatorModelConfig['metadata'];
|
||||
private readonly defaults: LocalSeparatorMdxModelConfig['defaults'];
|
||||
private readonly metadata: LocalSeparatorMdxModelConfig['metadata'];
|
||||
public onProgress: (progress: LocalSeparatorProgress) => void;
|
||||
private overlap: number;
|
||||
private runtimeBatchSize: number;
|
||||
@@ -97,7 +100,7 @@ class BrowserMdxSeparator {
|
||||
public static async create(
|
||||
session: ort.InferenceSession,
|
||||
runtimeProvider: LocalRuntimeProvider,
|
||||
config: LocalSeparatorModelConfig,
|
||||
config: LocalSeparatorMdxModelConfig,
|
||||
options: BrowserMdxSeparatorOptions = {},
|
||||
): Promise<BrowserMdxSeparator> {
|
||||
const timing = options.timing ?? new LocalSeparatorTimingCollector('mdx-separation');
|
||||
@@ -139,7 +142,7 @@ class BrowserMdxSeparator {
|
||||
private constructor(
|
||||
session: ort.InferenceSession,
|
||||
runtimeProvider: LocalRuntimeProvider,
|
||||
config: LocalSeparatorModelConfig,
|
||||
config: LocalSeparatorMdxModelConfig,
|
||||
options: BrowserMdxSeparatorOptions & {
|
||||
dsp: LocalSeparatorCpuDsp | LocalSeparatorGpuDsp;
|
||||
dspMode: 'cpu' | 'gpu-hybrid';
|
||||
@@ -470,10 +473,24 @@ function concatChannelPairs(chunks: StereoChannels[]): StereoChannels {
|
||||
return [concatFloat32(chunks.map(chunk => chunk[0])), concatFloat32(chunks.map(chunk => chunk[1]))];
|
||||
}
|
||||
|
||||
export async function runLocalSeparator(options: {
|
||||
function getProviderLabel(runtimeProvider: LocalRuntimeProvider): string {
|
||||
return runtimeProvider === 'webgpu' ? 'GPU/WebGPU' : 'CPU/wasm';
|
||||
}
|
||||
|
||||
function getChunkDurationSeconds(
|
||||
requestedChunkDurationSeconds: number | null,
|
||||
modelConfig: LocalSeparatorModelConfig,
|
||||
): number | null {
|
||||
if (requestedChunkDurationSeconds != null) {
|
||||
return requestedChunkDurationSeconds;
|
||||
}
|
||||
return modelConfig.defaultChunkDurationSeconds;
|
||||
}
|
||||
|
||||
async function runMdxLocalSeparator(options: {
|
||||
session: ort.InferenceSession;
|
||||
runtimeProvider: LocalRuntimeProvider;
|
||||
modelConfig: LocalSeparatorModelConfig;
|
||||
modelConfig: LocalSeparatorMdxModelConfig;
|
||||
audioBuffer: ArrayBuffer;
|
||||
chunkDurationSeconds: number | null;
|
||||
overlap: number;
|
||||
@@ -486,7 +503,7 @@ export async function runLocalSeparator(options: {
|
||||
}> {
|
||||
const timing = new LocalSeparatorTimingCollector('local-separation');
|
||||
const decoded = await timing.measureAsync('decode', () => decodeAudioToStereo(options.audioBuffer));
|
||||
log(`Running browser MDX separation on ${options.runtimeProvider === 'webgpu' ? 'GPU/WebGPU' : 'CPU/wasm'}...`);
|
||||
log(`Running browser MDX separation on ${getProviderLabel(options.runtimeProvider)}...`);
|
||||
|
||||
const separator = await BrowserMdxSeparator.create(
|
||||
options.session,
|
||||
@@ -502,11 +519,11 @@ export async function runLocalSeparator(options: {
|
||||
);
|
||||
|
||||
try {
|
||||
const outputs = await separateWithOptionalChunking(
|
||||
const outputs = await separateMdxWithOptionalChunking(
|
||||
separator,
|
||||
decoded,
|
||||
timing,
|
||||
options.chunkDurationSeconds,
|
||||
getChunkDurationSeconds(options.chunkDurationSeconds, options.modelConfig),
|
||||
options.modelConfig,
|
||||
options.onProgress,
|
||||
);
|
||||
@@ -518,7 +535,7 @@ export async function runLocalSeparator(options: {
|
||||
{ name: outputs.primaryStem, blob: primaryBlob },
|
||||
{ name: outputs.secondaryStem, blob: secondaryBlob },
|
||||
],
|
||||
providerLabel: options.runtimeProvider === 'webgpu' ? 'GPU/WebGPU' : 'CPU/wasm',
|
||||
providerLabel: getProviderLabel(options.runtimeProvider),
|
||||
debugSummary: separator.getDebugSummary({ model: options.modelConfig.filename }),
|
||||
};
|
||||
} finally {
|
||||
@@ -527,12 +544,12 @@ export async function runLocalSeparator(options: {
|
||||
}
|
||||
}
|
||||
|
||||
async function separateWithOptionalChunking(
|
||||
async function separateMdxWithOptionalChunking(
|
||||
separator: BrowserMdxSeparator,
|
||||
decoded: StereoChannels,
|
||||
timing: LocalSeparatorTimingCollector,
|
||||
chunkDurationSeconds: number | null,
|
||||
modelConfig: LocalSeparatorModelConfig,
|
||||
modelConfig: LocalSeparatorMdxModelConfig,
|
||||
onProgress: (progress: LocalSeparatorProgress) => void,
|
||||
): Promise<{
|
||||
stems: Record<string, StereoChannels>;
|
||||
@@ -594,3 +611,171 @@ async function separateWithOptionalChunking(
|
||||
secondaryStem,
|
||||
};
|
||||
}
|
||||
|
||||
type DemucsStemName = 'Vocals' | 'Drums' | 'Bass' | 'Others';
|
||||
|
||||
function toDemucsStemMap(result: Awaited<ReturnType<DemucsProcessor['separate']>>): Record<DemucsStemName, StereoChannels> {
|
||||
return {
|
||||
Vocals: [result.vocals.left, result.vocals.right],
|
||||
Drums: [result.drums.left, result.drums.right],
|
||||
Bass: [result.bass.left, result.bass.right],
|
||||
Others: [result.other.left, result.other.right],
|
||||
};
|
||||
}
|
||||
|
||||
async function runDemucsLocalSeparator(options: {
|
||||
session: ort.InferenceSession;
|
||||
runtimeProvider: LocalRuntimeProvider;
|
||||
modelConfig: LocalSeparatorDemucsModelConfig;
|
||||
audioBuffer: ArrayBuffer;
|
||||
chunkDurationSeconds: number | null;
|
||||
onProgress: (progress: LocalSeparatorProgress) => void;
|
||||
}): Promise<{
|
||||
stems: Array<{ name: string; blob: Blob }>;
|
||||
providerLabel: string;
|
||||
debugSummary: Record<string, unknown>;
|
||||
}> {
|
||||
const timing = new LocalSeparatorTimingCollector('local-demucs-separation');
|
||||
const decoded = await timing.measureAsync('decode', () => decodeAudioToStereo(options.audioBuffer));
|
||||
log(`Running browser Demucs separation on ${getProviderLabel(options.runtimeProvider)}...`);
|
||||
|
||||
const processor = new DemucsProcessor({
|
||||
ort,
|
||||
onProgress: ({ progress, currentSegment, totalSegments }) => {
|
||||
options.onProgress({
|
||||
stage: 'main',
|
||||
passLabel: 'Demucs pass',
|
||||
percent: progress * 100,
|
||||
processedChunks: currentSegment,
|
||||
totalChunks: totalSegments,
|
||||
});
|
||||
},
|
||||
onLog: (phase, message) => log(`demucs:${phase}`, message),
|
||||
});
|
||||
processor.session = options.session;
|
||||
|
||||
const outputMap = await separateDemucsWithOptionalChunking(
|
||||
processor,
|
||||
decoded,
|
||||
timing,
|
||||
getChunkDurationSeconds(options.chunkDurationSeconds, options.modelConfig),
|
||||
options.modelConfig,
|
||||
options.onProgress,
|
||||
);
|
||||
|
||||
const stems = options.modelConfig.outputStemNames.map(name => ({
|
||||
name,
|
||||
blob: timing.measureSync(`wavEncode:${name}`, () => channelsToWavBlob(outputMap[name as DemucsStemName])),
|
||||
}));
|
||||
|
||||
return {
|
||||
stems,
|
||||
providerLabel: getProviderLabel(options.runtimeProvider),
|
||||
debugSummary: timing.getSummary({
|
||||
model: options.modelConfig.filename,
|
||||
runtimeProvider: options.runtimeProvider,
|
||||
stemCount: stems.length,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async function separateDemucsWithOptionalChunking(
|
||||
processor: DemucsProcessor,
|
||||
decoded: StereoChannels,
|
||||
timing: LocalSeparatorTimingCollector,
|
||||
chunkDurationSeconds: number | null,
|
||||
modelConfig: LocalSeparatorDemucsModelConfig,
|
||||
onProgress: (progress: LocalSeparatorProgress) => void,
|
||||
): Promise<Record<DemucsStemName, StereoChannels>> {
|
||||
if (!chunkDurationSeconds) {
|
||||
return timing.measureAsync('demucsSeparate', async () => toDemucsStemMap(await processor.separate(decoded[0], decoded[1])));
|
||||
}
|
||||
|
||||
const chunkSamples = Math.max(1, Math.floor(chunkDurationSeconds * SAMPLE_RATE));
|
||||
if (decoded[0].length <= chunkSamples) {
|
||||
return timing.measureAsync('demucsSeparate', async () => toDemucsStemMap(await processor.separate(decoded[0], decoded[1])));
|
||||
}
|
||||
|
||||
const totalChunks = Math.ceil(decoded[0].length / chunkSamples);
|
||||
const chunkedOutputs: Record<DemucsStemName, StereoChannels[]> = {
|
||||
Vocals: [],
|
||||
Drums: [],
|
||||
Bass: [],
|
||||
Others: [],
|
||||
};
|
||||
|
||||
for (let index = 0; index < totalChunks; index += 1) {
|
||||
const start = index * chunkSamples;
|
||||
const end = Math.min(start + chunkSamples, decoded[0].length);
|
||||
const chunk = sliceChannels(decoded, start, end);
|
||||
|
||||
onProgress({
|
||||
stage: 'chunk-prep',
|
||||
passLabel: `Audio chunk ${index + 1}/${totalChunks}: preparing ${Math.round((end - start) / SAMPLE_RATE)}s chunk...`,
|
||||
percent: (index / totalChunks) * 100,
|
||||
processedChunks: index,
|
||||
totalChunks,
|
||||
});
|
||||
|
||||
const chunkResult = await timing.measureAsync(
|
||||
'demucsChunkedSeparate',
|
||||
async () => toDemucsStemMap(await processor.separate(chunk[0], chunk[1])),
|
||||
);
|
||||
|
||||
(Object.keys(chunkedOutputs) as DemucsStemName[]).forEach(stemName => {
|
||||
chunkedOutputs[stemName].push(chunkResult[stemName]);
|
||||
});
|
||||
|
||||
onProgress({
|
||||
stage: 'chunk-complete',
|
||||
passLabel: `Audio chunk ${index + 1}/${totalChunks}: Demucs pass`,
|
||||
percent: ((index + 1) / totalChunks) * 100,
|
||||
processedChunks: index + 1,
|
||||
totalChunks,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
Vocals: concatChannelPairs(chunkedOutputs.Vocals),
|
||||
Drums: concatChannelPairs(chunkedOutputs.Drums),
|
||||
Bass: concatChannelPairs(chunkedOutputs.Bass),
|
||||
Others: concatChannelPairs(chunkedOutputs.Others),
|
||||
};
|
||||
}
|
||||
|
||||
export async function runLocalSeparator(options: {
|
||||
session: ort.InferenceSession;
|
||||
runtimeProvider: LocalRuntimeProvider;
|
||||
modelConfig: LocalSeparatorModelConfig;
|
||||
audioBuffer: ArrayBuffer;
|
||||
chunkDurationSeconds: number | null;
|
||||
overlap: number;
|
||||
onProgress: (progress: LocalSeparatorProgress) => void;
|
||||
onProviderChange?: (provider: string) => void;
|
||||
}): Promise<{
|
||||
stems: Array<{ name: string; blob: Blob }>;
|
||||
providerLabel: string;
|
||||
debugSummary: Record<string, unknown>;
|
||||
}> {
|
||||
if (options.modelConfig.kind === 'demucs') {
|
||||
return runDemucsLocalSeparator({
|
||||
session: options.session,
|
||||
runtimeProvider: options.runtimeProvider,
|
||||
modelConfig: options.modelConfig,
|
||||
audioBuffer: options.audioBuffer,
|
||||
chunkDurationSeconds: options.chunkDurationSeconds,
|
||||
onProgress: options.onProgress,
|
||||
});
|
||||
}
|
||||
|
||||
return runMdxLocalSeparator({
|
||||
session: options.session,
|
||||
runtimeProvider: options.runtimeProvider,
|
||||
modelConfig: options.modelConfig,
|
||||
audioBuffer: options.audioBuffer,
|
||||
chunkDurationSeconds: options.chunkDurationSeconds,
|
||||
overlap: options.overlap,
|
||||
onProgress: options.onProgress,
|
||||
onProviderChange: options.onProviderChange,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
export type StereoChannels = [Float32Array, Float32Array];
|
||||
|
||||
export type LocalSeparatorModelId = 'UVR-MDX-NET-Inst_HQ_3.onnx' | 'htdemucs_4s.onnx';
|
||||
export type LocalSeparatorModelKind = 'mdx' | 'demucs';
|
||||
|
||||
export interface LocalSeparatorModelDefaults {
|
||||
sampleRate: number;
|
||||
hopLength: number;
|
||||
@@ -13,7 +16,7 @@ export interface LocalSeparatorModelDefaults {
|
||||
matchMixOverlap: number;
|
||||
}
|
||||
|
||||
export interface LocalSeparatorModelMetadata {
|
||||
export interface LocalSeparatorMdxMetadata {
|
||||
compensate: number;
|
||||
mdx_dim_f_set: number;
|
||||
mdx_dim_t_set: number;
|
||||
@@ -21,14 +24,36 @@ export interface LocalSeparatorModelMetadata {
|
||||
primary_stem: string;
|
||||
}
|
||||
|
||||
export interface LocalSeparatorModelConfig {
|
||||
export interface LocalSeparatorModelDownloadConfig {
|
||||
configKey: 'general.uvr5_web_runtime.mdx_net_model_url' | 'general.uvr5_web_runtime.htdemucs_4s_model_url';
|
||||
defaultUrl: string;
|
||||
expectedSizeBytes: number;
|
||||
}
|
||||
|
||||
interface LocalSeparatorModelConfigBase {
|
||||
id: LocalSeparatorModelId;
|
||||
filename: string;
|
||||
kind: LocalSeparatorModelKind;
|
||||
displayName: string;
|
||||
status: 'ready';
|
||||
outputStemNames: string[];
|
||||
defaultChunkDurationSeconds: number | null;
|
||||
defaults: LocalSeparatorModelDefaults;
|
||||
metadata: LocalSeparatorModelMetadata;
|
||||
download: LocalSeparatorModelDownloadConfig;
|
||||
}
|
||||
|
||||
export interface LocalSeparatorMdxModelConfig extends LocalSeparatorModelConfigBase {
|
||||
kind: 'mdx';
|
||||
metadata: LocalSeparatorMdxMetadata;
|
||||
}
|
||||
|
||||
export interface LocalSeparatorDemucsModelConfig extends LocalSeparatorModelConfigBase {
|
||||
kind: 'demucs';
|
||||
metadata: null;
|
||||
}
|
||||
|
||||
export type LocalSeparatorModelConfig = LocalSeparatorMdxModelConfig | LocalSeparatorDemucsModelConfig;
|
||||
|
||||
export interface LocalSeparatorProgress {
|
||||
stage: string;
|
||||
passLabel: string;
|
||||
|
||||
Reference in New Issue
Block a user