feat: added local separator model demucs_4s support
This commit is contained in:
@@ -328,4 +328,106 @@ SOFTWARE.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
### 11. htdemucs_4s
|
||||||
|
|
||||||
|
This project uses the htdemucs_4s model from the Demucs project by Facebook Research,
|
||||||
|
used for music source separation (stem extraction).
|
||||||
|
|
||||||
|
Original project: https://github.com/facebookresearch/demucs
|
||||||
|
Model weights (original, PyTorch): https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/955717e8-8726e21a.th
|
||||||
|
ONNX conversion (original): https://huggingface.co/timcsy/demucs-web-onnx/resolve/main/htdemucs_embedded.onnx
|
||||||
|
ONNX model downloaded by this project (unmodified copy of the above): https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx
|
||||||
|
|
||||||
|
```
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) Facebook, Inc. and its affiliates.
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this model and associated documentation files, to deal in the model without
|
||||||
|
restriction, including without limitation the rights to use, copy, modify,
|
||||||
|
merge, publish, distribute, sublicense, and/or sell copies of the model, and
|
||||||
|
to permit persons to whom the model is furnished to do so, subject to the
|
||||||
|
following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the model.
|
||||||
|
|
||||||
|
THE MODEL IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE MODEL OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
MODEL.
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 12. onnxruntime-web
|
||||||
|
|
||||||
|
onnxruntime-web is a JavaScript library for running ONNX models in the browser,
|
||||||
|
used for on-device neural network inference.
|
||||||
|
|
||||||
|
Original project: https://github.com/microsoft/onnxruntime
|
||||||
|
|
||||||
|
```
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 13. demucs-web
|
||||||
|
|
||||||
|
demucs-web is a browser-based wrapper for running Demucs stem separation models
|
||||||
|
in the browser via ONNX Runtime Web.
|
||||||
|
|
||||||
|
Original project: https://github.com/timcsy/demucs-web
|
||||||
|
|
||||||
|
```
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2024
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
Apache License Version 2.0, January 2004: http://www.apache.org/licenses/LICENSE-2.0
|
Apache License Version 2.0, January 2004: http://www.apache.org/licenses/LICENSE-2.0
|
||||||
@@ -391,4 +391,4 @@ Licensed under the Apache License, Version 2.0, with additional terms (see `LICE
|
|||||||
- No patent applications using this software or assets
|
- No patent applications using this software or assets
|
||||||
- Attribution required when used in public/commercial products (“Powered by K.G.Studio”)
|
- Attribution required when used in public/commercial products (“Powered by K.G.Studio”)
|
||||||
|
|
||||||
Third‑party notices (FluidR3_GM SoundFont, midi‑js‑soundfonts, VexFlow, prompt structure notes, Gemma 4 E4B, UVR-MDX-NET-Inst_HQ_3, MediaPipe, Meyda, web-audio-beat-detector, and tonal) are included in `LICENSE`.
|
Third‑party notices (FluidR3_GM SoundFont, midi‑js‑soundfonts, VexFlow, prompt structure notes, Gemma 4 E4B, UVR-MDX-NET-Inst_HQ_3, MediaPipe, Meyda, web-audio-beat-detector, tonal, htdemucs_4s, onnxruntime-web, and demucs-web) are included in `LICENSE`.
|
||||||
|
|||||||
Generated
+13
@@ -10,6 +10,7 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@breezystack/lamejs": "^1.2.7",
|
"@breezystack/lamejs": "^1.2.7",
|
||||||
"class-transformer": "^0.5.1",
|
"class-transformer": "^0.5.1",
|
||||||
|
"demucs-web": "^1.0.2",
|
||||||
"fft.js": "^4.0.4",
|
"fft.js": "^4.0.4",
|
||||||
"idb": "^8.0.3",
|
"idb": "^8.0.3",
|
||||||
"jszip": "^3.10.1",
|
"jszip": "^3.10.1",
|
||||||
@@ -4440,6 +4441,18 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/demucs-web": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/demucs-web/-/demucs-web-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-G/HWbbBDvfGzaxDZQhqTp4+6i8nlCIRGEd6E1+EZJJONx5Es3bBc+jHqhfyblTlsKo/7cAdylA8HfetBpbQ98w==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=16.0.0"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"onnxruntime-web": ">=1.17.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/deprecation": {
|
"node_modules/deprecation": {
|
||||||
"version": "2.3.1",
|
"version": "2.3.1",
|
||||||
"resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
|
"resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
|
||||||
|
|||||||
@@ -19,6 +19,7 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@breezystack/lamejs": "^1.2.7",
|
"@breezystack/lamejs": "^1.2.7",
|
||||||
"class-transformer": "^0.5.1",
|
"class-transformer": "^0.5.1",
|
||||||
|
"demucs-web": "^1.0.2",
|
||||||
"fft.js": "^4.0.4",
|
"fft.js": "^4.0.4",
|
||||||
"idb": "^8.0.3",
|
"idb": "^8.0.3",
|
||||||
"jszip": "^3.10.1",
|
"jszip": "^3.10.1",
|
||||||
|
|||||||
+2
-1
@@ -31,7 +31,8 @@
|
|||||||
"model_url": "https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task"
|
"model_url": "https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task"
|
||||||
},
|
},
|
||||||
"uvr5_web_runtime": {
|
"uvr5_web_runtime": {
|
||||||
"mdx_net_model_url": "https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx"
|
"mdx_net_model_url": "https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx",
|
||||||
|
"htdemucs_4s_model_url": "https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx"
|
||||||
},
|
},
|
||||||
"soundfont": {
|
"soundfont": {
|
||||||
"base_url": "https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/"
|
"base_url": "https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/"
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/react';
|
|||||||
import KGOnePanel from './KGOnePanel';
|
import KGOnePanel from './KGOnePanel';
|
||||||
import { KGAudioRegion } from '../core/region/KGAudioRegion';
|
import { KGAudioRegion } from '../core/region/KGAudioRegion';
|
||||||
import { KGAudioTrack } from '../core/track/KGAudioTrack';
|
import { KGAudioTrack } from '../core/track/KGAudioTrack';
|
||||||
|
import { LOCAL_SEPARATOR_MODEL_IDS } from '../util/local-separator/config';
|
||||||
|
|
||||||
const { mockLocalSeparatorDownload } = vi.hoisted(() => ({
|
const { mockLocalSeparatorDownload } = vi.hoisted(() => ({
|
||||||
mockLocalSeparatorDownload: vi.fn(async (_url?: string, _filename?: string, _onProgress?: unknown) => undefined),
|
mockLocalSeparatorDownload: vi.fn(async (_url?: string, _filename?: string, _onProgress?: unknown) => undefined),
|
||||||
@@ -11,7 +12,7 @@ const { mockLocalSeparatorDownload } = vi.hoisted(() => ({
|
|||||||
|
|
||||||
let kgoneEnabled = false;
|
let kgoneEnabled = false;
|
||||||
let selectedRegionIds: string[] = [];
|
let selectedRegionIds: string[] = [];
|
||||||
let localModelCached = false;
|
let localModelCached: Record<string, boolean> = {};
|
||||||
let localSeparationResult: Array<{ name: string; blob: Blob }> = [];
|
let localSeparationResult: Array<{ name: string; blob: Blob }> = [];
|
||||||
|
|
||||||
const mockRefreshProjectState = vi.fn();
|
const mockRefreshProjectState = vi.fn();
|
||||||
@@ -36,6 +37,7 @@ vi.mock('../core/config/ConfigManager', () => ({
|
|||||||
if (key === 'general.kgone.enabled') return kgoneEnabled;
|
if (key === 'general.kgone.enabled') return kgoneEnabled;
|
||||||
if (key === 'general.kgone.base_url') return 'http://127.0.0.1:8000';
|
if (key === 'general.kgone.base_url') return 'http://127.0.0.1:8000';
|
||||||
if (key === 'general.uvr5_web_runtime.mdx_net_model_url') return 'https://example.com/custom-uvr5.onnx';
|
if (key === 'general.uvr5_web_runtime.mdx_net_model_url') return 'https://example.com/custom-uvr5.onnx';
|
||||||
|
if (key === 'general.uvr5_web_runtime.htdemucs_4s_model_url') return 'https://example.com/custom-htdemucs.onnx';
|
||||||
return undefined;
|
return undefined;
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
@@ -83,13 +85,13 @@ vi.mock('../util/audioUtil', () => ({
|
|||||||
|
|
||||||
vi.mock('../util/local-separator/modelCache', () => ({
|
vi.mock('../util/local-separator/modelCache', () => ({
|
||||||
LocalSeparatorModelCache: {
|
LocalSeparatorModelCache: {
|
||||||
exists: vi.fn(async () => localModelCached),
|
exists: vi.fn(async (modelConfig: { id: string }) => localModelCached[modelConfig.id] ?? false),
|
||||||
download: vi.fn(async (url: string, filename: string, onProgress: (progress: unknown) => void) => {
|
download: vi.fn(async (modelConfig: { id: string; filename: string }, url: string, onProgress: (progress: unknown) => void) => {
|
||||||
localModelCached = true;
|
localModelCached[modelConfig.id] = true;
|
||||||
return mockLocalSeparatorDownload(url, filename, onProgress);
|
return mockLocalSeparatorDownload(modelConfig.filename, url, onProgress);
|
||||||
}),
|
}),
|
||||||
delete: vi.fn(async () => {
|
delete: vi.fn(async (modelConfig: { id: string }) => {
|
||||||
localModelCached = false;
|
localModelCached[modelConfig.id] = false;
|
||||||
}),
|
}),
|
||||||
getArrayBuffer: vi.fn(async () => new ArrayBuffer(16)),
|
getArrayBuffer: vi.fn(async () => new ArrayBuffer(16)),
|
||||||
},
|
},
|
||||||
@@ -125,7 +127,7 @@ describe('KGOnePanel local separator mode', () => {
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
kgoneEnabled = false;
|
kgoneEnabled = false;
|
||||||
selectedRegionIds = [];
|
selectedRegionIds = [];
|
||||||
localModelCached = false;
|
localModelCached = {};
|
||||||
localSeparationResult = [
|
localSeparationResult = [
|
||||||
{ name: 'Instrumental', blob: new Blob(['instrumental'], { type: 'audio/wav' }) },
|
{ name: 'Instrumental', blob: new Blob(['instrumental'], { type: 'audio/wav' }) },
|
||||||
{ name: 'Vocals', blob: new Blob(['vocals'], { type: 'audio/wav' }) },
|
{ name: 'Vocals', blob: new Blob(['vocals'], { type: 'audio/wav' }) },
|
||||||
@@ -143,41 +145,42 @@ describe('KGOnePanel local separator mode', () => {
|
|||||||
expect(screen.getByRole('button', { name: 'Remix' })).toBeDisabled();
|
expect(screen.getByRole('button', { name: 'Remix' })).toBeDisabled();
|
||||||
expect(screen.getByRole('button', { name: 'Repaint' })).toBeDisabled();
|
expect(screen.getByRole('button', { name: 'Repaint' })).toBeDisabled();
|
||||||
expect(screen.getByRole('button', { name: 'Separator' })).not.toBeDisabled();
|
expect(screen.getByRole('button', { name: 'Separator' })).not.toBeDisabled();
|
||||||
expect(screen.getByRole('button', { name: 'Download Model' })).toBeInTheDocument();
|
expect(screen.getByRole('button', { name: 'Download Selected Model' })).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('shows the single local separator model and advanced settings when the model is cached', async () => {
|
it('shows the single local separator model and advanced settings when the model is cached', async () => {
|
||||||
localModelCached = true;
|
localModelCached[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium] = true;
|
||||||
selectedRegionIds = ['audio-region-1'];
|
selectedRegionIds = ['audio-region-1'];
|
||||||
|
|
||||||
render(<KGOnePanel isVisible={true} />);
|
render(<KGOnePanel isVisible={true} />);
|
||||||
|
|
||||||
await screen.findByText('Selected Region');
|
await screen.findByText('Selected Region');
|
||||||
const options = await screen.findAllByRole('option');
|
const options = await screen.findAllByRole('option');
|
||||||
expect(options).toHaveLength(1);
|
expect(options).toHaveLength(2);
|
||||||
expect(options[0]).toHaveTextContent('Vocal and Instrument (Medium Accuracy)');
|
expect(options[0]).toHaveTextContent('Vocal and Instrument (Medium Accuracy)');
|
||||||
|
expect(options[1]).toHaveTextContent('Vocal, Drums, Bass, and Others');
|
||||||
|
|
||||||
fireEvent.click(screen.getByRole('button', { name: /Advanced Settings/i }));
|
fireEvent.click(screen.getByRole('button', { name: /Advanced Settings/i }));
|
||||||
expect(screen.getByLabelText('Optional audio chunk duration (seconds)')).toBeInTheDocument();
|
expect(screen.getByLabelText('Optional audio chunk duration (seconds)')).toBeInTheDocument();
|
||||||
expect(screen.getByLabelText('MDX overlap')).toBeInTheDocument();
|
expect(screen.getByLabelText('Model overlap')).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('uses the configured UVR5 model URL when downloading the local model', async () => {
|
it('uses the configured UVR5 model URL when downloading the local model', async () => {
|
||||||
render(<KGOnePanel isVisible={true} />);
|
render(<KGOnePanel isVisible={true} />);
|
||||||
|
|
||||||
fireEvent.click(await screen.findByRole('button', { name: 'Download Model' }));
|
fireEvent.click(await screen.findByRole('button', { name: 'Download Selected Model' }));
|
||||||
|
|
||||||
await waitFor(() => {
|
await waitFor(() => {
|
||||||
expect(mockLocalSeparatorDownload).toHaveBeenCalledWith(
|
expect(mockLocalSeparatorDownload).toHaveBeenCalledWith(
|
||||||
'https://example.com/custom-uvr5.onnx',
|
|
||||||
'UVR-MDX-NET-Inst_HQ_3.onnx',
|
'UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||||
|
'https://example.com/custom-uvr5.onnx',
|
||||||
expect.any(Function),
|
expect.any(Function),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('prompts for an audio region when the model is cached but nothing is selected', async () => {
|
it('prompts for an audio region when the model is cached but nothing is selected', async () => {
|
||||||
localModelCached = true;
|
localModelCached[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium] = true;
|
||||||
|
|
||||||
render(<KGOnePanel isVisible={true} />);
|
render(<KGOnePanel isVisible={true} />);
|
||||||
|
|
||||||
@@ -185,7 +188,7 @@ describe('KGOnePanel local separator mode', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('renders local separation outputs after processing completes', async () => {
|
it('renders local separation outputs after processing completes', async () => {
|
||||||
localModelCached = true;
|
localModelCached[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium] = true;
|
||||||
selectedRegionIds = ['audio-region-1'];
|
selectedRegionIds = ['audio-region-1'];
|
||||||
|
|
||||||
render(<KGOnePanel isVisible={true} />);
|
render(<KGOnePanel isVisible={true} />);
|
||||||
@@ -198,4 +201,36 @@ describe('KGOnePanel local separator mode', () => {
|
|||||||
expect(screen.getByRole('button', { name: 'Import All Stems to Timeline' })).toBeInTheDocument();
|
expect(screen.getByRole('button', { name: 'Import All Stems to Timeline' })).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('uses Demucs defaults and renders four local stem players', async () => {
|
||||||
|
localModelCached[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s] = true;
|
||||||
|
localSeparationResult = [
|
||||||
|
{ name: 'Vocals', blob: new Blob(['vocals'], { type: 'audio/wav' }) },
|
||||||
|
{ name: 'Drums', blob: new Blob(['drums'], { type: 'audio/wav' }) },
|
||||||
|
{ name: 'Bass', blob: new Blob(['bass'], { type: 'audio/wav' }) },
|
||||||
|
{ name: 'Others', blob: new Blob(['others'], { type: 'audio/wav' }) },
|
||||||
|
];
|
||||||
|
selectedRegionIds = ['audio-region-1'];
|
||||||
|
|
||||||
|
render(<KGOnePanel isVisible={true} />);
|
||||||
|
|
||||||
|
await screen.findByText('Selected Region');
|
||||||
|
fireEvent.change(screen.getByRole('combobox'), { target: { value: LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s } });
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: /Advanced Settings/i }));
|
||||||
|
|
||||||
|
await waitFor(() => {
|
||||||
|
expect((screen.getByLabelText('Optional audio chunk duration (seconds)') as HTMLInputElement).value).toBe('8');
|
||||||
|
expect((screen.getByLabelText('Model overlap') as HTMLInputElement).value).toBe('0.25');
|
||||||
|
});
|
||||||
|
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: 'Separate Stems' }));
|
||||||
|
|
||||||
|
await waitFor(() => {
|
||||||
|
expect(screen.getByText('Vocals')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('Drums')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('Bass')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('Others')).toBeInTheDocument();
|
||||||
|
expect(screen.getByRole('button', { name: 'Import All Stems to Timeline' })).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -16,13 +16,15 @@ import { ImportStemsCommand } from '../core/commands';
|
|||||||
import type { StemImportEntry } from '../core/commands';
|
import type { StemImportEntry } from '../core/commands';
|
||||||
import { showAlert } from '../util/dialogUtil';
|
import { showAlert } from '../util/dialogUtil';
|
||||||
import {
|
import {
|
||||||
LOCAL_SEPARATOR_MODEL_CONFIG,
|
getLocalSeparatorModelConfig,
|
||||||
LOCAL_SEPARATOR_MODEL_FILENAME,
|
LOCAL_SEPARATOR_MODELS,
|
||||||
LOCAL_SEPARATOR_DEFAULT_MODEL_URL,
|
LOCAL_SEPARATOR_MODEL_CONFIGS,
|
||||||
|
LOCAL_SEPARATOR_MODEL_IDS,
|
||||||
} from '../util/local-separator/config';
|
} from '../util/local-separator/config';
|
||||||
import { LocalSeparatorModelCache } from '../util/local-separator/modelCache';
|
import { LocalSeparatorModelCache } from '../util/local-separator/modelCache';
|
||||||
import { runLocalSeparator } from '../util/local-separator/runner';
|
import { runLocalSeparator } from '../util/local-separator/runner';
|
||||||
import { LocalOrtRuntimeManager, detectLocalRuntimeSupport } from '../util/local-separator/runtime';
|
import { LocalOrtRuntimeManager, detectLocalRuntimeSupport } from '../util/local-separator/runtime';
|
||||||
|
import type { LocalSeparatorModelConfig, LocalSeparatorModelId } from '../util/local-separator/types';
|
||||||
|
|
||||||
// ─── Types ────────────────────────────────────────────────────────────────────
|
// ─── Types ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -36,9 +38,10 @@ const SERVER_SEPARATOR_MODELS = [
|
|||||||
{ label: 'Vocal and Instrument (High Accuracy)', value: 'MDX23C-8KFFT-InstVoc_HQ.ckpt' },
|
{ label: 'Vocal and Instrument (High Accuracy)', value: 'MDX23C-8KFFT-InstVoc_HQ.ckpt' },
|
||||||
{ label: 'Vocal, Drums, Bass, Guitar, Piano, and Others', value: 'htdemucs_6s.yaml' },
|
{ label: 'Vocal, Drums, Bass, Guitar, Piano, and Others', value: 'htdemucs_6s.yaml' },
|
||||||
] as const;
|
] as const;
|
||||||
const LOCAL_SEPARATOR_MODELS = [
|
const LOCAL_SEPARATOR_MODEL_OPTIONS = LOCAL_SEPARATOR_MODELS.map(modelConfig => ({
|
||||||
{ label: LOCAL_SEPARATOR_MODEL_CONFIG.displayName, value: LOCAL_SEPARATOR_MODEL_FILENAME },
|
label: modelConfig.displayName,
|
||||||
] as const;
|
value: modelConfig.id,
|
||||||
|
})) as ReadonlyArray<{ label: string; value: LocalSeparatorModelId }>;
|
||||||
const KGONE_TABS = ['fullsong', 'remix', 'repaint', 'separator'] as const;
|
const KGONE_TABS = ['fullsong', 'remix', 'repaint', 'separator'] as const;
|
||||||
|
|
||||||
const CLIP_NOTES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'];
|
const CLIP_NOTES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'];
|
||||||
@@ -862,8 +865,11 @@ function countRepaintTracks(sourceTrackName: string): number {
|
|||||||
const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
||||||
const { selectedRegionIds, projectName, bpm, timeSignature, maxBars, refreshProjectState } = useProjectStore();
|
const { selectedRegionIds, projectName, bpm, timeSignature, maxBars, refreshProjectState } = useProjectStore();
|
||||||
const localOnlyMode = mode === 'local-separator';
|
const localOnlyMode = mode === 'local-separator';
|
||||||
const availableSeparatorModels = localOnlyMode ? LOCAL_SEPARATOR_MODELS : SERVER_SEPARATOR_MODELS;
|
const availableSeparatorModels = localOnlyMode ? LOCAL_SEPARATOR_MODEL_OPTIONS : SERVER_SEPARATOR_MODELS;
|
||||||
const [model, setModel] = useState<typeof SERVER_SEPARATOR_MODELS[number]['value']>(availableSeparatorModels[0].value);
|
const [model, setModel] = useState<string>(availableSeparatorModels[0].value);
|
||||||
|
const currentLocalModelConfig = useMemo<LocalSeparatorModelConfig>(() => {
|
||||||
|
return getLocalSeparatorModelConfig(model);
|
||||||
|
}, [model]);
|
||||||
|
|
||||||
// Generation state
|
// Generation state
|
||||||
const [genStatus, setGenStatus] = useState<GenStatus>('idle');
|
const [genStatus, setGenStatus] = useState<GenStatus>('idle');
|
||||||
@@ -881,7 +887,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
const [localProgressPercent, setLocalProgressPercent] = useState(0);
|
const [localProgressPercent, setLocalProgressPercent] = useState(0);
|
||||||
const [localProgressText, setLocalProgressText] = useState('');
|
const [localProgressText, setLocalProgressText] = useState('');
|
||||||
const [localChunkDurationSeconds, setLocalChunkDurationSeconds] = useState('');
|
const [localChunkDurationSeconds, setLocalChunkDurationSeconds] = useState('');
|
||||||
const [localOverlap, setLocalOverlap] = useState(String(LOCAL_SEPARATOR_MODEL_CONFIG.defaults.overlap));
|
const [localOverlap, setLocalOverlap] = useState(String(currentLocalModelConfig.defaults.overlap));
|
||||||
|
|
||||||
const abortRef = useRef<AbortController | null>(null);
|
const abortRef = useRef<AbortController | null>(null);
|
||||||
const taskIdRef = useRef<string>('');
|
const taskIdRef = useRef<string>('');
|
||||||
@@ -922,18 +928,28 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
setModel(availableSeparatorModels[0].value);
|
setModel(availableSeparatorModels[0].value);
|
||||||
}, [availableSeparatorModels]);
|
}, [availableSeparatorModels]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!localOnlyMode) return;
|
||||||
|
setLocalChunkDurationSeconds(
|
||||||
|
currentLocalModelConfig.defaultChunkDurationSeconds == null
|
||||||
|
? ''
|
||||||
|
: String(currentLocalModelConfig.defaultChunkDurationSeconds),
|
||||||
|
);
|
||||||
|
setLocalOverlap(String(currentLocalModelConfig.defaults.overlap));
|
||||||
|
}, [currentLocalModelConfig, localOnlyMode]);
|
||||||
|
|
||||||
const refreshLocalModelCacheState = useCallback(async () => {
|
const refreshLocalModelCacheState = useCallback(async () => {
|
||||||
if (!localOnlyMode) return;
|
if (!localOnlyMode) return;
|
||||||
setIsCheckingLocalModel(true);
|
setIsCheckingLocalModel(true);
|
||||||
try {
|
try {
|
||||||
setIsLocalModelCached(await LocalSeparatorModelCache.exists());
|
setIsLocalModelCached(await LocalSeparatorModelCache.exists(currentLocalModelConfig));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('[KGOne] Local model cache check failed:', err);
|
console.error('[KGOne] Local model cache check failed:', err);
|
||||||
setErrorMsg(err instanceof Error ? err.message : String(err));
|
setErrorMsg(err instanceof Error ? err.message : String(err));
|
||||||
} finally {
|
} finally {
|
||||||
setIsCheckingLocalModel(false);
|
setIsCheckingLocalModel(false);
|
||||||
}
|
}
|
||||||
}, [localOnlyMode]);
|
}, [currentLocalModelConfig, localOnlyMode]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!localOnlyMode) return;
|
if (!localOnlyMode) return;
|
||||||
@@ -960,11 +976,11 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
}, [selectedRegionIds]);
|
}, [selectedRegionIds]);
|
||||||
|
|
||||||
const getConfiguredLocalSeparatorModelUrl = useCallback(() => {
|
const getConfiguredLocalSeparatorModelUrl = useCallback(() => {
|
||||||
const configured = ConfigManager.instance().get('general.uvr5_web_runtime.mdx_net_model_url');
|
const configured = ConfigManager.instance().get(currentLocalModelConfig.download.configKey);
|
||||||
return typeof configured === 'string' && configured.trim()
|
return typeof configured === 'string' && configured.trim()
|
||||||
? configured
|
? configured
|
||||||
: LOCAL_SEPARATOR_DEFAULT_MODEL_URL;
|
: currentLocalModelConfig.download.defaultUrl;
|
||||||
}, []);
|
}, [currentLocalModelConfig]);
|
||||||
|
|
||||||
const isGenerating = genStatus !== 'idle' && genStatus !== 'done' && genStatus !== 'error';
|
const isGenerating = genStatus !== 'idle' && genStatus !== 'done' && genStatus !== 'error';
|
||||||
|
|
||||||
@@ -972,25 +988,25 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
setIsDownloadingLocalModel(true);
|
setIsDownloadingLocalModel(true);
|
||||||
setErrorMsg('');
|
setErrorMsg('');
|
||||||
setLocalProgressPercent(0);
|
setLocalProgressPercent(0);
|
||||||
setLocalProgressText('Downloading local separator model...');
|
setLocalProgressText(`Downloading ${currentLocalModelConfig.displayName}...`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await LocalSeparatorModelCache.download(
|
await LocalSeparatorModelCache.download(
|
||||||
|
currentLocalModelConfig,
|
||||||
getConfiguredLocalSeparatorModelUrl(),
|
getConfiguredLocalSeparatorModelUrl(),
|
||||||
LOCAL_SEPARATOR_MODEL_FILENAME,
|
|
||||||
progress => {
|
progress => {
|
||||||
const receivedMb = (progress.receivedBytes / (1024 * 1024)).toFixed(1);
|
const receivedMb = (progress.receivedBytes / (1024 * 1024)).toFixed(1);
|
||||||
const totalMb = progress.totalBytes ? (progress.totalBytes / (1024 * 1024)).toFixed(1) : null;
|
const totalMb = progress.totalBytes ? (progress.totalBytes / (1024 * 1024)).toFixed(1) : null;
|
||||||
setLocalProgressPercent(progress.totalBytes ? progress.percent : 0);
|
setLocalProgressPercent(progress.totalBytes ? progress.percent : 0);
|
||||||
setLocalProgressText(
|
setLocalProgressText(
|
||||||
totalMb
|
totalMb
|
||||||
? `Downloading local separator model... ${receivedMb} / ${totalMb} MB`
|
? `Downloading ${currentLocalModelConfig.displayName}... ${receivedMb} / ${totalMb} MB`
|
||||||
: `Downloading local separator model... ${receivedMb} MB`,
|
: `Downloading ${currentLocalModelConfig.displayName}... ${receivedMb} MB`,
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
setLocalProgressPercent(100);
|
setLocalProgressPercent(100);
|
||||||
setLocalProgressText('Local separator model is ready.');
|
setLocalProgressText(`${currentLocalModelConfig.displayName} is ready.`);
|
||||||
await refreshLocalModelCacheState();
|
await refreshLocalModelCacheState();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setLocalProgressPercent(0);
|
setLocalProgressPercent(0);
|
||||||
@@ -999,13 +1015,13 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
} finally {
|
} finally {
|
||||||
setIsDownloadingLocalModel(false);
|
setIsDownloadingLocalModel(false);
|
||||||
}
|
}
|
||||||
}, [getConfiguredLocalSeparatorModelUrl, refreshLocalModelCacheState]);
|
}, [currentLocalModelConfig, getConfiguredLocalSeparatorModelUrl, refreshLocalModelCacheState]);
|
||||||
|
|
||||||
const handleDeleteLocalModel = useCallback(async () => {
|
const handleDeleteLocalModel = useCallback(async () => {
|
||||||
setIsDeletingLocalModel(true);
|
setIsDeletingLocalModel(true);
|
||||||
setErrorMsg('');
|
setErrorMsg('');
|
||||||
try {
|
try {
|
||||||
await LocalSeparatorModelCache.delete();
|
await LocalSeparatorModelCache.delete(currentLocalModelConfig);
|
||||||
localRuntimeManagerRef.current?.reset();
|
localRuntimeManagerRef.current?.reset();
|
||||||
setLocalProviderLabel(runtimeSupport.webgpuExposed ? 'webgpu available' : 'cpu/wasm only');
|
setLocalProviderLabel(runtimeSupport.webgpuExposed ? 'webgpu available' : 'cpu/wasm only');
|
||||||
setLocalProgressPercent(0);
|
setLocalProgressPercent(0);
|
||||||
@@ -1016,7 +1032,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
} finally {
|
} finally {
|
||||||
setIsDeletingLocalModel(false);
|
setIsDeletingLocalModel(false);
|
||||||
}
|
}
|
||||||
}, [refreshLocalModelCacheState, runtimeSupport.webgpuExposed]);
|
}, [currentLocalModelConfig, refreshLocalModelCacheState, runtimeSupport.webgpuExposed]);
|
||||||
|
|
||||||
const handleSeparateServer = useCallback(async () => {
|
const handleSeparateServer = useCallback(async () => {
|
||||||
if (!selectedAudioRegion) return;
|
if (!selectedAudioRegion) return;
|
||||||
@@ -1212,12 +1228,12 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
setLocalProviderLabel(runtimeSupport.webgpuExposed ? 'webgpu available' : 'cpu/wasm only');
|
setLocalProviderLabel(runtimeSupport.webgpuExposed ? 'webgpu available' : 'cpu/wasm only');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const modelBuffer = await LocalSeparatorModelCache.getArrayBuffer();
|
const modelBuffer = await LocalSeparatorModelCache.getArrayBuffer(currentLocalModelConfig);
|
||||||
const runtimeManager = localRuntimeManagerRef.current ?? new LocalOrtRuntimeManager({
|
const runtimeManager = localRuntimeManagerRef.current ?? new LocalOrtRuntimeManager({
|
||||||
onProviderChange: provider => setLocalProviderLabel(provider),
|
onProviderChange: provider => setLocalProviderLabel(provider),
|
||||||
});
|
});
|
||||||
localRuntimeManagerRef.current = runtimeManager;
|
localRuntimeManagerRef.current = runtimeManager;
|
||||||
const runtime = await runtimeManager.ensureRuntime(LOCAL_SEPARATOR_MODEL_CONFIG, new Uint8Array(modelBuffer));
|
const runtime = await runtimeManager.ensureRuntime(currentLocalModelConfig, new Uint8Array(modelBuffer));
|
||||||
|
|
||||||
setGenStatus('generating');
|
setGenStatus('generating');
|
||||||
setLocalProgressPercent(3);
|
setLocalProgressPercent(3);
|
||||||
@@ -1247,10 +1263,10 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
const result = await runLocalSeparator({
|
const result = await runLocalSeparator({
|
||||||
session: runtime.session,
|
session: runtime.session,
|
||||||
runtimeProvider: runtime.provider,
|
runtimeProvider: runtime.provider,
|
||||||
modelConfig: LOCAL_SEPARATOR_MODEL_CONFIG,
|
modelConfig: currentLocalModelConfig,
|
||||||
audioBuffer: inputBuffer,
|
audioBuffer: inputBuffer,
|
||||||
chunkDurationSeconds: Number.isFinite(chunkDuration) && (chunkDuration ?? 0) > 0 ? chunkDuration : null,
|
chunkDurationSeconds: Number.isFinite(chunkDuration) && (chunkDuration ?? 0) > 0 ? chunkDuration : null,
|
||||||
overlap: Number.isFinite(overlapValue) ? overlapValue : LOCAL_SEPARATOR_MODEL_CONFIG.defaults.overlap,
|
overlap: Number.isFinite(overlapValue) ? overlapValue : currentLocalModelConfig.defaults.overlap,
|
||||||
onProviderChange: provider => setLocalProviderLabel(provider),
|
onProviderChange: provider => setLocalProviderLabel(provider),
|
||||||
onProgress: progress => {
|
onProgress: progress => {
|
||||||
setLocalProgressPercent(progress.percent);
|
setLocalProgressPercent(progress.percent);
|
||||||
@@ -1283,6 +1299,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
projectName,
|
projectName,
|
||||||
localChunkDurationSeconds,
|
localChunkDurationSeconds,
|
||||||
localOverlap,
|
localOverlap,
|
||||||
|
currentLocalModelConfig,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const handleSeparate = useCallback(async () => {
|
const handleSeparate = useCallback(async () => {
|
||||||
@@ -1381,14 +1398,19 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
<div className="kgone-local-mode-card">
|
<div className="kgone-local-mode-card">
|
||||||
<div className="kgone-local-mode-title">Local Separator Mode</div>
|
<div className="kgone-local-mode-title">Local Separator Mode</div>
|
||||||
<div className="kgone-local-mode-text">
|
<div className="kgone-local-mode-text">
|
||||||
Only Vocal and Instrument (Medium Accuracy) is available while not integrated with K.G.One Music Studio server.
|
If K.G.One Music Studio is unavailable, Local Separator Mode provides a built-in alternative for extracting stems
|
||||||
Processing in local may take long time depending on your hardware. When fallback to CPU happens, the webpage may
|
directly in your browser. Two local models are available: Vocal and Instrument (Medium Accuracy), and Vocal,
|
||||||
temporarily hang with little or no UI response until processing advances.{' '}
|
Drums, Bass, and Others. Download status below reflects the currently selected model. Vocal and Instrument
|
||||||
|
(Medium Accuracy) usually takes longer to process than Vocal, Drums, Bass, and Others, and total processing time
|
||||||
|
will still depend on your hardware. If processing falls back to CPU, the page may become temporarily less
|
||||||
|
responsive while separation is running.{' '}
|
||||||
<a href="https://github.com/KGAudioLab/K.G.One" target="_blank" rel="noopener noreferrer">Learn more about K.G.One Music Studio server integration.</a>
|
<a href="https://github.com/KGAudioLab/K.G.One" target="_blank" rel="noopener noreferrer">Learn more about K.G.One Music Studio server integration.</a>
|
||||||
</div>
|
</div>
|
||||||
<div className="kgone-runtime-row">
|
<div className="kgone-runtime-row">
|
||||||
<div className="kgone-provider-chip">Provider: {localProviderLabel}</div>
|
<div className="kgone-provider-chip">Provider: {localProviderLabel}</div>
|
||||||
<div className="kgone-provider-chip">Model: {isLocalModelCached ? 'downloaded' : 'not downloaded'}</div>
|
<div className="kgone-provider-chip">
|
||||||
|
Model: {currentLocalModelConfig.displayName} ({isLocalModelCached ? 'downloaded' : 'not downloaded'})
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{(localProgressText || isCheckingLocalModel) && (
|
{(localProgressText || isCheckingLocalModel) && (
|
||||||
<div className="kgone-progress-block">
|
<div className="kgone-progress-block">
|
||||||
@@ -1414,7 +1436,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
disabled={isCheckingLocalModel || isDownloadingLocalModel || isDeletingLocalModel || isGenerating}
|
disabled={isCheckingLocalModel || isDownloadingLocalModel || isDeletingLocalModel || isGenerating}
|
||||||
onClick={() => void handleDownloadLocalModel()}
|
onClick={() => void handleDownloadLocalModel()}
|
||||||
>
|
>
|
||||||
{isDownloadingLocalModel ? 'Downloading Model...' : 'Download Model'}
|
{isDownloadingLocalModel ? 'Downloading Model...' : 'Download Selected Model'}
|
||||||
</button>
|
</button>
|
||||||
) : (
|
) : (
|
||||||
<>
|
<>
|
||||||
@@ -1451,7 +1473,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
|
|
||||||
<div className="kgone-field">
|
<div className="kgone-field">
|
||||||
<label className="kgone-label">Separation Model</label>
|
<label className="kgone-label">Separation Model</label>
|
||||||
<select className="kgone-select" value={model} onChange={e => setModel(e.target.value as typeof SERVER_SEPARATOR_MODELS[number]['value'])}>
|
<select className="kgone-select" value={model} onChange={e => setModel(e.target.value)}>
|
||||||
{availableSeparatorModels.map(m => (
|
{availableSeparatorModels.map(m => (
|
||||||
<option key={m.value} value={m.value}>{m.label}</option>
|
<option key={m.value} value={m.value}>{m.label}</option>
|
||||||
))}
|
))}
|
||||||
@@ -1474,10 +1496,10 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
<div className="kgone-field">
|
<div className="kgone-field">
|
||||||
<label className="kgone-label">MDX overlap</label>
|
<label className="kgone-label">Model overlap</label>
|
||||||
<input
|
<input
|
||||||
className="kgone-input"
|
className="kgone-input"
|
||||||
aria-label="MDX overlap"
|
aria-label="Model overlap"
|
||||||
type="number"
|
type="number"
|
||||||
min={0.001}
|
min={0.001}
|
||||||
max={0.999}
|
max={0.999}
|
||||||
@@ -1552,7 +1574,7 @@ const SeparatorTab: React.FC<{ mode: KGOneMode }> = ({ mode }) => {
|
|||||||
) : (
|
) : (
|
||||||
<div className="kgone-separator-hint">
|
<div className="kgone-separator-hint">
|
||||||
{localOnlyMode && !isLocalModelCached
|
{localOnlyMode && !isLocalModelCached
|
||||||
? 'Download the local separator model, then select an audio region on the timeline to extract stems from it.'
|
? `Download ${currentLocalModelConfig.displayName}, then select an audio region on the timeline to extract stems from it.`
|
||||||
: 'Select an audio region on the timeline to extract stems from it. Only audio regions are supported — MIDI regions cannot be separated.'}
|
: 'Select an audio region on the timeline to extract stems from it. Only audio regions are supported — MIDI regions cannot be separated.'}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ const configState = new Map<string, unknown>([
|
|||||||
['general.local_browser.context_length', 65536],
|
['general.local_browser.context_length', 65536],
|
||||||
['general.local_browser.model_url', 'https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task'],
|
['general.local_browser.model_url', 'https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task'],
|
||||||
['general.uvr5_web_runtime.mdx_net_model_url', 'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx'],
|
['general.uvr5_web_runtime.mdx_net_model_url', 'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx'],
|
||||||
|
['general.uvr5_web_runtime.htdemucs_4s_model_url', 'https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx'],
|
||||||
['general.soundfont.base_url', 'https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/'],
|
['general.soundfont.base_url', 'https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/'],
|
||||||
['general.kgone.enabled', false],
|
['general.kgone.enabled', false],
|
||||||
['general.kgone.base_url', 'http://127.0.0.1:8000'],
|
['general.kgone.base_url', 'http://127.0.0.1:8000'],
|
||||||
@@ -140,6 +141,7 @@ describe('GeneralSettings', () => {
|
|||||||
|
|
||||||
expect(await screen.findByDisplayValue('https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task')).toBeTruthy();
|
expect(await screen.findByDisplayValue('https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task')).toBeTruthy();
|
||||||
expect(screen.getByDisplayValue('https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx')).toBeTruthy();
|
expect(screen.getByDisplayValue('https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx')).toBeTruthy();
|
||||||
|
expect(screen.getByDisplayValue('https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx')).toBeTruthy();
|
||||||
|
|
||||||
const inputs = screen.getAllByRole('textbox');
|
const inputs = screen.getAllByRole('textbox');
|
||||||
const gemmaUrlInput = inputs.find(input =>
|
const gemmaUrlInput = inputs.find(input =>
|
||||||
@@ -148,22 +150,30 @@ describe('GeneralSettings', () => {
|
|||||||
const uvr5UrlInput = inputs.find(input =>
|
const uvr5UrlInput = inputs.find(input =>
|
||||||
(input as HTMLInputElement).value.includes('UVR-MDX-NET-Inst_HQ_3.onnx'),
|
(input as HTMLInputElement).value.includes('UVR-MDX-NET-Inst_HQ_3.onnx'),
|
||||||
) as HTMLInputElement | undefined;
|
) as HTMLInputElement | undefined;
|
||||||
|
const htdemucsUrlInput = inputs.find(input =>
|
||||||
|
(input as HTMLInputElement).value.includes('htdemucs_embedded.onnx'),
|
||||||
|
) as HTMLInputElement | undefined;
|
||||||
|
|
||||||
expect(gemmaUrlInput).toBeTruthy();
|
expect(gemmaUrlInput).toBeTruthy();
|
||||||
expect(uvr5UrlInput).toBeTruthy();
|
expect(uvr5UrlInput).toBeTruthy();
|
||||||
|
expect(htdemucsUrlInput).toBeTruthy();
|
||||||
|
|
||||||
fireEvent.change(gemmaUrlInput!, { target: { value: 'https://example.com/gemma.task' } });
|
fireEvent.change(gemmaUrlInput!, { target: { value: 'https://example.com/gemma.task' } });
|
||||||
fireEvent.change(uvr5UrlInput!, { target: { value: 'https://example.com/uvr5.onnx' } });
|
fireEvent.change(uvr5UrlInput!, { target: { value: 'https://example.com/uvr5.onnx' } });
|
||||||
|
fireEvent.change(htdemucsUrlInput!, { target: { value: 'https://example.com/htdemucs.onnx' } });
|
||||||
|
|
||||||
await waitFor(() => {
|
await waitFor(() => {
|
||||||
expect(configManagerMock.set).toHaveBeenCalledWith('general.local_browser.model_url', 'https://example.com/gemma.task');
|
expect(configManagerMock.set).toHaveBeenCalledWith('general.local_browser.model_url', 'https://example.com/gemma.task');
|
||||||
expect(configManagerMock.set).toHaveBeenCalledWith('general.uvr5_web_runtime.mdx_net_model_url', 'https://example.com/uvr5.onnx');
|
expect(configManagerMock.set).toHaveBeenCalledWith('general.uvr5_web_runtime.mdx_net_model_url', 'https://example.com/uvr5.onnx');
|
||||||
|
expect(configManagerMock.set).toHaveBeenCalledWith('general.uvr5_web_runtime.htdemucs_4s_model_url', 'https://example.com/htdemucs.onnx');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('restores default download URLs and deletes the UVR5 model cache', async () => {
|
it('restores default download URLs and deletes the UVR5 model cache', async () => {
|
||||||
localSeparatorModelCacheMock.exists
|
localSeparatorModelCacheMock.exists
|
||||||
.mockResolvedValueOnce(true)
|
.mockResolvedValueOnce(true)
|
||||||
|
.mockResolvedValueOnce(true)
|
||||||
|
.mockResolvedValueOnce(false)
|
||||||
.mockResolvedValueOnce(false);
|
.mockResolvedValueOnce(false);
|
||||||
|
|
||||||
render(<GeneralSettings />);
|
render(<GeneralSettings />);
|
||||||
@@ -173,6 +183,7 @@ describe('GeneralSettings', () => {
|
|||||||
const restoreLinks = screen.getAllByText('Restore default');
|
const restoreLinks = screen.getAllByText('Restore default');
|
||||||
fireEvent.click(restoreLinks[0]);
|
fireEvent.click(restoreLinks[0]);
|
||||||
fireEvent.click(restoreLinks[1]);
|
fireEvent.click(restoreLinks[1]);
|
||||||
|
fireEvent.click(restoreLinks[2]);
|
||||||
const uvr5DeleteButton = screen.getAllByRole('button', { name: 'Delete Cached Model' })[1];
|
const uvr5DeleteButton = screen.getAllByRole('button', { name: 'Delete Cached Model' })[1];
|
||||||
expect(uvr5DeleteButton).not.toBeDisabled();
|
expect(uvr5DeleteButton).not.toBeDisabled();
|
||||||
fireEvent.click(uvr5DeleteButton);
|
fireEvent.click(uvr5DeleteButton);
|
||||||
@@ -186,6 +197,10 @@ describe('GeneralSettings', () => {
|
|||||||
'general.uvr5_web_runtime.mdx_net_model_url',
|
'general.uvr5_web_runtime.mdx_net_model_url',
|
||||||
'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx',
|
'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||||
);
|
);
|
||||||
|
expect(configManagerMock.set).toHaveBeenCalledWith(
|
||||||
|
'general.uvr5_web_runtime.htdemucs_4s_model_url',
|
||||||
|
'https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx',
|
||||||
|
);
|
||||||
expect(localSeparatorModelCacheMock.delete).toHaveBeenCalled();
|
expect(localSeparatorModelCacheMock.delete).toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,10 @@ import {
|
|||||||
normalizeLocalLLMContextLength,
|
normalizeLocalLLMContextLength,
|
||||||
type LocalLLMContextLength,
|
type LocalLLMContextLength,
|
||||||
} from '../../../util/localLLMConfig';
|
} from '../../../util/localLLMConfig';
|
||||||
import { LOCAL_SEPARATOR_DEFAULT_MODEL_URL } from '../../../util/local-separator/config';
|
import {
|
||||||
|
LOCAL_SEPARATOR_MODEL_CONFIGS,
|
||||||
|
LOCAL_SEPARATOR_MODEL_IDS,
|
||||||
|
} from '../../../util/local-separator/config';
|
||||||
|
|
||||||
const GeneralSettings: React.FC = () => {
|
const GeneralSettings: React.FC = () => {
|
||||||
const [llmProvider, setLlmProvider] = useState<string>(LOCAL_LLM_PROVIDER_KEY);
|
const [llmProvider, setLlmProvider] = useState<string>(LOCAL_LLM_PROVIDER_KEY);
|
||||||
@@ -39,9 +42,12 @@ const GeneralSettings: React.FC = () => {
|
|||||||
const [localModelState, setLocalModelState] = useState<LocalLLMModelState>(LocalLLMModelManager.getState());
|
const [localModelState, setLocalModelState] = useState<LocalLLMModelState>(LocalLLMModelManager.getState());
|
||||||
const [localModelUrl, setLocalModelUrl] = useState<string>('');
|
const [localModelUrl, setLocalModelUrl] = useState<string>('');
|
||||||
const [uvr5ModelUrl, setUvr5ModelUrl] = useState<string>('');
|
const [uvr5ModelUrl, setUvr5ModelUrl] = useState<string>('');
|
||||||
|
const [htdemucsModelUrl, setHtdemucsModelUrl] = useState<string>('');
|
||||||
const [isUvr5ModelCached, setIsUvr5ModelCached] = useState<boolean>(false);
|
const [isUvr5ModelCached, setIsUvr5ModelCached] = useState<boolean>(false);
|
||||||
const [isCheckingUvr5ModelCache, setIsCheckingUvr5ModelCache] = useState<boolean>(false);
|
const [isCheckingUvr5ModelCache, setIsCheckingUvr5ModelCache] = useState<boolean>(false);
|
||||||
const [isDeletingUvr5Model, setIsDeletingUvr5Model] = useState<boolean>(false);
|
const [isDeletingUvr5Model, setIsDeletingUvr5Model] = useState<boolean>(false);
|
||||||
|
const [isHtdemucsModelCached, setIsHtdemucsModelCached] = useState<boolean>(false);
|
||||||
|
const [isDeletingHtdemucsModel, setIsDeletingHtdemucsModel] = useState<boolean>(false);
|
||||||
|
|
||||||
const configManager = ConfigManager.instance();
|
const configManager = ConfigManager.instance();
|
||||||
|
|
||||||
@@ -62,10 +68,16 @@ const GeneralSettings: React.FC = () => {
|
|||||||
const refreshUvr5ModelCacheState = useCallback(async () => {
|
const refreshUvr5ModelCacheState = useCallback(async () => {
|
||||||
setIsCheckingUvr5ModelCache(true);
|
setIsCheckingUvr5ModelCache(true);
|
||||||
try {
|
try {
|
||||||
setIsUvr5ModelCached(await LocalSeparatorModelCache.exists());
|
const [mdxCached, demucsCached] = await Promise.all([
|
||||||
|
LocalSeparatorModelCache.exists(LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium]),
|
||||||
|
LocalSeparatorModelCache.exists(LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s]),
|
||||||
|
]);
|
||||||
|
setIsUvr5ModelCached(mdxCached);
|
||||||
|
setIsHtdemucsModelCached(demucsCached);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to check UVR5 cached model state:', error);
|
console.error('Failed to check UVR5 cached model state:', error);
|
||||||
setIsUvr5ModelCached(false);
|
setIsUvr5ModelCached(false);
|
||||||
|
setIsHtdemucsModelCached(false);
|
||||||
} finally {
|
} finally {
|
||||||
setIsCheckingUvr5ModelCache(false);
|
setIsCheckingUvr5ModelCache(false);
|
||||||
}
|
}
|
||||||
@@ -95,7 +107,14 @@ const GeneralSettings: React.FC = () => {
|
|||||||
setCompatibleModel((configManager.get('general.openai_compatible.model') as string) || '');
|
setCompatibleModel((configManager.get('general.openai_compatible.model') as string) || '');
|
||||||
setLocalContextLength(normalizeLocalLLMContextLength(configManager.get('general.local_browser.context_length')));
|
setLocalContextLength(normalizeLocalLLMContextLength(configManager.get('general.local_browser.context_length')));
|
||||||
setLocalModelUrl((configManager.get('general.local_browser.model_url') as string) || LOCAL_LLM_DEFAULT_MODEL_URL);
|
setLocalModelUrl((configManager.get('general.local_browser.model_url') as string) || LOCAL_LLM_DEFAULT_MODEL_URL);
|
||||||
setUvr5ModelUrl((configManager.get('general.uvr5_web_runtime.mdx_net_model_url') as string) || LOCAL_SEPARATOR_DEFAULT_MODEL_URL);
|
setUvr5ModelUrl(
|
||||||
|
(configManager.get('general.uvr5_web_runtime.mdx_net_model_url') as string)
|
||||||
|
|| LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium].download.defaultUrl,
|
||||||
|
);
|
||||||
|
setHtdemucsModelUrl(
|
||||||
|
(configManager.get('general.uvr5_web_runtime.htdemucs_4s_model_url') as string)
|
||||||
|
|| LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s].download.defaultUrl,
|
||||||
|
);
|
||||||
setSoundfontBaseUrl((configManager.get('general.soundfont.base_url') as string) || '');
|
setSoundfontBaseUrl((configManager.get('general.soundfont.base_url') as string) || '');
|
||||||
setKgoneEnabled((configManager.get('general.kgone.enabled') as boolean) ?? false);
|
setKgoneEnabled((configManager.get('general.kgone.enabled') as boolean) ?? false);
|
||||||
setKgoneBaseUrl((configManager.get('general.kgone.base_url') as string) || '');
|
setKgoneBaseUrl((configManager.get('general.kgone.base_url') as string) || '');
|
||||||
@@ -245,6 +264,11 @@ const GeneralSettings: React.FC = () => {
|
|||||||
debouncedSave('general.uvr5_web_runtime.mdx_net_model_url', value);
|
debouncedSave('general.uvr5_web_runtime.mdx_net_model_url', value);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const handleHtdemucsModelUrlChange = (value: string) => {
|
||||||
|
setHtdemucsModelUrl(value);
|
||||||
|
debouncedSave('general.uvr5_web_runtime.htdemucs_4s_model_url', value);
|
||||||
|
};
|
||||||
|
|
||||||
const handleDeleteLocalModel = async () => {
|
const handleDeleteLocalModel = async () => {
|
||||||
try {
|
try {
|
||||||
await LocalLLMModelManager.deleteCachedModel();
|
await LocalLLMModelManager.deleteCachedModel();
|
||||||
@@ -256,7 +280,7 @@ const GeneralSettings: React.FC = () => {
|
|||||||
const handleDeleteUvr5Model = async () => {
|
const handleDeleteUvr5Model = async () => {
|
||||||
setIsDeletingUvr5Model(true);
|
setIsDeletingUvr5Model(true);
|
||||||
try {
|
try {
|
||||||
await LocalSeparatorModelCache.delete();
|
await LocalSeparatorModelCache.delete(LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium]);
|
||||||
setIsUvr5ModelCached(false);
|
setIsUvr5ModelCached(false);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to delete UVR5 cached model:', error);
|
console.error('Failed to delete UVR5 cached model:', error);
|
||||||
@@ -266,6 +290,19 @@ const GeneralSettings: React.FC = () => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const handleDeleteHtdemucsModel = async () => {
|
||||||
|
setIsDeletingHtdemucsModel(true);
|
||||||
|
try {
|
||||||
|
await LocalSeparatorModelCache.delete(LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s]);
|
||||||
|
setIsHtdemucsModelCached(false);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to delete HTDemucs cached model:', error);
|
||||||
|
} finally {
|
||||||
|
setIsDeletingHtdemucsModel(false);
|
||||||
|
await refreshUvr5ModelCacheState();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const handleLocalContextLengthChange = async (value: string) => {
|
const handleLocalContextLengthChange = async (value: string) => {
|
||||||
const parsed = Number(value);
|
const parsed = Number(value);
|
||||||
const normalized = normalizeLocalLLMContextLength(parsed);
|
const normalized = normalizeLocalLLMContextLength(parsed);
|
||||||
@@ -448,7 +485,7 @@ const GeneralSettings: React.FC = () => {
|
|||||||
<input
|
<input
|
||||||
type="text"
|
type="text"
|
||||||
className="settings-input"
|
className="settings-input"
|
||||||
placeholder={`e.g. ${LOCAL_SEPARATOR_DEFAULT_MODEL_URL}`}
|
placeholder={`e.g. ${LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium].download.defaultUrl}`}
|
||||||
value={uvr5ModelUrl}
|
value={uvr5ModelUrl}
|
||||||
onChange={(e) => handleUvr5ModelUrlChange(e.target.value)}
|
onChange={(e) => handleUvr5ModelUrlChange(e.target.value)}
|
||||||
/>
|
/>
|
||||||
@@ -458,7 +495,9 @@ const GeneralSettings: React.FC = () => {
|
|||||||
href="#"
|
href="#"
|
||||||
onClick={(e) => {
|
onClick={(e) => {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
handleUvr5ModelUrlChange(LOCAL_SEPARATOR_DEFAULT_MODEL_URL);
|
handleUvr5ModelUrlChange(
|
||||||
|
LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium].download.defaultUrl,
|
||||||
|
);
|
||||||
}}
|
}}
|
||||||
style={{ color: '#5a9fd4', textDecoration: 'underline', cursor: 'pointer' }}
|
style={{ color: '#5a9fd4', textDecoration: 'underline', cursor: 'pointer' }}
|
||||||
>
|
>
|
||||||
@@ -477,6 +516,45 @@ const GeneralSettings: React.FC = () => {
|
|||||||
{isDeletingUvr5Model ? 'Deleting...' : 'Delete Cached Model'}
|
{isDeletingUvr5Model ? 'Deleting...' : 'Delete Cached Model'}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div className="settings-item">
|
||||||
|
<label className="settings-label">
|
||||||
|
htdemucs_4s Download URL
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
className="settings-input"
|
||||||
|
placeholder={`e.g. ${LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s].download.defaultUrl}`}
|
||||||
|
value={htdemucsModelUrl}
|
||||||
|
onChange={(e) => handleHtdemucsModelUrlChange(e.target.value)}
|
||||||
|
/>
|
||||||
|
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
|
||||||
|
Changing this URL may break downloads or point to an incompatible model file.{' '}
|
||||||
|
<a
|
||||||
|
href="#"
|
||||||
|
onClick={(e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
handleHtdemucsModelUrlChange(
|
||||||
|
LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s].download.defaultUrl,
|
||||||
|
);
|
||||||
|
}}
|
||||||
|
style={{ color: '#5a9fd4', textDecoration: 'underline', cursor: 'pointer' }}
|
||||||
|
>
|
||||||
|
Restore default
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="settings-item" style={{ marginTop: '12px' }}>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="settings-btn settings-btn-danger"
|
||||||
|
onClick={() => void handleDeleteHtdemucsModel()}
|
||||||
|
disabled={isCheckingUvr5ModelCache || isDeletingHtdemucsModel || !isHtdemucsModelCached}
|
||||||
|
>
|
||||||
|
{isDeletingHtdemucsModel ? 'Deleting...' : 'Delete Cached Model'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="settings-group">
|
<div className="settings-group">
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ interface AppConfig {
|
|||||||
};
|
};
|
||||||
uvr5_web_runtime: {
|
uvr5_web_runtime: {
|
||||||
mdx_net_model_url: string;
|
mdx_net_model_url: string;
|
||||||
|
htdemucs_4s_model_url: string;
|
||||||
};
|
};
|
||||||
openai: {
|
openai: {
|
||||||
api_key: string;
|
api_key: string;
|
||||||
@@ -224,7 +225,8 @@ export class ConfigManager {
|
|||||||
model_url: 'https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task'
|
model_url: 'https://huggingface.co/notabilia/gemma-4-E4B-it-litert-lm/resolve/main/gemma-4-E4B-it-web.task'
|
||||||
},
|
},
|
||||||
uvr5_web_runtime: {
|
uvr5_web_runtime: {
|
||||||
mdx_net_model_url: 'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx'
|
mdx_net_model_url: 'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||||
|
htdemucs_4s_model_url: 'https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx',
|
||||||
},
|
},
|
||||||
soundfont: {
|
soundfont: {
|
||||||
base_url: 'https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/'
|
base_url: 'https://cdn.jsdelivr.net/npm/soundfont-for-samplers/FluidR3_GM/'
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
import { LocalSeparatorModelCache } from '../../util/local-separator/modelCache';
|
import { LocalSeparatorModelCache } from '../../util/local-separator/modelCache';
|
||||||
import {
|
import {
|
||||||
LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES,
|
LOCAL_SEPARATOR_MODEL_CONFIGS,
|
||||||
LOCAL_SEPARATOR_MODEL_FILENAME,
|
LOCAL_SEPARATOR_MODEL_IDS,
|
||||||
} from '../../util/local-separator/config';
|
} from '../../util/local-separator/config';
|
||||||
|
|
||||||
class MockWritableFileStream {
|
class MockWritableFileStream {
|
||||||
@@ -113,7 +113,9 @@ vi.stubGlobal('navigator', {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('LocalSeparatorModelCache', () => {
|
describe('LocalSeparatorModelCache', () => {
|
||||||
const makeModelBytes = (fill: number): Uint8Array => new Uint8Array(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES).fill(fill);
|
const mdxConfig = LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium];
|
||||||
|
const demucsConfig = LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s];
|
||||||
|
const makeModelBytes = (size: number, fill: number): Uint8Array => new Uint8Array(size).fill(fill);
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
mockRoot.clear();
|
mockRoot.clear();
|
||||||
@@ -121,64 +123,64 @@ describe('LocalSeparatorModelCache', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('downloads and stores a model in OPFS cache', async () => {
|
it('downloads and stores a model in OPFS cache', async () => {
|
||||||
const bytes = makeModelBytes(1);
|
const bytes = makeModelBytes(mdxConfig.download.expectedSizeBytes, 1);
|
||||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(bytes, {
|
vi.stubGlobal('fetch', vi.fn(async () => new Response(bytes, {
|
||||||
status: 200,
|
status: 200,
|
||||||
headers: { 'Content-Length': String(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES) },
|
headers: { 'Content-Length': String(mdxConfig.download.expectedSizeBytes) },
|
||||||
})));
|
})));
|
||||||
|
|
||||||
await LocalSeparatorModelCache.download('https://example.com/model.onnx');
|
await LocalSeparatorModelCache.download(mdxConfig, 'https://example.com/model.onnx');
|
||||||
|
|
||||||
expect(await LocalSeparatorModelCache.exists()).toBe(true);
|
expect(await LocalSeparatorModelCache.exists(mdxConfig)).toBe(true);
|
||||||
const buffer = await LocalSeparatorModelCache.getArrayBuffer();
|
const buffer = await LocalSeparatorModelCache.getArrayBuffer(mdxConfig);
|
||||||
expect(buffer.byteLength).toBe(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES);
|
expect(buffer.byteLength).toBe(mdxConfig.download.expectedSizeBytes);
|
||||||
expect(new Uint8Array(buffer)[0]).toBe(1);
|
expect(new Uint8Array(buffer)[0]).toBe(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('replaces a broken cached file on redownload', async () => {
|
it('replaces a broken cached file on redownload', async () => {
|
||||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(1), {
|
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(mdxConfig.download.expectedSizeBytes, 1), {
|
||||||
status: 200,
|
status: 200,
|
||||||
headers: { 'Content-Length': String(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES) },
|
headers: { 'Content-Length': String(mdxConfig.download.expectedSizeBytes) },
|
||||||
})));
|
})));
|
||||||
await LocalSeparatorModelCache.download('https://example.com/model.onnx');
|
await LocalSeparatorModelCache.download(mdxConfig, 'https://example.com/model.onnx');
|
||||||
|
|
||||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(9), {
|
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(mdxConfig.download.expectedSizeBytes, 9), {
|
||||||
status: 200,
|
status: 200,
|
||||||
headers: { 'Content-Length': String(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES) },
|
headers: { 'Content-Length': String(mdxConfig.download.expectedSizeBytes) },
|
||||||
})));
|
})));
|
||||||
await LocalSeparatorModelCache.download('https://example.com/model.onnx');
|
await LocalSeparatorModelCache.download(mdxConfig, 'https://example.com/model.onnx');
|
||||||
|
|
||||||
const buffer = await LocalSeparatorModelCache.getArrayBuffer();
|
const buffer = await LocalSeparatorModelCache.getArrayBuffer(mdxConfig);
|
||||||
expect(buffer.byteLength).toBe(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES);
|
expect(buffer.byteLength).toBe(mdxConfig.download.expectedSizeBytes);
|
||||||
expect(new Uint8Array(buffer)[0]).toBe(9);
|
expect(new Uint8Array(buffer)[0]).toBe(9);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('deletes the cached model file', async () => {
|
it('deletes the cached model file', async () => {
|
||||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(2), {
|
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(mdxConfig.download.expectedSizeBytes, 2), {
|
||||||
status: 200,
|
status: 200,
|
||||||
headers: { 'Content-Length': String(LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES) },
|
headers: { 'Content-Length': String(mdxConfig.download.expectedSizeBytes) },
|
||||||
})));
|
})));
|
||||||
await LocalSeparatorModelCache.download('https://example.com/model.onnx');
|
await LocalSeparatorModelCache.download(mdxConfig, 'https://example.com/model.onnx');
|
||||||
|
|
||||||
await LocalSeparatorModelCache.delete();
|
await LocalSeparatorModelCache.delete(mdxConfig);
|
||||||
|
|
||||||
expect(await LocalSeparatorModelCache.exists()).toBe(false);
|
expect(await LocalSeparatorModelCache.exists(mdxConfig)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('rejects and deletes a cached file when the size is wrong', async () => {
|
it('rejects and deletes a cached file when the size is wrong', async () => {
|
||||||
const dir = await navigator.storage.getDirectory();
|
const dir = await navigator.storage.getDirectory();
|
||||||
const modelsDir = await dir.getDirectoryHandle('models', { create: true });
|
const modelsDir = await dir.getDirectoryHandle('models', { create: true });
|
||||||
const fileHandle = await modelsDir.getFileHandle(LOCAL_SEPARATOR_MODEL_FILENAME, { create: true });
|
const fileHandle = await modelsDir.getFileHandle(mdxConfig.filename, { create: true });
|
||||||
const fileWritable = await fileHandle.createWritable();
|
const fileWritable = await fileHandle.createWritable();
|
||||||
await fileWritable.write(new Uint8Array([1, 2, 3]));
|
await fileWritable.write(new Uint8Array([1, 2, 3]));
|
||||||
await fileWritable.close();
|
await fileWritable.close();
|
||||||
|
|
||||||
const sizeHandle = await modelsDir.getFileHandle(`${LOCAL_SEPARATOR_MODEL_FILENAME}.size`, { create: true });
|
const sizeHandle = await modelsDir.getFileHandle(`${mdxConfig.filename}.size`, { create: true });
|
||||||
const sizeWritable = await sizeHandle.createWritable();
|
const sizeWritable = await sizeHandle.createWritable();
|
||||||
await sizeWritable.write(String(3));
|
await sizeWritable.write(String(3));
|
||||||
await sizeWritable.close();
|
await sizeWritable.close();
|
||||||
|
|
||||||
expect(await LocalSeparatorModelCache.exists()).toBe(false);
|
expect(await LocalSeparatorModelCache.exists(mdxConfig)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('fails a download when the final size does not match the expected model size', async () => {
|
it('fails a download when the final size does not match the expected model size', async () => {
|
||||||
@@ -187,7 +189,19 @@ describe('LocalSeparatorModelCache', () => {
|
|||||||
headers: { 'Content-Length': '3' },
|
headers: { 'Content-Length': '3' },
|
||||||
})));
|
})));
|
||||||
|
|
||||||
await expect(LocalSeparatorModelCache.download('https://example.com/model.onnx')).rejects.toThrow(/size mismatch/i);
|
await expect(LocalSeparatorModelCache.download(mdxConfig, 'https://example.com/model.onnx')).rejects.toThrow(/size mismatch/i);
|
||||||
expect(await LocalSeparatorModelCache.exists()).toBe(false);
|
expect(await LocalSeparatorModelCache.exists(mdxConfig)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('tracks cached files independently per model', async () => {
|
||||||
|
vi.stubGlobal('fetch', vi.fn(async () => new Response(makeModelBytes(demucsConfig.download.expectedSizeBytes, 7), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Length': String(demucsConfig.download.expectedSizeBytes) },
|
||||||
|
})));
|
||||||
|
|
||||||
|
await LocalSeparatorModelCache.download(demucsConfig, 'https://example.com/htdemucs.onnx');
|
||||||
|
|
||||||
|
expect(await LocalSeparatorModelCache.exists(demucsConfig)).toBe(true);
|
||||||
|
expect(await LocalSeparatorModelCache.exists(mdxConfig)).toBe(false);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Vendored
+35
@@ -0,0 +1,35 @@
|
|||||||
|
declare module 'demucs-web' {
|
||||||
|
export interface DemucsTrackOutput {
|
||||||
|
left: Float32Array;
|
||||||
|
right: Float32Array;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DemucsSeparationResult {
|
||||||
|
drums: DemucsTrackOutput;
|
||||||
|
bass: DemucsTrackOutput;
|
||||||
|
other: DemucsTrackOutput;
|
||||||
|
vocals: DemucsTrackOutput;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DemucsProgressInfo {
|
||||||
|
progress: number;
|
||||||
|
currentSegment: number;
|
||||||
|
totalSegments: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DemucsProcessorOptions {
|
||||||
|
ort: typeof import('onnxruntime-web/webgpu');
|
||||||
|
modelPath?: string;
|
||||||
|
sessionOptions?: import('onnxruntime-web/webgpu').InferenceSession.SessionOptions;
|
||||||
|
onProgress?: (info: DemucsProgressInfo) => void;
|
||||||
|
onLog?: (phase: string, message: string) => void;
|
||||||
|
onDownloadProgress?: (loaded: number, total: number) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DemucsProcessor {
|
||||||
|
constructor(options?: DemucsProcessorOptions);
|
||||||
|
session: import('onnxruntime-web/webgpu').InferenceSession | null;
|
||||||
|
loadModel(pathOrBuffer?: string | ArrayBuffer): Promise<import('onnxruntime-web/webgpu').InferenceSession>;
|
||||||
|
separate(left: Float32Array, right: Float32Array): Promise<DemucsSeparationResult>;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,32 +1,81 @@
|
|||||||
import type { LocalSeparatorModelConfig } from './types';
|
import type { LocalSeparatorModelConfig, LocalSeparatorModelId } from './types';
|
||||||
|
|
||||||
export const LOCAL_SEPARATOR_DEFAULT_MODEL_URL =
|
export const LOCAL_SEPARATOR_MODEL_IDS = {
|
||||||
'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx';
|
mdxMedium: 'UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||||
|
htdemucs4s: 'htdemucs_4s.onnx',
|
||||||
|
} as const satisfies Record<string, LocalSeparatorModelId>;
|
||||||
|
|
||||||
export const LOCAL_SEPARATOR_MODEL_FILENAME = 'UVR-MDX-NET-Inst_HQ_3.onnx';
|
export const LOCAL_SEPARATOR_DEFAULT_MODEL_ID = LOCAL_SEPARATOR_MODEL_IDS.mdxMedium;
|
||||||
export const LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES = 66759214;
|
|
||||||
|
|
||||||
export const LOCAL_SEPARATOR_MODEL_CONFIG: LocalSeparatorModelConfig = {
|
export const LOCAL_SEPARATOR_MODEL_CONFIGS: Record<LocalSeparatorModelId, LocalSeparatorModelConfig> = {
|
||||||
filename: LOCAL_SEPARATOR_MODEL_FILENAME,
|
[LOCAL_SEPARATOR_MODEL_IDS.mdxMedium]: {
|
||||||
displayName: 'Vocal and Instrument (Medium Accuracy)',
|
id: LOCAL_SEPARATOR_MODEL_IDS.mdxMedium,
|
||||||
status: 'ready',
|
filename: 'UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||||
defaults: {
|
kind: 'mdx',
|
||||||
sampleRate: 44100,
|
displayName: 'Vocal and Instrument (Medium Accuracy)',
|
||||||
hopLength: 1024,
|
status: 'ready',
|
||||||
segmentSize: 256,
|
outputStemNames: ['Instrumental', 'Vocals'],
|
||||||
overlap: 0.25,
|
defaultChunkDurationSeconds: null,
|
||||||
batchSize: 1,
|
defaults: {
|
||||||
enableDenoise: false,
|
sampleRate: 44100,
|
||||||
invertUsingSpec: false,
|
hopLength: 1024,
|
||||||
normalizationThreshold: 0.9,
|
segmentSize: 256,
|
||||||
amplificationThreshold: 0,
|
overlap: 0.25,
|
||||||
matchMixOverlap: 0.02,
|
batchSize: 1,
|
||||||
|
enableDenoise: false,
|
||||||
|
invertUsingSpec: false,
|
||||||
|
normalizationThreshold: 0.9,
|
||||||
|
amplificationThreshold: 0,
|
||||||
|
matchMixOverlap: 0.02,
|
||||||
|
},
|
||||||
|
metadata: {
|
||||||
|
compensate: 1.021,
|
||||||
|
mdx_dim_f_set: 3072,
|
||||||
|
mdx_dim_t_set: 8,
|
||||||
|
mdx_n_fft_scale_set: 7680,
|
||||||
|
primary_stem: 'Instrumental',
|
||||||
|
},
|
||||||
|
download: {
|
||||||
|
configKey: 'general.uvr5_web_runtime.mdx_net_model_url',
|
||||||
|
defaultUrl: 'https://huggingface.co/notabilia/uvr5-models/resolve/main/UVR-MDX-NET-Inst_HQ_3.onnx',
|
||||||
|
expectedSizeBytes: 66759214,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
metadata: {
|
[LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s]: {
|
||||||
compensate: 1.021,
|
id: LOCAL_SEPARATOR_MODEL_IDS.htdemucs4s,
|
||||||
mdx_dim_f_set: 3072,
|
filename: 'htdemucs_4s.onnx',
|
||||||
mdx_dim_t_set: 8,
|
kind: 'demucs',
|
||||||
mdx_n_fft_scale_set: 7680,
|
displayName: 'Vocal, Drums, Bass, and Others',
|
||||||
primary_stem: 'Instrumental',
|
status: 'ready',
|
||||||
|
outputStemNames: ['Vocals', 'Drums', 'Bass', 'Others'],
|
||||||
|
defaultChunkDurationSeconds: 8,
|
||||||
|
defaults: {
|
||||||
|
sampleRate: 44100,
|
||||||
|
hopLength: 1024,
|
||||||
|
segmentSize: 256,
|
||||||
|
overlap: 0.25,
|
||||||
|
batchSize: 1,
|
||||||
|
enableDenoise: false,
|
||||||
|
invertUsingSpec: false,
|
||||||
|
normalizationThreshold: 0.9,
|
||||||
|
amplificationThreshold: 0,
|
||||||
|
matchMixOverlap: 0.02,
|
||||||
|
},
|
||||||
|
metadata: null,
|
||||||
|
download: {
|
||||||
|
configKey: 'general.uvr5_web_runtime.htdemucs_4s_model_url',
|
||||||
|
defaultUrl: 'https://huggingface.co/notabilia/uvr5-models/resolve/main/htdemucs_embedded.onnx',
|
||||||
|
expectedSizeBytes: 180534758,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const LOCAL_SEPARATOR_MODELS = Object.values(LOCAL_SEPARATOR_MODEL_CONFIGS);
|
||||||
|
|
||||||
|
export function getLocalSeparatorModelConfig(modelId: string): LocalSeparatorModelConfig {
|
||||||
|
const config = LOCAL_SEPARATOR_MODEL_CONFIGS[modelId as LocalSeparatorModelId];
|
||||||
|
if (!config) {
|
||||||
|
return LOCAL_SEPARATOR_MODEL_CONFIGS[LOCAL_SEPARATOR_DEFAULT_MODEL_ID];
|
||||||
|
}
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import type { LocalSeparatorModelConfig, StereoChannels } from './types';
|
import type { LocalSeparatorMdxModelConfig, StereoChannels } from './types';
|
||||||
import { FFT, createWindowCache, getHannPeriodic, index4d, reflectPad } from './shared';
|
import { FFT, createWindowCache, getHannPeriodic, index4d, reflectPad } from './shared';
|
||||||
|
|
||||||
interface SpectrogramPayload {
|
interface SpectrogramPayload {
|
||||||
@@ -18,7 +18,7 @@ export class LocalSeparatorCpuDsp {
|
|||||||
private readonly numFreqBins: number;
|
private readonly numFreqBins: number;
|
||||||
private readonly inverseFft: FFT;
|
private readonly inverseFft: FFT;
|
||||||
|
|
||||||
constructor(config: LocalSeparatorModelConfig) {
|
constructor(config: LocalSeparatorMdxModelConfig) {
|
||||||
this.nFft = config.metadata.mdx_n_fft_scale_set;
|
this.nFft = config.metadata.mdx_n_fft_scale_set;
|
||||||
this.hopLength = config.defaults.hopLength;
|
this.hopLength = config.defaults.hopLength;
|
||||||
this.dimF = config.metadata.mdx_dim_f_set;
|
this.dimF = config.metadata.mdx_dim_f_set;
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import type { LocalSeparatorModelConfig } from './types';
|
import type { LocalSeparatorMdxModelConfig } from './types';
|
||||||
import { LocalSeparatorCpuDsp } from './cpuDsp';
|
import { LocalSeparatorCpuDsp } from './cpuDsp';
|
||||||
import { reflectPad } from './shared';
|
import { reflectPad } from './shared';
|
||||||
|
|
||||||
@@ -140,7 +140,7 @@ export class LocalSeparatorGpuDsp {
|
|||||||
private paramBuffer: GPUBufferLike | null = null;
|
private paramBuffer: GPUBufferLike | null = null;
|
||||||
private readonly pipeline: GPUComputePipelineLike;
|
private readonly pipeline: GPUComputePipelineLike;
|
||||||
|
|
||||||
public static async create(config: LocalSeparatorModelConfig): Promise<LocalSeparatorGpuDsp> {
|
public static async create(config: LocalSeparatorMdxModelConfig): Promise<LocalSeparatorGpuDsp> {
|
||||||
if (!('gpu' in navigator)) {
|
if (!('gpu' in navigator)) {
|
||||||
throw new Error('WebGPU is not available for GPU DSP.');
|
throw new Error('WebGPU is not available for GPU DSP.');
|
||||||
}
|
}
|
||||||
@@ -164,7 +164,7 @@ export class LocalSeparatorGpuDsp {
|
|||||||
return new LocalSeparatorGpuDsp(config, device);
|
return new LocalSeparatorGpuDsp(config, device);
|
||||||
}
|
}
|
||||||
|
|
||||||
private constructor(config: LocalSeparatorModelConfig, device: GPUDeviceLike) {
|
private constructor(config: LocalSeparatorMdxModelConfig, device: GPUDeviceLike) {
|
||||||
this.device = device;
|
this.device = device;
|
||||||
this.cpuDsp = new LocalSeparatorCpuDsp(config);
|
this.cpuDsp = new LocalSeparatorCpuDsp(config);
|
||||||
this.nFft = config.metadata.mdx_n_fft_scale_set;
|
this.nFft = config.metadata.mdx_n_fft_scale_set;
|
||||||
|
|||||||
@@ -1,7 +1,4 @@
|
|||||||
import {
|
import type { LocalSeparatorModelConfig } from './types';
|
||||||
LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES,
|
|
||||||
LOCAL_SEPARATOR_MODEL_FILENAME,
|
|
||||||
} from './config';
|
|
||||||
import { OpfsModelCache, type ModelDownloadProgress } from '../opfsModelCache';
|
import { OpfsModelCache, type ModelDownloadProgress } from '../opfsModelCache';
|
||||||
|
|
||||||
const cache = new OpfsModelCache({ directoryName: 'models' });
|
const cache = new OpfsModelCache({ directoryName: 'models' });
|
||||||
@@ -9,34 +6,34 @@ const cache = new OpfsModelCache({ directoryName: 'models' });
|
|||||||
export { type ModelDownloadProgress };
|
export { type ModelDownloadProgress };
|
||||||
|
|
||||||
export class LocalSeparatorModelCache {
|
export class LocalSeparatorModelCache {
|
||||||
public static async exists(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<boolean> {
|
public static async exists(modelConfig: LocalSeparatorModelConfig): Promise<boolean> {
|
||||||
return cache.exists(filename, {
|
return cache.exists(modelConfig.filename, {
|
||||||
expectedSizeBytes: LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES,
|
expectedSizeBytes: modelConfig.download.expectedSizeBytes,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async getFile(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<File> {
|
public static async getFile(modelConfig: LocalSeparatorModelConfig): Promise<File> {
|
||||||
return cache.getFile(filename);
|
return cache.getFile(modelConfig.filename);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async getArrayBuffer(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<ArrayBuffer> {
|
public static async getArrayBuffer(modelConfig: LocalSeparatorModelConfig): Promise<ArrayBuffer> {
|
||||||
return cache.getArrayBuffer(filename);
|
return cache.getArrayBuffer(modelConfig.filename);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async delete(filename: string = LOCAL_SEPARATOR_MODEL_FILENAME): Promise<void> {
|
public static async delete(modelConfig: LocalSeparatorModelConfig): Promise<void> {
|
||||||
await cache.delete(filename);
|
await cache.delete(modelConfig.filename);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async download(
|
public static async download(
|
||||||
|
modelConfig: LocalSeparatorModelConfig,
|
||||||
sourceUrl: string,
|
sourceUrl: string,
|
||||||
filename: string = LOCAL_SEPARATOR_MODEL_FILENAME,
|
|
||||||
onProgress?: (progress: ModelDownloadProgress) => void,
|
onProgress?: (progress: ModelDownloadProgress) => void,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
await cache.download(
|
await cache.download(
|
||||||
sourceUrl,
|
sourceUrl,
|
||||||
filename,
|
modelConfig.filename,
|
||||||
{
|
{
|
||||||
expectedSizeBytes: LOCAL_SEPARATOR_MODEL_EXPECTED_SIZE_BYTES,
|
expectedSizeBytes: modelConfig.download.expectedSizeBytes,
|
||||||
},
|
},
|
||||||
onProgress,
|
onProgress,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,10 +1,13 @@
|
|||||||
import * as ort from 'onnxruntime-web/webgpu';
|
import * as ort from 'onnxruntime-web/webgpu';
|
||||||
|
import { DemucsProcessor } from 'demucs-web';
|
||||||
import { LocalSeparatorCpuDsp } from './cpuDsp';
|
import { LocalSeparatorCpuDsp } from './cpuDsp';
|
||||||
import { LocalSeparatorGpuDsp } from './gpuDsp';
|
import { LocalSeparatorGpuDsp } from './gpuDsp';
|
||||||
import { LocalSeparatorTimingCollector } from './timing';
|
import { LocalSeparatorTimingCollector } from './timing';
|
||||||
import type {
|
import type {
|
||||||
LocalRuntimeProvider,
|
LocalRuntimeProvider,
|
||||||
|
LocalSeparatorDemucsModelConfig,
|
||||||
LocalSeparatorModelConfig,
|
LocalSeparatorModelConfig,
|
||||||
|
LocalSeparatorMdxModelConfig,
|
||||||
LocalSeparatorProgress,
|
LocalSeparatorProgress,
|
||||||
StereoChannels,
|
StereoChannels,
|
||||||
} from './types';
|
} from './types';
|
||||||
@@ -76,8 +79,8 @@ function unpackBatchOutput(outputData: Float32Array, batchInfo: { dims: number[]
|
|||||||
class BrowserMdxSeparator {
|
class BrowserMdxSeparator {
|
||||||
private readonly session: ort.InferenceSession;
|
private readonly session: ort.InferenceSession;
|
||||||
private readonly runtimeProvider: LocalRuntimeProvider;
|
private readonly runtimeProvider: LocalRuntimeProvider;
|
||||||
private readonly defaults: LocalSeparatorModelConfig['defaults'];
|
private readonly defaults: LocalSeparatorMdxModelConfig['defaults'];
|
||||||
private readonly metadata: LocalSeparatorModelConfig['metadata'];
|
private readonly metadata: LocalSeparatorMdxModelConfig['metadata'];
|
||||||
public onProgress: (progress: LocalSeparatorProgress) => void;
|
public onProgress: (progress: LocalSeparatorProgress) => void;
|
||||||
private overlap: number;
|
private overlap: number;
|
||||||
private runtimeBatchSize: number;
|
private runtimeBatchSize: number;
|
||||||
@@ -97,7 +100,7 @@ class BrowserMdxSeparator {
|
|||||||
public static async create(
|
public static async create(
|
||||||
session: ort.InferenceSession,
|
session: ort.InferenceSession,
|
||||||
runtimeProvider: LocalRuntimeProvider,
|
runtimeProvider: LocalRuntimeProvider,
|
||||||
config: LocalSeparatorModelConfig,
|
config: LocalSeparatorMdxModelConfig,
|
||||||
options: BrowserMdxSeparatorOptions = {},
|
options: BrowserMdxSeparatorOptions = {},
|
||||||
): Promise<BrowserMdxSeparator> {
|
): Promise<BrowserMdxSeparator> {
|
||||||
const timing = options.timing ?? new LocalSeparatorTimingCollector('mdx-separation');
|
const timing = options.timing ?? new LocalSeparatorTimingCollector('mdx-separation');
|
||||||
@@ -139,7 +142,7 @@ class BrowserMdxSeparator {
|
|||||||
private constructor(
|
private constructor(
|
||||||
session: ort.InferenceSession,
|
session: ort.InferenceSession,
|
||||||
runtimeProvider: LocalRuntimeProvider,
|
runtimeProvider: LocalRuntimeProvider,
|
||||||
config: LocalSeparatorModelConfig,
|
config: LocalSeparatorMdxModelConfig,
|
||||||
options: BrowserMdxSeparatorOptions & {
|
options: BrowserMdxSeparatorOptions & {
|
||||||
dsp: LocalSeparatorCpuDsp | LocalSeparatorGpuDsp;
|
dsp: LocalSeparatorCpuDsp | LocalSeparatorGpuDsp;
|
||||||
dspMode: 'cpu' | 'gpu-hybrid';
|
dspMode: 'cpu' | 'gpu-hybrid';
|
||||||
@@ -470,10 +473,24 @@ function concatChannelPairs(chunks: StereoChannels[]): StereoChannels {
|
|||||||
return [concatFloat32(chunks.map(chunk => chunk[0])), concatFloat32(chunks.map(chunk => chunk[1]))];
|
return [concatFloat32(chunks.map(chunk => chunk[0])), concatFloat32(chunks.map(chunk => chunk[1]))];
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function runLocalSeparator(options: {
|
function getProviderLabel(runtimeProvider: LocalRuntimeProvider): string {
|
||||||
|
return runtimeProvider === 'webgpu' ? 'GPU/WebGPU' : 'CPU/wasm';
|
||||||
|
}
|
||||||
|
|
||||||
|
function getChunkDurationSeconds(
|
||||||
|
requestedChunkDurationSeconds: number | null,
|
||||||
|
modelConfig: LocalSeparatorModelConfig,
|
||||||
|
): number | null {
|
||||||
|
if (requestedChunkDurationSeconds != null) {
|
||||||
|
return requestedChunkDurationSeconds;
|
||||||
|
}
|
||||||
|
return modelConfig.defaultChunkDurationSeconds;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runMdxLocalSeparator(options: {
|
||||||
session: ort.InferenceSession;
|
session: ort.InferenceSession;
|
||||||
runtimeProvider: LocalRuntimeProvider;
|
runtimeProvider: LocalRuntimeProvider;
|
||||||
modelConfig: LocalSeparatorModelConfig;
|
modelConfig: LocalSeparatorMdxModelConfig;
|
||||||
audioBuffer: ArrayBuffer;
|
audioBuffer: ArrayBuffer;
|
||||||
chunkDurationSeconds: number | null;
|
chunkDurationSeconds: number | null;
|
||||||
overlap: number;
|
overlap: number;
|
||||||
@@ -486,7 +503,7 @@ export async function runLocalSeparator(options: {
|
|||||||
}> {
|
}> {
|
||||||
const timing = new LocalSeparatorTimingCollector('local-separation');
|
const timing = new LocalSeparatorTimingCollector('local-separation');
|
||||||
const decoded = await timing.measureAsync('decode', () => decodeAudioToStereo(options.audioBuffer));
|
const decoded = await timing.measureAsync('decode', () => decodeAudioToStereo(options.audioBuffer));
|
||||||
log(`Running browser MDX separation on ${options.runtimeProvider === 'webgpu' ? 'GPU/WebGPU' : 'CPU/wasm'}...`);
|
log(`Running browser MDX separation on ${getProviderLabel(options.runtimeProvider)}...`);
|
||||||
|
|
||||||
const separator = await BrowserMdxSeparator.create(
|
const separator = await BrowserMdxSeparator.create(
|
||||||
options.session,
|
options.session,
|
||||||
@@ -502,11 +519,11 @@ export async function runLocalSeparator(options: {
|
|||||||
);
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const outputs = await separateWithOptionalChunking(
|
const outputs = await separateMdxWithOptionalChunking(
|
||||||
separator,
|
separator,
|
||||||
decoded,
|
decoded,
|
||||||
timing,
|
timing,
|
||||||
options.chunkDurationSeconds,
|
getChunkDurationSeconds(options.chunkDurationSeconds, options.modelConfig),
|
||||||
options.modelConfig,
|
options.modelConfig,
|
||||||
options.onProgress,
|
options.onProgress,
|
||||||
);
|
);
|
||||||
@@ -518,7 +535,7 @@ export async function runLocalSeparator(options: {
|
|||||||
{ name: outputs.primaryStem, blob: primaryBlob },
|
{ name: outputs.primaryStem, blob: primaryBlob },
|
||||||
{ name: outputs.secondaryStem, blob: secondaryBlob },
|
{ name: outputs.secondaryStem, blob: secondaryBlob },
|
||||||
],
|
],
|
||||||
providerLabel: options.runtimeProvider === 'webgpu' ? 'GPU/WebGPU' : 'CPU/wasm',
|
providerLabel: getProviderLabel(options.runtimeProvider),
|
||||||
debugSummary: separator.getDebugSummary({ model: options.modelConfig.filename }),
|
debugSummary: separator.getDebugSummary({ model: options.modelConfig.filename }),
|
||||||
};
|
};
|
||||||
} finally {
|
} finally {
|
||||||
@@ -527,12 +544,12 @@ export async function runLocalSeparator(options: {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function separateWithOptionalChunking(
|
async function separateMdxWithOptionalChunking(
|
||||||
separator: BrowserMdxSeparator,
|
separator: BrowserMdxSeparator,
|
||||||
decoded: StereoChannels,
|
decoded: StereoChannels,
|
||||||
timing: LocalSeparatorTimingCollector,
|
timing: LocalSeparatorTimingCollector,
|
||||||
chunkDurationSeconds: number | null,
|
chunkDurationSeconds: number | null,
|
||||||
modelConfig: LocalSeparatorModelConfig,
|
modelConfig: LocalSeparatorMdxModelConfig,
|
||||||
onProgress: (progress: LocalSeparatorProgress) => void,
|
onProgress: (progress: LocalSeparatorProgress) => void,
|
||||||
): Promise<{
|
): Promise<{
|
||||||
stems: Record<string, StereoChannels>;
|
stems: Record<string, StereoChannels>;
|
||||||
@@ -594,3 +611,171 @@ async function separateWithOptionalChunking(
|
|||||||
secondaryStem,
|
secondaryStem,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type DemucsStemName = 'Vocals' | 'Drums' | 'Bass' | 'Others';
|
||||||
|
|
||||||
|
function toDemucsStemMap(result: Awaited<ReturnType<DemucsProcessor['separate']>>): Record<DemucsStemName, StereoChannels> {
|
||||||
|
return {
|
||||||
|
Vocals: [result.vocals.left, result.vocals.right],
|
||||||
|
Drums: [result.drums.left, result.drums.right],
|
||||||
|
Bass: [result.bass.left, result.bass.right],
|
||||||
|
Others: [result.other.left, result.other.right],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runDemucsLocalSeparator(options: {
|
||||||
|
session: ort.InferenceSession;
|
||||||
|
runtimeProvider: LocalRuntimeProvider;
|
||||||
|
modelConfig: LocalSeparatorDemucsModelConfig;
|
||||||
|
audioBuffer: ArrayBuffer;
|
||||||
|
chunkDurationSeconds: number | null;
|
||||||
|
onProgress: (progress: LocalSeparatorProgress) => void;
|
||||||
|
}): Promise<{
|
||||||
|
stems: Array<{ name: string; blob: Blob }>;
|
||||||
|
providerLabel: string;
|
||||||
|
debugSummary: Record<string, unknown>;
|
||||||
|
}> {
|
||||||
|
const timing = new LocalSeparatorTimingCollector('local-demucs-separation');
|
||||||
|
const decoded = await timing.measureAsync('decode', () => decodeAudioToStereo(options.audioBuffer));
|
||||||
|
log(`Running browser Demucs separation on ${getProviderLabel(options.runtimeProvider)}...`);
|
||||||
|
|
||||||
|
const processor = new DemucsProcessor({
|
||||||
|
ort,
|
||||||
|
onProgress: ({ progress, currentSegment, totalSegments }) => {
|
||||||
|
options.onProgress({
|
||||||
|
stage: 'main',
|
||||||
|
passLabel: 'Demucs pass',
|
||||||
|
percent: progress * 100,
|
||||||
|
processedChunks: currentSegment,
|
||||||
|
totalChunks: totalSegments,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
onLog: (phase, message) => log(`demucs:${phase}`, message),
|
||||||
|
});
|
||||||
|
processor.session = options.session;
|
||||||
|
|
||||||
|
const outputMap = await separateDemucsWithOptionalChunking(
|
||||||
|
processor,
|
||||||
|
decoded,
|
||||||
|
timing,
|
||||||
|
getChunkDurationSeconds(options.chunkDurationSeconds, options.modelConfig),
|
||||||
|
options.modelConfig,
|
||||||
|
options.onProgress,
|
||||||
|
);
|
||||||
|
|
||||||
|
const stems = options.modelConfig.outputStemNames.map(name => ({
|
||||||
|
name,
|
||||||
|
blob: timing.measureSync(`wavEncode:${name}`, () => channelsToWavBlob(outputMap[name as DemucsStemName])),
|
||||||
|
}));
|
||||||
|
|
||||||
|
return {
|
||||||
|
stems,
|
||||||
|
providerLabel: getProviderLabel(options.runtimeProvider),
|
||||||
|
debugSummary: timing.getSummary({
|
||||||
|
model: options.modelConfig.filename,
|
||||||
|
runtimeProvider: options.runtimeProvider,
|
||||||
|
stemCount: stems.length,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function separateDemucsWithOptionalChunking(
|
||||||
|
processor: DemucsProcessor,
|
||||||
|
decoded: StereoChannels,
|
||||||
|
timing: LocalSeparatorTimingCollector,
|
||||||
|
chunkDurationSeconds: number | null,
|
||||||
|
modelConfig: LocalSeparatorDemucsModelConfig,
|
||||||
|
onProgress: (progress: LocalSeparatorProgress) => void,
|
||||||
|
): Promise<Record<DemucsStemName, StereoChannels>> {
|
||||||
|
if (!chunkDurationSeconds) {
|
||||||
|
return timing.measureAsync('demucsSeparate', async () => toDemucsStemMap(await processor.separate(decoded[0], decoded[1])));
|
||||||
|
}
|
||||||
|
|
||||||
|
const chunkSamples = Math.max(1, Math.floor(chunkDurationSeconds * SAMPLE_RATE));
|
||||||
|
if (decoded[0].length <= chunkSamples) {
|
||||||
|
return timing.measureAsync('demucsSeparate', async () => toDemucsStemMap(await processor.separate(decoded[0], decoded[1])));
|
||||||
|
}
|
||||||
|
|
||||||
|
const totalChunks = Math.ceil(decoded[0].length / chunkSamples);
|
||||||
|
const chunkedOutputs: Record<DemucsStemName, StereoChannels[]> = {
|
||||||
|
Vocals: [],
|
||||||
|
Drums: [],
|
||||||
|
Bass: [],
|
||||||
|
Others: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
for (let index = 0; index < totalChunks; index += 1) {
|
||||||
|
const start = index * chunkSamples;
|
||||||
|
const end = Math.min(start + chunkSamples, decoded[0].length);
|
||||||
|
const chunk = sliceChannels(decoded, start, end);
|
||||||
|
|
||||||
|
onProgress({
|
||||||
|
stage: 'chunk-prep',
|
||||||
|
passLabel: `Audio chunk ${index + 1}/${totalChunks}: preparing ${Math.round((end - start) / SAMPLE_RATE)}s chunk...`,
|
||||||
|
percent: (index / totalChunks) * 100,
|
||||||
|
processedChunks: index,
|
||||||
|
totalChunks,
|
||||||
|
});
|
||||||
|
|
||||||
|
const chunkResult = await timing.measureAsync(
|
||||||
|
'demucsChunkedSeparate',
|
||||||
|
async () => toDemucsStemMap(await processor.separate(chunk[0], chunk[1])),
|
||||||
|
);
|
||||||
|
|
||||||
|
(Object.keys(chunkedOutputs) as DemucsStemName[]).forEach(stemName => {
|
||||||
|
chunkedOutputs[stemName].push(chunkResult[stemName]);
|
||||||
|
});
|
||||||
|
|
||||||
|
onProgress({
|
||||||
|
stage: 'chunk-complete',
|
||||||
|
passLabel: `Audio chunk ${index + 1}/${totalChunks}: Demucs pass`,
|
||||||
|
percent: ((index + 1) / totalChunks) * 100,
|
||||||
|
processedChunks: index + 1,
|
||||||
|
totalChunks,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
Vocals: concatChannelPairs(chunkedOutputs.Vocals),
|
||||||
|
Drums: concatChannelPairs(chunkedOutputs.Drums),
|
||||||
|
Bass: concatChannelPairs(chunkedOutputs.Bass),
|
||||||
|
Others: concatChannelPairs(chunkedOutputs.Others),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runLocalSeparator(options: {
|
||||||
|
session: ort.InferenceSession;
|
||||||
|
runtimeProvider: LocalRuntimeProvider;
|
||||||
|
modelConfig: LocalSeparatorModelConfig;
|
||||||
|
audioBuffer: ArrayBuffer;
|
||||||
|
chunkDurationSeconds: number | null;
|
||||||
|
overlap: number;
|
||||||
|
onProgress: (progress: LocalSeparatorProgress) => void;
|
||||||
|
onProviderChange?: (provider: string) => void;
|
||||||
|
}): Promise<{
|
||||||
|
stems: Array<{ name: string; blob: Blob }>;
|
||||||
|
providerLabel: string;
|
||||||
|
debugSummary: Record<string, unknown>;
|
||||||
|
}> {
|
||||||
|
if (options.modelConfig.kind === 'demucs') {
|
||||||
|
return runDemucsLocalSeparator({
|
||||||
|
session: options.session,
|
||||||
|
runtimeProvider: options.runtimeProvider,
|
||||||
|
modelConfig: options.modelConfig,
|
||||||
|
audioBuffer: options.audioBuffer,
|
||||||
|
chunkDurationSeconds: options.chunkDurationSeconds,
|
||||||
|
onProgress: options.onProgress,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return runMdxLocalSeparator({
|
||||||
|
session: options.session,
|
||||||
|
runtimeProvider: options.runtimeProvider,
|
||||||
|
modelConfig: options.modelConfig,
|
||||||
|
audioBuffer: options.audioBuffer,
|
||||||
|
chunkDurationSeconds: options.chunkDurationSeconds,
|
||||||
|
overlap: options.overlap,
|
||||||
|
onProgress: options.onProgress,
|
||||||
|
onProviderChange: options.onProviderChange,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
export type StereoChannels = [Float32Array, Float32Array];
|
export type StereoChannels = [Float32Array, Float32Array];
|
||||||
|
|
||||||
|
export type LocalSeparatorModelId = 'UVR-MDX-NET-Inst_HQ_3.onnx' | 'htdemucs_4s.onnx';
|
||||||
|
export type LocalSeparatorModelKind = 'mdx' | 'demucs';
|
||||||
|
|
||||||
export interface LocalSeparatorModelDefaults {
|
export interface LocalSeparatorModelDefaults {
|
||||||
sampleRate: number;
|
sampleRate: number;
|
||||||
hopLength: number;
|
hopLength: number;
|
||||||
@@ -13,7 +16,7 @@ export interface LocalSeparatorModelDefaults {
|
|||||||
matchMixOverlap: number;
|
matchMixOverlap: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LocalSeparatorModelMetadata {
|
export interface LocalSeparatorMdxMetadata {
|
||||||
compensate: number;
|
compensate: number;
|
||||||
mdx_dim_f_set: number;
|
mdx_dim_f_set: number;
|
||||||
mdx_dim_t_set: number;
|
mdx_dim_t_set: number;
|
||||||
@@ -21,14 +24,36 @@ export interface LocalSeparatorModelMetadata {
|
|||||||
primary_stem: string;
|
primary_stem: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LocalSeparatorModelConfig {
|
export interface LocalSeparatorModelDownloadConfig {
|
||||||
|
configKey: 'general.uvr5_web_runtime.mdx_net_model_url' | 'general.uvr5_web_runtime.htdemucs_4s_model_url';
|
||||||
|
defaultUrl: string;
|
||||||
|
expectedSizeBytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface LocalSeparatorModelConfigBase {
|
||||||
|
id: LocalSeparatorModelId;
|
||||||
filename: string;
|
filename: string;
|
||||||
|
kind: LocalSeparatorModelKind;
|
||||||
displayName: string;
|
displayName: string;
|
||||||
status: 'ready';
|
status: 'ready';
|
||||||
|
outputStemNames: string[];
|
||||||
|
defaultChunkDurationSeconds: number | null;
|
||||||
defaults: LocalSeparatorModelDefaults;
|
defaults: LocalSeparatorModelDefaults;
|
||||||
metadata: LocalSeparatorModelMetadata;
|
download: LocalSeparatorModelDownloadConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface LocalSeparatorMdxModelConfig extends LocalSeparatorModelConfigBase {
|
||||||
|
kind: 'mdx';
|
||||||
|
metadata: LocalSeparatorMdxMetadata;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LocalSeparatorDemucsModelConfig extends LocalSeparatorModelConfigBase {
|
||||||
|
kind: 'demucs';
|
||||||
|
metadata: null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type LocalSeparatorModelConfig = LocalSeparatorMdxModelConfig | LocalSeparatorDemucsModelConfig;
|
||||||
|
|
||||||
export interface LocalSeparatorProgress {
|
export interface LocalSeparatorProgress {
|
||||||
stage: string;
|
stage: string;
|
||||||
passLabel: string;
|
passLabel: string;
|
||||||
|
|||||||
Reference in New Issue
Block a user