diff --git a/.gitignore b/.gitignore index 6d41809..1d5cf58 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,10 @@ app/storage/processed/* !app/storage/uploads/.gitkeep !app/storage/processed/.gitkeep .DS_Store + +# VST3 and sample library directories (proprietary binaries) +vst_plugins/ +samples/ .idea/ .vscode/ *.log diff --git a/Dockerfile b/Dockerfile index bcefc13..06c0ccd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,18 +19,25 @@ RUN apt-get update && apt-get install -y \ ffmpeg \ libsndfile1 \ libfluidsynth3 \ + libcurl4 \ build-essential \ && rm -rf /var/lib/apt/lists/* # Thiết lập biến môi trường hiển thị cho X11 ảo ENV DISPLAY=:99 +# Create VST3 and sample directories +RUN mkdir -p /opt/daw_engine/vst3 /opt/daw_engine/soundfonts /opt/daw_engine/samples/pianobook + WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY . . +# Copy VST3 plugins if present +COPY ./vst_plugins/ /opt/daw_engine/vst3/ + # Tạo thư mục chứa file nhạc và cấp quyền ghi RUN mkdir -p /app/app/storage/uploads /app/app/storage/processed && chmod -R 777 /app/app/storage diff --git a/app/api/v1/plugins.py b/app/api/v1/plugins.py index 8f25a25..a01020d 100644 --- a/app/api/v1/plugins.py +++ b/app/api/v1/plugins.py @@ -1,10 +1,11 @@ import os, uuid, json, tempfile -from fastapi import APIRouter, HTTPException, Depends, UploadFile, File +from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, BackgroundTasks from pydantic import BaseModel from typing import Optional, Any from app.config import settings from app.core.vst_engine import PluginManager, HAS_PEDALBOARD, HAS_PYFLUIDSYNTH from app.core.render_engine import PythonRenderEngine +from app.core.soundfont_inspector import SoundFontInspector from app.api.v1.auth import get_current_user router = APIRouter() @@ -12,6 +13,16 @@ router = APIRouter() UPLOAD_SF_DIR = os.path.join(settings.STORAGE_DIR, "soundfonts") os.makedirs(UPLOAD_SF_DIR, exist_ok=True) +SYSTEM_SF_DIR = "/opt/daw_engine/soundfonts" + +_inspector = None + +def get_inspector(): + global _inspector + if _inspector is None: + _inspector = SoundFontInspector(system_sf_dir=SYSTEM_SF_DIR, upload_sf_dir=UPLOAD_SF_DIR) + return _inspector + @router.get("/available") async def list_plugins(current_user: dict = Depends(get_current_user)): @@ -35,6 +46,14 @@ async def list_default_soundfonts(): return result +@router.get("/soundfonts/catalog") +async def soundfont_catalog(current_user: dict = Depends(get_current_user)): + inspector = get_inspector() + full_catalog = inspector.get_catalog() + condensed_catalog = inspector.get_condensed_catalog_summary() + return {"full_catalog": full_catalog, "condensed_catalog": condensed_catalog} + + @router.get("/soundfont-instruments/{sf_id}") async def list_soundfont_instruments(sf_id: str, current_user: dict = Depends(get_current_user)): pm = PluginManager(upload_sf_dir=UPLOAD_SF_DIR) @@ -45,6 +64,7 @@ async def list_soundfont_instruments(sf_id: str, current_user: dict = Depends(ge @router.post("/upload-soundfont") async def upload_soundfont( file: UploadFile = File(...), + background_tasks: BackgroundTasks = None, current_user: dict = Depends(get_current_user) ): if not (file.filename and (file.filename.endswith(".sf2") or file.filename.endswith(".sf3"))): @@ -69,6 +89,11 @@ async def upload_soundfont( import json json.dump({"original_name": file.filename, "uuid": file_uuid, "file": file_id}, f) + inspector = get_inspector() + inspector.invalidate_catalog_cache() + if background_tasks: + background_tasks.add_task(inspector.generate_full_catalog) + return {"id": file_id, "name": file.filename, "path": dest_path, "size_bytes": len(contents)} @@ -93,6 +118,8 @@ async def delete_soundfont(sf_id: str, current_user: dict = Depends(get_current_ break if not deleted: raise HTTPException(status_code=404, detail="SoundFont not found") + inspector = get_inspector() + inspector.invalidate_catalog_cache() return {"deleted": True, "sf_id": sf_id} diff --git a/app/core/render_engine.py b/app/core/render_engine.py index 6954c30..88ce5eb 100644 --- a/app/core/render_engine.py +++ b/app/core/render_engine.py @@ -6,6 +6,7 @@ from app.config import settings from app.core.vst_engine import ( render_midi_events_to_audio, PluginManager, + DecentSamplerManager, HAS_PEDALBOARD, HAS_PYFLUIDSYNTH, ) @@ -51,10 +52,19 @@ class PythonRenderEngine: def render_session_container(self, session: dict, section_store: dict, bpm: float, time_sig_num: int, total_samples: int) -> np.ndarray: session_buffer = np.zeros((2, total_samples), dtype=np.float32) + _channel_counter = 0 + for track in session.get("tracks", []): track_type = track.get("type", "AUDIO") track_buffer = np.zeros((2, total_samples), dtype=np.float32) + soundfont_bank = track.get("soundfont_bank", 0) + soundfont_program = track.get("soundfont_program", 0) + is_percussion = track.get("is_percussion", False) or (soundfont_bank == 128) + midi_channel = 9 if is_percussion else (_channel_counter % 9) + if not is_percussion: + _channel_counter += 1 + for item in track.get("items", []): start_sample = self.bars_to_samples(item["start_bar"], bpm, time_sig_num) dur_samples = self.bars_to_samples(item["duration_bars"], bpm, time_sig_num) @@ -124,11 +134,43 @@ class PythonRenderEngine: plugin_mgr = PluginManager() vst = plugin_mgr.load_vst(instrument_id) if instrument_id else None - if vst and HAS_PEDALBOARD: + instrument_source = track.get("instrument_source", "soundfont") + + if instrument_source == "pianobook": + dspreset_path = track.get("dspreset_path", "") + if dspreset_path and os.path.exists(dspreset_path) and HAS_PEDALBOARD: + from pedalboard import Pedalboard + ds_manager = DecentSamplerManager() + try: + vst = ds_manager.create_decent_sampler_instance(dspreset_path) + midi_messages = PluginManager.midi_events_to_messages( + midi_events, bpm, self.sample_rate + ) + total_needed = 0 + for ev in midi_events: + end_sec = (ev.get("start_beat", 0) + ev.get("duration_beats", 1)) * (60.0 / bpm) + dur_samples = int(end_sec * self.sample_rate) + if dur_samples > total_needed: + total_needed = dur_samples + total_needed = max(total_needed, 1024) + silent = np.zeros((2, total_needed), dtype=np.float32) + board = Pedalboard([vst]) + synth_buffer = board(silent, sample_rate=self.sample_rate, midi_messages=midi_messages) + except Exception as e: + print(f"[RenderEngine] DecentSampler/Pianobook error: {e}") + synth_buffer = render_midi_events_to_audio( + midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth' + ) + else: + synth_buffer = render_midi_events_to_audio( + midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth' + ) + elif vst and HAS_PEDALBOARD: from pedalboard import Pedalboard # Convert MIDI events with precise sample offset midi_messages = PluginManager.midi_events_to_messages( - midi_events, bpm, self.sample_rate + midi_events, bpm, self.sample_rate, + bank=soundfont_bank, program=soundfont_program ) total_needed = 0 for ev in midi_events: @@ -149,7 +191,7 @@ class PythonRenderEngine: import fluidsynth fl = fluidsynth.FluidSynth(sample_rate=self.sample_rate, gain=0.5) fid = fl.sfload(sf_path) - fl.program_select(0, fid, 0, 0) + fl.program_select(midi_channel, fid, soundfont_bank, soundfont_program) beat_sec = 60.0 / bpm total_sec = 0 for ev in midi_events: @@ -165,11 +207,11 @@ class PythonRenderEngine: dur_beats = ev.get("duration_beats", 1.0) start_sec = start_beat * beat_sec dur_sec = dur_beats * beat_sec - fl.noteon(0, note, velocity) + fl.noteon(midi_channel, note, velocity) start_s = int(start_sec * self.sample_rate) dur_s = int(dur_sec * self.sample_rate) block = fl.get_samples(int(dur_s)) if hasattr(fl, 'get_samples') else np.zeros((2, dur_s), dtype=np.float32) - fl.noteoff(0, note) + fl.noteoff(midi_channel, note) if block.shape[1] > 0: end_s = min(start_s + block.shape[1], total_samples) actual = end_s - start_s diff --git a/app/core/soundfont_inspector.py b/app/core/soundfont_inspector.py new file mode 100644 index 0000000..f0d18b5 --- /dev/null +++ b/app/core/soundfont_inspector.py @@ -0,0 +1,155 @@ +import os +import json +import logging + +logger = logging.getLogger(__name__) + +try: + from sf2utils.sf2parse import Sf2File + HAS_SF2UTILS = True +except ImportError: + HAS_SF2UTILS = False + +GM_CATEGORIES = [ + ("Piano", range(0, 8)), + ("Chromatic Percussion", range(8, 16)), + ("Organ", range(16, 24)), + ("Guitar", range(24, 32)), + ("Bass", range(32, 40)), + ("Strings", range(40, 48)), + ("Ensemble", range(48, 56)), + ("Brass", range(56, 64)), + ("Reed", range(64, 72)), + ("Pipe", range(72, 80)), + ("Synth Lead", range(80, 90)), + ("Synth Pad", range(90, 104)), +] + +MAX_CONDENSED_ENTRIES = 50 + + +class SoundFontInspector: + def __init__(self, system_sf_dir="/opt/daw_engine/soundfonts", upload_sf_dir=None): + self.system_sf_dir = system_sf_dir + self.upload_sf_dir = upload_sf_dir + self._catalog_cache = None + + def invalidate_catalog_cache(self): + self._catalog_cache = None + + def inspect_sf2_file(self, filepath: str) -> dict: + if not HAS_SF2UTILS: + logger.warning("sf2utils not installed, cannot inspect .sf2 files") + return {} + if not os.path.exists(filepath): + return {} + + try: + sf_name = os.path.basename(filepath) + sf_id = os.path.splitext(sf_name)[0].lower() + + instruments = [] + with open(filepath, 'rb') as f: + sf2 = Sf2File(f) + for preset in sf2.presets: + name = preset.name.strip() + if name == "EOP" or (preset.bank == 128 and preset.preset == 127): + continue + instruments.append({ + "bank": preset.bank, + "program": preset.preset, + "name": name, + "is_percussion": (preset.bank == 128) + }) + + return { + "soundfont_id": sf_id, + "filename": sf_name, + "total_instruments": len(instruments), + "instruments": instruments + } + except Exception as e: + logger.warning(f"Skipping corrupted .sf2 file {filepath}: {e}") + return {} + + def _scan_directory(self, directory: str) -> dict: + catalog = {} + if not os.path.isdir(directory): + return catalog + for fname in os.listdir(directory): + if not fname.lower().endswith(('.sf2', '.sf3')): + continue + full_path = os.path.join(directory, fname) + sf_info = self.inspect_sf2_file(full_path) + if sf_info and sf_info.get("soundfont_id"): + catalog[sf_info["soundfont_id"]] = sf_info + return catalog + + def generate_full_catalog(self, output_json_path: str = None) -> dict: + catalog = {} + catalog.update(self._scan_directory(self.system_sf_dir)) + if self.upload_sf_dir and os.path.isdir(self.upload_sf_dir): + catalog.update(self._scan_directory(self.upload_sf_dir)) + + if output_json_path: + os.makedirs(os.path.dirname(output_json_path), exist_ok=True) + with open(output_json_path, 'w', encoding='utf-8') as f: + json.dump(catalog, f, ensure_ascii=False, indent=2) + + self._catalog_cache = catalog + return catalog + + def get_catalog(self) -> dict: + if self._catalog_cache is not None: + return self._catalog_cache + return self.generate_full_catalog() + + def get_condensed_catalog_summary(self) -> dict: + catalog = self.get_catalog() + condensed = {} + for sf_id, sf_info in catalog.items(): + instruments = sf_info.get("instruments", []) + if not instruments: + continue + + selected = [] + used_programs = set() + for cat_name, prog_range in GM_CATEGORIES: + cat_members = [ + inst for inst in instruments + if inst["program"] in prog_range and not inst["is_percussion"] + ] + if cat_members: + representative = cat_members[0] + key = (representative["program"], representative["bank"]) + if key not in used_programs: + used_programs.add(key) + selected.append(representative) + + drum_kits = [inst for inst in instruments if inst["is_percussion"]] + for dk in drum_kits[:3]: + key = (dk["program"], dk["bank"]) + if key not in used_programs: + used_programs.add(key) + selected.append(dk) + + if len(selected) > MAX_CONDENSED_ENTRIES: + selected = selected[:MAX_CONDENSED_ENTRIES] + + condensed[sf_id] = { + "soundfont_id": sf_info["soundfont_id"], + "filename": sf_info["filename"], + "total_instruments": sf_info["total_instruments"], + "condensed_count": len(selected), + "instruments": selected + } + return condensed + + def format_condensed_for_prompt(self) -> str: + condensed = self.get_condensed_catalog_summary() + lines = [] + for sf_id, info in condensed.items(): + lines.append(f"SoundFont ID: '{sf_id}' (File: {info['filename']}):") + for inst in info["instruments"]: + lines.append(f" - {inst['name']}: bank={inst['bank']}, program={inst['program']}") + return "\n".join(lines) diff --git a/app/core/vst_engine.py b/app/core/vst_engine.py index 8d0fa2f..1831162 100644 --- a/app/core/vst_engine.py +++ b/app/core/vst_engine.py @@ -286,11 +286,15 @@ class PluginManager: } @staticmethod - def midi_events_to_messages(midi_events: list, bpm: float, sr: int) -> list: + def midi_events_to_messages(midi_events: list, bpm: float, sr: int, bank: int = None, program: int = None) -> list: if not HAS_PEDALBOARD: return [] beat_duration_sec = 60.0 / max(30.0, bpm) messages = [] + if bank is not None: + messages.append(MidiMessage(control_change=0, value=bank, sample_offset=0)) + if program is not None: + messages.append(MidiMessage(program_change=program, sample_offset=0)) for ev in midi_events: note = ev.get("note", 60) velocity = ev.get("velocity", 100) @@ -313,3 +317,29 @@ class PluginManager: if data[8:12] != b'sfbk': return False return True + + +class DecentSamplerManager: + def __init__(self, vst_path="/opt/daw_engine/vst3/DecentSampler.vst3"): + self.vst_path = vst_path + + def create_decent_sampler_instance(self, dspreset_path: str): + if not HAS_PEDALBOARD: + raise RuntimeError("pedalboard not available") + if not os.path.exists(self.vst_path): + raise FileNotFoundError(f"DecentSampler VST3 not found at {self.vst_path}") + if not os.path.exists(dspreset_path): + raise FileNotFoundError(f"Preset file not found at {dspreset_path}") + + plugin = VST3Plugin(self.vst_path) + + abs_preset = os.path.abspath(dspreset_path) + preset_dir = os.path.dirname(abs_preset) + cwd_before = os.getcwd() + try: + os.chdir(preset_dir) + plugin.load_preset(abs_preset) + finally: + os.chdir(cwd_before) + + return plugin diff --git a/app/static/js/app.jsx b/app/static/js/app.jsx index 39db074..94a0390 100644 --- a/app/static/js/app.jsx +++ b/app/static/js/app.jsx @@ -3351,9 +3351,13 @@ const PluginManagerModal = ({ isOpen, onClose, pluginsData }) => { try { const result = await window.SonicAPI.uploadSoundFont(file); setSfUploadStatus('Uploaded: ' + result.name); - // Refresh plugin list + // Refresh plugin list and catalog const data = await window.SonicAPI.listPlugins(); setLocalData(data); + try { + const cat = await window.SonicAPI.getSoundfontCatalog(); + window.__soundfontCatalog = cat; + } catch (_) {} } catch (err) { setSfUploadStatus('Error: ' + err.message); } @@ -7027,6 +7031,11 @@ const App = () => { if (active) setSelectedProviderId(active.id); } } catch (e) { } + try { + window.SonicAPI.getSoundfontCatalog().then(cat => { + window.__soundfontCatalog = cat; + }).catch(() => {}); + } catch (e) { } })(); } }; @@ -13802,6 +13811,13 @@ const App = () => { })) }; targetTrack.midiItems = [...(targetTrack.midiItems || []), newMidiItem]; + if (aiTrack.soundfont_bank !== undefined && aiTrack.soundfont_program !== undefined) { + targetTrack.soundfont_bank = aiTrack.soundfont_bank; + targetTrack.soundfont_program = aiTrack.soundfont_program; + if (window.SonicSF && window.SonicSF.applyAITrackInstrument) { + window.SonicSF.applyAITrackInstrument(aiTrack.soundfont_bank, aiTrack.soundfont_program); + } + } }); return updatedTracks; }); diff --git a/app/static/js/services/aiGateway.js b/app/static/js/services/aiGateway.js index e1e8d76..31a5307 100644 --- a/app/static/js/services/aiGateway.js +++ b/app/static/js/services/aiGateway.js @@ -48,7 +48,7 @@ const AIGateway = (function() { name: 'fade_out', description: 'Fade-out clip (0.5s đến max)', parameters: { type: 'object', properties: { track_id: { type: 'string' }, duration_seconds: { type: 'number' }, clip_index: { type: 'number', description: 'Chỉ số của clip trên track (1-based, ví dụ: 1 cho clip 1, 2 cho clip 2)' }, clip_id: { type: 'string', description: 'ID của clip cụ thể' } } } }, { name: 'generate_multitrack_midi', - description: 'Generates multi-track MIDI data based on genre, bar duration, and requested instruments list.', + description: 'Generates multi-track MIDI data along with SoundFont Program configurations for each track.', parameters: { type: 'object', properties: { @@ -57,12 +57,15 @@ const AIGateway = (function() { total_bars: { type: 'integer' }, tracks: { type: 'array', - description: 'Array of instrument tracks along with their corresponding MIDI notes', + description: 'Array of instrument tracks with MIDI notes and SoundFont instrument selection', items: { type: 'object', properties: { track_name: { type: 'string', description: 'Track name (e.g., String Ensemble, Epic Brass, Taiko Drums)' }, instrument_type: { type: 'string', enum: ['STRINGS', 'BRASS', 'SYNTH', 'PERCUSSION', 'DRUMS'] }, + soundfont_id: { type: 'string', description: "ID of the SoundFont to use (e.g. 'generaluser_gs')" }, + soundfont_bank: { type: 'integer', default: 0, description: 'MIDI Bank code: 0 for melodic instruments, 128 for Drums/Percussion' }, + soundfont_program: { type: 'integer', description: 'MIDI Program Number 0-127 matching the instrument name in the SoundFont catalog' }, notes: { type: 'array', items: { @@ -77,7 +80,7 @@ const AIGateway = (function() { } } }, - required: ['track_name', 'instrument_type', 'notes'] + required: ['track_name', 'instrument_type', 'soundfont_id', 'soundfont_bank', 'soundfont_program', 'notes'] } } }, @@ -176,13 +179,29 @@ const AIGateway = (function() { return calls; } + function buildCatalogPromptSection() { + const catalog = window.__soundfontCatalog; + if (!catalog || !catalog.condensed_catalog) return ''; + const lines = []; + for (const [sfId, info] of Object.entries(catalog.condensed_catalog)) { + lines.push(`SoundFont ID: '${sfId}' (File: ${info.filename}):`); + for (const inst of info.instruments || []) { + lines.push(` - ${inst.name}: bank=${inst.bank}, program=${inst.program}`); + } + } + if (lines.length === 0) return ''; + return `\n=== SOUNDFONT INSTRUMENT CATALOG ===\nYou have the following SoundFont instruments available on the system:\n${lines.join('\n')}\n\nMANDATORY RULES WHEN CREATING TRACKS WITH generate_multitrack_midi:\n1. You MUST look up the catalog above and fill in the correct soundfont_id, soundfont_bank, and soundfont_program for each track.\n2. Melodic instruments (Piano, Strings, Brass, etc.) MUST use soundfont_bank=0.\n3. Drums and Percussion MUST use soundfont_bank=128.\n4. Example: For \"Brass horns\", use soundfont_id="generaluser_gs", soundfont_bank=0, soundfont_program=56.\n5. Example: For \"Drum kit\", use soundfont_id="generaluser_gs", soundfont_bank=128, soundfont_program=0.\n`; + } + function buildUserMessage(prompt, context, systemInstruction = '') { const contextStr = JSON.stringify(context, null, 2); const toolNames = DEFAULT_TOOLS.map(t => ` - ${t.name}: ${t.description}`).join('\n'); + const catalogSection = buildCatalogPromptSection(); return [ { role: 'system', content: `Bạn là trợ lý điều khiển DAW chuyên nghiệp. Nhiệm vụ của bạn là phân tích yêu cầu của người dùng và chuyển đổi thành danh sách các function calls tương ứng. ${systemInstruction ? `\nHướng dẫn tạo nhạc đặc biệt từ Preset:\n${systemInstruction}\n` : ''} +${catalogSection} QUAN TRỌNG: 1. Bạn đang hoạt động ở chế độ một lượt (one-shot). Hãy trả về TẤT CẢ các function calls cần thiết để thực hiện toàn bộ các bước trong yêu cầu của người dùng trong một phản hồi duy nhất. Đừng thực hiện từng bước qua nhiều lượt chat. 2. Có thể gọi nhiều function cùng một lúc (gọi song song/nối tiếp). Chúng sẽ được thực thi theo thứ tự bạn trả về. diff --git a/app/static/js/services/api.js b/app/static/js/services/api.js index 8976fb7..4d0e429 100644 --- a/app/static/js/services/api.js +++ b/app/static/js/services/api.js @@ -63,6 +63,7 @@ window.API_BASE_URL = window.API_BASE_URL || window.location.origin; savePreferences: (prefs) => apiRequest('/api/v1/user/preferences', { method: 'POST', body: JSON.stringify({ preferences: prefs }) }), listPlugins: () => apiRequest('/api/v1/plugins/available', { method: 'GET' }), + getSoundfontCatalog: () => apiRequest('/api/v1/plugins/soundfonts/catalog', { method: 'GET' }), listDefaultSoundfonts: () => apiRequest('/api/v1/plugins/default-soundfonts', { method: 'GET' }), listSoundfontInstruments: (sfId) => apiRequest(`/api/v1/plugins/soundfont-instruments/${sfId}`, { method: 'GET' }), renderProject: (projectJson, outputFilename) => apiRequest('/api/v1/plugins/render', { method: 'POST', body: JSON.stringify({ project_json: projectJson, output_filename: outputFilename }) }), diff --git a/app/static/js/services/soundfontPlayer.js b/app/static/js/services/soundfontPlayer.js index 4e411e0..c60875d 100644 --- a/app/static/js/services/soundfontPlayer.js +++ b/app/static/js/services/soundfontPlayer.js @@ -30,9 +30,45 @@ return window.__sharedAudioCtx; }; + // ── Per-channel MIDI state (16 GM channels) ── + const _channels = Array.from({ length: 16 }, () => ({ bank: 0, program: 0, isPercussion: false })); + let _nextMelodicChannel = 0; + const SonicSF = { loadedFonts: {}, + controllerChange: function (channel, controller, value) { + if (channel < 0 || channel > 15) return; + if (controller === 0) { + _channels[channel].bank = value; + _channels[channel].isPercussion = (value === 128); + } + }, + + programChange: function (channel, program) { + if (channel < 0 || channel > 15) return; + _channels[channel].program = program; + }, + + allocateChannel: function (bank) { + if (bank === 128) return 9; + const ch = _nextMelodicChannel % 9; + _nextMelodicChannel = (_nextMelodicChannel + 1) % 9; + return ch; + }, + + applyAITrackInstrument: function (bank, program) { + const channel = this.allocateChannel(bank); + this.controllerChange(channel, 0, bank); + this.programChange(channel, program); + return channel; + }, + + getChannelState: function (channel) { + if (channel < 0 || channel > 15) return null; + return { ..._channels[channel] }; + }, + // Load SoundFont from URL → ArrayBuffer → store in memory loadSoundFont: async function (url) { if (this.loadedFonts[url]) return this.loadedFonts[url]; @@ -43,7 +79,7 @@ return buffer; }, - playNote: function (note, velocity, durationMs, startTime, program, destinationNode) { + playNote: function (note, velocity, durationMs, startTime, program, destinationNode, channel) { const ctx = getCtx(); const freq = 440 * Math.pow(2, (note - 69) / 12); if (freq <= 0 || isNaN(freq)) return null; @@ -59,7 +95,11 @@ let releaseTime = 0.2; let volFactor = 0.25; - const prog = program !== undefined ? parseInt(program) : 0; + let prog = program !== undefined ? parseInt(program) : 0; + if (channel !== undefined && channel >= 0 && channel < 16) { + const chState = _channels[channel]; + prog = chState.program || prog; + } if (prog >= 0 && prog <= 7) { // Pianos oscType = 'sine'; decayTime = 0.3; diff --git a/requirements.txt b/requirements.txt index 1caf9cc..7575a29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ jsonschema>=4.18.0 pedalboard>=0.8.0 mido>=1.3.0 pyfluidsynth>=1.3.0 +sf2utils>=0.9.0 diff --git a/wiki.md b/wiki.md index a030adf..2fb4a40 100644 --- a/wiki.md +++ b/wiki.md @@ -270,3 +270,9 @@ - **Tóm tắt thay đổi:** (1) Sub-tab loop selection chỉ active khi nút Loop (st.isLooping) bật — không còn bị ảnh hưởng bởi global isLoopingSelection. (2) Stop dừng ngay lập tức: `SonicSF.stopAll()` set gain 0 và stop oscillator tại ctx.currentTime, không ramp. - **Các file ảnh hưởng:** `app/static/js/app.jsx`, `app/static/js/services/soundfontPlayer.js`, `app/static/js/app.precompiled.js` - **Ghi chú/Test (nếu có):** `npm run build` pass. + +### [2026-07-26 12:31] Task: VST/SoundFont inspection engine + AI schema +- **Summary:** New `SoundFontInspector` (sf2utils) scans .sf2 files, generates full/condensed catalog. Added `/soundfonts/catalog` API. Updated AI tool schema with `soundfont_id/bank/program`. Condensed catalog auto-injected into system prompt. Server render now passes bank/program + MIDI channel routing (drums → ch9). Added `DecentSamplerManager` + Pianobook render path. Client `SonicSF`: `applyAITrackInstrument`, `controllerChange`, `programChange`. Catalog cache invalidated on upload. +- **Files:** `requirements.txt`, `app/core/soundfont_inspector.py` (new), `app/api/v1/plugins.py`, `app/core/vst_engine.py`, `app/core/render_engine.py`, `app/static/js/services/aiGateway.js`, `app/static/js/services/api.js`, `app/static/js/services/soundfontPlayer.js`, `app/static/js/app.jsx`, `Dockerfile`, `.gitignore`, `vst_plugins/` (new), `samples/pianobook/` (new) +- **Tests:** `python3 -m pytest tests/ -v` → 61 passed, 1 pre-existing failure. Python syntax check OK on all modified files. +---