feat: add SoundFont inspection engine + AI instrument schema

- SoundFontInspector (sf2utils) scans .sf2, generates full/condensed catalog
- GET /api/v1/plugins/soundfonts/catalog with lazy init + cache invalidation
- AI tool generate_multitrack_midi now requires soundfont_id/bank/program
- Condensed catalog auto-injected into AI system prompt with bank rules
- Server render: FluidSynth program_select uses bank/program + channel routing (drums→ch9)
- VST3 pedalboard path inserts CC0 bank select + program change before notes
- DecentSamplerManager loads .dspreset with CWD fix for relative sample paths
- Pianobook render branch in render_engine.py
- Client SonicSF: controllerChange, programChange, applyAITrackInstrument
- Post-AI track creation applies instrument via applyAITrackInstrument
- Background cache rescan on .sf2 upload, frontend re-fetches catalog
- libcurl4 + VST3 dirs in Dockerfile

.gitignore

@@ -18,6 +18,10 @@ app/storage/processed/*
 !app/storage/uploads/.gitkeep
 !app/storage/processed/.gitkeep
 .DS_Store
+
+# VST3 and sample library directories (proprietary binaries)
+vst_plugins/
+samples/
 .idea/
 .vscode/
 *.log

Dockerfile

@@ -19,18 +19,25 @@ RUN apt-get update && apt-get install -y \
     ffmpeg \
     libsndfile1 \
     libfluidsynth3 \
+    libcurl4 \
     build-essential \
     && rm -rf /var/lib/apt/lists/*
 
 # Thiết lập biến môi trường hiển thị cho X11 ảo
 ENV DISPLAY=:99
 
+# Create VST3 and sample directories
+RUN mkdir -p /opt/daw_engine/vst3 /opt/daw_engine/soundfonts /opt/daw_engine/samples/pianobook
+
 WORKDIR /app
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . .
 
+# Copy VST3 plugins if present
+COPY ./vst_plugins/ /opt/daw_engine/vst3/
+
 # Tạo thư mục chứa file nhạc và cấp quyền ghi
 RUN mkdir -p /app/app/storage/uploads /app/app/storage/processed && chmod -R 777 /app/app/storage
 

app/api/v1/plugins.py

@@ -1,10 +1,11 @@
 import os, uuid, json, tempfile
-from fastapi import APIRouter, HTTPException, Depends, UploadFile, File
+from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, BackgroundTasks
 from pydantic import BaseModel
 from typing import Optional, Any
 from app.config import settings
 from app.core.vst_engine import PluginManager, HAS_PEDALBOARD, HAS_PYFLUIDSYNTH
 from app.core.render_engine import PythonRenderEngine
+from app.core.soundfont_inspector import SoundFontInspector
 from app.api.v1.auth import get_current_user
 
 router = APIRouter()
@@ -12,6 +13,16 @@ router = APIRouter()
 UPLOAD_SF_DIR = os.path.join(settings.STORAGE_DIR, "soundfonts")
 os.makedirs(UPLOAD_SF_DIR, exist_ok=True)
 
+SYSTEM_SF_DIR = "/opt/daw_engine/soundfonts"
+
+_inspector = None
+
+def get_inspector():
+    global _inspector
+    if _inspector is None:
+        _inspector = SoundFontInspector(system_sf_dir=SYSTEM_SF_DIR, upload_sf_dir=UPLOAD_SF_DIR)
+    return _inspector
+
 
 @router.get("/available")
 async def list_plugins(current_user: dict = Depends(get_current_user)):
@@ -35,6 +46,14 @@ async def list_default_soundfonts():
     return result
 
 
+@router.get("/soundfonts/catalog")
+async def soundfont_catalog(current_user: dict = Depends(get_current_user)):
+    inspector = get_inspector()
+    full_catalog = inspector.get_catalog()
+    condensed_catalog = inspector.get_condensed_catalog_summary()
+    return {"full_catalog": full_catalog, "condensed_catalog": condensed_catalog}
+
+
 @router.get("/soundfont-instruments/{sf_id}")
 async def list_soundfont_instruments(sf_id: str, current_user: dict = Depends(get_current_user)):
     pm = PluginManager(upload_sf_dir=UPLOAD_SF_DIR)
@@ -45,6 +64,7 @@ async def list_soundfont_instruments(sf_id: str, current_user: dict = Depends(ge
 @router.post("/upload-soundfont")
 async def upload_soundfont(
     file: UploadFile = File(...),
+    background_tasks: BackgroundTasks = None,
     current_user: dict = Depends(get_current_user)
 ):
     if not (file.filename and (file.filename.endswith(".sf2") or file.filename.endswith(".sf3"))):
@@ -69,6 +89,11 @@ async def upload_soundfont(
         import json
         json.dump({"original_name": file.filename, "uuid": file_uuid, "file": file_id}, f)
 
+    inspector = get_inspector()
+    inspector.invalidate_catalog_cache()
+    if background_tasks:
+        background_tasks.add_task(inspector.generate_full_catalog)
+
     return {"id": file_id, "name": file.filename, "path": dest_path, "size_bytes": len(contents)}
 
 
@@ -93,6 +118,8 @@ async def delete_soundfont(sf_id: str, current_user: dict = Depends(get_current_
             break
     if not deleted:
         raise HTTPException(status_code=404, detail="SoundFont not found")
+    inspector = get_inspector()
+    inspector.invalidate_catalog_cache()
     return {"deleted": True, "sf_id": sf_id}
 
 

app/core/render_engine.py

@@ -6,6 +6,7 @@ from app.config import settings
 from app.core.vst_engine import (
     render_midi_events_to_audio,
     PluginManager,
+    DecentSamplerManager,
     HAS_PEDALBOARD,
     HAS_PYFLUIDSYNTH,
 )
@@ -51,10 +52,19 @@ class PythonRenderEngine:
     def render_session_container(self, session: dict, section_store: dict, bpm: float, time_sig_num: int, total_samples: int) -> np.ndarray:
         session_buffer = np.zeros((2, total_samples), dtype=np.float32)
 
+        _channel_counter = 0
+
         for track in session.get("tracks", []):
             track_type = track.get("type", "AUDIO")
             track_buffer = np.zeros((2, total_samples), dtype=np.float32)
 
+            soundfont_bank = track.get("soundfont_bank", 0)
+            soundfont_program = track.get("soundfont_program", 0)
+            is_percussion = track.get("is_percussion", False) or (soundfont_bank == 128)
+            midi_channel = 9 if is_percussion else (_channel_counter % 9)
+            if not is_percussion:
+                _channel_counter += 1
+
             for item in track.get("items", []):
                 start_sample = self.bars_to_samples(item["start_bar"], bpm, time_sig_num)
                 dur_samples = self.bars_to_samples(item["duration_bars"], bpm, time_sig_num)
@@ -124,11 +134,43 @@ class PythonRenderEngine:
                             plugin_mgr = PluginManager()
                             vst = plugin_mgr.load_vst(instrument_id) if instrument_id else None
 
-                            if vst and HAS_PEDALBOARD:
+                            instrument_source = track.get("instrument_source", "soundfont")
+
+                            if instrument_source == "pianobook":
+                                dspreset_path = track.get("dspreset_path", "")
+                                if dspreset_path and os.path.exists(dspreset_path) and HAS_PEDALBOARD:
+                                    from pedalboard import Pedalboard
+                                    ds_manager = DecentSamplerManager()
+                                    try:
+                                        vst = ds_manager.create_decent_sampler_instance(dspreset_path)
+                                        midi_messages = PluginManager.midi_events_to_messages(
+                                            midi_events, bpm, self.sample_rate
+                                        )
+                                        total_needed = 0
+                                        for ev in midi_events:
+                                            end_sec = (ev.get("start_beat", 0) + ev.get("duration_beats", 1)) * (60.0 / bpm)
+                                            dur_samples = int(end_sec * self.sample_rate)
+                                            if dur_samples > total_needed:
+                                                total_needed = dur_samples
+                                        total_needed = max(total_needed, 1024)
+                                        silent = np.zeros((2, total_needed), dtype=np.float32)
+                                        board = Pedalboard([vst])
+                                        synth_buffer = board(silent, sample_rate=self.sample_rate, midi_messages=midi_messages)
+                                    except Exception as e:
+                                        print(f"[RenderEngine] DecentSampler/Pianobook error: {e}")
+                                        synth_buffer = render_midi_events_to_audio(
+                                            midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth'
+                                        )
+                                else:
+                                    synth_buffer = render_midi_events_to_audio(
+                                        midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth'
+                                    )
+                            elif vst and HAS_PEDALBOARD:
                                 from pedalboard import Pedalboard
                                 # Convert MIDI events with precise sample offset
                                 midi_messages = PluginManager.midi_events_to_messages(
-                                    midi_events, bpm, self.sample_rate
+                                    midi_events, bpm, self.sample_rate,
+                                    bank=soundfont_bank, program=soundfont_program
                                 )
                                 total_needed = 0
                                 for ev in midi_events:
@@ -149,7 +191,7 @@ class PythonRenderEngine:
                                     import fluidsynth
                                     fl = fluidsynth.FluidSynth(sample_rate=self.sample_rate, gain=0.5)
                                     fid = fl.sfload(sf_path)
-                                    fl.program_select(0, fid, 0, 0)
+                                    fl.program_select(midi_channel, fid, soundfont_bank, soundfont_program)
                                     beat_sec = 60.0 / bpm
                                     total_sec = 0
                                     for ev in midi_events:
@@ -165,11 +207,11 @@ class PythonRenderEngine:
                                         dur_beats = ev.get("duration_beats", 1.0)
                                         start_sec = start_beat * beat_sec
                                         dur_sec = dur_beats * beat_sec
-                                        fl.noteon(0, note, velocity)
+                                        fl.noteon(midi_channel, note, velocity)
                                         start_s = int(start_sec * self.sample_rate)
                                         dur_s = int(dur_sec * self.sample_rate)
                                         block = fl.get_samples(int(dur_s)) if hasattr(fl, 'get_samples') else np.zeros((2, dur_s), dtype=np.float32)
-                                        fl.noteoff(0, note)
+                                        fl.noteoff(midi_channel, note)
                                         if block.shape[1] > 0:
                                             end_s = min(start_s + block.shape[1], total_samples)
                                             actual = end_s - start_s

app/core/soundfont_inspector.py

@@ -0,0 +1,155 @@
+import os
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+try:
+    from sf2utils.sf2parse import Sf2File
+    HAS_SF2UTILS = True
+except ImportError:
+    HAS_SF2UTILS = False
+
+GM_CATEGORIES = [
+    ("Piano", range(0, 8)),
+    ("Chromatic Percussion", range(8, 16)),
+    ("Organ", range(16, 24)),
+    ("Guitar", range(24, 32)),
+    ("Bass", range(32, 40)),
+    ("Strings", range(40, 48)),
+    ("Ensemble", range(48, 56)),
+    ("Brass", range(56, 64)),
+    ("Reed", range(64, 72)),
+    ("Pipe", range(72, 80)),
+    ("Synth Lead", range(80, 90)),
+    ("Synth Pad", range(90, 104)),
+]
+
+MAX_CONDENSED_ENTRIES = 50
+
+
+class SoundFontInspector:
+    def __init__(self, system_sf_dir="/opt/daw_engine/soundfonts", upload_sf_dir=None):
+        self.system_sf_dir = system_sf_dir
+        self.upload_sf_dir = upload_sf_dir
+        self._catalog_cache = None
+
+    def invalidate_catalog_cache(self):
+        self._catalog_cache = None
+
+    def inspect_sf2_file(self, filepath: str) -> dict:
+        if not HAS_SF2UTILS:
+            logger.warning("sf2utils not installed, cannot inspect .sf2 files")
+            return {}
+        if not os.path.exists(filepath):
+            return {}
+
+        try:
+            sf_name = os.path.basename(filepath)
+            sf_id = os.path.splitext(sf_name)[0].lower()
+
+            instruments = []
+            with open(filepath, 'rb') as f:
+                sf2 = Sf2File(f)
+                for preset in sf2.presets:
+                    name = preset.name.strip()
+                    if name == "EOP" or (preset.bank == 128 and preset.preset == 127):
+                        continue
+                    instruments.append({
+                        "bank": preset.bank,
+                        "program": preset.preset,
+                        "name": name,
+                        "is_percussion": (preset.bank == 128)
+                    })
+
+            return {
+                "soundfont_id": sf_id,
+                "filename": sf_name,
+                "total_instruments": len(instruments),
+                "instruments": instruments
+            }
+        except Exception as e:
+            logger.warning(f"Skipping corrupted .sf2 file {filepath}: {e}")
+            return {}
+
+    def _scan_directory(self, directory: str) -> dict:
+        catalog = {}
+        if not os.path.isdir(directory):
+            return catalog
+        for fname in os.listdir(directory):
+            if not fname.lower().endswith(('.sf2', '.sf3')):
+                continue
+            full_path = os.path.join(directory, fname)
+            sf_info = self.inspect_sf2_file(full_path)
+            if sf_info and sf_info.get("soundfont_id"):
+                catalog[sf_info["soundfont_id"]] = sf_info
+        return catalog
+
+    def generate_full_catalog(self, output_json_path: str = None) -> dict:
+        catalog = {}
+        catalog.update(self._scan_directory(self.system_sf_dir))
+        if self.upload_sf_dir and os.path.isdir(self.upload_sf_dir):
+            catalog.update(self._scan_directory(self.upload_sf_dir))
+
+        if output_json_path:
+            os.makedirs(os.path.dirname(output_json_path), exist_ok=True)
+            with open(output_json_path, 'w', encoding='utf-8') as f:
+                json.dump(catalog, f, ensure_ascii=False, indent=2)
+
+        self._catalog_cache = catalog
+        return catalog
+
+    def get_catalog(self) -> dict:
+        if self._catalog_cache is not None:
+            return self._catalog_cache
+        return self.generate_full_catalog()
+
+    def get_condensed_catalog_summary(self) -> dict:
+        catalog = self.get_catalog()
+        condensed = {}
+        for sf_id, sf_info in catalog.items():
+            instruments = sf_info.get("instruments", [])
+            if not instruments:
+                continue
+
+            selected = []
+            used_programs = set()
+            for cat_name, prog_range in GM_CATEGORIES:
+                cat_members = [
+                    inst for inst in instruments
+                    if inst["program"] in prog_range and not inst["is_percussion"]
+                ]
+                if cat_members:
+                    representative = cat_members[0]
+                    key = (representative["program"], representative["bank"])
+                    if key not in used_programs:
+                        used_programs.add(key)
+                        selected.append(representative)
+
+            drum_kits = [inst for inst in instruments if inst["is_percussion"]]
+            for dk in drum_kits[:3]:
+                key = (dk["program"], dk["bank"])
+                if key not in used_programs:
+                    used_programs.add(key)
+                    selected.append(dk)
+
+            if len(selected) > MAX_CONDENSED_ENTRIES:
+                selected = selected[:MAX_CONDENSED_ENTRIES]
+
+            condensed[sf_id] = {
+                "soundfont_id": sf_info["soundfont_id"],
+                "filename": sf_info["filename"],
+                "total_instruments": sf_info["total_instruments"],
+                "condensed_count": len(selected),
+                "instruments": selected
+            }
+        return condensed
+
+    def format_condensed_for_prompt(self) -> str:
+        condensed = self.get_condensed_catalog_summary()
+        lines = []
+        for sf_id, info in condensed.items():
+            lines.append(f"SoundFont ID: '{sf_id}' (File: {info['filename']}):")
+            for inst in info["instruments"]:
+                lines.append(f"  - {inst['name']}: bank={inst['bank']}, program={inst['program']}")
+        return "\n".join(lines)

app/core/vst_engine.py

@@ -286,11 +286,15 @@ class PluginManager:
         }
 
     @staticmethod
-    def midi_events_to_messages(midi_events: list, bpm: float, sr: int) -> list:
+    def midi_events_to_messages(midi_events: list, bpm: float, sr: int, bank: int = None, program: int = None) -> list:
         if not HAS_PEDALBOARD:
             return []
         beat_duration_sec = 60.0 / max(30.0, bpm)
         messages = []
+        if bank is not None:
+            messages.append(MidiMessage(control_change=0, value=bank, sample_offset=0))
+        if program is not None:
+            messages.append(MidiMessage(program_change=program, sample_offset=0))
         for ev in midi_events:
             note = ev.get("note", 60)
             velocity = ev.get("velocity", 100)
@@ -313,3 +317,29 @@ class PluginManager:
         if data[8:12] != b'sfbk':
             return False
         return True
+
+
+class DecentSamplerManager:
+    def __init__(self, vst_path="/opt/daw_engine/vst3/DecentSampler.vst3"):
+        self.vst_path = vst_path
+
+    def create_decent_sampler_instance(self, dspreset_path: str):
+        if not HAS_PEDALBOARD:
+            raise RuntimeError("pedalboard not available")
+        if not os.path.exists(self.vst_path):
+            raise FileNotFoundError(f"DecentSampler VST3 not found at {self.vst_path}")
+        if not os.path.exists(dspreset_path):
+            raise FileNotFoundError(f"Preset file not found at {dspreset_path}")
+
+        plugin = VST3Plugin(self.vst_path)
+
+        abs_preset = os.path.abspath(dspreset_path)
+        preset_dir = os.path.dirname(abs_preset)
+        cwd_before = os.getcwd()
+        try:
+            os.chdir(preset_dir)
+            plugin.load_preset(abs_preset)
+        finally:
+            os.chdir(cwd_before)
+
+        return plugin

app/static/js/app.jsx

@@ -3351,9 +3351,13 @@ const PluginManagerModal = ({ isOpen, onClose, pluginsData }) => {
     try {
       const result = await window.SonicAPI.uploadSoundFont(file);
       setSfUploadStatus('Uploaded: ' + result.name);
-      // Refresh plugin list
+      // Refresh plugin list and catalog
       const data = await window.SonicAPI.listPlugins();
       setLocalData(data);
+      try {
+        const cat = await window.SonicAPI.getSoundfontCatalog();
+        window.__soundfontCatalog = cat;
+      } catch (_) {}
     } catch (err) {
       setSfUploadStatus('Error: ' + err.message);
     }
@@ -7027,6 +7031,11 @@ const App = () => {
             if (active) setSelectedProviderId(active.id);
           }
         } catch (e) { }
+        try {
+          window.SonicAPI.getSoundfontCatalog().then(cat => {
+            window.__soundfontCatalog = cat;
+          }).catch(() => {});
+        } catch (e) { }
       })();
     }
   };
@@ -13802,6 +13811,13 @@ const App = () => {
               }))
             };
             targetTrack.midiItems = [...(targetTrack.midiItems || []), newMidiItem];
+            if (aiTrack.soundfont_bank !== undefined && aiTrack.soundfont_program !== undefined) {
+              targetTrack.soundfont_bank = aiTrack.soundfont_bank;
+              targetTrack.soundfont_program = aiTrack.soundfont_program;
+              if (window.SonicSF && window.SonicSF.applyAITrackInstrument) {
+                window.SonicSF.applyAITrackInstrument(aiTrack.soundfont_bank, aiTrack.soundfont_program);
+              }
+            }
           });
           return updatedTracks;
         });

app/static/js/services/aiGateway.js

@@ -48,7 +48,7 @@ const AIGateway = (function() {
     name: 'fade_out', description: 'Fade-out clip (0.5s đến max)', parameters: { type: 'object', properties: { track_id: { type: 'string' }, duration_seconds: { type: 'number' }, clip_index: { type: 'number', description: 'Chỉ số của clip trên track (1-based, ví dụ: 1 cho clip 1, 2 cho clip 2)' }, clip_id: { type: 'string', description: 'ID của clip cụ thể' } } }
   }, {
     name: 'generate_multitrack_midi',
-    description: 'Generates multi-track MIDI data based on genre, bar duration, and requested instruments list.',
+    description: 'Generates multi-track MIDI data along with SoundFont Program configurations for each track.',
     parameters: {
       type: 'object',
       properties: {
@@ -57,12 +57,15 @@ const AIGateway = (function() {
         total_bars: { type: 'integer' },
         tracks: {
           type: 'array',
-          description: 'Array of instrument tracks along with their corresponding MIDI notes',
+          description: 'Array of instrument tracks with MIDI notes and SoundFont instrument selection',
           items: {
             type: 'object',
             properties: {
               track_name: { type: 'string', description: 'Track name (e.g., String Ensemble, Epic Brass, Taiko Drums)' },
               instrument_type: { type: 'string', enum: ['STRINGS', 'BRASS', 'SYNTH', 'PERCUSSION', 'DRUMS'] },
+              soundfont_id: { type: 'string', description: "ID of the SoundFont to use (e.g. 'generaluser_gs')" },
+              soundfont_bank: { type: 'integer', default: 0, description: 'MIDI Bank code: 0 for melodic instruments, 128 for Drums/Percussion' },
+              soundfont_program: { type: 'integer', description: 'MIDI Program Number 0-127 matching the instrument name in the SoundFont catalog' },
               notes: {
                 type: 'array',
                 items: {
@@ -77,7 +80,7 @@ const AIGateway = (function() {
                 }
               }
             },
-            required: ['track_name', 'instrument_type', 'notes']
+            required: ['track_name', 'instrument_type', 'soundfont_id', 'soundfont_bank', 'soundfont_program', 'notes']
           }
         }
       },
@@ -176,13 +179,29 @@ const AIGateway = (function() {
     return calls;
   }
 
+  function buildCatalogPromptSection() {
+    const catalog = window.__soundfontCatalog;
+    if (!catalog || !catalog.condensed_catalog) return '';
+    const lines = [];
+    for (const [sfId, info] of Object.entries(catalog.condensed_catalog)) {
+      lines.push(`SoundFont ID: '${sfId}' (File: ${info.filename}):`);
+      for (const inst of info.instruments || []) {
+        lines.push(`  - ${inst.name}: bank=${inst.bank}, program=${inst.program}`);
+      }
+    }
+    if (lines.length === 0) return '';
+    return `\n=== SOUNDFONT INSTRUMENT CATALOG ===\nYou have the following SoundFont instruments available on the system:\n${lines.join('\n')}\n\nMANDATORY RULES WHEN CREATING TRACKS WITH generate_multitrack_midi:\n1. You MUST look up the catalog above and fill in the correct soundfont_id, soundfont_bank, and soundfont_program for each track.\n2. Melodic instruments (Piano, Strings, Brass, etc.) MUST use soundfont_bank=0.\n3. Drums and Percussion MUST use soundfont_bank=128.\n4. Example: For \"Brass horns\", use soundfont_id="generaluser_gs", soundfont_bank=0, soundfont_program=56.\n5. Example: For \"Drum kit\", use soundfont_id="generaluser_gs", soundfont_bank=128, soundfont_program=0.\n`;
+  }
+
   function buildUserMessage(prompt, context, systemInstruction = '') {
     const contextStr = JSON.stringify(context, null, 2);
     const toolNames = DEFAULT_TOOLS.map(t => `  - ${t.name}: ${t.description}`).join('\n');
+    const catalogSection = buildCatalogPromptSection();
     return [
       { role: 'system', content: `Bạn là trợ lý điều khiển DAW chuyên nghiệp.
 Nhiệm vụ của bạn là phân tích yêu cầu của người dùng và chuyển đổi thành danh sách các function calls tương ứng.
 ${systemInstruction ? `\nHướng dẫn tạo nhạc đặc biệt từ Preset:\n${systemInstruction}\n` : ''}
+${catalogSection}
 QUAN TRỌNG:
 1. Bạn đang hoạt động ở chế độ một lượt (one-shot). Hãy trả về TẤT CẢ các function calls cần thiết để thực hiện toàn bộ các bước trong yêu cầu của người dùng trong một phản hồi duy nhất. Đừng thực hiện từng bước qua nhiều lượt chat.
 2. Có thể gọi nhiều function cùng một lúc (gọi song song/nối tiếp). Chúng sẽ được thực thi theo thứ tự bạn trả về.

app/static/js/services/api.js

@@ -63,6 +63,7 @@ window.API_BASE_URL = window.API_BASE_URL || window.location.origin;
         savePreferences: (prefs) => apiRequest('/api/v1/user/preferences', { method: 'POST', body: JSON.stringify({ preferences: prefs }) }),
 
         listPlugins: () => apiRequest('/api/v1/plugins/available', { method: 'GET' }),
+        getSoundfontCatalog: () => apiRequest('/api/v1/plugins/soundfonts/catalog', { method: 'GET' }),
         listDefaultSoundfonts: () => apiRequest('/api/v1/plugins/default-soundfonts', { method: 'GET' }),
         listSoundfontInstruments: (sfId) => apiRequest(`/api/v1/plugins/soundfont-instruments/${sfId}`, { method: 'GET' }),
         renderProject: (projectJson, outputFilename) => apiRequest('/api/v1/plugins/render', { method: 'POST', body: JSON.stringify({ project_json: projectJson, output_filename: outputFilename }) }),

app/static/js/services/soundfontPlayer.js

@@ -30,9 +30,45 @@
         return window.__sharedAudioCtx;
     };
 
+    // ── Per-channel MIDI state (16 GM channels) ──
+    const _channels = Array.from({ length: 16 }, () => ({ bank: 0, program: 0, isPercussion: false }));
+    let _nextMelodicChannel = 0;
+
     const SonicSF = {
         loadedFonts: {},
 
+        controllerChange: function (channel, controller, value) {
+            if (channel < 0 || channel > 15) return;
+            if (controller === 0) {
+                _channels[channel].bank = value;
+                _channels[channel].isPercussion = (value === 128);
+            }
+        },
+
+        programChange: function (channel, program) {
+            if (channel < 0 || channel > 15) return;
+            _channels[channel].program = program;
+        },
+
+        allocateChannel: function (bank) {
+            if (bank === 128) return 9;
+            const ch = _nextMelodicChannel % 9;
+            _nextMelodicChannel = (_nextMelodicChannel + 1) % 9;
+            return ch;
+        },
+
+        applyAITrackInstrument: function (bank, program) {
+            const channel = this.allocateChannel(bank);
+            this.controllerChange(channel, 0, bank);
+            this.programChange(channel, program);
+            return channel;
+        },
+
+        getChannelState: function (channel) {
+            if (channel < 0 || channel > 15) return null;
+            return { ..._channels[channel] };
+        },
+
         // Load SoundFont from URL → ArrayBuffer → store in memory
         loadSoundFont: async function (url) {
             if (this.loadedFonts[url]) return this.loadedFonts[url];
@@ -43,7 +79,7 @@
             return buffer;
         },
 
-        playNote: function (note, velocity, durationMs, startTime, program, destinationNode) {
+        playNote: function (note, velocity, durationMs, startTime, program, destinationNode, channel) {
             const ctx = getCtx();
             const freq = 440 * Math.pow(2, (note - 69) / 12);
             if (freq <= 0 || isNaN(freq)) return null;
@@ -59,7 +95,11 @@
             let releaseTime = 0.2;
             let volFactor = 0.25;
 
-            const prog = program !== undefined ? parseInt(program) : 0;
+            let prog = program !== undefined ? parseInt(program) : 0;
+            if (channel !== undefined && channel >= 0 && channel < 16) {
+                const chState = _channels[channel];
+                prog = chState.program || prog;
+            }
             if (prog >= 0 && prog <= 7) { // Pianos
                 oscType = 'sine';
                 decayTime = 0.3;

requirements.txt

@@ -14,3 +14,4 @@ jsonschema>=4.18.0
 pedalboard>=0.8.0
 mido>=1.3.0
 pyfluidsynth>=1.3.0
+sf2utils>=0.9.0

wiki.md

@@ -270,3 +270,9 @@
 - **Tóm tắt thay đổi:** (1) Sub-tab loop selection chỉ active khi nút Loop (st.isLooping) bật — không còn bị ảnh hưởng bởi global isLoopingSelection. (2) Stop dừng ngay lập tức: `SonicSF.stopAll()` set gain 0 và stop oscillator tại ctx.currentTime, không ramp.
 - **Các file ảnh hưởng:** `app/static/js/app.jsx`, `app/static/js/services/soundfontPlayer.js`, `app/static/js/app.precompiled.js`
 - **Ghi chú/Test (nếu có):** `npm run build` pass.
+
+### [2026-07-26 12:31] Task: VST/SoundFont inspection engine + AI schema
+- **Summary:** New `SoundFontInspector` (sf2utils) scans .sf2 files, generates full/condensed catalog. Added `/soundfonts/catalog` API. Updated AI tool schema with `soundfont_id/bank/program`. Condensed catalog auto-injected into system prompt. Server render now passes bank/program + MIDI channel routing (drums → ch9). Added `DecentSamplerManager` + Pianobook render path. Client `SonicSF`: `applyAITrackInstrument`, `controllerChange`, `programChange`. Catalog cache invalidated on upload.
+- **Files:** `requirements.txt`, `app/core/soundfont_inspector.py` (new), `app/api/v1/plugins.py`, `app/core/vst_engine.py`, `app/core/render_engine.py`, `app/static/js/services/aiGateway.js`, `app/static/js/services/api.js`, `app/static/js/services/soundfontPlayer.js`, `app/static/js/app.jsx`, `Dockerfile`, `.gitignore`, `vst_plugins/` (new), `samples/pianobook/` (new)
+- **Tests:** `python3 -m pytest tests/ -v` → 61 passed, 1 pre-existing failure. Python syntax check OK on all modified files.
+---