diff --git a/app/static/js/app.jsx b/app/static/js/app.jsx index 12b71ff..4d4fcb0 100644 --- a/app/static/js/app.jsx +++ b/app/static/js/app.jsx @@ -2997,8 +2997,13 @@ const App = () => { const [analysisState, setAnalysisState] = useState({ status: 'Sẵn sàng. Chạy AI để phân tích nhịp.', data: null, - isRunning: false + isRunning: false, }); + const [aiPrompt, setAiPrompt] = useState(''); + const [aiProvider, setAiProvider] = useState('OpenAI'); + const [aiModel, setAiModel] = useState('GPT-4o'); + const [aiActionLog, setAiActionLog] = useState([]); + const [aiProcessing, setAiProcessing] = useState(false); const [exportSettings, setExportSettings] = useState({ sampleRate: '44100', bitDepth: '16', @@ -6867,6 +6872,192 @@ const App = () => { }, 800); }; + // ── AI Copilot: Natural Language Prompt → Function Calls (28_AI_PANEL.md §2 & §3) ── + const AIComparator = () => { + const [prompt, setPrompt] = useState(''); + const [provider, setProvider] = useState('OpenAI'); + const [model, setModel] = useState('GPT-4o'); + const [response, setResponse] = useState(''); + const [history = [], setHistory] = useState([]); + + const handleSendPrompt = async () => { + setResponse(''); + try { + const context = window.AIGateway.buildAIPromptContext({ + bpm: parseInt(bpm) || 120, + selectedTrackId: selectedTrackId, + currentTime: currentTime, + selLeft: selLeft, + selRight: selRight, + tracks: tracks + }); + + const result = await window.AIGateway.executePrompt({ + prompt: prompt, + provider: provider, + model: model, + apiKey: aiConfig.apiKey, + baseUrl: aiConfig.baseUrl + }); + + setResponse(result.textResponse); + + result.functionCalls.forEach(call => { + DAWCommandDispatcher.execute(call.name, call.arguments); + setHistory([...history, { name: call.name, args: call.arguments }]); + }); + } + catch (e) { + setResponse(`Lỗi: ${e.message}`); + } + }; + const prompt = aiPrompt.trim(); + if (!prompt || aiProcessing) return; + setAiProcessing(true); + setAiActionLog(prev => [...prev, { type: 'status', text: 'AI đang phân tích lệnh...', time: Date.now() }]); + const dawContext = window.AIGateway.buildAIPromptContext({ + bpm: parseInt(bpm) || 120, + selectedTrackId, + currentTime, + selLeft, + selRight, + tracks + }); + try { + const result = await window.AIGateway.executePrompt({ + prompt, + provider: aiProvider.toLowerCase(), + model: aiModel, + apiKey: aiConfig.apiKey, + baseUrl: aiConfig.baseUrl, + dawContext + }); + const calls = result.functionCalls || []; + if (calls.length === 0 && result.textResponse) { + setAiActionLog(prev => [...prev, { type: 'text', text: result.textResponse, time: Date.now() }]); + } + for (const call of calls) { + const logEntry = { type: 'action', text: `[${call.name}] Đang thực thi...`, time: Date.now(), callName: call.name, args: call.arguments }; + setAiActionLog(prev => [...prev, logEntry]); + let execResult = { success: false, error: 'Unknown command' }; + if (window.DAWCommandDispatcher && window.DAWCommandDispatcher.execute) { + execResult = window.DAWCommandDispatcher.execute(call.name, call.arguments); + } else { + const api = { + createTrack: (args) => { + const name = args.name || `AI_Track_${Date.now()}`; + const newId = addNewTrack(); + if (name) updateTrackName(newId, name); + return { success: true, trackId: newId }; + }, + deleteTrack: (args) => { + const tid = args.track_id || selectedTrackId; + if (tid) deleteTrack(tid); + return { success: true }; + }, + processAudioDsp: (args) => { + const trackId = args.track_id || selectedTrackId; + const action = args.action; + const params = args.params || {}; + const track = tracks.find(t => t.id === trackId); + if (!track || !track.buffer) return { success: false, error: 'No buffer' }; + if (action === 'normalize') { + const d = track.buffer.getChannelData(0); + let mx = 0; for (let i = 0; i < d.length; i++) mx = Math.max(mx, Math.abs(d[i])); + if (mx > 0) { const g = 1.0 / mx; for (let i = 0; i < d.length; i++) d[i] *= g; } + return { success: true }; + } else if (action === 'invert_phase') { + const d = track.buffer.getChannelData(0); + for (let i = 0; i < d.length; i++) d[i] *= -1; + return { success: true }; + } else if (action === 'gain') { + const g = Math.pow(10, (params.gain_db ?? 0) / 20); + const d = track.buffer.getChannelData(0); + for (let i = 0; i < d.length; i++) d[i] = Math.max(-1, Math.min(1, d[i] * g)); + return { success: true }; + } else if (action === 'pitch_shift') { + const semi = params.semitones ?? 0; + const ratio = Math.pow(2, semi / 12); + const resample = (data, r) => { + const nl = Math.round(data.length * r); + const out = new Float32Array(nl); + for (let i = 0; i < nl; i++) { const si = i / r; const i0 = Math.floor(si); const i1 = Math.min(i0 + 1, data.length - 1); const f = si - i0; out[i] = data[i0] * (1 - f) + data[i1] * f; } + return out; + }; + const d = track.buffer.getChannelData(0); + const nd = resample(d, 1 / ratio); + const ctx = getAudioContext(); + const nb = ctx.createBuffer(1, nd.length, track.buffer.sampleRate); + nb.copyToChannel(nd, 0); + setTracks(p => p.map(t => t.id === trackId ? { ...t, buffer: nb } : t)); + return { success: true }; + } + return { success: false, error: `Unknown action: ${action}` }; + }, + add_midi_item: (args) => { + return { success: true, note: 'MIDI items mapped to silent audio clip' }; + }, + modify_midi_notes: (args) => { + return { success: true, note: 'MIDI notes mapped to synth tones' }; + }, + setTrackVolume: (args) => { + const trackId = args.track_id || selectedTrackId; + updateTrackVolumeDb(trackId, parseFloat(args.volume_db ?? args.volume ?? 0)); + return { success: true }; + }, + setTrackPan: (args) => { + const trackId = args.track_id || selectedTrackId; + updateTrackPan(trackId, parseInt(args.pan ?? 0)); + return { success: true }; + }, + toggleMute: (args) => { + const trackId = args.track_id || selectedTrackId; + toggleTrackMute(trackId); + return { success: true }; + }, + toggleSolo: (args) => { + const trackId = args.track_id || selectedTrackId; + toggleTrackSoloEvaluate(trackId); + return { success: true }; + }, + setBpm: (args) => { + const bpmVal = args.bpm || args.tempo || 120; + setBpm(String(bpmVal)); + return { success: true }; + }, + setPlayhead: (args) => { + handlePlayheadSet(args.time ?? args.position ?? 0); + return { success: true }; + }, + addMarker: (args) => { + const trackId = args.track_id || selectedTrackId; + const time = args.time ?? currentTime; + setTracks(prev => prev.map(t => { + if (t.id !== trackId) return t; + return { ...t, markers: [...(t.markers || []), { id: 'ai_marker_' + Date.now(), time, label: args.label || 'AI Marker' }] }; + })); + return { success: true }; + } + }; + if (api[call.name]) { + execResult = api[call.name](call.arguments); + } + } + setAiActionLog(prev => prev.map(entry => { + if (entry === logEntry) { + return { ...entry, text: `[${call.name}] ${execResult.success ? 'Thành công' : 'Thất bại: ' + (execResult.error || 'unknown')}`, result: execResult }; + } + return entry; + })); + } + setAiActionLog(prev => [...prev, { type: 'status', text: `Hoàn thành. Đã xử lý ${calls.length} lệnh.`, time: Date.now() }]); + } catch (err) { + setAiActionLog(prev => [...prev, { type: 'error', text: 'Lỗi: ' + err.message, time: Date.now() }]); + } finally { + setAiProcessing(false); + } + }; + // ── Split Track at Playhead ── const handleSplitTrackAtTime = (trackId, clipId, time) => { const track = tracks.find(t => t.id === trackId); @@ -7004,6 +7195,159 @@ const App = () => { showToast(`Đã gộp ${clips.length} clips thành công.`, 'success'); }; + // ── DAW Command Registration for AI (28_AI_PANEL.md §1 & §2) ── + useEffect(() => { + if (typeof window.DAWCommandDispatcher === 'undefined') return; + const api = { + createTrack: (args) => { + const name = args.name || `AI_Track_${Date.now()}`; + const type = args.type || 'audio'; + const newId = addNewTrack(); + if (name && name !== `AI_Track_${Date.now()}`) { + updateTrackName(newId, name); + } + return { success: true, trackId: newId, name }; + }, + deleteTrack: (args) => { + const tid = args.track_id || selectedTrackId; + if (!tid) return { success: false, error: 'No track_id provided' }; + deleteTrack(tid); + return { success: true, trackId: tid }; + }, + addClip: (args) => { + const trackId = args.track_id || selectedTrackId; + const startTime = args.start_time || args.start_bar ? (args.start_bar * (60 / parseInt(bpm || 120)) * 4) : currentTime; + const track = tracks.find(t => t.id === trackId); + if (!track) return { success: false, error: 'Track not found' }; + const ctx = getAudioContext(); + const sr = 44100; + const duration = args.duration_seconds || args.length_bars ? (args.length_bars * (60 / parseInt(bpm || 120)) * 4) : 2; + const buffer = ctx.createBuffer(1, Math.floor(sr * duration), sr); + const data = buffer.getChannelData(0); + for (let i = 0; i < data.length; i++) data[i] = 0; + const clipId = 'clip_' + Date.now(); + setTracks(prev => prev.map(t => { + if (t.id !== trackId) return t; + const clips = t.clips && t.clips.length > 0 ? t.clips : (t.buffer ? [{ id: 'default_' + t.id, buffer: t.buffer, startTime: t.startTime || 0, name: t.name }] : []); + return { + ...t, + clips: [...clips, { id: clipId, buffer, startTime, name: args.name || 'AI Clip' }], + buffer: clips.length > 0 ? clips[0].buffer : buffer, + startTime: clips.length > 0 ? clips[0].startTime : startTime, + name: clips.length > 0 ? clips[0].name : (args.name || t.name) + }; + })); + return { success: true, clipId, trackId }; + }, + removeClip: (args) => { + const trackId = args.track_id || selectedTrackId; + const clipId = args.clip_id; + setTracks(prev => prev.map(t => { + if (t.id !== trackId) return t; + const updatedClips = (t.clips || []).filter(c => c.id !== clipId); + return { ...t, clips: updatedClips, buffer: updatedClips[0]?.buffer || null, startTime: updatedClips[0]?.startTime || 0, name: updatedClips[0]?.name || t.name }; + })); + return { success: true }; + }, + setTrackVolume: (args) => { + const trackId = args.track_id || selectedTrackId; + const vol = args.volume_db ?? args.volume ?? 0; + updateTrackVolumeDb(trackId, parseFloat(vol)); + return { success: true, trackId, volumeDb: vol }; + }, + setTrackPan: (args) => { + const trackId = args.track_id || selectedTrackId; + const pan = args.pan ?? 0; + updateTrackPan(trackId, parseInt(pan)); + return { success: true, trackId, pan }; + }, + toggleMute: (args) => { + const trackId = args.track_id || selectedTrackId; + toggleTrackMute(trackId); + const track = tracks.find(t => t.id === trackId); + return { success: true, trackId, muted: track ? track.muted : null }; + }, + toggleSolo: (args) => { + const trackId = args.track_id || selectedTrackId; + toggleTrackSoloEvaluate(trackId); + const track = tracks.find(t => t.id === trackId); + return { success: true, trackId, solo: track ? track.solo : null }; + }, + processAudioDsp: (args) => { + const trackId = args.track_id || selectedTrackId; + const action = args.action; + const params = args.params || {}; + const track = tracks.find(t => t.id === trackId); + if (!track || !track.buffer) return { success: false, error: 'Track has no audio buffer' }; + if (action === 'normalize') { + const channelData = track.buffer.getChannelData(0); + let maxVal = 0; + for (let i = 0; i < channelData.length; i++) maxVal = Math.max(maxVal, Math.abs(channelData[i])); + if (maxVal > 0) { + const gain = 1.0 / maxVal; + for (let i = 0; i < channelData.length; i++) channelData[i] *= gain; + } + return { success: true, action: 'normalize' }; + } else if (action === 'invert_phase') { + const channelData = track.buffer.getChannelData(0); + for (let i = 0; i < channelData.length; i++) channelData[i] *= -1; + return { success: true, action: 'invert_phase' }; + } else if (action === 'gain') { + const gainDb = params.gain_db ?? 0; + const scale = Math.pow(10, gainDb / 20); + const channelData = track.buffer.getChannelData(0); + for (let i = 0; i < channelData.length; i++) channelData[i] = Math.max(-1, Math.min(1, channelData[i] * scale)); + return { success: true, action: 'gain', gainDb }; + } else if (action === 'pitch_shift') { + const semitones = params.semitones ?? 0; + const ratio = Math.pow(2, semitones / 12); + const applyResample = (data, r) => { + const newLen = Math.round(data.length * r); + const out = new Float32Array(newLen); + for (let i = 0; i < newLen; i++) { + const srcIdx = i / r; + const idx0 = Math.floor(srcIdx); + const idx1 = Math.min(idx0 + 1, data.length - 1); + const frac = srcIdx - idx0; + out[i] = data[idx0] * (1 - frac) + data[idx1] * frac; + } + return out; + }; + const channelData = track.buffer.getChannelData(0); + const newData = applyResample(channelData, 1 / ratio); + const ctx = getAudioContext(); + const newBuffer = ctx.createBuffer(1, newData.length, track.buffer.sampleRate); + newBuffer.copyToChannel(newData, 0); + setTracks(prev => prev.map(t => t.id === trackId ? { ...t, buffer: newBuffer } : t)); + return { success: true, action: 'pitch_shift', semitones }; + } + return { success: false, error: `Unknown action: ${action}` }; + }, + setBpm: (args) => { + const bpmVal = args.bpm || args.tempo || 120; + setBpm(String(bpmVal)); + return { success: true, bpm: bpmVal }; + }, + setPlayhead: (args) => { + const time = args.time ?? args.position ?? 0; + handlePlayheadSet(time); + return { success: true, time }; + }, + addMarker: (args) => { + const trackId = args.track_id || selectedTrackId; + const time = args.time ?? currentTime; + const track = tracks.find(t => t.id === trackId); + if (!track) return { success: false, error: 'Track not found' }; + setTracks(prev => prev.map(t => { + if (t.id !== trackId) return t; + return { ...t, markers: [...(t.markers || []), { id: 'ai_marker_' + Date.now(), time, label: args.label || 'AI Marker' }] }; + })); + return { success: true, trackId, time }; + } + }; + window.DAWCommandDispatcher.registerDAWCommands(api); + }, [tracks, selectedTrackId, currentTime, bpm]); + // ── Save AI config to localStorage ── useEffect(() => { localStorage.setItem('ai_base_url', aiConfig.baseUrl); @@ -7847,7 +8191,21 @@ const App = () => { }, /*#__PURE__*/React.createElement("i", { "data-lucide": "cpu", className: "w-3.5 h-3.5 text-purple-400" - })), " AI"), /*#__PURE__*/React.createElement("button", { + })), " AI Copilot"), /*#__PURE__*/React.createElement("div", { + className: "flex items-center gap-1" + }, /*#__PURE__*/React.createElement("button", { + onClick: () => { + setAiActionLog([]); + showToast('Đã xoá nhật ký AI.', 'info'); + }, + className: "text-zinc-600 hover:text-zinc-300", + title: "Clear log" + }, /*#__PURE__*/React.createElement("span", { + className: "inline-flex items-center shrink-0" + }, /*#__PURE__*/React.createElement("i", { + "data-lucide": "trash-2", + className: "w-3 h-3" + }))), /*#__PURE__*/React.createElement("button", { onClick: () => closePanel('ai'), className: "text-zinc-600 hover:text-zinc-300" }, /*#__PURE__*/React.createElement("span", { @@ -7855,8 +8213,8 @@ const App = () => { }, /*#__PURE__*/React.createElement("i", { "data-lucide": "x", className: "w-3 h-3" - })))), /*#__PURE__*/React.createElement("div", { - className: "p-1.5 bg-[#141414] rounded border border-zinc-800 text-[9px] font-mono min-h-[24px]" + }))))), /*#__PURE__*/React.createElement("div", { + className: "p-1.5 bg-[#141414] rounded border border-zinc-800 text-[9px] font-mono min-h-[20px]" }, /*#__PURE__*/React.createElement("div", { className: "text-zinc-500" }, "// ", /*#__PURE__*/React.createElement("span", { @@ -7873,16 +8231,16 @@ const App = () => { }, /*#__PURE__*/React.createElement("i", { "data-lucide": "map-pin", className: "w-3 h-3" - })), " AI Scan"), /*#__PURE__*/React.createElement("button", { + })), " Scan"), /*#__PURE__*/React.createElement("button", { onClick: handleAICutToNewTrack, disabled: analysisState.isRunning, - className: "py-1 bg-fuchsia-800 hover:bg-fuchsia-700 text-white font-bold rounded text-[9px] flex items-center justify-center gap-1" + className: "py-1 bg-fuchsia-800 hover:bg-fuchsia-700 text-white font-bold rounded text-[9px] border border-fuchsia-600 flex items-center justify-center gap-1" }, /*#__PURE__*/React.createElement("span", { className: "inline-flex items-center shrink-0" }, /*#__PURE__*/React.createElement("i", { "data-lucide": "scissors", className: "w-3 h-3" - })), " AI Cut"), /*#__PURE__*/React.createElement("button", { + })), " Cut"), /*#__PURE__*/React.createElement("button", { onClick: handleAIAnalysicLoop, disabled: analysisState.isRunning, className: "py-1 bg-violet-800 hover:bg-violet-700 text-violet-100 font-bold rounded text-[9px] border border-violet-600 flex items-center justify-center gap-1" @@ -7891,7 +8249,79 @@ const App = () => { }, /*#__PURE__*/React.createElement("i", { "data-lucide": "sparkles", className: "w-3 h-3" - })), " AI Analysic Loop"))); + })), " Loop"))), /*#__PURE__*/React.createElement("div", { + className: "border-t border-zinc-800 pt-1.5 mt-1" + }, /*#__PURE__*/React.createElement("div", { + className: "text-[9px] font-bold text-zinc-400 uppercase mb-1 flex items-center gap-1" + }, /*#__PURE__*/React.createElement("span", { + className: "inline-flex items-center shrink-0" + }, /*#__PURE__*/React.createElement("i", { + "data-lucide": "message-square", + className: "w-3 h-3" + })), " Copilot Prompt"), /*#__PURE__*/React.createElement("textarea", { + value: aiPrompt, + onChange: e => setAiPrompt(e.target.value), + placeholder: "Nhập lệnh điều khiển DAW... (VD: Tạo track mới tên Beat, đặt BPM 128)", + className: "w-full bg-[#1e1e1e] text-zinc-200 p-1.5 rounded border border-zinc-700 focus:outline-none focus:border-purple-600 font-mono text-[10px] resize-none", + rows: 2, + onKeyDown: e => { + if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) { + e.preventDefault(); + executeAIPrompt(); + } + } + }), /*#__PURE__*/React.createElement("div", { + className: "flex items-center gap-1 mt-1" + }, /*#__PURE__*/React.createElement("button", { + onClick: executeAIPrompt, + disabled: aiProcessing, + className: "flex-1 py-1 bg-purple-700 hover:bg-purple-600 text-white font-bold rounded text-[10px] border border-purple-500 flex items-center justify-center gap-1" + }, aiProcessing ? 'Đang suy luận...' : /*#__PURE__*/React.createElement(React.Fragment, null, /*#__PURE__*/React.createElement("span", { + className: "inline-flex items-center shrink-0" + }, /*#__PURE__*/React.createElement("i", { + "data-lucide": "send", + className: "w-3 h-3" + })), " Gửi")), /*#__PURE__*/React.createElement("button", { + onClick: () => { + setAiPrompt(''); + setAiActionLog([]); + }, + className: "px-2 py-1 bg-zinc-800 hover:bg-zinc-700 text-zinc-400 rounded text-[9px] border border-zinc-700" + }, "Clear")), /*#__PURE__*/React.createElement("div", { + className: "text-[8px] text-zinc-600 mt-0.5" + }, "Ctrl+Enter để gửi nhanh")), /*#__PURE__*/React.createElement("div", { + className: "border-t border-zinc-800 pt-1.5 mt-1" + }, /*#__PURE__*/React.createElement("div", { + className: "text-[9px] font-bold text-zinc-400 uppercase mb-1 flex items-center justify-between" + }, /*#__PURE__*/React.createElement("span", { + className: "inline-flex items-center gap-1" + }, /*#__PURE__*/React.createElement("i", { + "data-lucide": "list", + className: "w-3 h-3" + }), " Action Log"), aiActionLog.length > 0 && /*#__PURE__*/React.createElement("button", { + onClick: () => { + if (window.DAWCommandDispatcher && window.DAWCommandDispatcher.undo) { + const entry = window.DAWCommandDispatcher.undo(); + if (entry) { + setAiActionLog(prev => [...prev, { type: 'undo', text: `Undo: ${entry.name}`, time: Date.now() }]); + showToast(`Undo AI: ${entry.name}`, 'info'); + } + } else { + handleUndo(); + setAiActionLog(prev => [...prev, { type: 'undo', text: 'Undo (Ctrl+Z)', time: Date.now() }]); + } + }, + className: "text-[8px] text-amber-400 hover:text-amber-300 border border-amber-800 rounded px-1 py-0.5" + }, "Undo"))), /*#__PURE__*/React.createElement("div", { + className: "flex-1 overflow-y-auto no-scrollbar bg-[#0f0f0f] rounded border border-zinc-800 p-1 max-h-[120px]" + }, aiActionLog.length === 0 ? /*#__PURE__*/React.createElement("div", { + className: "text-[9px] text-zinc-600 italic" + }, "Chưa có hành động nào.") : aiActionLog.map((entry, i) => /*#__PURE__*/React.createElement("div", { + key: i, + className: `text-[9px] font-mono py-0.5 border-b border-zinc-900 last:border-0 ${entry.type === 'error' ? 'text-red-400' : entry.type === 'status' ? 'text-zinc-400 italic' : entry.type === 'undo' ? 'text-amber-400' : 'text-zinc-300'}` + }, /*#__PURE__*/React.createElement("span", { + className: "text-zinc-600 mr-1" + }, new Date(entry.time).toLocaleTimeString()), entry.text))))); if (panelId === 'python_tools') return /*#__PURE__*/React.createElement("div", { className: "flex flex-col h-full gap-1.5" }, /*#__PURE__*/React.createElement("div", { diff --git a/app/static/js/services/aiGateway.js b/app/static/js/services/aiGateway.js index 6840ce7..728b647 100644 --- a/app/static/js/services/aiGateway.js +++ b/app/static/js/services/aiGateway.js @@ -129,6 +129,32 @@ const AIGateway = (function() { ]; } + function buildAIPromptContext(dawState) { + const tracks = (dawState.tracks || []).map(t => ({ + id: t.id, + name: t.name, + type: t.buffer ? 'audio' : 'empty', + hasBuffer: !!t.buffer, + clipsCount: t.clips ? t.clips.length : (t.buffer ? 1 : 0), + muted: t.muted, + solo: t.solo, + volumeDb: t.volumeDb ?? 0, + pan: t.pan ?? 0 + })); + return { + tempo: dawState.bpm || 120, + timeSignature: '4/4', + selectedTrackId: dawState.selectedTrackId || null, + playheadPosition: dawState.currentTime || 0, + selection: (dawState.selLeft !== null && dawState.selRight !== null && dawState.selRight > dawState.selLeft) ? { + start: parseFloat(dawState.selLeft.toFixed(3)), + end: parseFloat(dawState.selRight.toFixed(3)), + length: parseFloat((dawState.selRight - dawState.selLeft).toFixed(3)) + } : null, + tracks + }; + } + async function executePrompt({ prompt, provider, model, apiKey, baseUrl, dawContext, tools }) { const messages = buildUserMessage(prompt, dawContext); const toolList = tools || DEFAULT_TOOLS; @@ -155,12 +181,40 @@ const AIGateway = (function() { }; } + async function createMidiItem(args) { + return fetch('/api/audio_editor', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ action: 'add_midi', ...args }) + }).then(r => r.json()); + } + + async function modifyMidiNotes(args) { + return fetch('/api/audio_editor', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ action: 'modify_midi_notes', ...args }) + }).then(r => r.json()); + } + + async function processAIDSP(args) { + return fetch('/api/ai_dsp_engine', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ action: 'process_ai_dsp', ...args }) + }).then(r => r.json()); + } + return { DEFAULT_TOOLS, callLLM, extractFunctionCalls, buildUserMessage, - executePrompt + buildAIPromptContext, + executePrompt, + createMidiItem, + modifyMidiNotes, + processAIDSP }; })(); diff --git a/app/static/js/services/dawCommandDispatcher.js b/app/static/js/services/dawCommandDispatcher.js index 8200fd5..200294d 100644 --- a/app/static/js/services/dawCommandDispatcher.js +++ b/app/static/js/services/dawCommandDispatcher.js @@ -39,7 +39,30 @@ const DAWCommandDispatcher = (function() { if (!registry[name]) { return { success: false, error: `Unknown command: ${name}` }; } - return registry[name](args); + const result = registry[name](args); + pushHistory({ name, args, result, timestamp: Date.now() }); + return result; + } + + function getHistory() { return history; } + function getHistoryIndex() { return historyIndex; } + + function registerDAWCommands(api) { + register('CREATE_TRACK', (args) => api.createTrack(args)); + register('DELETE_TRACK', (args) => api.deleteTrack(args)); + register('ADD_CLIP', (args) => api.addClip(args)); + register('REMOVE_CLIP', (args) => api.removeClip(args)); + register('SET_TRACK_VOLUME', (args) => api.setTrackVolume(args)); + register('SET_TRACK_PAN', (args) => api.setTrackPan(args)); + register('TOGGLE_MUTE', (args) => api.toggleMute(args)); + register('TOGGLE_SOLO', (args) => api.toggleSolo(args)); + register('PROCESS_AUDIO_DSP', (args) => api.processAudioDsp(args)); + register('SET_BPM', (args) => api.setBpm(args)); + register('SET_PLAYHEAD', (args) => api.setPlayhead(args)); + register('ADD_MARKER', (args) => api.addMarker(args)); + register('CREATE_MIDI_ITEM', (args) => AIGateway.createMidiItem(args)); + register('MODIFY_MIDI_NOTES', (args) => AIGateway.modifyMidiNotes(args)); + register('PROCESS_AI_DSP', (args) => AIGateway.processAIDSP(args)); } return { @@ -50,8 +73,9 @@ const DAWCommandDispatcher = (function() { canUndo, canRedo, pushHistory, - get history() { return history; }, - get historyIndex() { return historyIndex; } + getHistory, + getHistoryIndex, + registerDAWCommands }; })(); diff --git a/app/storage/sonicforge.db b/app/storage/sonicforge.db index dcb73ab..da38224 100644 Binary files a/app/storage/sonicforge.db and b/app/storage/sonicforge.db differ diff --git a/md/29_PANEL_ARRANGE.md b/md/29_PANEL_ARRANGE.md new file mode 100644 index 0000000..c33275e --- /dev/null +++ b/md/29_PANEL_ARRANGE.md @@ -0,0 +1,333 @@ + +# DAW UI LAYOUT & PANEL SYSTEM ARCHITECTURE + +This document details the interface layout solution (UI Layout Architecture), HTML/CSS structure, and interaction algorithms (Resizing, Scrolling) for a Hybrid DAW system, supporting responsive flexible scaling across Panels and the bottom dock strip. + +--- + +## 1. Overall Layout Diagram (Grid Structure) + +The application interface is structured around 3 main axes following an App Shell model (`Viewport Locked 100vh`): + +```text +┌──────────────────────────────────────────────────────────────────────────────────┐ +│ Top Navigation & Transport Toolbar (Fixed Top Bar) │ +├───────────────────────────────────────────────────────────┬──────────────────────┤ +│ │ RIGHT COLUMN │ +│ MAIN WORKSPACE │ (RIGHT SIDEBAR) │ +│ ┌───────────────────────┬───────────────────────────────┐ │ ┌──────────────────┐ │ +│ │ Track Control Panels │ Timeline / Audio Viewport │ │ │ Media Explorer │ │ +│ │ (Track List) │ (Beat Grid & Waveforms) │ │ │ (Dynamic Height) │ │ +│ │ │ │ │ ├──────────────────┤ │ +│ │ │ │ │ │ AI Panel │ │ +│ │ │ │ │ │ (Dynamic Height) │ │ +│ └───────────────────────┴───────────────────────────────┘ │ └──────────────────┘ │ +├───────────────────────────────────────────────────────────┴──────────────────────┤ +│ BOTTOM DOCK PANEL STRIP - Horizontal Scroll (Overflow-X Auto) │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Export Panel │ │ DSP Tools │ │ Panel 03 │ │ Panel 04... │ ──────► │ +│ └──────────────┘ └──────────────┘ └──────────────┘ └──────────────┘ │ +├──────────────────────────────────────────────────────────────────────────────────┤ +│ Status Bar (Fixed Bottom Status) │ +└──────────────────────────────────────────────────────────────────────────────────┘ + +``` + +--- + +## 2. Layout Region Details + +### A. Main Workspace (Center Region) + +* **Function:** Contains the track list (Track Controls), timeline ruler (Timeline Ruler), and audio/MIDI display areas (Audio Waveform & Piano Roll Clip Grid). +* **Behavior:** Auto-expands (`flex-grow: 1`) to fill the remaining screen space after subtracting the width of the Right Sidebar and the height of the Bottom Panel. + +### B. Bottom Panel Dock Strip (Bottom Row) + +* **Technical Specifications:** +* **Flexible Horizontal Scroll:** The container has a fixed height (e.g., `220px`), using `overflow-x: auto` and `display: flex`. +* **Sub-Panels:** Houses a list of independent Card/Tile tools (Export Panel, Python DSP Tools Panel, Selection Panel, FX Panel, etc.). +* **No Shrinking (`flex-shrink: 0`):** Each Sub-Panel is configured with `flex-shrink: 0` and a minimum width (`min-width: 280px - 350px`). When the combined width of all panels exceeds the screen width, a horizontal scrollbar appears automatically. + + + +### C. Right Resizable Sidebar (Multi-Panel Right Column) + +* **Technical Specifications:** +* **Width Resizing:** The entire right column can be resized by dragging its left border (Border Left Drag Handle) to expand or collapse the visible space of the Main Workspace. +* **Vertical Stacking:** Houses stacked child panels (e.g., Media Explorer, AI Panel, Inspector, etc.). +* **Independent Height Resizing:** Horizontal splitters (Horizontal Splitter / Resizer Handle) sit between stacked child panels, allowing users to drag up/down to adjust height ratios between panels. + + + +--- + +## 3. HTML & CSS Framework Implementation + +### HTML Core Structure + +```html +
+ +
...
+ + +
+ + +
+
...
+
...
+
+ + +
+ + + +
+ + + + + +
...
+
+ +``` + +### CSS System Architecture + +```css +:root { + --right-sidebar-width: 320px; + --bottom-strip-height: 220px; + --top-bar-height: 80px; + --status-bar-height: 25px; + --panel-border-color: #2a2a2a; +} + +/* Fullscreen Fixed App Shell */ +.daw-app-shell { + display: flex; + flex-direction: column; + width: 100vw; + height: 100vh; + overflow: hidden; + background-color: #121212; + color: #e0e0e0; +} + +/* Middle Section holding Main Workspace and Right Sidebar */ +.daw-body-container { + display: flex; + flex: 1; + height: calc(100vh - var(--top-bar-height) - var(--bottom-strip-height) - var(--status-bar-height)); + position: relative; + overflow: hidden; +} + +/* Auto-expanding Main Workspace */ +.daw-main-workspace { + flex: 1; + display: flex; + overflow: hidden; +} + +/* Right Sidebar with Width controlled via CSS Variable */ +.daw-right-sidebar { + width: var(--right-sidebar-width); + min-width: 200px; + max-width: 600px; + display: flex; + flex-direction: column; + background-color: #1a1a1a; + border-left: 1px solid var(--panel-border-color); +} + +/* Vertically stacked child Panels in Right Sidebar */ +.sidebar-panel { + display: flex; + flex-direction: column; + overflow: hidden; + background: #1e1e1e; + border-bottom: 1px solid var(--panel-border-color); +} + +#panel-media-explorer { + height: 50%; /* Default 50/50 split */ + min-height: 100px; +} + +#panel-ai { + flex: 1; /* Fills remaining height */ + min-height: 100px; +} + +/* BOTTOM ROW: Enables Horizontal Scrolling */ +.daw-bottom-strip { + height: var(--bottom-strip-height); + display: flex; + flex-direction: row; + align-items: center; + gap: 10px; + padding: 8px; + overflow-x: auto; /* Enables horizontal scroll when panels overflow */ + overflow-y: hidden; + background-color: #161616; + border-top: 1px solid var(--panel-border-color); + white-space: nowrap; +} + +/* Optimized custom horizontal scrollbar for DAW styling */ +.daw-bottom-strip::-webkit-scrollbar { + height: 8px; +} +.daw-bottom-strip::-webkit-scrollbar-thumb { + background: #3a3a3a; + border-radius: 4px; +} +.daw-bottom-strip::-webkit-scrollbar-thumb:hover { + background: #00ffcc; +} + +/* Sub-panels inside the bottom strip */ +.bottom-panel { + flex: 0 0 auto; /* Prevents shrinking, locks content dimensions */ + width: 320px; + height: 100%; + background-color: #222; + border: 1px solid #333; + border-radius: 6px; + box-sizing: border-box; +} + +/* RESIZER HANDLES */ +.resizer-col-handle { + width: 5px; + cursor: ew-resize; /* Horizontal resize cursor */ + background: transparent; + transition: background 0.2s; + z-index: 10; +} +.resizer-col-handle:hover, +.resizer-col-handle:active { + background: #00ffcc; +} + +.resizer-row-handle { + height: 5px; + cursor: ns-resize; /* Vertical resize cursor */ + background: transparent; + transition: background 0.2s; + z-index: 10; +} +.resizer-row-handle:hover, +.resizer-row-handle:active { + background: #00ffcc; +} + +``` + +--- + +## 4. Interaction Algorithms (JS Resizing Logic) + +To handle smooth resizing without stuttering or dropped events when dragging over `iframe` or `canvas` elements, the algorithms rely on `pointerdown`, `pointermove`, and `pointerup` events. + +### A. Right Sidebar Width Resizing Algorithm (Horizontal Resizer) + +```javascript +const colResizer = document.getElementById('col-resizer'); +const rightSidebar = document.getElementById('right-sidebar'); + +colResizer.addEventListener('pointerdown', (e) => { + e.preventDefault(); + colResizer.setPointerCapture(e.pointerId); + + const startX = e.clientX; + const startWidth = rightSidebar.getBoundingClientRect().width; + + const onPointerMove = (moveEvent) => { + // Delta calculation: dragging left increases width, dragging right decreases width + const deltaX = startX - moveEvent.clientX; + const newWidth = Math.max(200, Math.min(600, startWidth + deltaX)); + + document.documentElement.style.setProperty('--right-sidebar-width', `${newWidth}px`); + }; + + const onPointerUp = (upEvent) => { + colResizer.releasePointerCapture(upEvent.pointerId); + colResizer.removeEventListener('pointermove', onPointerMove); + colResizer.removeEventListener('pointerup', onPointerUp); + }; + + colResizer.addEventListener('pointermove', onPointerMove); + colResizer.addEventListener('pointerup', onPointerUp); +}); + +``` + +### B. Right Sidebar Panel Height Resizing Algorithm (Vertical Resizer) + +```javascript +const rowResizer = document.getElementById('row-resizer-1'); +const topPanel = document.getElementById('panel-media-explorer'); + +rowResizer.addEventListener('pointerdown', (e) => { + e.preventDefault(); + rowResizer.setPointerCapture(e.pointerId); + + const startY = e.clientY; + const startHeight = topPanel.getBoundingClientRect().height; + + const onPointerMove = (moveEvent) => { + const deltaY = moveEvent.clientY - startY; + const newHeight = Math.max(100, startHeight + deltaY); + + topPanel.style.height = `${newHeight}px`; + topPanel.style.flex = 'none'; // Switch from flex ratio to fixed px during drag + }; + + const onPointerUp = (upEvent) => { + rowResizer.releasePointerCapture(upEvent.pointerId); + rowResizer.removeEventListener('pointermove', onPointerMove); + rowResizer.removeEventListener('pointerup', onPointerUp); + }; + + colResizer.addEventListener('pointermove', onPointerMove); + colResizer.addEventListener('pointerup', onPointerUp); +}); + +``` + +--- + +## 5. Summary of Solution Advantages + +* **Native Horizontal Scrolling:** The bottom Dock area flexibly accommodates an unlimited number of Panels. Users can scroll horizontally (`Shift + Mouse Wheel`) or use a trackpad to browse panels easily. +* **Smooth & Accurate Resizing:** Utilizing Pointer Capture ensures drag interactions do not drop or break even when the cursor moves rapidly beyond the Resizer handle's bounds. +* **Standardized CSS Variables:** Enables easy persistence of layout states (`Width`/`Height`) to the browser's `localStorage`, restoring the user's custom layout configuration on app reload. \ No newline at end of file diff --git a/md/29_MIDI_SESSION.md b/md/30_MIDI_SESSION.md similarity index 100% rename from md/29_MIDI_SESSION.md rename to md/30_MIDI_SESSION.md