feat: bổ sung MIDI

This commit is contained in:
2026-07-23 08:32:23 +07:00
parent 6545e1746e
commit 0e44e44ecb
15 changed files with 3304 additions and 12680 deletions
+146 -6
View File
@@ -1,14 +1,119 @@
import time
import json
import uuid
import os
from fastapi import APIRouter, HTTPException, Depends, Header
from pydantic import BaseModel
from typing import Optional, Any, Dict
from jsonschema import validate, ValidationError
from app.models.user import get_db_connection
from app.api.v1.auth import get_current_user, decode_token
router = APIRouter()
SCHEMA_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "models", "project_schema.json")
def upgrade_project_json_if_needed(project_data: dict) -> dict:
if "main_session" in project_data:
return project_data
tracks = project_data.get("tracks", [])
upgraded_tracks = []
for t in tracks:
track_id = str(t.get("id", ""))
track_name = t.get("name", "Track")
vol = t.get("volumeDb", 0.0)
pan = t.get("pan", 0.0)
muted = t.get("muted", False)
solo = t.get("solo", False)
items = []
for c in t.get("clips", []):
items.append({
"id": c.get("id"),
"name": c.get("name", "Audio Clip"),
"type": "AUDIO_ITEM",
"start_bar": c.get("startTime", 0.0) / 4.0,
"duration_bars": 4.0,
"clip_start_offset_bars": 0.0,
"source_data": {
"audio_file_url": f"/static/audio/uploads/{t.get('serverFileId')}" if t.get("serverFileId") else "",
"sample_rate": 44100,
"channels": 2,
"gain": 1.0
}
})
for m in t.get("midiItems", []):
items.append({
"id": m.get("id"),
"name": m.get("name", "MIDI Item"),
"type": "MIDI_ITEM",
"start_bar": m.get("startTime", 0.0) / 4.0,
"duration_bars": m.get("duration", 4.0),
"clip_start_offset_bars": 0.0,
"source_data": {
"total_buffer_bars": m.get("duration", 8.0),
"notes": m.get("notes", [])
}
})
upgraded_tracks.append({
"id": track_id,
"name": track_name,
"type": "MIDI" if t.get("midiItems") else "AUDIO",
"volume_db": vol,
"pan": pan,
"mute": muted,
"solo": solo,
"fx_chain": [],
"synth_engine": {
"plugin_id": "synth",
"preset_id": "default",
"parameters": {}
},
"items": items
})
return {
"project_id": project_data.get("id", "temp_project"),
"metadata": {
"title": project_data.get("name", "Dự án mới"),
"bpm": 120.0,
"time_signature_numerator": 4,
"time_signature_denominator": 4,
"sample_rate": 44100
},
"main_session": {
"id": "main",
"name": "MAIN SESSION",
"is_root": True,
"length_bars": 16.0,
"auto_compute_length": True,
"tracks": upgraded_tracks
},
"section_store": {}
}
def validate_project_data(data_json: str) -> str:
try:
data = json.loads(data_json)
if "main_session" not in data:
data = upgrade_project_json_if_needed(data)
data_json = json.dumps(data)
if os.path.exists(SCHEMA_PATH):
with open(SCHEMA_PATH, "r") as f:
schema = json.load(f)
validate(instance=data, schema=schema)
return data_json
except json.JSONDecodeError as e:
raise HTTPException(status_code=400, detail=f"Cấu trúc JSON không hợp lệ: {str(e)}")
except ValidationError as e:
path = " -> ".join(str(p) for p in e.path)
raise HTTPException(status_code=400, detail=f"Lỗi xác thực project schema tại [{path}]: {e.message}")
except Exception as e:
raise HTTPException(status_code=400, detail=f"Lỗi xác thực dự án: {str(e)}")
class SaveProjectRequest(BaseModel):
name: str
data_json: str
@@ -24,11 +129,12 @@ def get_optional_user(authorization: Optional[str] = Header(None)) -> Optional[d
@router.post("/temp")
async def save_temp_project(req: SaveTempProjectRequest, current_user: Optional[dict] = Depends(get_optional_user)):
validated_data_json = validate_project_data(req.data_json)
user_id = current_user["user_id"] if current_user else "anonymous"
conn = get_db_connection()
cursor = conn.cursor()
size_bytes = len(req.data_json.encode("utf-8"))
size_bytes = len(validated_data_json.encode("utf-8"))
now = time.time()
temp_id = f"temp_{user_id}"
@@ -36,7 +142,7 @@ async def save_temp_project(req: SaveTempProjectRequest, current_user: Optional[
INSERT INTO projects (id, user_id, name, data_json, is_temp, size_bytes, updated_at)
VALUES (?, ?, 'Dự án tạm chưa lưu', ?, 1, ?, ?)
ON CONFLICT(id) DO UPDATE SET data_json = excluded.data_json, size_bytes = excluded.size_bytes, updated_at = excluded.updated_at
""", (temp_id, user_id, req.data_json, size_bytes, now))
""", (temp_id, user_id, validated_data_json, size_bytes, now))
conn.commit()
conn.close()
@@ -64,6 +170,7 @@ async def get_temp_project(current_user: Optional[dict] = Depends(get_optional_u
@router.post("/cloud")
async def save_cloud_project(req: SaveProjectRequest, current_user: dict = Depends(get_current_user)):
validated_data_json = validate_project_data(req.data_json)
user_id = current_user["user_id"]
conn = get_db_connection()
cursor = conn.cursor()
@@ -76,7 +183,7 @@ async def save_cloud_project(req: SaveProjectRequest, current_user: dict = Depen
used_row = cursor.fetchone()
used_bytes = used_row["total_used"] if used_row and used_row["total_used"] else 0
new_size_bytes = len(req.data_json.encode("utf-8"))
new_size_bytes = len(validated_data_json.encode("utf-8"))
max_bytes = storage_limit_mb * 1024 * 1024
if used_bytes + new_size_bytes > max_bytes:
@@ -92,7 +199,7 @@ async def save_cloud_project(req: SaveProjectRequest, current_user: dict = Depen
cursor.execute("""
INSERT INTO projects (id, user_id, name, data_json, is_temp, size_bytes, updated_at)
VALUES (?, ?, ?, ?, 0, ?, ?)
""", (project_id, user_id, req.name, req.data_json, new_size_bytes, now))
""", (project_id, user_id, req.name, validated_data_json, new_size_bytes, now))
temp_id = f"temp_{user_id}"
cursor.execute("DELETE FROM projects WHERE id = ? AND is_temp = 1", (temp_id,))
@@ -157,6 +264,7 @@ async def delete_cloud_project(project_id: str, current_user: dict = Depends(get
@router.put("/cloud/{project_id}")
async def update_cloud_project(project_id: str, req: SaveProjectRequest, current_user: dict = Depends(get_current_user)):
validated_data_json = validate_project_data(req.data_json)
user_id = current_user["user_id"]
conn = get_db_connection()
cursor = conn.cursor()
@@ -167,15 +275,47 @@ async def update_cloud_project(project_id: str, req: SaveProjectRequest, current
conn.close()
raise HTTPException(status_code=404, detail="Không tìm thấy dự án để cập nhật")
new_size_bytes = len(req.data_json.encode("utf-8"))
new_size_bytes = len(validated_data_json.encode("utf-8"))
now = time.time()
cursor.execute("""
UPDATE projects
SET name = ?, data_json = ?, size_bytes = ?, updated_at = ?
WHERE id = ? AND user_id = ?
""", (req.name, req.data_json, new_size_bytes, now, project_id, user_id))
""", (req.name, validated_data_json, new_size_bytes, now, project_id, user_id))
conn.commit()
conn.close()
return {"success": True, "message": "Đã cập nhật dự án thành công"}
class RenderProjectRequest(BaseModel):
sample_rate: Optional[int] = 44100
@router.post("/cloud/{project_id}/render")
async def render_project_endpoint(project_id: str, req: RenderProjectRequest, current_user: dict = Depends(get_current_user)):
user_id = current_user["user_id"]
conn = get_db_connection()
cursor = conn.cursor()
cursor.execute("SELECT name, data_json FROM projects WHERE id = ? AND user_id = ? AND is_temp = 0", (project_id, user_id))
row = cursor.fetchone()
conn.close()
if not row:
raise HTTPException(status_code=404, detail="Không tìm thấy dự án để kết xuất")
# Validate schema
validate_project_data(row["data_json"])
# Trigger Celery task
from app.tasks.worker import render_project_task
task = render_project_task.delay(
project_id=project_id,
project_name=row["name"],
project_json_str=row["data_json"],
sample_rate=req.sample_rate or 44100
)
return {
"task_id": task.id,
"status": "processing"
}
+229
View File
@@ -0,0 +1,229 @@
import os
import numpy as np
import soundfile as sf
from app.config import settings
from app.core.vst_engine import render_midi_events_to_audio
import subprocess
import sys
def check_pedalboard_safe():
try:
res = subprocess.run(
[sys.executable, "-c", "import pedalboard"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=2.0
)
return res.returncode == 0
except Exception:
return False
HAS_PEDALBOARD = check_pedalboard_safe()
if HAS_PEDALBOARD:
try:
from pedalboard import Pedalboard, Gain
except Exception:
HAS_PEDALBOARD = False
class PythonRenderEngine:
def __init__(self, sample_rate=44100):
self.sample_rate = sample_rate
def bars_to_samples(self, bars: float, bpm: float, time_sig_num: int) -> int:
seconds_per_beat = 60.0 / max(20.0, bpm)
seconds_per_bar = seconds_per_beat * time_sig_num
return int(bars * seconds_per_bar * self.sample_rate)
def resolve_file_path(self, url_or_id: str) -> str:
if not url_or_id:
return ""
base = os.path.basename(url_or_id)
# Check uploads directory
p_uploads = os.path.join(settings.UPLOADS_DIR, base)
if os.path.exists(p_uploads):
return p_uploads
# Check processed directory
p_processed = os.path.join(settings.PROCESSED_DIR, base)
if os.path.exists(p_processed):
return p_processed
# Check general storage directory
p_storage = os.path.join(settings.STORAGE_DIR, base)
if os.path.exists(p_storage):
return p_storage
# Direct check
if os.path.exists(url_or_id):
return url_or_id
return url_or_id
def render_session_container(self, session: dict, section_store: dict, bpm: float, time_sig_num: int, total_samples: int) -> np.ndarray:
session_buffer = np.zeros((2, total_samples), dtype=np.float32)
for track in session.get("tracks", []):
track_type = track.get("type", "AUDIO")
track_buffer = np.zeros((2, total_samples), dtype=np.float32)
for item in track.get("items", []):
start_sample = self.bars_to_samples(item["start_bar"], bpm, time_sig_num)
dur_samples = self.bars_to_samples(item["duration_bars"], bpm, time_sig_num)
offset_sample = self.bars_to_samples(item["clip_start_offset_bars"], bpm, time_sig_num)
item_type = item.get("type")
if item_type == "AUDIO_ITEM":
source_data = item.get("source_data", {})
audio_url = source_data.get("audio_file_url", "")
resolved_path = self.resolve_file_path(audio_url)
if resolved_path and os.path.exists(resolved_path):
try:
audio_data, sr = sf.read(resolved_path, dtype='float32')
if sr != self.sample_rate:
# Resampling fallback if simple, otherwise skip
pass
# Handle channel mapping (Mono/Stereo)
if len(audio_data.shape) == 1:
audio_data = np.vstack([audio_data, audio_data])
else:
audio_data = audio_data.T # Shape: (channels, samples)
# Trim source offset & duration
src_len = audio_data.shape[1]
if offset_sample < src_len:
actual_dur = min(dur_samples, src_len - offset_sample)
sliced_audio = audio_data[:, offset_sample : offset_sample + actual_dur]
# Apply gain
gain_val = source_data.get("gain", 1.0)
sliced_audio = sliced_audio * gain_val
# Write to track buffer with boundaries
write_end = min(start_sample + sliced_audio.shape[1], total_samples)
actual_len = write_end - start_sample
if actual_len > 0:
track_buffer[:, start_sample:write_end] += sliced_audio[:, :actual_len]
except Exception as e:
print(f"[RenderEngine] Error reading audio file {resolved_path}: {e}")
elif item_type == "MIDI_ITEM":
source_data = item.get("source_data", {})
notes = source_data.get("notes", [])
# Convert to midi events required by vst_engine
midi_events = []
for note in notes:
note_start_bar = note["start_beat"] / time_sig_num
# Filter notes within the non-destructive visible window
offset_bar = item["clip_start_offset_bars"]
dur_bar = item["duration_bars"]
if note_start_bar >= offset_bar and note_start_bar < (offset_bar + dur_bar):
rel_bar_in_item = note_start_bar - offset_bar
target_global_bar = item["start_bar"] + rel_bar_in_item
midi_events.append({
"note": note["pitch"],
"start_beat": target_global_bar * time_sig_num,
"duration_beats": note["duration_beats"],
"velocity": int(note.get("velocity", 0.8) * 127)
})
if midi_events:
try:
# Synthesize MIDI track notes
synth_buffer = render_midi_events_to_audio(
midi_events=midi_events,
sr=self.sample_rate,
bpm=bpm,
instrument='synth'
)
# Add to track buffer
actual_len = min(synth_buffer.shape[1], total_samples)
track_buffer[:, :actual_len] += synth_buffer[:, :actual_len]
except Exception as e:
print(f"[RenderEngine] Error rendering MIDI: {e}")
elif item_type == "SECTION_ITEM":
source_data = item.get("source_data", {})
sec_id = source_data.get("referenced_section_id", "")
if sec_id and sec_id in section_store:
# Render nested section recursively
sec_container = section_store[sec_id]
sec_buffer = self.render_session_container(
session=sec_container,
section_store=section_store,
bpm=bpm,
time_sig_num=time_sig_num,
total_samples=total_samples
)
# Apply non-destructive crop/slicing on section buffer
if offset_sample < total_samples:
actual_dur = min(dur_samples, total_samples - offset_sample)
sliced_sec = sec_buffer[:, offset_sample : offset_sample + actual_dur]
# Write to track buffer
write_end = min(start_sample + sliced_sec.shape[1], total_samples)
actual_len = write_end - start_sample
if actual_len > 0:
track_buffer[:, start_sample:write_end] += sliced_sec[:, :actual_len]
# Apply Track Gain (via Pedalboard or fallback)
vol_db = track.get("volume_db", 0.0)
pan = track.get("pan", 0.0)
mute = track.get("mute", False)
if mute:
continue
# Process track volume
if HAS_PEDALBOARD:
try:
board = Pedalboard([Gain(gain_db=vol_db)])
processed_track = board(track_buffer, sample_rate=self.sample_rate)
except Exception:
gain_linear = 10 ** (vol_db / 20.0)
processed_track = track_buffer * gain_linear
else:
gain_linear = 10 ** (vol_db / 20.0)
processed_track = track_buffer * gain_linear
# Apply Track Pan
if pan != 0.0:
# Constant power panning
theta = ((np.clip(pan, -1.0, 1.0) + 1.0) / 2.0) * (np.pi / 2.0)
processed_track[0, :] *= np.cos(theta)
processed_track[1, :] *= np.sin(theta)
# Mix track to session
session_buffer += processed_track
return session_buffer
def render_project(self, project_json: dict, output_filepath: str):
bpm = project_json["metadata"]["bpm"]
time_sig_num = project_json["metadata"].get("time_signature_numerator", 4)
main_session = project_json["main_session"]
section_store = project_json.get("section_store", {})
# Compute total project samples
total_bars = main_session.get("length_bars", 16.0)
total_samples = self.bars_to_samples(total_bars, bpm, time_sig_num)
# Render main session
master_buffer = self.render_session_container(
session=main_session,
section_store=section_store,
bpm=bpm,
time_sig_num=time_sig_num,
total_samples=total_samples
)
# Normalization to prevent clipping
max_peak = np.max(np.abs(master_buffer))
if max_peak > 1.0:
master_buffer /= max_peak
# Write final output file
sf.write(output_filepath, master_buffer.T, self.sample_rate)
return output_filepath
+142
View File
@@ -0,0 +1,142 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "DAWProject",
"type": "object",
"properties": {
"project_id": { "type": "string" },
"metadata": {
"type": "object",
"properties": {
"title": { "type": "string" },
"bpm": { "type": "number", "minimum": 20.0, "maximum": 999.0, "default": 120.0 },
"time_signature_numerator": { "type": "integer", "default": 4 },
"time_signature_denominator": { "type": "integer", "default": 4 },
"sample_rate": { "type": "integer", "default": 44100 }
},
"required": ["title", "bpm", "time_signature_numerator", "time_signature_denominator", "sample_rate"]
},
"main_session": { "$ref": "#/definitions/SessionContainer" },
"section_store": {
"type": "object",
"description": "Auxiliary registry mapping section_id to sub-session containers",
"additionalProperties": { "$ref": "#/definitions/SessionContainer" }
}
},
"required": ["project_id", "metadata", "main_session", "section_store"],
"definitions": {
"SessionContainer": {
"type": "object",
"properties": {
"id": { "type": "string" },
"name": { "type": "string" },
"is_root": { "type": "boolean" },
"length_bars": { "type": "number", "description": "Computed or manually set total length in bars" },
"auto_compute_length": { "type": "boolean", "default": true },
"tracks": {
"type": "array",
"items": { "$ref": "#/definitions/Track" }
}
},
"required": ["id", "is_root", "tracks"]
},
"Track": {
"type": "object",
"properties": {
"id": { "type": "string" },
"name": { "type": "string" },
"type": { "type": "string", "enum": ["AUDIO", "MIDI", "SECTION"] },
"volume_db": { "type": "number", "default": 0.0 },
"pan": { "type": "number", "minimum": -1.0, "maximum": 1.0, "default": 0.0 },
"mute": { "type": "boolean", "default": false },
"solo": { "type": "boolean", "default": false },
"fx_chain": {
"type": "array",
"items": { "$ref": "#/definitions/FXPlugin" }
},
"synth_engine": { "$ref": "#/definitions/SynthPlugin" },
"items": {
"type": "array",
"items": { "$ref": "#/definitions/TimelineItem" }
}
},
"required": ["id", "name", "type", "items"]
},
"TimelineItem": {
"type": "object",
"properties": {
"id": { "type": "string" },
"name": { "type": "string" },
"type": { "type": "string", "enum": ["AUDIO_ITEM", "MIDI_ITEM", "SECTION_ITEM"] },
"start_bar": { "type": "number", "description": "Global timeline position where the item starts" },
"duration_bars": { "type": "number", "description": "Visible duration on the track timeline in bars" },
"clip_start_offset_bars": { "type": "number", "description": "Internal start offset inside the source buffer/item" },
"source_data": {
"type": "object",
"oneOf": [
{ "$ref": "#/definitions/AudioSourceData" },
{ "$ref": "#/definitions/MIDISourceData" },
{ "$ref": "#/definitions/SectionSourceData" }
]
}
},
"required": ["id", "type", "start_bar", "duration_bars", "clip_start_offset_bars", "source_data"]
},
"AudioSourceData": {
"type": "object",
"properties": {
"audio_file_url": { "type": "string" },
"sample_rate": { "type": "integer" },
"channels": { "type": "integer" },
"gain": { "type": "number", "default": 1.0 }
},
"required": ["audio_file_url"]
},
"MIDISourceData": {
"type": "object",
"properties": {
"total_buffer_bars": { "type": "number", "default": 8.0 },
"notes": {
"type": "array",
"items": { "$ref": "#/definitions/MIDINote" }
}
},
"required": ["total_buffer_bars", "notes"]
},
"SectionSourceData": {
"type": "object",
"properties": {
"referenced_section_id": { "type": "string", "description": "Pointer to section_store key" }
},
"required": ["referenced_section_id"]
},
"MIDINote": {
"type": "object",
"properties": {
"id": { "type": "string" },
"pitch": { "type": "integer", "minimum": 0, "maximum": 127 },
"start_beat": { "type": "number", "description": "Beat offset relative to the start of the source buffer (bar 0)" },
"duration_beats": { "type": "number" },
"velocity": { "type": "number", "minimum": 0.0, "maximum": 1.0, "default": 0.8 },
"pan": { "type": "number", "minimum": -1.0, "maximum": 1.0, "default": 0.0 }
},
"required": ["id", "pitch", "start_beat", "duration_beats", "velocity"]
},
"FXPlugin": {
"type": "object",
"properties": {
"plugin_id": { "type": "string" },
"name": { "type": "string" },
"bypass": { "type": "boolean", "default": false },
"parameters": { "type": "object" }
}
},
"SynthPlugin": {
"type": "object",
"properties": {
"plugin_id": { "type": "string" },
"preset_id": { "type": "string" },
"parameters": { "type": "object" }
}
}
}
}
+1034 -152
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because one or more lines are too long
+17 -9
View File
@@ -3,12 +3,11 @@
const SFS_VERSION = "1.0.0";
function exportProjectToSFS(projectState) {
const sfsBundle = {
format: "SONICFORGE_STUDIO_PROJECT",
version: SFS_VERSION,
timestamp: Date.now(),
domain: window.location.origin,
project: {
let projectObj = {};
if (projectState.main_session) {
projectObj = projectState;
} else {
projectObj = {
id: projectState.id || `proj_${Date.now()}`,
name: projectState.name || "Dự án mới",
tracks: (projectState.tracks || []).map(t => ({
@@ -24,7 +23,15 @@
markers: t.markers || [],
serverFileId: t.serverFileId || null
}))
}
};
}
const sfsBundle = {
format: "SONICFORGE_STUDIO_PROJECT",
version: SFS_VERSION,
timestamp: Date.now(),
domain: window.location.origin,
project: projectObj
};
const jsonStr = JSON.stringify(sfsBundle, null, 2);
@@ -33,7 +40,8 @@
const a = document.createElement('a');
a.href = url;
a.download = `${(projectState.name || 'project').replace(/\s+/g, '_')}.sfs`;
const displayName = projectObj.metadata?.title || projectObj.name || 'project';
a.download = `${displayName.replace(/\s+/g, '_')}.sfs`;
a.click();
URL.revokeObjectURL(url);
}
@@ -53,7 +61,7 @@
autoSaveTimer = setTimeout(async () => {
try {
const state = getProjectStateCallback();
if (!state || !state.tracks || state.tracks.length === 0) return;
if (!state || (!state.tracks && !state.main_session)) return;
const dataJson = JSON.stringify(state);
localStorage.setItem('sonic_temp_project', dataJson);
if (window.SonicAPI && localStorage.getItem('sonic_token')) {
Binary file not shown.
+24
View File
@@ -303,3 +303,27 @@ def cleanup_expired_files_task(max_age_hours: int = 24):
"cleaned_size_mb": round(cleaned_size / (1024 * 1024), 2),
"max_age_hours": max_age_hours
}
@celery_app.task
def render_project_task(project_id: str, project_name: str, project_json_str: str, sample_rate: int = 44100):
"""
Task Celery để kết xuất dự án ngoại tuyến (Offline Project Mixdown) áp dụng specs 30_DAW_ARCHITECT.md.
"""
import json
from app.core.render_engine import PythonRenderEngine
project_json = json.loads(project_json_str)
output_filename = f"{project_id}_render.wav"
output_path = os.path.join(settings.PROCESSED_DIR, output_filename)
engine = PythonRenderEngine(sample_rate=sample_rate)
engine.render_project(project_json, output_path)
return {
"project_id": project_id,
"project_name": project_name,
"success": True,
"output_file_id": output_filename
}