Compare commits
6 Commits
6545e1746e
...
0b2382573f
| Author | SHA1 | Date | |
|---|---|---|---|
| 0b2382573f | |||
| b460d40824 | |||
| 92877d02b7 | |||
| 0bffe2d0d9 | |||
| af5d20d2e5 | |||
| 0e44e44ecb |
+146
-6
@@ -1,14 +1,119 @@
|
|||||||
import time
|
import time
|
||||||
import json
|
import json
|
||||||
import uuid
|
import uuid
|
||||||
|
import os
|
||||||
from fastapi import APIRouter, HTTPException, Depends, Header
|
from fastapi import APIRouter, HTTPException, Depends, Header
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from typing import Optional, Any, Dict
|
from typing import Optional, Any, Dict
|
||||||
|
from jsonschema import validate, ValidationError
|
||||||
from app.models.user import get_db_connection
|
from app.models.user import get_db_connection
|
||||||
from app.api.v1.auth import get_current_user, decode_token
|
from app.api.v1.auth import get_current_user, decode_token
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
SCHEMA_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "models", "project_schema.json")
|
||||||
|
|
||||||
|
def upgrade_project_json_if_needed(project_data: dict) -> dict:
|
||||||
|
if "main_session" in project_data:
|
||||||
|
return project_data
|
||||||
|
|
||||||
|
tracks = project_data.get("tracks", [])
|
||||||
|
upgraded_tracks = []
|
||||||
|
for t in tracks:
|
||||||
|
track_id = str(t.get("id", ""))
|
||||||
|
track_name = t.get("name", "Track")
|
||||||
|
vol = t.get("volumeDb", 0.0)
|
||||||
|
pan = t.get("pan", 0.0)
|
||||||
|
muted = t.get("muted", False)
|
||||||
|
solo = t.get("solo", False)
|
||||||
|
|
||||||
|
items = []
|
||||||
|
for c in t.get("clips", []):
|
||||||
|
items.append({
|
||||||
|
"id": c.get("id"),
|
||||||
|
"name": c.get("name", "Audio Clip"),
|
||||||
|
"type": "AUDIO_ITEM",
|
||||||
|
"start_bar": c.get("startTime", 0.0) / 4.0,
|
||||||
|
"duration_bars": 4.0,
|
||||||
|
"clip_start_offset_bars": 0.0,
|
||||||
|
"source_data": {
|
||||||
|
"audio_file_url": f"/static/audio/uploads/{t.get('serverFileId')}" if t.get("serverFileId") else "",
|
||||||
|
"sample_rate": 44100,
|
||||||
|
"channels": 2,
|
||||||
|
"gain": 1.0
|
||||||
|
}
|
||||||
|
})
|
||||||
|
for m in t.get("midiItems", []):
|
||||||
|
items.append({
|
||||||
|
"id": m.get("id"),
|
||||||
|
"name": m.get("name", "MIDI Item"),
|
||||||
|
"type": "MIDI_ITEM",
|
||||||
|
"start_bar": m.get("startTime", 0.0) / 4.0,
|
||||||
|
"duration_bars": m.get("duration", 4.0),
|
||||||
|
"clip_start_offset_bars": 0.0,
|
||||||
|
"source_data": {
|
||||||
|
"total_buffer_bars": m.get("duration", 8.0),
|
||||||
|
"notes": m.get("notes", [])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
upgraded_tracks.append({
|
||||||
|
"id": track_id,
|
||||||
|
"name": track_name,
|
||||||
|
"type": "MIDI" if t.get("midiItems") else "AUDIO",
|
||||||
|
"volume_db": vol,
|
||||||
|
"pan": pan,
|
||||||
|
"mute": muted,
|
||||||
|
"solo": solo,
|
||||||
|
"fx_chain": [],
|
||||||
|
"synth_engine": {
|
||||||
|
"plugin_id": "synth",
|
||||||
|
"preset_id": "default",
|
||||||
|
"parameters": {}
|
||||||
|
},
|
||||||
|
"items": items
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"project_id": project_data.get("id", "temp_project"),
|
||||||
|
"metadata": {
|
||||||
|
"title": project_data.get("name", "Dự án mới"),
|
||||||
|
"bpm": 120.0,
|
||||||
|
"time_signature_numerator": 4,
|
||||||
|
"time_signature_denominator": 4,
|
||||||
|
"sample_rate": 44100
|
||||||
|
},
|
||||||
|
"main_session": {
|
||||||
|
"id": "main",
|
||||||
|
"name": "MAIN SESSION",
|
||||||
|
"is_root": True,
|
||||||
|
"length_bars": 16.0,
|
||||||
|
"auto_compute_length": True,
|
||||||
|
"tracks": upgraded_tracks
|
||||||
|
},
|
||||||
|
"section_store": {}
|
||||||
|
}
|
||||||
|
|
||||||
|
def validate_project_data(data_json: str) -> str:
|
||||||
|
try:
|
||||||
|
data = json.loads(data_json)
|
||||||
|
if "main_session" not in data:
|
||||||
|
data = upgrade_project_json_if_needed(data)
|
||||||
|
data_json = json.dumps(data)
|
||||||
|
|
||||||
|
if os.path.exists(SCHEMA_PATH):
|
||||||
|
with open(SCHEMA_PATH, "r") as f:
|
||||||
|
schema = json.load(f)
|
||||||
|
validate(instance=data, schema=schema)
|
||||||
|
return data_json
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
raise HTTPException(status_code=400, detail=f"Cấu trúc JSON không hợp lệ: {str(e)}")
|
||||||
|
except ValidationError as e:
|
||||||
|
path = " -> ".join(str(p) for p in e.path)
|
||||||
|
raise HTTPException(status_code=400, detail=f"Lỗi xác thực project schema tại [{path}]: {e.message}")
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(status_code=400, detail=f"Lỗi xác thực dự án: {str(e)}")
|
||||||
|
|
||||||
class SaveProjectRequest(BaseModel):
|
class SaveProjectRequest(BaseModel):
|
||||||
name: str
|
name: str
|
||||||
data_json: str
|
data_json: str
|
||||||
@@ -24,11 +129,12 @@ def get_optional_user(authorization: Optional[str] = Header(None)) -> Optional[d
|
|||||||
|
|
||||||
@router.post("/temp")
|
@router.post("/temp")
|
||||||
async def save_temp_project(req: SaveTempProjectRequest, current_user: Optional[dict] = Depends(get_optional_user)):
|
async def save_temp_project(req: SaveTempProjectRequest, current_user: Optional[dict] = Depends(get_optional_user)):
|
||||||
|
validated_data_json = validate_project_data(req.data_json)
|
||||||
user_id = current_user["user_id"] if current_user else "anonymous"
|
user_id = current_user["user_id"] if current_user else "anonymous"
|
||||||
conn = get_db_connection()
|
conn = get_db_connection()
|
||||||
cursor = conn.cursor()
|
cursor = conn.cursor()
|
||||||
|
|
||||||
size_bytes = len(req.data_json.encode("utf-8"))
|
size_bytes = len(validated_data_json.encode("utf-8"))
|
||||||
now = time.time()
|
now = time.time()
|
||||||
temp_id = f"temp_{user_id}"
|
temp_id = f"temp_{user_id}"
|
||||||
|
|
||||||
@@ -36,7 +142,7 @@ async def save_temp_project(req: SaveTempProjectRequest, current_user: Optional[
|
|||||||
INSERT INTO projects (id, user_id, name, data_json, is_temp, size_bytes, updated_at)
|
INSERT INTO projects (id, user_id, name, data_json, is_temp, size_bytes, updated_at)
|
||||||
VALUES (?, ?, 'Dự án tạm chưa lưu', ?, 1, ?, ?)
|
VALUES (?, ?, 'Dự án tạm chưa lưu', ?, 1, ?, ?)
|
||||||
ON CONFLICT(id) DO UPDATE SET data_json = excluded.data_json, size_bytes = excluded.size_bytes, updated_at = excluded.updated_at
|
ON CONFLICT(id) DO UPDATE SET data_json = excluded.data_json, size_bytes = excluded.size_bytes, updated_at = excluded.updated_at
|
||||||
""", (temp_id, user_id, req.data_json, size_bytes, now))
|
""", (temp_id, user_id, validated_data_json, size_bytes, now))
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
conn.close()
|
conn.close()
|
||||||
@@ -64,6 +170,7 @@ async def get_temp_project(current_user: Optional[dict] = Depends(get_optional_u
|
|||||||
|
|
||||||
@router.post("/cloud")
|
@router.post("/cloud")
|
||||||
async def save_cloud_project(req: SaveProjectRequest, current_user: dict = Depends(get_current_user)):
|
async def save_cloud_project(req: SaveProjectRequest, current_user: dict = Depends(get_current_user)):
|
||||||
|
validated_data_json = validate_project_data(req.data_json)
|
||||||
user_id = current_user["user_id"]
|
user_id = current_user["user_id"]
|
||||||
conn = get_db_connection()
|
conn = get_db_connection()
|
||||||
cursor = conn.cursor()
|
cursor = conn.cursor()
|
||||||
@@ -76,7 +183,7 @@ async def save_cloud_project(req: SaveProjectRequest, current_user: dict = Depen
|
|||||||
used_row = cursor.fetchone()
|
used_row = cursor.fetchone()
|
||||||
used_bytes = used_row["total_used"] if used_row and used_row["total_used"] else 0
|
used_bytes = used_row["total_used"] if used_row and used_row["total_used"] else 0
|
||||||
|
|
||||||
new_size_bytes = len(req.data_json.encode("utf-8"))
|
new_size_bytes = len(validated_data_json.encode("utf-8"))
|
||||||
max_bytes = storage_limit_mb * 1024 * 1024
|
max_bytes = storage_limit_mb * 1024 * 1024
|
||||||
|
|
||||||
if used_bytes + new_size_bytes > max_bytes:
|
if used_bytes + new_size_bytes > max_bytes:
|
||||||
@@ -92,7 +199,7 @@ async def save_cloud_project(req: SaveProjectRequest, current_user: dict = Depen
|
|||||||
cursor.execute("""
|
cursor.execute("""
|
||||||
INSERT INTO projects (id, user_id, name, data_json, is_temp, size_bytes, updated_at)
|
INSERT INTO projects (id, user_id, name, data_json, is_temp, size_bytes, updated_at)
|
||||||
VALUES (?, ?, ?, ?, 0, ?, ?)
|
VALUES (?, ?, ?, ?, 0, ?, ?)
|
||||||
""", (project_id, user_id, req.name, req.data_json, new_size_bytes, now))
|
""", (project_id, user_id, req.name, validated_data_json, new_size_bytes, now))
|
||||||
|
|
||||||
temp_id = f"temp_{user_id}"
|
temp_id = f"temp_{user_id}"
|
||||||
cursor.execute("DELETE FROM projects WHERE id = ? AND is_temp = 1", (temp_id,))
|
cursor.execute("DELETE FROM projects WHERE id = ? AND is_temp = 1", (temp_id,))
|
||||||
@@ -157,6 +264,7 @@ async def delete_cloud_project(project_id: str, current_user: dict = Depends(get
|
|||||||
|
|
||||||
@router.put("/cloud/{project_id}")
|
@router.put("/cloud/{project_id}")
|
||||||
async def update_cloud_project(project_id: str, req: SaveProjectRequest, current_user: dict = Depends(get_current_user)):
|
async def update_cloud_project(project_id: str, req: SaveProjectRequest, current_user: dict = Depends(get_current_user)):
|
||||||
|
validated_data_json = validate_project_data(req.data_json)
|
||||||
user_id = current_user["user_id"]
|
user_id = current_user["user_id"]
|
||||||
conn = get_db_connection()
|
conn = get_db_connection()
|
||||||
cursor = conn.cursor()
|
cursor = conn.cursor()
|
||||||
@@ -167,15 +275,47 @@ async def update_cloud_project(project_id: str, req: SaveProjectRequest, current
|
|||||||
conn.close()
|
conn.close()
|
||||||
raise HTTPException(status_code=404, detail="Không tìm thấy dự án để cập nhật")
|
raise HTTPException(status_code=404, detail="Không tìm thấy dự án để cập nhật")
|
||||||
|
|
||||||
new_size_bytes = len(req.data_json.encode("utf-8"))
|
new_size_bytes = len(validated_data_json.encode("utf-8"))
|
||||||
now = time.time()
|
now = time.time()
|
||||||
|
|
||||||
cursor.execute("""
|
cursor.execute("""
|
||||||
UPDATE projects
|
UPDATE projects
|
||||||
SET name = ?, data_json = ?, size_bytes = ?, updated_at = ?
|
SET name = ?, data_json = ?, size_bytes = ?, updated_at = ?
|
||||||
WHERE id = ? AND user_id = ?
|
WHERE id = ? AND user_id = ?
|
||||||
""", (req.name, req.data_json, new_size_bytes, now, project_id, user_id))
|
""", (req.name, validated_data_json, new_size_bytes, now, project_id, user_id))
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
conn.close()
|
conn.close()
|
||||||
return {"success": True, "message": "Đã cập nhật dự án thành công"}
|
return {"success": True, "message": "Đã cập nhật dự án thành công"}
|
||||||
|
|
||||||
|
class RenderProjectRequest(BaseModel):
|
||||||
|
sample_rate: Optional[int] = 44100
|
||||||
|
|
||||||
|
@router.post("/cloud/{project_id}/render")
|
||||||
|
async def render_project_endpoint(project_id: str, req: RenderProjectRequest, current_user: dict = Depends(get_current_user)):
|
||||||
|
user_id = current_user["user_id"]
|
||||||
|
conn = get_db_connection()
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute("SELECT name, data_json FROM projects WHERE id = ? AND user_id = ? AND is_temp = 0", (project_id, user_id))
|
||||||
|
row = cursor.fetchone()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
if not row:
|
||||||
|
raise HTTPException(status_code=404, detail="Không tìm thấy dự án để kết xuất")
|
||||||
|
|
||||||
|
# Validate schema
|
||||||
|
validate_project_data(row["data_json"])
|
||||||
|
|
||||||
|
# Trigger Celery task
|
||||||
|
from app.tasks.worker import render_project_task
|
||||||
|
task = render_project_task.delay(
|
||||||
|
project_id=project_id,
|
||||||
|
project_name=row["name"],
|
||||||
|
project_json_str=row["data_json"],
|
||||||
|
sample_rate=req.sample_rate or 44100
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"task_id": task.id,
|
||||||
|
"status": "processing"
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,229 @@
|
|||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
import soundfile as sf
|
||||||
|
from app.config import settings
|
||||||
|
from app.core.vst_engine import render_midi_events_to_audio
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
def check_pedalboard_safe():
|
||||||
|
try:
|
||||||
|
res = subprocess.run(
|
||||||
|
[sys.executable, "-c", "import pedalboard"],
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
timeout=2.0
|
||||||
|
)
|
||||||
|
return res.returncode == 0
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
HAS_PEDALBOARD = check_pedalboard_safe()
|
||||||
|
if HAS_PEDALBOARD:
|
||||||
|
try:
|
||||||
|
from pedalboard import Pedalboard, Gain
|
||||||
|
except Exception:
|
||||||
|
HAS_PEDALBOARD = False
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class PythonRenderEngine:
|
||||||
|
def __init__(self, sample_rate=44100):
|
||||||
|
self.sample_rate = sample_rate
|
||||||
|
|
||||||
|
def bars_to_samples(self, bars: float, bpm: float, time_sig_num: int) -> int:
|
||||||
|
seconds_per_beat = 60.0 / max(20.0, bpm)
|
||||||
|
seconds_per_bar = seconds_per_beat * time_sig_num
|
||||||
|
return int(bars * seconds_per_bar * self.sample_rate)
|
||||||
|
|
||||||
|
def resolve_file_path(self, url_or_id: str) -> str:
|
||||||
|
if not url_or_id:
|
||||||
|
return ""
|
||||||
|
base = os.path.basename(url_or_id)
|
||||||
|
# Check uploads directory
|
||||||
|
p_uploads = os.path.join(settings.UPLOADS_DIR, base)
|
||||||
|
if os.path.exists(p_uploads):
|
||||||
|
return p_uploads
|
||||||
|
# Check processed directory
|
||||||
|
p_processed = os.path.join(settings.PROCESSED_DIR, base)
|
||||||
|
if os.path.exists(p_processed):
|
||||||
|
return p_processed
|
||||||
|
# Check general storage directory
|
||||||
|
p_storage = os.path.join(settings.STORAGE_DIR, base)
|
||||||
|
if os.path.exists(p_storage):
|
||||||
|
return p_storage
|
||||||
|
# Direct check
|
||||||
|
if os.path.exists(url_or_id):
|
||||||
|
return url_or_id
|
||||||
|
return url_or_id
|
||||||
|
|
||||||
|
def render_session_container(self, session: dict, section_store: dict, bpm: float, time_sig_num: int, total_samples: int) -> np.ndarray:
|
||||||
|
session_buffer = np.zeros((2, total_samples), dtype=np.float32)
|
||||||
|
|
||||||
|
for track in session.get("tracks", []):
|
||||||
|
track_type = track.get("type", "AUDIO")
|
||||||
|
track_buffer = np.zeros((2, total_samples), dtype=np.float32)
|
||||||
|
|
||||||
|
for item in track.get("items", []):
|
||||||
|
start_sample = self.bars_to_samples(item["start_bar"], bpm, time_sig_num)
|
||||||
|
dur_samples = self.bars_to_samples(item["duration_bars"], bpm, time_sig_num)
|
||||||
|
offset_sample = self.bars_to_samples(item["clip_start_offset_bars"], bpm, time_sig_num)
|
||||||
|
|
||||||
|
item_type = item.get("type")
|
||||||
|
if item_type == "AUDIO_ITEM":
|
||||||
|
source_data = item.get("source_data", {})
|
||||||
|
audio_url = source_data.get("audio_file_url", "")
|
||||||
|
resolved_path = self.resolve_file_path(audio_url)
|
||||||
|
|
||||||
|
if resolved_path and os.path.exists(resolved_path):
|
||||||
|
try:
|
||||||
|
audio_data, sr = sf.read(resolved_path, dtype='float32')
|
||||||
|
if sr != self.sample_rate:
|
||||||
|
# Resampling fallback if simple, otherwise skip
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Handle channel mapping (Mono/Stereo)
|
||||||
|
if len(audio_data.shape) == 1:
|
||||||
|
audio_data = np.vstack([audio_data, audio_data])
|
||||||
|
else:
|
||||||
|
audio_data = audio_data.T # Shape: (channels, samples)
|
||||||
|
|
||||||
|
# Trim source offset & duration
|
||||||
|
src_len = audio_data.shape[1]
|
||||||
|
if offset_sample < src_len:
|
||||||
|
actual_dur = min(dur_samples, src_len - offset_sample)
|
||||||
|
sliced_audio = audio_data[:, offset_sample : offset_sample + actual_dur]
|
||||||
|
|
||||||
|
# Apply gain
|
||||||
|
gain_val = source_data.get("gain", 1.0)
|
||||||
|
sliced_audio = sliced_audio * gain_val
|
||||||
|
|
||||||
|
# Write to track buffer with boundaries
|
||||||
|
write_end = min(start_sample + sliced_audio.shape[1], total_samples)
|
||||||
|
actual_len = write_end - start_sample
|
||||||
|
if actual_len > 0:
|
||||||
|
track_buffer[:, start_sample:write_end] += sliced_audio[:, :actual_len]
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[RenderEngine] Error reading audio file {resolved_path}: {e}")
|
||||||
|
|
||||||
|
elif item_type == "MIDI_ITEM":
|
||||||
|
source_data = item.get("source_data", {})
|
||||||
|
notes = source_data.get("notes", [])
|
||||||
|
|
||||||
|
# Convert to midi events required by vst_engine
|
||||||
|
midi_events = []
|
||||||
|
for note in notes:
|
||||||
|
note_start_bar = note["start_beat"] / time_sig_num
|
||||||
|
# Filter notes within the non-destructive visible window
|
||||||
|
offset_bar = item["clip_start_offset_bars"]
|
||||||
|
dur_bar = item["duration_bars"]
|
||||||
|
if note_start_bar >= offset_bar and note_start_bar < (offset_bar + dur_bar):
|
||||||
|
rel_bar_in_item = note_start_bar - offset_bar
|
||||||
|
target_global_bar = item["start_bar"] + rel_bar_in_item
|
||||||
|
midi_events.append({
|
||||||
|
"note": note["pitch"],
|
||||||
|
"start_beat": target_global_bar * time_sig_num,
|
||||||
|
"duration_beats": note["duration_beats"],
|
||||||
|
"velocity": int(note.get("velocity", 0.8) * 127)
|
||||||
|
})
|
||||||
|
|
||||||
|
if midi_events:
|
||||||
|
try:
|
||||||
|
# Synthesize MIDI track notes
|
||||||
|
synth_buffer = render_midi_events_to_audio(
|
||||||
|
midi_events=midi_events,
|
||||||
|
sr=self.sample_rate,
|
||||||
|
bpm=bpm,
|
||||||
|
instrument='synth'
|
||||||
|
)
|
||||||
|
# Add to track buffer
|
||||||
|
actual_len = min(synth_buffer.shape[1], total_samples)
|
||||||
|
track_buffer[:, :actual_len] += synth_buffer[:, :actual_len]
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[RenderEngine] Error rendering MIDI: {e}")
|
||||||
|
|
||||||
|
elif item_type == "SECTION_ITEM":
|
||||||
|
source_data = item.get("source_data", {})
|
||||||
|
sec_id = source_data.get("referenced_section_id", "")
|
||||||
|
if sec_id and sec_id in section_store:
|
||||||
|
# Render nested section recursively
|
||||||
|
sec_container = section_store[sec_id]
|
||||||
|
sec_buffer = self.render_session_container(
|
||||||
|
session=sec_container,
|
||||||
|
section_store=section_store,
|
||||||
|
bpm=bpm,
|
||||||
|
time_sig_num=time_sig_num,
|
||||||
|
total_samples=total_samples
|
||||||
|
)
|
||||||
|
|
||||||
|
# Apply non-destructive crop/slicing on section buffer
|
||||||
|
if offset_sample < total_samples:
|
||||||
|
actual_dur = min(dur_samples, total_samples - offset_sample)
|
||||||
|
sliced_sec = sec_buffer[:, offset_sample : offset_sample + actual_dur]
|
||||||
|
|
||||||
|
# Write to track buffer
|
||||||
|
write_end = min(start_sample + sliced_sec.shape[1], total_samples)
|
||||||
|
actual_len = write_end - start_sample
|
||||||
|
if actual_len > 0:
|
||||||
|
track_buffer[:, start_sample:write_end] += sliced_sec[:, :actual_len]
|
||||||
|
|
||||||
|
# Apply Track Gain (via Pedalboard or fallback)
|
||||||
|
vol_db = track.get("volume_db", 0.0)
|
||||||
|
pan = track.get("pan", 0.0)
|
||||||
|
mute = track.get("mute", False)
|
||||||
|
|
||||||
|
if mute:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Process track volume
|
||||||
|
if HAS_PEDALBOARD:
|
||||||
|
try:
|
||||||
|
board = Pedalboard([Gain(gain_db=vol_db)])
|
||||||
|
processed_track = board(track_buffer, sample_rate=self.sample_rate)
|
||||||
|
except Exception:
|
||||||
|
gain_linear = 10 ** (vol_db / 20.0)
|
||||||
|
processed_track = track_buffer * gain_linear
|
||||||
|
else:
|
||||||
|
gain_linear = 10 ** (vol_db / 20.0)
|
||||||
|
processed_track = track_buffer * gain_linear
|
||||||
|
|
||||||
|
# Apply Track Pan
|
||||||
|
if pan != 0.0:
|
||||||
|
# Constant power panning
|
||||||
|
theta = ((np.clip(pan, -1.0, 1.0) + 1.0) / 2.0) * (np.pi / 2.0)
|
||||||
|
processed_track[0, :] *= np.cos(theta)
|
||||||
|
processed_track[1, :] *= np.sin(theta)
|
||||||
|
|
||||||
|
# Mix track to session
|
||||||
|
session_buffer += processed_track
|
||||||
|
|
||||||
|
return session_buffer
|
||||||
|
|
||||||
|
def render_project(self, project_json: dict, output_filepath: str):
|
||||||
|
bpm = project_json["metadata"]["bpm"]
|
||||||
|
time_sig_num = project_json["metadata"].get("time_signature_numerator", 4)
|
||||||
|
main_session = project_json["main_session"]
|
||||||
|
section_store = project_json.get("section_store", {})
|
||||||
|
|
||||||
|
# Compute total project samples
|
||||||
|
total_bars = main_session.get("length_bars", 16.0)
|
||||||
|
total_samples = self.bars_to_samples(total_bars, bpm, time_sig_num)
|
||||||
|
|
||||||
|
# Render main session
|
||||||
|
master_buffer = self.render_session_container(
|
||||||
|
session=main_session,
|
||||||
|
section_store=section_store,
|
||||||
|
bpm=bpm,
|
||||||
|
time_sig_num=time_sig_num,
|
||||||
|
total_samples=total_samples
|
||||||
|
)
|
||||||
|
|
||||||
|
# Normalization to prevent clipping
|
||||||
|
max_peak = np.max(np.abs(master_buffer))
|
||||||
|
if max_peak > 1.0:
|
||||||
|
master_buffer /= max_peak
|
||||||
|
|
||||||
|
# Write final output file
|
||||||
|
sf.write(output_filepath, master_buffer.T, self.sample_rate)
|
||||||
|
return output_filepath
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "DAWProject",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"project_id": { "type": "string" },
|
||||||
|
"metadata": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"title": { "type": "string" },
|
||||||
|
"bpm": { "type": "number", "minimum": 20.0, "maximum": 999.0, "default": 120.0 },
|
||||||
|
"time_signature_numerator": { "type": "integer", "default": 4 },
|
||||||
|
"time_signature_denominator": { "type": "integer", "default": 4 },
|
||||||
|
"sample_rate": { "type": "integer", "default": 44100 }
|
||||||
|
},
|
||||||
|
"required": ["title", "bpm", "time_signature_numerator", "time_signature_denominator", "sample_rate"]
|
||||||
|
},
|
||||||
|
"main_session": { "$ref": "#/definitions/SessionContainer" },
|
||||||
|
"section_store": {
|
||||||
|
"type": "object",
|
||||||
|
"description": "Auxiliary registry mapping section_id to sub-session containers",
|
||||||
|
"additionalProperties": { "$ref": "#/definitions/SessionContainer" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["project_id", "metadata", "main_session", "section_store"],
|
||||||
|
"definitions": {
|
||||||
|
"SessionContainer": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string" },
|
||||||
|
"name": { "type": "string" },
|
||||||
|
"is_root": { "type": "boolean" },
|
||||||
|
"length_bars": { "type": "number", "description": "Computed or manually set total length in bars" },
|
||||||
|
"auto_compute_length": { "type": "boolean", "default": true },
|
||||||
|
"tracks": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/Track" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["id", "is_root", "tracks"]
|
||||||
|
},
|
||||||
|
"Track": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string" },
|
||||||
|
"name": { "type": "string" },
|
||||||
|
"type": { "type": "string", "enum": ["AUDIO", "MIDI", "SECTION"] },
|
||||||
|
"volume_db": { "type": "number", "default": 0.0 },
|
||||||
|
"pan": { "type": "number", "minimum": -1.0, "maximum": 1.0, "default": 0.0 },
|
||||||
|
"mute": { "type": "boolean", "default": false },
|
||||||
|
"solo": { "type": "boolean", "default": false },
|
||||||
|
"fx_chain": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/FXPlugin" }
|
||||||
|
},
|
||||||
|
"synth_engine": { "$ref": "#/definitions/SynthPlugin" },
|
||||||
|
"items": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/TimelineItem" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["id", "name", "type", "items"]
|
||||||
|
},
|
||||||
|
"TimelineItem": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string" },
|
||||||
|
"name": { "type": "string" },
|
||||||
|
"type": { "type": "string", "enum": ["AUDIO_ITEM", "MIDI_ITEM", "SECTION_ITEM"] },
|
||||||
|
"start_bar": { "type": "number", "description": "Global timeline position where the item starts" },
|
||||||
|
"duration_bars": { "type": "number", "description": "Visible duration on the track timeline in bars" },
|
||||||
|
"clip_start_offset_bars": { "type": "number", "description": "Internal start offset inside the source buffer/item" },
|
||||||
|
"source_data": {
|
||||||
|
"type": "object",
|
||||||
|
"oneOf": [
|
||||||
|
{ "$ref": "#/definitions/AudioSourceData" },
|
||||||
|
{ "$ref": "#/definitions/MIDISourceData" },
|
||||||
|
{ "$ref": "#/definitions/SectionSourceData" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["id", "type", "start_bar", "duration_bars", "clip_start_offset_bars", "source_data"]
|
||||||
|
},
|
||||||
|
"AudioSourceData": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"audio_file_url": { "type": "string" },
|
||||||
|
"sample_rate": { "type": "integer" },
|
||||||
|
"channels": { "type": "integer" },
|
||||||
|
"gain": { "type": "number", "default": 1.0 }
|
||||||
|
},
|
||||||
|
"required": ["audio_file_url"]
|
||||||
|
},
|
||||||
|
"MIDISourceData": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"total_buffer_bars": { "type": "number", "default": 8.0 },
|
||||||
|
"notes": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/MIDINote" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["total_buffer_bars", "notes"]
|
||||||
|
},
|
||||||
|
"SectionSourceData": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"referenced_section_id": { "type": "string", "description": "Pointer to section_store key" }
|
||||||
|
},
|
||||||
|
"required": ["referenced_section_id"]
|
||||||
|
},
|
||||||
|
"MIDINote": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string" },
|
||||||
|
"pitch": { "type": "integer", "minimum": 0, "maximum": 127 },
|
||||||
|
"start_beat": { "type": "number", "description": "Beat offset relative to the start of the source buffer (bar 0)" },
|
||||||
|
"duration_beats": { "type": "number" },
|
||||||
|
"velocity": { "type": "number", "minimum": 0.0, "maximum": 1.0, "default": 0.8 },
|
||||||
|
"pan": { "type": "number", "minimum": -1.0, "maximum": 1.0, "default": 0.0 }
|
||||||
|
},
|
||||||
|
"required": ["id", "pitch", "start_beat", "duration_beats", "velocity"]
|
||||||
|
},
|
||||||
|
"FXPlugin": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"plugin_id": { "type": "string" },
|
||||||
|
"name": { "type": "string" },
|
||||||
|
"bypass": { "type": "boolean", "default": false },
|
||||||
|
"parameters": { "type": "object" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"SynthPlugin": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"plugin_id": { "type": "string" },
|
||||||
|
"preset_id": { "type": "string" },
|
||||||
|
"parameters": { "type": "object" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+2509
-418
File diff suppressed because it is too large
Load Diff
+302
-12152
File diff suppressed because one or more lines are too long
@@ -3,12 +3,11 @@
|
|||||||
const SFS_VERSION = "1.0.0";
|
const SFS_VERSION = "1.0.0";
|
||||||
|
|
||||||
function exportProjectToSFS(projectState) {
|
function exportProjectToSFS(projectState) {
|
||||||
const sfsBundle = {
|
let projectObj = {};
|
||||||
format: "SONICFORGE_STUDIO_PROJECT",
|
if (projectState.main_session) {
|
||||||
version: SFS_VERSION,
|
projectObj = projectState;
|
||||||
timestamp: Date.now(),
|
} else {
|
||||||
domain: window.location.origin,
|
projectObj = {
|
||||||
project: {
|
|
||||||
id: projectState.id || `proj_${Date.now()}`,
|
id: projectState.id || `proj_${Date.now()}`,
|
||||||
name: projectState.name || "Dự án mới",
|
name: projectState.name || "Dự án mới",
|
||||||
tracks: (projectState.tracks || []).map(t => ({
|
tracks: (projectState.tracks || []).map(t => ({
|
||||||
@@ -24,7 +23,15 @@
|
|||||||
markers: t.markers || [],
|
markers: t.markers || [],
|
||||||
serverFileId: t.serverFileId || null
|
serverFileId: t.serverFileId || null
|
||||||
}))
|
}))
|
||||||
}
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const sfsBundle = {
|
||||||
|
format: "SONICFORGE_STUDIO_PROJECT",
|
||||||
|
version: SFS_VERSION,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
domain: window.location.origin,
|
||||||
|
project: projectObj
|
||||||
};
|
};
|
||||||
|
|
||||||
const jsonStr = JSON.stringify(sfsBundle, null, 2);
|
const jsonStr = JSON.stringify(sfsBundle, null, 2);
|
||||||
@@ -33,7 +40,8 @@
|
|||||||
|
|
||||||
const a = document.createElement('a');
|
const a = document.createElement('a');
|
||||||
a.href = url;
|
a.href = url;
|
||||||
a.download = `${(projectState.name || 'project').replace(/\s+/g, '_')}.sfs`;
|
const displayName = projectObj.metadata?.title || projectObj.name || 'project';
|
||||||
|
a.download = `${displayName.replace(/\s+/g, '_')}.sfs`;
|
||||||
a.click();
|
a.click();
|
||||||
URL.revokeObjectURL(url);
|
URL.revokeObjectURL(url);
|
||||||
}
|
}
|
||||||
@@ -53,7 +61,7 @@
|
|||||||
autoSaveTimer = setTimeout(async () => {
|
autoSaveTimer = setTimeout(async () => {
|
||||||
try {
|
try {
|
||||||
const state = getProjectStateCallback();
|
const state = getProjectStateCallback();
|
||||||
if (!state || !state.tracks || state.tracks.length === 0) return;
|
if (!state || (!state.tracks && !state.main_session)) return;
|
||||||
const dataJson = JSON.stringify(state);
|
const dataJson = JSON.stringify(state);
|
||||||
localStorage.setItem('sonic_temp_project', dataJson);
|
localStorage.setItem('sonic_temp_project', dataJson);
|
||||||
if (window.SonicAPI && localStorage.getItem('sonic_token')) {
|
if (window.SonicAPI && localStorage.getItem('sonic_token')) {
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
class PCMRecorderProcessor extends AudioWorkletProcessor {
|
||||||
|
constructor() {
|
||||||
|
super();
|
||||||
|
this.bufferSize = 4096;
|
||||||
|
this.buffer = new Float32Array(this.bufferSize);
|
||||||
|
this.bufferIndex = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
process(inputs, outputs, parameters) {
|
||||||
|
const input = inputs[0];
|
||||||
|
if (input && input.length > 0) {
|
||||||
|
const inputChannel = input[0]; // Mono Channel 0
|
||||||
|
|
||||||
|
for (let i = 0; i < inputChannel.length; i++) {
|
||||||
|
this.buffer[this.bufferIndex++] = inputChannel[i];
|
||||||
|
|
||||||
|
// When Ring-Buffer fills, send Float32Array to Main Thread
|
||||||
|
if (this.bufferIndex >= this.bufferSize) {
|
||||||
|
this.port.postMessage({
|
||||||
|
type: 'PCM_DATA',
|
||||||
|
buffer: this.buffer.slice(0, this.bufferSize)
|
||||||
|
});
|
||||||
|
this.bufferIndex = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true; // Keep worklet active
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
registerProcessor('pcm-recorder-processor', PCMRecorderProcessor);
|
||||||
Binary file not shown.
@@ -303,3 +303,27 @@ def cleanup_expired_files_task(max_age_hours: int = 24):
|
|||||||
"cleaned_size_mb": round(cleaned_size / (1024 * 1024), 2),
|
"cleaned_size_mb": round(cleaned_size / (1024 * 1024), 2),
|
||||||
"max_age_hours": max_age_hours
|
"max_age_hours": max_age_hours
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task
|
||||||
|
def render_project_task(project_id: str, project_name: str, project_json_str: str, sample_rate: int = 44100):
|
||||||
|
"""
|
||||||
|
Task Celery để kết xuất dự án ngoại tuyến (Offline Project Mixdown) áp dụng specs 30_DAW_ARCHITECT.md.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
from app.core.render_engine import PythonRenderEngine
|
||||||
|
|
||||||
|
project_json = json.loads(project_json_str)
|
||||||
|
output_filename = f"{project_id}_render.wav"
|
||||||
|
output_path = os.path.join(settings.PROCESSED_DIR, output_filename)
|
||||||
|
|
||||||
|
engine = PythonRenderEngine(sample_rate=sample_rate)
|
||||||
|
engine.render_project(project_json, output_path)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"project_id": project_id,
|
||||||
|
"project_name": project_name,
|
||||||
|
"success": True,
|
||||||
|
"output_file_id": output_filename
|
||||||
|
}
|
||||||
|
|
||||||
|
|||||||
+79
-21
@@ -1,5 +1,6 @@
|
|||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="vi">
|
<html lang="vi">
|
||||||
|
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
@@ -14,7 +15,7 @@
|
|||||||
<script src="/static/js/services/storage.js"></script>
|
<script src="/static/js/services/storage.js"></script>
|
||||||
<script src="/static/js/services/aiGateway.js"></script>
|
<script src="/static/js/services/aiGateway.js"></script>
|
||||||
<script src="/static/js/services/dawCommandDispatcher.js"></script>
|
<script src="/static/js/services/dawCommandDispatcher.js"></script>
|
||||||
<script src="/static/js/app.precompiled.js" defer></script>
|
<script src="/static/js/app.precompiled.js?v=202607231122" defer></script>
|
||||||
<style>
|
<style>
|
||||||
:root {
|
:root {
|
||||||
--right-sidebar-width: 320px;
|
--right-sidebar-width: 320px;
|
||||||
@@ -31,26 +32,71 @@
|
|||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
user-select: none;
|
user-select: none;
|
||||||
}
|
}
|
||||||
.daw-bg { background-color: #1e1e1e; }
|
|
||||||
.daw-panel { background-color: #262626; }
|
.daw-bg {
|
||||||
.daw-header { background-color: #2e2e2e; }
|
background-color: #1e1e1e;
|
||||||
.daw-border { border-color: #181818; }
|
}
|
||||||
.daw-track-active { background-color: #333333; }
|
|
||||||
::-webkit-scrollbar { width: 10px; height: 10px; }
|
.daw-panel {
|
||||||
::-webkit-scrollbar-track { background: #141414; }
|
background-color: #262626;
|
||||||
::-webkit-scrollbar-thumb { background: #3a3a3a; border: 2px solid #141414; border-radius: 4px; }
|
}
|
||||||
::-webkit-scrollbar-thumb:hover { background: #4a4a4a; }
|
|
||||||
.knob-container { position: relative; width: 28px; height: 28px; }
|
.daw-header {
|
||||||
.knob-dial { transform-origin: center; transition: transform 0.1s ease; }
|
background-color: #2e2e2e;
|
||||||
|
}
|
||||||
|
|
||||||
|
.daw-border {
|
||||||
|
border-color: #181818;
|
||||||
|
}
|
||||||
|
|
||||||
|
.daw-track-active {
|
||||||
|
background-color: #333333;
|
||||||
|
}
|
||||||
|
|
||||||
|
::-webkit-scrollbar {
|
||||||
|
width: 10px;
|
||||||
|
height: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
::-webkit-scrollbar-track {
|
||||||
|
background: #141414;
|
||||||
|
}
|
||||||
|
|
||||||
|
::-webkit-scrollbar-thumb {
|
||||||
|
background: #3a3a3a;
|
||||||
|
border: 2px solid #141414;
|
||||||
|
border-radius: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
::-webkit-scrollbar-thumb:hover {
|
||||||
|
background: #4a4a4a;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knob-container {
|
||||||
|
position: relative;
|
||||||
|
width: 28px;
|
||||||
|
height: 28px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knob-dial {
|
||||||
|
transform-origin: center;
|
||||||
|
transition: transform 0.1s ease;
|
||||||
|
}
|
||||||
|
|
||||||
.selection-interactive-box {
|
.selection-interactive-box {
|
||||||
min-width: 4px;
|
min-width: 4px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.no-scrollbar {
|
.no-scrollbar {
|
||||||
scrollbar-width: none; /* Firefox */
|
scrollbar-width: none;
|
||||||
-ms-overflow-style: none; /* IE 10+ */
|
/* Firefox */
|
||||||
|
-ms-overflow-style: none;
|
||||||
|
/* IE 10+ */
|
||||||
}
|
}
|
||||||
|
|
||||||
.no-scrollbar::-webkit-scrollbar {
|
.no-scrollbar::-webkit-scrollbar {
|
||||||
display: none; /* Safari and Chrome */
|
display: none;
|
||||||
|
/* Safari and Chrome */
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Fullscreen Fixed App Shell */
|
/* Fullscreen Fixed App Shell */
|
||||||
@@ -101,12 +147,14 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
#panel-media-explorer {
|
#panel-media-explorer {
|
||||||
height: 50%; /* Default 50/50 split */
|
height: 50%;
|
||||||
|
/* Default 50/50 split */
|
||||||
min-height: 100px;
|
min-height: 100px;
|
||||||
}
|
}
|
||||||
|
|
||||||
#panel-ai {
|
#panel-ai {
|
||||||
flex: 1; /* Fills remaining height */
|
flex: 1;
|
||||||
|
/* Fills remaining height */
|
||||||
min-height: 100px;
|
min-height: 100px;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,7 +166,8 @@
|
|||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
padding: 8px;
|
padding: 8px;
|
||||||
overflow-x: auto; /* Enables horizontal scroll when panels overflow */
|
overflow-x: auto;
|
||||||
|
/* Enables horizontal scroll when panels overflow */
|
||||||
overflow-y: hidden;
|
overflow-y: hidden;
|
||||||
background-color: #161616;
|
background-color: #161616;
|
||||||
border-top: 1px solid var(--panel-border-color);
|
border-top: 1px solid var(--panel-border-color);
|
||||||
@@ -129,17 +178,20 @@
|
|||||||
.daw-bottom-strip::-webkit-scrollbar {
|
.daw-bottom-strip::-webkit-scrollbar {
|
||||||
height: 8px;
|
height: 8px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.daw-bottom-strip::-webkit-scrollbar-thumb {
|
.daw-bottom-strip::-webkit-scrollbar-thumb {
|
||||||
background: #3a3a3a;
|
background: #3a3a3a;
|
||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.daw-bottom-strip::-webkit-scrollbar-thumb:hover {
|
.daw-bottom-strip::-webkit-scrollbar-thumb:hover {
|
||||||
background: #00ffcc;
|
background: #00ffcc;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Sub-panels inside the bottom strip */
|
/* Sub-panels inside the bottom strip */
|
||||||
.bottom-panel {
|
.bottom-panel {
|
||||||
flex: 0 0 auto; /* Prevents shrinking, locks content dimensions */
|
flex: 0 0 auto;
|
||||||
|
/* Prevents shrinking, locks content dimensions */
|
||||||
width: 320px;
|
width: 320px;
|
||||||
height: 100%;
|
height: 100%;
|
||||||
background-color: #222;
|
background-color: #222;
|
||||||
@@ -151,11 +203,13 @@
|
|||||||
/* RESIZER HANDLES */
|
/* RESIZER HANDLES */
|
||||||
.resizer-col-handle {
|
.resizer-col-handle {
|
||||||
width: 5px;
|
width: 5px;
|
||||||
cursor: ew-resize; /* Horizontal resize cursor */
|
cursor: ew-resize;
|
||||||
|
/* Horizontal resize cursor */
|
||||||
background: transparent;
|
background: transparent;
|
||||||
transition: background 0.2s;
|
transition: background 0.2s;
|
||||||
z-index: 10;
|
z-index: 10;
|
||||||
}
|
}
|
||||||
|
|
||||||
.resizer-col-handle:hover,
|
.resizer-col-handle:hover,
|
||||||
.resizer-col-handle:active {
|
.resizer-col-handle:active {
|
||||||
background: #00ffcc;
|
background: #00ffcc;
|
||||||
@@ -163,18 +217,22 @@
|
|||||||
|
|
||||||
.resizer-row-handle {
|
.resizer-row-handle {
|
||||||
height: 5px;
|
height: 5px;
|
||||||
cursor: ns-resize; /* Vertical resize cursor */
|
cursor: ns-resize;
|
||||||
|
/* Vertical resize cursor */
|
||||||
background: transparent;
|
background: transparent;
|
||||||
transition: background 0.2s;
|
transition: background 0.2s;
|
||||||
z-index: 10;
|
z-index: 10;
|
||||||
}
|
}
|
||||||
|
|
||||||
.resizer-row-handle:hover,
|
.resizer-row-handle:hover,
|
||||||
.resizer-row-handle:active {
|
.resizer-row-handle:active {
|
||||||
background: #00ffcc;
|
background: #00ffcc;
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
|
|
||||||
<body class="h-screen w-screen flex flex-col">
|
<body class="h-screen w-screen flex flex-col">
|
||||||
<div id="root" class="h-full w-full flex flex-col"></div>
|
<div id="root" class="h-full w-full flex flex-col"></div>
|
||||||
</body>
|
</body>
|
||||||
|
|
||||||
</html>
|
</html>
|
||||||
@@ -0,0 +1,760 @@
|
|||||||
|
Here is the complete translation and conversion of the document into a clean, professionally formatted Markdown layout:
|
||||||
|
|
||||||
|
# ARCHITECTURAL, TECHNICAL, AND ALGORITHMIC SPECIFICATION
|
||||||
|
|
||||||
|
## Hybrid Web-Based Digital Audio Workstation (DAW) with Nested Section Architecture and Non-Destructive Timeline Mechanics
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 1. System Overview & Architecture Design
|
||||||
|
|
||||||
|
#### 1.1 High-Level Architecture Topology
|
||||||
|
|
||||||
|
The system follows a hybrid Client-Server architecture designed for real-time Web-based audio production, composition, and high-performance offline DSP rendering.
|
||||||
|
|
||||||
|
* **Frontend Client (HTML5 / Vanilla JS / Web Audio API / HTML5 Canvas)**
|
||||||
|
* **UI Layer:** HTML5 Canvas / Web Components for high-FPS multi-lane timeline rendering, Piano Roll canvas, Sample Editor, and Sub-Tab navigation.
|
||||||
|
* **Audio Engine Layer:** Web Audio API `AudioContext` graph, Custom `AudioWorklet` Processors (WebAssembly/JS) for real-time synthesis, playback scheduling, sample playback, and latency-compensated signal routing.
|
||||||
|
* **State Management Engine:** Immutable/Reactive Central State Store handling Session tree hierarchy, Section Store registries, Undo/Redo stack, and view-state context isolation.
|
||||||
|
|
||||||
|
|
||||||
|
* **Backend Server (Python Engine)**
|
||||||
|
* **RESTful / WebSocket API:** Event-driven client communication layer (FastAPI or AIOHTTP).
|
||||||
|
* **DSP / Rendering Engine:** Python-based audio processing (`numpy`, `scipy`, `pyo`, `pedalboard`) for offline stem bouncing, high-fidelity export, sample processing, and optional VST/VSTi hosting/bridging.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
```text
|
||||||
|
+-----------------------------------------------------------------------------------+
|
||||||
|
| FRONTEND (HTML5/JS) |
|
||||||
|
| |
|
||||||
|
| +-----------------------------------------------------------------------------+ |
|
||||||
|
| | UI & View State System | |
|
||||||
|
| | +---------------------+ +----------------------+ +--------------------+ | |
|
||||||
|
| | | Main Session Canvas | | Section-Tab View | | Piano Roll View | | |
|
||||||
|
| | +---------------------+ +----------------------+ +--------------------+ | |
|
||||||
|
| +-----------------------------------------------------------------------------+ |
|
||||||
|
| | |
|
||||||
|
| +-----------------------------------------------------------------------------+ |
|
||||||
|
| | Central Data State Store | |
|
||||||
|
| | [Project Model] ---> [Section Store] ---> [Item Clip Metadata] | |
|
||||||
|
| +-----------------------------------------------------------------------------+ |
|
||||||
|
| | |
|
||||||
|
| +-----------------------------------------------------------------------------+ |
|
||||||
|
| | Audio & Clock Engine | |
|
||||||
|
| | +------------------------+ +------------------+ +---------------------+ | |
|
||||||
|
| | | Precision Scheduler | | Web Audio Graph | | AudioWorklet Synth | | |
|
||||||
|
| | | (Lookahead Timer) | | AudioNode Router | | / WebAssembly Core | | |
|
||||||
|
| | +------------------------+ +------------------+ +---------------------+ | |
|
||||||
|
| +-----------------------------------------------------------------------------+ |
|
||||||
|
+------------------------------------------^----------------------------------------+
|
||||||
|
| WebSocket / REST API
|
||||||
|
+------------------------------------------v----------------------------------------+
|
||||||
|
| BACKEND SERVER (PYTHON) |
|
||||||
|
| +-----------------------------------------------------------------------------+ |
|
||||||
|
| | FastAPI / WebSocket Handler | |
|
||||||
|
| +-----------------------------------------------------------------------------+ |
|
||||||
|
| | DSP Engine (Pedalboard / Numpy / Scipy) - Offline Render, Audio Export | |
|
||||||
|
| +-----------------------------------------------------------------------------+ |
|
||||||
|
| | VST / VSTi Hosting Bridge & Plugin State Persistence | |
|
||||||
|
| +-----------------------------------------------------------------------------+ |
|
||||||
|
+-----------------------------------------------------------------------------------+
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2. Detailed Data Schemas (JSON Specification)
|
||||||
|
|
||||||
|
#### 2.1 Project Root Schema (`project_schema.json`)
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "DAWProject",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"project_id": { "type": "string", "format": "uuid" },
|
||||||
|
"metadata": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"title": { "type": "string" },
|
||||||
|
"bpm": { "type": "number", "minimum": 20.0, "maximum": 999.0, "default": 120.0 },
|
||||||
|
"time_signature_numerator": { "type": "integer", "default": 4 },
|
||||||
|
"time_signature_denominator": { "type": "integer", "default": 4 },
|
||||||
|
"sample_rate": { "type": "integer", "default": 44100 }
|
||||||
|
},
|
||||||
|
"required": ["title", "bpm", "time_signature_numerator", "time_signature_denominator", "sample_rate"]
|
||||||
|
},
|
||||||
|
"main_session": { "$ref": "#/definitions/SessionContainer" },
|
||||||
|
"section_store": {
|
||||||
|
"type": "object",
|
||||||
|
"description": "Auxiliary registry mapping section_id to sub-session containers",
|
||||||
|
"additionalProperties": { "$ref": "#/definitions/SessionContainer" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["project_id", "metadata", "main_session", "section_store"],
|
||||||
|
"definitions": {
|
||||||
|
"SessionContainer": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string" },
|
||||||
|
"name": { "type": "string" },
|
||||||
|
"is_root": { "type": "boolean" },
|
||||||
|
"length_bars": { "type": "number", "description": "Computed or manually set total length in bars" },
|
||||||
|
"auto_compute_length": { "type": "boolean", "default": true },
|
||||||
|
"tracks": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/Track" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["id", "is_root", "tracks"]
|
||||||
|
},
|
||||||
|
"Track": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string" },
|
||||||
|
"name": { "type": "string" },
|
||||||
|
"type": { "type": "string", "enum": ["AUDIO", "MIDI", "SECTION"] },
|
||||||
|
"volume_db": { "type": "number", "default": 0.0 },
|
||||||
|
"pan": { "type": "number", "minimum": -1.0, "maximum": 1.0, "default": 0.0 },
|
||||||
|
"mute": { "type": "boolean", "default": false },
|
||||||
|
"solo": { "type": "boolean", "default": false },
|
||||||
|
"fx_chain": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/FXPlugin" }
|
||||||
|
},
|
||||||
|
"synth_engine": { "$ref": "#/definitions/SynthPlugin" },
|
||||||
|
"items": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/TimelineItem" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["id", "name", "type", "items"]
|
||||||
|
},
|
||||||
|
"TimelineItem": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string" },
|
||||||
|
"name": { "type": "string" },
|
||||||
|
"type": { "type": "string", "enum": ["AUDIO_ITEM", "MIDI_ITEM", "SECTION_ITEM"] },
|
||||||
|
"start_bar": { "type": "number", "description": "Global timeline position where the item starts" },
|
||||||
|
"duration_bars": { "type": "number", "description": "Visible duration on the track timeline in bars" },
|
||||||
|
"clip_start_offset_bars": { "type": "number", "description": "Internal start offset inside the source buffer/item" },
|
||||||
|
"source_data": {
|
||||||
|
"type": "object",
|
||||||
|
"oneOf": [
|
||||||
|
{ "$ref": "#/definitions/AudioSourceData" },
|
||||||
|
{ "$ref": "#/definitions/MIDISourceData" },
|
||||||
|
{ "$ref": "#/definitions/SectionSourceData" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["id", "type", "start_bar", "duration_bars", "clip_start_offset_bars", "source_data"]
|
||||||
|
},
|
||||||
|
"AudioSourceData": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"audio_file_url": { "type": "string" },
|
||||||
|
"sample_rate": { "type": "integer" },
|
||||||
|
"channels": { "type": "integer" },
|
||||||
|
"gain": { "type": "number", "default": 1.0 }
|
||||||
|
},
|
||||||
|
"required": ["audio_file_url"]
|
||||||
|
},
|
||||||
|
"MIDISourceData": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"total_buffer_bars": { "type": "number", "default": 8.0 },
|
||||||
|
"notes": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "$ref": "#/definitions/MIDINote" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["total_buffer_bars", "notes"]
|
||||||
|
},
|
||||||
|
"SectionSourceData": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"referenced_section_id": { "type": "string", "description": "Pointer to section_store key" }
|
||||||
|
},
|
||||||
|
"required": ["referenced_section_id"]
|
||||||
|
},
|
||||||
|
"MIDINote": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string" },
|
||||||
|
"pitch": { "type": "integer", "minimum": 0, "maximum": 127 },
|
||||||
|
"start_beat": { "type": "number", "description": "Beat offset relative to the start of the source buffer (bar 0)" },
|
||||||
|
"duration_beats": { "type": "number" },
|
||||||
|
"velocity": { "type": "number", "minimum": 0.0, "maximum": 1.0, "default": 0.8 },
|
||||||
|
"pan": { "type": "number", "minimum": -1.0, "maximum": 1.0, "default": 0.0 }
|
||||||
|
},
|
||||||
|
"required": ["id", "pitch", "start_beat", "duration_beats", "velocity"]
|
||||||
|
},
|
||||||
|
"FXPlugin": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"plugin_id": { "type": "string" },
|
||||||
|
"name": { "type": "string" },
|
||||||
|
"bypass": { "type": "boolean", "default": false },
|
||||||
|
"parameters": { "type": "object" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"SynthPlugin": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"plugin_id": { "type": "string" },
|
||||||
|
"preset_id": { "type": "string" },
|
||||||
|
"parameters": { "type": "object" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3. UI, Tab Navigation & View State Management
|
||||||
|
|
||||||
|
#### 3.1 Tab Context Model, Pinning Rules & Close Prevention Hierarchy
|
||||||
|
|
||||||
|
The application manages view tabs dynamically while maintaining strict lifecycle integrity:
|
||||||
|
|
||||||
|
* **Main Session Tab (Fixed / Pinned):** Always pinned at index 0 (`is_closeable: false`). It cannot be closed under any circumstances.
|
||||||
|
* **Sub-Tabs (Section-Tab, Piano Roll Tab, Audio Sample Editor Sub-Tab):** Dynamic views (`is_closeable: true`).
|
||||||
|
* **Parent-Child Tab Dependency Rules:**
|
||||||
|
* A Section-Tab represents an intermediate sub-session.
|
||||||
|
* When a user opens a child item (e.g., a `MIDIItem` or `AudioItem` inside a Section-Tab) into a Piano Roll Tab or Audio Sample Editor Sub-Tab, a parent-child context lineage is registered.
|
||||||
|
* **Close Block Rule:** A Section-Tab cannot be closed while any of its child items are currently open in active sub-tabs. Attempting to close the parent Section-Tab displays a block notice highlighting open child editors.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
```text
|
||||||
|
+---------------------------------------+
|
||||||
|
| Tab Navigation Controller |
|
||||||
|
+-------------------+-------------------+
|
||||||
|
|
|
||||||
|
+--------------------------------+--------------------------------+
|
||||||
|
| (Pinned / Uncloseable) | (Dynamic / Closable) | (Dynamic / Closable)
|
||||||
|
+--------v--------+ +--------v--------+ +--------v--------+
|
||||||
|
| MAIN SESSION | | SECTION TAB | | PIANO ROLL TAB |
|
||||||
|
| (Root Context) | | (Sub-Session) | | (Item Context) |
|
||||||
|
| | | [Parent Context] | [Child Context]|
|
||||||
|
+-----------------+ +--------+--------+ +--------+--------+
|
||||||
|
| |
|
||||||
|
+---- Depends on child closure ---+
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
##### State Object Schema with Tab Dependency Tracking:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"active_tab_id": "tab_pr_1",
|
||||||
|
"open_tabs": [
|
||||||
|
{
|
||||||
|
"tab_id": "tab_root",
|
||||||
|
"title": "MAIN SESSION",
|
||||||
|
"type": "MAIN_SESSION",
|
||||||
|
"target_id": "main",
|
||||||
|
"is_closeable": false,
|
||||||
|
"parent_tab_id": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"tab_id": "tab_sec_1",
|
||||||
|
"title": "Section: Verse 1",
|
||||||
|
"type": "SECTION_TAB",
|
||||||
|
"target_id": "Section_01",
|
||||||
|
"is_closeable": true,
|
||||||
|
"parent_tab_id": "tab_root"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"tab_id": "tab_pr_1",
|
||||||
|
"title": "Piano Roll: Bassline",
|
||||||
|
"type": "PIANO_ROLL",
|
||||||
|
"target_id": "ItemMIDI_Bassline",
|
||||||
|
"is_closeable": true,
|
||||||
|
"parent_tab_id": "tab_sec_1"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"piano_roll_state": {
|
||||||
|
"target_item_id": "ItemMIDI_Bassline",
|
||||||
|
"viewport_start_bar": 0.0,
|
||||||
|
"viewport_bar_width": 8.0,
|
||||||
|
"scroll_y_pitch": 60,
|
||||||
|
"snap_resolution": "1/16",
|
||||||
|
"note_selection": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 3.2 Piano Roll View Canvas Layout & Interaction Spec
|
||||||
|
|
||||||
|
* **Top Navigation Rule Pane (Bars/Beats Bar):**
|
||||||
|
* Displays bars from $0$ to $N$ (where $N = \text{total\_buffer\_bars}$, e.g., 8 bars).
|
||||||
|
* Highlights active clip visibility bounds (e.g., Bar 4.0 to Bar 6.0 shaded with active overlay, exterior bars dimmed).
|
||||||
|
|
||||||
|
|
||||||
|
* **Left Piano Keybed:**
|
||||||
|
* Anchored vertically, spans pitches $0$ (C-1) through $127$ (G9).
|
||||||
|
* Draws standard 88 key / 128 key pattern with distinct black key visually offset bars and pitch labeling ($C3$, $C4$, etc.).
|
||||||
|
|
||||||
|
|
||||||
|
* **Note Grid Canvas (Right Pane):**
|
||||||
|
* Synced to vertical pitch scroll and horizontal beat zoom.
|
||||||
|
* **Row Background Rendering:** Black key rows are assigned darker background fill color `#1A1A1E`, white key rows use `#25252A`.
|
||||||
|
* **Snap Grid Lines:** Rendered dynamically based on selected snap mode: Free, 1/1 Bar, 1/2 Beat, 1/4 Beat, 1/8 Beat, 1/16 Beat, 1/32 Beat.
|
||||||
|
|
||||||
|
|
||||||
|
* **Bottom Controller Pane (CC / Velocity / Pan Lane):**
|
||||||
|
* Synchronized horizontally with note grid.
|
||||||
|
* Displays vertical stem bars per note representing properties (Velocity, Pan). Allows click-and-drag line shaping or direct stem adjustment.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4. Audio & Synth Engine Routing Architecture (Web Audio API)
|
||||||
|
|
||||||
|
#### 4.1 Real-Time Signal Flow Graph
|
||||||
|
|
||||||
|
```text
|
||||||
|
[MIDI Scheduler] ---> [AudioWorklet / Virtual Synth Engine]
|
||||||
|
|
|
||||||
|
v (Audio Buffer / Stream)
|
||||||
|
[Audio Sample Playback Node] ----> [Track Channel FX Chain]
|
||||||
|
|
|
||||||
|
v
|
||||||
|
[Track Gain / Pan Node]
|
||||||
|
|
|
||||||
|
v
|
||||||
|
+---------------------+---------------------+
|
||||||
|
| |
|
||||||
|
v (If inside Section) v (If Direct Track)
|
||||||
|
[Section Sub-Mix Bus] [Main Master Mixer Bus]
|
||||||
|
| |
|
||||||
|
+-------------------->----------------------+
|
||||||
|
|
|
||||||
|
v
|
||||||
|
[Web Audio Destination]
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 4.2 Web Audio Node Architecture Specifications
|
||||||
|
|
||||||
|
* **AudioTrack Node Structure:**
|
||||||
|
```javascript
|
||||||
|
TrackAudioGraph = {
|
||||||
|
inputNode: GainNode,
|
||||||
|
fxChain: [ BiquadFilterNode, DelayNode, ConvolverNode ],
|
||||||
|
panNode: StereoPannerNode,
|
||||||
|
outputGainNode: GainNode,
|
||||||
|
connect(destination) { ... }
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
* **Section Bus Graph Routing:**
|
||||||
|
* Each Section in Section-tab Store instantiates an intermediate `GainNode` sub-mixer (`SectionBus`).
|
||||||
|
* Tracks within the Section connect their final outputs to `SectionBus`.
|
||||||
|
* When a `SectionItem` is placed on a Main Session track, the `SectionBus` output is routed into the Main Session track's input node, preserving non-destructive DSP processing hierarchies.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5. Core Mathematical & Technical Algorithms
|
||||||
|
|
||||||
|
#### 5.1 Algorithm 1: Non-Destructive Item Slicing & Offset Playback Math
|
||||||
|
|
||||||
|
##### Mathematical Formulation
|
||||||
|
|
||||||
|
Let:
|
||||||
|
|
||||||
|
* $T_{\text{global}}$ = Current global playback time in seconds on the main timeline.
|
||||||
|
* $\text{BPM}$ = Beats Per Minute of the project.
|
||||||
|
* $\text{TS}_{\text{num}}$ = Time Signature Numerator (e.g., 4 beats per bar).
|
||||||
|
* $S_{\text{item}}$ = Item start position in global bars ($\text{start\_bar}$).
|
||||||
|
* $L_{\text{item}}$ = Item visible length on timeline in bars ($\text{duration\_bars}$).
|
||||||
|
* $O_{\text{item}}$ = Source internal start offset in bars ($\text{clip\_start\_offset\_bars}$).
|
||||||
|
|
||||||
|
Bar to Time Conversion Factor:
|
||||||
|
|
||||||
|
$$\text{SecondsPerBeat} = \frac{60.0}{\text{BPM}}$$
|
||||||
|
|
||||||
|
$$\text{SecondsPerBar} = \text{SecondsPerBeat} \times \text{TS}_{\text{num}}$$
|
||||||
|
|
||||||
|
Item Global Time Bounds:
|
||||||
|
|
||||||
|
$$T_{\text{start}} = S_{\text{item}} \times \text{SecondsPerBar}$$
|
||||||
|
|
||||||
|
$$T_{\text{end}} = (S_{\text{item}} + L_{\text{item}}) \times \text{SecondsPerBar}$$
|
||||||
|
|
||||||
|
Active Playback Slicing Condition: An item is active if and only if:
|
||||||
|
|
||||||
|
$$T_{\text{start}} \le T_{\text{global}} < T_{\text{end}}$$
|
||||||
|
|
||||||
|
Local Item Buffer Time Mapping ($T_{\text{local}}$): When $T_{\text{global}}$ falls within $[T_{\text{start}}, T_{\text{end}}]$, the corresponding time $T_{\text{local\_bars}}$ relative to the internal source clip buffer (0 to $\text{BufferLength}$) is:
|
||||||
|
|
||||||
|
$$T_{\text{local\_bars}} = \frac{T_{\text{global}} - T_{\text{start}}}{\text{SecondsPerBar}} + O_{\text{item}}$$
|
||||||
|
|
||||||
|
MIDI Note Slicing & Filtering Rule: For a MIDI note $N$ inside the item source with start beat $N_{\text{start\_beat}}$ and length $N_{\text{dur\_beat}}$ (converted to internal bar metric $N_{\text{bar\_start}} = \frac{N_{\text{start\_beat}}}{\text{TS}_{\text{num}}}$, $N_{\text{bar\_dur}} = \frac{N_{\text{dur\_beat}}}{\text{TS}_{\text{num}}}$):
|
||||||
|
|
||||||
|
The note is triggered during main playback if and only if:
|
||||||
|
|
||||||
|
$$N_{\text{bar\_start}} \ge O_{\text{item}} \quad \text{AND} \quad N_{\text{bar\_start}} < (O_{\text{item}} + L_{\text{item}})$$
|
||||||
|
|
||||||
|
##### Pseudocode Implementation
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
function getActiveMIDINotesForPlayback(item, currentGlobalBar, timeSigNum) {
|
||||||
|
const itemStartBar = item.start_bar;
|
||||||
|
const itemEndBar = item.start_bar + item.duration_bars;
|
||||||
|
const offsetBar = item.clip_start_offset_bars;
|
||||||
|
|
||||||
|
// Check if playback cursor is inside visible item clip
|
||||||
|
if (currentGlobalBar < itemStartBar || currentGlobalBar >= itemEndBar) {
|
||||||
|
return []; // Item inactive
|
||||||
|
}
|
||||||
|
|
||||||
|
const activeNotes = [];
|
||||||
|
const internalWindowStartBar = offsetBar;
|
||||||
|
const internalWindowEndBar = offsetBar + item.duration_bars;
|
||||||
|
|
||||||
|
for (const note of item.source_data.notes) {
|
||||||
|
const noteStartBar = note.start_beat / timeSigNum;
|
||||||
|
const noteEndBar = noteStartBar + (note.duration_beats / timeSigNum);
|
||||||
|
|
||||||
|
// Filter notes outside the non-destructive visible window
|
||||||
|
if (noteStartBar >= internalWindowStartBar && noteStartBar < internalWindowEndBar) {
|
||||||
|
// Calculate playback time relative to global session
|
||||||
|
const relativeBarInItem = noteStartBar - internalWindowStartBar;
|
||||||
|
const targetGlobalBar = itemStartBar + relativeBarInItem;
|
||||||
|
|
||||||
|
activeNotes.push({
|
||||||
|
note: note,
|
||||||
|
scheduledGlobalBar: targetGlobalBar
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return activeNotes;
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 5.2 Algorithm 2: Dynamic Section Length Calculation Algorithm
|
||||||
|
|
||||||
|
When `auto_compute_length` is enabled for a Section, its total duration in bars $L_{\text{section}}$ is dynamically evaluated from the boundary bounds of all child items across all tracks inside that Section.
|
||||||
|
|
||||||
|
##### Mathematical Formulation
|
||||||
|
|
||||||
|
Let $T$ be the set of tracks in the section, and $I(t)$ be the set of items in track $t$.
|
||||||
|
|
||||||
|
$$L_{\text{section}} = \max_{t \in T} \left( \max_{i \in I(t)} \left( i.\text{start\_bar} + i.\text{duration\_bars} \right) \right)$$
|
||||||
|
|
||||||
|
If $I(t)$ is empty for all $t$, then $L_{\text{section}} = 4.0$ (default baseline minimum).
|
||||||
|
|
||||||
|
##### Implementation Architecture
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
function recomputeSectionLength(sectionContainer) {
|
||||||
|
if (!sectionContainer.auto_compute_length) {
|
||||||
|
return sectionContainer.length_bars;
|
||||||
|
}
|
||||||
|
|
||||||
|
let maxEndBar = 0.0;
|
||||||
|
|
||||||
|
for (const track of sectionContainer.tracks) {
|
||||||
|
for (const item of track.items) {
|
||||||
|
const itemEndBar = item.start_bar + item.duration_bars;
|
||||||
|
if (itemEndBar > maxEndBar) {
|
||||||
|
maxEndBar = itemEndBar;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enforce baseline grid quantization rounding (e.g. minimum 1 bar)
|
||||||
|
const computedLength = Math.max(1.0, Math.ceil(maxEndBar));
|
||||||
|
sectionContainer.length_bars = computedLength;
|
||||||
|
|
||||||
|
return computedLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 5.3 Algorithm 3: Piano Roll Grid Mapping & Quantization Math
|
||||||
|
|
||||||
|
##### Grid Coordinate Transformation Formulae
|
||||||
|
|
||||||
|
Let:
|
||||||
|
|
||||||
|
* $X_{\text{px}}$ = Pixel X-coordinate on Piano Roll Canvas.
|
||||||
|
* $Y_{\text{px}}$ = Pixel Y-coordinate on Piano Roll Canvas.
|
||||||
|
* $\text{Zoom}_x$ = Pixels per Beat.
|
||||||
|
* $\text{NoteHeight}$ = Height in pixels per pitch key row (e.g., 18px).
|
||||||
|
* $\text{Scroll}_x$ = Horizontal scroll offset in beats.
|
||||||
|
* $\text{Scroll}_y$ = Vertical scroll top note pitch (e.g., pitch 127 down to 0).
|
||||||
|
|
||||||
|
Beat to Canvas Pixel Conversion:
|
||||||
|
|
||||||
|
$$X_{\text{px}} = (\text{Beat} - \text{Scroll}_x) \times \text{Zoom}_x$$
|
||||||
|
|
||||||
|
$$\text{Beat} = \frac{X_{\text{px}}}{\text{Zoom}_x} + \text{Scroll}_x$$
|
||||||
|
|
||||||
|
Pitch to Canvas Pixel Conversion:
|
||||||
|
|
||||||
|
$$Y_{\text{px}} = (127 - \text{Pitch} - \text{Scroll}_y) \times \text{NoteHeight}$$
|
||||||
|
|
||||||
|
$$\text{Pitch} = 127 - \left\lfloor \frac{Y_{\text{px}}}{\text{NoteHeight}} \right\rfloor - \text{Scroll}_y$$
|
||||||
|
|
||||||
|
##### Quantization (Snap To Grid) Math
|
||||||
|
|
||||||
|
Let $Q$ be the snap unit in beats (e.g., $1/4 \text{ bar} = 1.0 \text{ beat}$, $1/16 \text{ note} = 0.25 \text{ beat}$). Given raw unquantized beat $B_{\text{raw}}$:
|
||||||
|
|
||||||
|
$$B_{\text{quantized}} = \text{round}\left(\frac{B_{\text{raw}}}{Q}\right) \times Q$$
|
||||||
|
|
||||||
|
#### 5.4 Algorithm 4: Tab Close Dependency & Lifecycle Validation Algorithm
|
||||||
|
|
||||||
|
This algorithm validates whether a tab close request can be fulfilled, enforcing the fixed Main Session constraint and preventing parent Section tab closures while child editor sub-tabs remain active.
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
function requestCloseTab(tabIdToClose, stateStore) {
|
||||||
|
const targetTab = stateStore.open_tabs.find(tab => tab.tab_id === tabIdToClose);
|
||||||
|
if (!targetTab) {
|
||||||
|
return { success: false, reason: "TAB_NOT_FOUND" };
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Rule: Main Session cannot be closed
|
||||||
|
if (!targetTab.is_closeable || targetTab.type === 'MAIN_SESSION') {
|
||||||
|
return { success: false, reason: "CANNOT_CLOSE_MAIN_SESSION" };
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Rule: Section Tab cannot be closed if child tabs are active
|
||||||
|
if (targetTab.type === 'SECTION_TAB') {
|
||||||
|
const activeChildTabs = stateStore.open_tabs.filter(
|
||||||
|
tab => tab.parent_tab_id === targetTab.tab_id
|
||||||
|
);
|
||||||
|
|
||||||
|
if (activeChildTabs.length > 0) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
reason: "SECTION_HAS_ACTIVE_CHILD_EDITORS",
|
||||||
|
activeChildTabs: activeChildTabs.map(t => ({ id: t.tab_id, title: t.title }))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Execution: Perform clean tab shutdown and update active context
|
||||||
|
const updatedTabs = stateStore.open_tabs.filter(tab => tab.tab_id !== tabIdToClose);
|
||||||
|
|
||||||
|
// Fallback active tab selection if current active tab is being closed
|
||||||
|
let nextActiveTabId = stateStore.active_tab_id;
|
||||||
|
if (stateStore.active_tab_id === tabIdToClose) {
|
||||||
|
// Fallback to parent tab, or default to main session (index 0)
|
||||||
|
nextActiveTabId = targetTab.parent_tab_id || updatedTabs[0].tab_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
stateStore.open_tabs = updatedTabs;
|
||||||
|
stateStore.active_tab_id = nextActiveTabId;
|
||||||
|
|
||||||
|
return { success: true, nextActiveTabId: nextActiveTabId };
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 5.5 Algorithm 5: Sample-Accurate Lookahead MIDI & Audio Scheduler
|
||||||
|
|
||||||
|
Web Audio API timing operates on a high-precision hardware audio clock (`audioContext.currentTime`). JavaScript timers (`setTimeout`/`setInterval`) lack frame accuracy. The Lookahead Scheduler combines JS interval ticks with Web Audio precision scheduling.
|
||||||
|
|
||||||
|
```text
|
||||||
|
Lookahead Window (e.g. 100ms)
|
||||||
|
|-------------------------------------------|
|
||||||
|
| AudioContext Time: 10.0s |
|
||||||
|
| Schedule horizon: 10.1s |
|
||||||
|
| |
|
||||||
|
| [Event 1 @ 10.02s] -> Scheduled in WebAudio
|
||||||
|
| [Event 2 @ 10.08s] -> Scheduled in WebAudio
|
||||||
|
|___________________________________________|
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Scheduler Specification
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
class PrecisionAudioScheduler {
|
||||||
|
constructor(audioCtx, lookaheadMs = 25.0, scheduleAheadTimeSec = 0.1) {
|
||||||
|
this.audioCtx = audioCtx;
|
||||||
|
this.lookaheadMs = lookaheadMs; // Frequency of timer evaluation
|
||||||
|
this.scheduleAheadTime = scheduleAheadTimeSec; // How far ahead to queue WebAudio events
|
||||||
|
this.nextNoteBeat = 0.0;
|
||||||
|
this.currentBeat = 0.0;
|
||||||
|
this.bpm = 120.0;
|
||||||
|
this.timerId = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
beatToTime(beat) {
|
||||||
|
const secondsPerBeat = 60.0 / this.bpm;
|
||||||
|
return beat * secondsPerBeat;
|
||||||
|
}
|
||||||
|
|
||||||
|
timeToBeat(timeSec) {
|
||||||
|
const secondsPerBeat = 60.0 / this.bpm;
|
||||||
|
return timeSec / secondsPerBeat;
|
||||||
|
}
|
||||||
|
|
||||||
|
schedulerTick(activeSession) {
|
||||||
|
const currentTime = this.audioCtx.currentTime;
|
||||||
|
const horizonTime = currentTime + this.scheduleAheadTime;
|
||||||
|
|
||||||
|
// Traverse session items and find notes falling within [currentTime, horizonTime]
|
||||||
|
const pendingEvents = activeSession.getEventsInTimeRange(
|
||||||
|
this.timeToBeat(currentTime),
|
||||||
|
this.timeToBeat(horizonTime)
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const evt of pendingEvents) {
|
||||||
|
if (!evt.scheduled) {
|
||||||
|
const preciseAudioTime = currentTime + this.beatToTime(evt.targetBeat - this.currentBeat);
|
||||||
|
this.triggerWebAudioEvent(evt, preciseAudioTime);
|
||||||
|
evt.scheduled = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
triggerWebAudioEvent(evt, exactAudioTime) {
|
||||||
|
if (evt.type === 'MIDI_NOTE_ON') {
|
||||||
|
const synthNode = evt.trackSynthNode;
|
||||||
|
synthNode.noteOn(evt.note.pitch, evt.note.velocity, exactAudioTime);
|
||||||
|
synthNode.noteOff(evt.note.pitch, exactAudioTime + this.beatToTime(evt.note.duration_beats));
|
||||||
|
} else if (evt.type === 'AUDIO_CLIP') {
|
||||||
|
const sourceNode = this.audioCtx.createBufferSource();
|
||||||
|
sourceNode.buffer = evt.audioBuffer;
|
||||||
|
sourceNode.connect(evt.trackGainNode);
|
||||||
|
sourceNode.start(exactAudioTime, evt.offsetSec, evt.durationSec);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
start(session) {
|
||||||
|
this.timerId = setInterval(() => this.schedulerTick(session), this.lookaheadMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
stop() {
|
||||||
|
if (this.timerId) clearInterval(this.timerId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 5.6 Algorithm 6: Playhead UI Rendering Sync Loop
|
||||||
|
|
||||||
|
UI Playhead rendering uses `requestAnimationFrame` and queries `audioContext.currentTime` directly to prevent visual jitter or lag.
|
||||||
|
|
||||||
|
$$\text{Current Beat UI} = \frac{\text{audioCtx.currentTime} - \text{PlaybackStartTimeSec}}{\text{SecondsPerBeat}}$$
|
||||||
|
|
||||||
|
$$\text{Pixel Position X} = (\text{Current Beat UI} - \text{ViewportStartBeat}) \times \text{Zoom}_x$$
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6. Backend Python Server Architecture & Offline Render Spec
|
||||||
|
|
||||||
|
#### 6.1 Server Architecture Framework
|
||||||
|
|
||||||
|
* **Framework:** FastAPI with Async WebSocket endpoints for real-time state synchronization.
|
||||||
|
* **DSP Engine:** `pedalboard` (Spotify's Python Audio Processing Library) and `numpy` for multi-track mixing, high-quality audio resampling, and plugin hosting.
|
||||||
|
|
||||||
|
#### 6.2 Python Offline Stem Bouncing Engine Specification (`render_engine.py`)
|
||||||
|
|
||||||
|
```python
|
||||||
|
import numpy as np
|
||||||
|
from pedalboard import Pedalboard, Gain, Reverb, Compressor
|
||||||
|
import soundfile as sf
|
||||||
|
|
||||||
|
class PythonRenderEngine:
|
||||||
|
def __init__(self, sample_rate=44100):
|
||||||
|
self.sample_rate = sample_rate
|
||||||
|
|
||||||
|
def bars_to_samples(self, bars: float, bpm: float, time_sig_num: int) -> int:
|
||||||
|
seconds_per_beat = 60.0 / bpm
|
||||||
|
seconds_per_bar = seconds_per_beat * time_sig_num
|
||||||
|
return int(bars * seconds_per_bar * self.sample_rate)
|
||||||
|
|
||||||
|
def render_project(self, project_json: dict, output_filepath: str):
|
||||||
|
bpm = project_json["metadata"]["bpm"]
|
||||||
|
time_sig_num = project_json["metadata"]["time_signature_numerator"]
|
||||||
|
main_session = project_json["main_session"]
|
||||||
|
|
||||||
|
# 1. Compute total project samples
|
||||||
|
total_bars = main_session.get("length_bars", 16.0)
|
||||||
|
total_samples = self.bars_to_samples(total_bars, bpm, time_sig_num)
|
||||||
|
|
||||||
|
# Stereo Master Buffer
|
||||||
|
master_buffer = np.zeros((2, total_samples), dtype=np.float32)
|
||||||
|
|
||||||
|
# 2. Iterate and process main tracks
|
||||||
|
for track in main_session["tracks"]:
|
||||||
|
track_type = track["type"]
|
||||||
|
track_buffer = np.zeros((2, total_samples), dtype=np.float32)
|
||||||
|
|
||||||
|
for item in track["items"]:
|
||||||
|
start_sample = self.bars_to_samples(item["start_bar"], bpm, time_sig_num)
|
||||||
|
dur_samples = self.bars_to_samples(item["duration_bars"], bpm, time_sig_num)
|
||||||
|
offset_sample = self.bars_to_samples(item["clip_start_offset_bars"], bpm, time_sig_num)
|
||||||
|
|
||||||
|
if item["type"] == "AUDIO_ITEM":
|
||||||
|
# Load audio source sample array
|
||||||
|
audio_data, sr = sf.read(item["source_data"]["audio_file_url"], dtype='float32')
|
||||||
|
audio_data = audio_data.T # Shape: (channels, samples)
|
||||||
|
|
||||||
|
# Apply non-destructive trimming offset
|
||||||
|
sliced_audio = audio_data[:, offset_sample : offset_sample + dur_samples]
|
||||||
|
|
||||||
|
# Accumulate into track buffer with bounds checks
|
||||||
|
end_sample = min(start_sample + sliced_audio.shape[1], total_samples)
|
||||||
|
actual_len = end_sample - start_sample
|
||||||
|
track_buffer[:, start_sample:end_sample] += sliced_audio[:, :actual_len]
|
||||||
|
|
||||||
|
# Apply Track Gain and FX Chain via Pedalboard
|
||||||
|
board = Pedalboard([Gain(gain_db=track.get("volume_db", 0.0))])
|
||||||
|
processed_track = board(track_buffer, sample_rate=self.sample_rate)
|
||||||
|
|
||||||
|
# Mix down to Master
|
||||||
|
master_buffer += processed_track
|
||||||
|
|
||||||
|
# 3. Write final output file
|
||||||
|
sf.write(output_filepath, master_buffer.T, self.sample_rate)
|
||||||
|
return output_filepath
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 7. Execution Context & Sub-Tab Lifecycle Matrix
|
||||||
|
|
||||||
|
| Context Tab Type | Scope Identifier | View Boundaries | Is Closeable | Close Dependency Conditions | Audio Routing Target |
|
||||||
|
| --- | --- | --- | --- | --- | --- |
|
||||||
|
| **MAIN SESSION** | Root | Full Master Timeline ($0 \to N$ Bars) | No | Pinned permanently; cannot be closed | WebAudio Hardware Destination |
|
||||||
|
| **SECTION TAB** | Section_ID | Dynamic Section Bounds ($0 \to L_{\text{section}}$) | Yes | Blocked if any child editor sub-tabs are open | Target Section Bus Gain Node |
|
||||||
|
| **PIANO ROLL** | MIDIItem_ID | Item Source Length Bounds ($0 \to N_{\text{buffer}}$) | Yes | Can close freely; notifies parent Section tab | Track Instrument Synth Engine |
|
||||||
|
| **SAMPLE EDITOR** | AudioItem_ID | Sample Buffer Waveform ($0 \to T_{\text{sample}}$) | Yes | Can close freely; notifies parent Section tab | Track Audio Node Router |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 8. Summary of Non-Destructive Slice & Tab Lifecycle Validation
|
||||||
|
|
||||||
|
* **Tab Close Prevention Test:**
|
||||||
|
1. `MAIN SESSION` close request is rejected immediately (`CANNOT_CLOSE_MAIN_SESSION`).
|
||||||
|
2. `Section_01` tab has an active child editor tab (`Piano Roll: Bassline`).
|
||||||
|
3. Request to close `Section_01` tab returns `SECTION_HAS_ACTIVE_CHILD_EDITORS`.
|
||||||
|
4. User closes `Piano Roll: Bassline` tab first.
|
||||||
|
5. Subsequent close request for `Section_01` succeeds and cleans up UI context.
|
||||||
|
|
||||||
|
|
||||||
|
* **8-Bar Source with 2-Bar Visible Crop Test:**
|
||||||
|
1. Given `MIDIItem` length = 8 bars ($0 \dots 8$).
|
||||||
|
2. User drags left boundary to Bar 4 and right boundary to Bar 6.
|
||||||
|
3. `start_bar = 4.0` (Global Session Placement), `duration_bars = 2.0`, `clip_start_offset_bars = 4.0`.
|
||||||
|
4. Transport reaches global Bar 4.0 $\to$ scheduler evaluates internal bounds $[4.0, 6.0)$ and triggers only visible notes while preserving complete 8-bar non-destructive source.
|
||||||
@@ -1,361 +0,0 @@
|
|||||||
Here is the conversion of the document into a professional English Markdown format:
|
|
||||||
|
|
||||||
# ARCHITECTURAL, TECHNICAL, AND ALGORITHMIC SPECIFICATION
|
|
||||||
|
|
||||||
## Sub-Session System, Section Arrangement & Piano Roll Tab (Hybrid DAW)
|
|
||||||
|
|
||||||
This document details the technical solution for building a Hierarchical DAW Engine. This architecture enables nesting Sub-Sessions (Sections) inside the Main Session, alongside a Sub-Tab Editor system (including Piano Roll and Audio Sample Editor) to precisely edit MIDI and Audio Items.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 0. Non-Breaking Modular Principles (Integration & Backward Compatibility)
|
|
||||||
|
|
||||||
To guarantee that new features do not disrupt the DAW's existing core logic and codebase, the entire extension architecture is designed according to these principles:
|
|
||||||
|
|
||||||
* **Extensibility & Encapsulation:**
|
|
||||||
* The current Session architecture serves directly as the **Project Root / Main Session**.
|
|
||||||
* `SectionItem`, `ItemMIDI`, and `ItemAudio` operate as **Polymorphic Item Types** inheriting from the existing base `Item` class/interface. Existing Item logic (e.g., drag-and-drop, timeline trimming) remains $100\%$ untouched.
|
|
||||||
|
|
||||||
|
|
||||||
* **Decoupled State Pipeline:**
|
|
||||||
* The logic governing the Playhead, Transport controls (Play/Pause/Stop), and the global Audio Context of the Main Session remains unmodified.
|
|
||||||
* **Nested Time Mapping** acts solely as an intermediate Transformation Layer when passing time coordinates down into Sub-Sessions. It does not overwrite or mutate the beat synchronization loop of the Main Timeline.
|
|
||||||
|
|
||||||
|
|
||||||
* **Plugin Style Architecture (Audio & MIDI Engine):**
|
|
||||||
* Synth Tracks, Audio Clip Processors, and Sub-Session Sub-Mix Buses plug into the existing AudioNode Graph as auxiliary nodes. They route directly back to the current Master Node without breaking pre-established Gain/Pan/FX pipelines.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 1. Hierarchical Data Model
|
|
||||||
|
|
||||||
To support embedding Sessions within Sessions as well as isolated Clip/Sample-level editing, the data state model expands into an encapsulated Tree Graph structure.
|
|
||||||
|
|
||||||
```text
|
|
||||||
Project Root
|
|
||||||
├── Main Session (Root Session - Current Session Structure)
|
|
||||||
│ ├── Track 01 (Audio Track)
|
|
||||||
│ │ └── ItemAudio: "Vocals.wav" ──► [Opens Audio Sample Editor Sub-Tab]
|
|
||||||
│ ├── Track 02 (MIDI Track + Synth Engine)
|
|
||||||
│ │ └── ItemMIDI: "Melody_Main" ──► [Opens Piano Roll Sub-Tab]
|
|
||||||
│ └── Track 03 (Section Track - New Track Type)
|
|
||||||
│ └── Item: Section_A (Referencing SubSession_01)
|
|
||||||
│
|
|
||||||
├── Sub-Sessions Store (Auxiliary Memory Registry)
|
|
||||||
│ ├── SubSession_01 ("Verse 1")
|
|
||||||
│ │ ├── Computed Length: Dynamic Bars (Auto-calculated from longest Item)
|
|
||||||
│ │ ├── Track 1.1 (Audio Track)
|
|
||||||
│ │ │ └── ItemAudio: "Guitar_Riff.wav" ──► [Opens Audio Sample Editor Sub-Tab]
|
|
||||||
│ │ └── Track 1.2 (MIDI Track)
|
|
||||||
│ │ └── ItemMIDI: "Bassline" ─────────► [Opens Piano Roll Sub-Tab]
|
|
||||||
│ └── SubSession_02 ("Chorus")
|
|
||||||
│
|
|
||||||
└── Active Editor Views / Sub-Tabs (Isolated Editing Contexts)
|
|
||||||
├── Audio Sample Editor Sub-Tab (Edits Audio Clips from Main Session or Sub-Session)
|
|
||||||
└── Piano Roll Sub-Tab (Edits MIDI Items from Main Session or Sub-Session)
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Detailed Data Schemas (JSON Specs)
|
|
||||||
|
|
||||||
**a. Schema: `NoteMIDI**`
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface NoteMIDI {
|
|
||||||
id: string;
|
|
||||||
pitch: number; // 0 - 127 (Midi Note Number, e.g., 60 = C4)
|
|
||||||
startTick: number; // Time coordinate based on Pulses Per Quarter note (PPQ, e.g., 960 PPQ)
|
|
||||||
durationTicks: number;
|
|
||||||
velocity: number; // 0 - 127
|
|
||||||
selected?: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
**b. Schema: `ItemMIDI` (Belongs to MIDI Track - Inherits from Base Item)**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface ItemMIDI {
|
|
||||||
id: string;
|
|
||||||
type: 'MIDI';
|
|
||||||
name: string;
|
|
||||||
parentSessionId: string; // Target Session ID (Main or Sub-Session)
|
|
||||||
startBar: number; // Start position on the Timeline (Bar)
|
|
||||||
lengthBars: number; // Item duration in Bars
|
|
||||||
offsetTick: number; // Internal trim offset
|
|
||||||
notes: NoteMIDI[]; // Array tracking MIDI Notes
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
**c. Schema: `ItemAudio` (Belongs to Audio Track - Inherits from Base Item)**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface ItemAudio {
|
|
||||||
id: string;
|
|
||||||
type: 'AUDIO';
|
|
||||||
name: string;
|
|
||||||
parentSessionId: string; // Target Session ID (Main or Sub-Session)
|
|
||||||
startBar: number;
|
|
||||||
lengthBars: number;
|
|
||||||
samplePath: string; // Audio file path or Buffer Key
|
|
||||||
sampleOffsetSec: number; // Playback start point offset (Trim In)
|
|
||||||
gain: number; // Clip Gain
|
|
||||||
pitchShiftSemi: number; // Pitch Shift (Semitones)
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
**d. Schema: `SectionItem` (Represents a Sub-Session inside the Main Session)**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface SectionItem {
|
|
||||||
id: string;
|
|
||||||
type: 'SECTION';
|
|
||||||
subSessionId: string; // Reference ID pointing to SubSession inside Memory Store
|
|
||||||
name: string;
|
|
||||||
startBar: number;
|
|
||||||
lengthBars: number; // Defaults to SubSession.computedLengthBars unless trimmed/cropped
|
|
||||||
loop: boolean; // Enables repetition if lengthBars > SubSession.computedLengthBars
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
**e. Schema: `Session` (Unified structure for both Main Session and Sub-Session)**
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface Session {
|
|
||||||
id: string;
|
|
||||||
name: string;
|
|
||||||
isMain: boolean;
|
|
||||||
timeSignature: [number, number]; // e.g., [4, 4]
|
|
||||||
bpm: number;
|
|
||||||
tracks: Track[];
|
|
||||||
|
|
||||||
// Dynamically calculated derived state; never assigned manually
|
|
||||||
get computedLengthBars(): number;
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 2. Audio & Synth Engine Routing Architecture (Web Audio API)
|
|
||||||
|
|
||||||
For MIDI tracks to output audio, each is bound to an Instrument/Synth Instance. When a Section is placed onto the Main Session, all audio generated by its child tracks is bussed directly into the existing Gain/Pan matrix.
|
|
||||||
|
|
||||||
#### Audio Node Graph Diagram
|
|
||||||
|
|
||||||
```text
|
|
||||||
[MIDI Items] ──(Triggers)──► [Synth Engine / Soundfont / WebAssembly VSTi]
|
|
||||||
│
|
|
||||||
[Audio Items] ──(Buffer Source)──────────┤
|
|
||||||
▼
|
|
||||||
[Track Gain / Pan Node]
|
|
||||||
│
|
|
||||||
▼
|
|
||||||
[Sub-Session Sub-Mix Bus Node]
|
|
||||||
│
|
|
||||||
┌──────────────────────┴──────────────────────┐
|
|
||||||
▼ ▼
|
|
||||||
[Main Session Audio Graph] [Solo / Mute Logic]
|
|
||||||
(Current Audio Processing Logic)
|
|
||||||
│
|
|
||||||
▼
|
|
||||||
[Master Destination]
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
**Instrument Engine Processing Logic for MIDI Tracks:**
|
|
||||||
|
|
||||||
* **Virtual Instrument Binding:** Every MIDI Track instantiates a synthesis `AudioNode` (e.g., Web Audio API Soundfont Player, WebSynth JS, or WASM Synthesizer).
|
|
||||||
* **Dynamic Polyphony Engine:** As playback scans across MIDI Notes, the system triggers `noteOn(pitch, velocity, time)` and `noteOff(pitch, time)` events. These are scheduled ahead of time ($100\text{ms} - 200\text{ms}$ Lookahead) via the `AudioContext.currentTime` clock.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 3. Tab UI Management & Event Processing Flow (Tab Navigation Stack)
|
|
||||||
|
|
||||||
The graphical interface expands on a Tab Manager & Navigation Stack model to handle isolated views (Views/Sub-tabs) for specific data entities.
|
|
||||||
|
|
||||||
```text
|
|
||||||
[ Tabs Bar ] ── [ Main Session ] │ [ Sub-Session: Verse 1 ] │ [ Piano Roll: Bassline ] │ [ Sample Edit: Vocals.wav ]
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Interaction & Navigation Mechanics:
|
|
||||||
|
|
||||||
* **Opening a Sub-Session Tab:**
|
|
||||||
* *Action:* User double-clicks a `SectionItem` on a Main Track.
|
|
||||||
* *Result:*
|
|
||||||
* Instantiates a new Tab using `ID = SubSession.id`.
|
|
||||||
* Maps the Timeline Viewport rendering context to the SubSession.
|
|
||||||
* Enables adding, editing, or deleting child tracks (Audio & MIDI) within the Sub-Session boundary.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
* **Opening the Piano Roll Sub-Tab:**
|
|
||||||
* *Action:* User double-clicks an `ItemMIDI` inside the Main Session OR a Sub-Session.
|
|
||||||
* *Result:*
|
|
||||||
* Instantiates a Sub-tab labeled: `Piano Roll - [Item Name]`.
|
|
||||||
* Caches context references: `{ itemId, parentSessionId }`.
|
|
||||||
* Passes the `ItemMIDI.notes` array directly into the Canvas/Piano Roll Grid.
|
|
||||||
* Any add/edit/delete actions executed on notes inside the Piano Roll instantly update the native `ItemMIDI` in the target Session via Mutable/Immutable References.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
* **Opening the Audio Sample Editor Sub-Tab (Session Edit Audio Sample):**
|
|
||||||
* *Action:* User double-clicks OR right-clicks and selects "Edit" on an `ItemAudio` inside the Main Session or a Sub-Session.
|
|
||||||
* *Result:*
|
|
||||||
* Instantiates a Sub-tab labeled: `Audio Editor - [Clip Name]`.
|
|
||||||
* Loads the high-resolution Waveform of the target `ItemAudio` onto the sample editing Viewport.
|
|
||||||
* Provides access to tools: Trim start/end, Normalized Peak, Pitch Shift, Reverse, Fade In/Out, or DSP slicing.
|
|
||||||
* When clicking *Save / Apply Changes*: The system updates the `ItemAudio` attributes (or dispatches a DSP processing request to the Python Server for heavy tasks) and forces a visual refresh of the Clip on the Main Session / Sub-Session timeline.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### Data Persistence & Dynamic Sub-Session Length Updates:
|
|
||||||
|
|
||||||
* Because JavaScript handles array/object data passing by **Reference**, modifications made to Notes in the Piano Roll Tab or Clips in the Audio Editor directly update the origin State of the corresponding Session.
|
|
||||||
* Any add/remove/move/stretch operation targeting an Item inside a Sub-session will immediately trigger the **Dynamic Length Recalculation** algorithm to update the temporal boundary of the Sub-Session.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 4. Core Algorithms
|
|
||||||
|
|
||||||
#### Algorithm 1: Dynamic Sub-Session Length Calculation
|
|
||||||
|
|
||||||
Sub-sessions do not enforce rigid length constraints. Instead, they dynamically map their duration ($L_{\text{bars}}$) to match the furthest end-point of all encapsulated Items.
|
|
||||||
|
|
||||||
**Formula:**
|
|
||||||
Given a Sub-Session containing a list of $T$ tracks, where each track $t$ holds a list of $I_t$ items (Audio, MIDI, etc.):
|
|
||||||
|
|
||||||
|
|
||||||
$$\text{ItemEndBar}(item) = item.\text{startBar} + item.\text{lengthBars}$$
|
|
||||||
|
|
||||||
$$L_{\text{bars}} = \max_{t \in T} \left( \max_{i \in I_t} (\text{ItemEndBar}(i)) \right)$$
|
|
||||||
|
|
||||||
*If the Sub-session is entirely empty (contains no Items), $L_{\text{bars}}$ defaults to $1$ Bar (or the default duration of a single grid bar).*
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
function calculateSubSessionLength(subSession) {
|
|
||||||
let maxEndBar = 1; // Minimum duration fallback for empty sub-sessions
|
|
||||||
|
|
||||||
for (const track of subSession.tracks) {
|
|
||||||
for (const item of track.items) {
|
|
||||||
const itemEndBar = item.startBar + item.lengthBars;
|
|
||||||
if (itemEndBar > maxEndBar) {
|
|
||||||
maxEndBar = itemEndBar;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return maxEndBar;
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Algorithm 2: Nested Time Mapping
|
|
||||||
|
|
||||||
When the Main Session Playhead tracks time $T_{\text{main}}$ (seconds), the engine must calculate the relative time coordinate $T_{\text{sub}}$ inside the active Sub-Session.
|
|
||||||
|
|
||||||
**Formula:**
|
|
||||||
Assume:
|
|
||||||
|
|
||||||
* $S_{\text{bar}}$: The starting Bar of the Section Item on the Main Timeline.
|
|
||||||
* $L_{\text{bars}}$: The dynamically evaluated length of the root Sub-Session ($L_{\text{bars}} = \text{calculateSubSessionLength}(\text{SubSession})$).
|
|
||||||
* $BPM$: Beats Per Minute.
|
|
||||||
* $TimeSig$: Beats per Bar (e.g., 4 beats).
|
|
||||||
|
|
||||||
$$\text{SecondsPerBar} = \frac{60}{\text{BPM}} \times \text{TimeSig}$$
|
|
||||||
|
|
||||||
$$\text{OffsetSeconds} = (T_{\text{main}} - (S_{\text{bar}} - 1) \times \text{SecondsPerBar})$$
|
|
||||||
|
|
||||||
If `SectionItem.loop = true`:
|
|
||||||
|
|
||||||
|
|
||||||
$$T_{\text{sub}} = \text{OffsetSeconds} \pmod{L_{\text{bars}} \times \text{SecondsPerBar}}$$
|
|
||||||
|
|
||||||
If `SectionItem.loop = false`:
|
|
||||||
|
|
||||||
|
|
||||||
$$T_{\text{sub}} = \begin{cases} \text{OffsetSeconds} & \text{if } 0 \le \text{OffsetSeconds} \le (L_{\text{bars}} \times \text{SecondsPerBar}) \\ \text{undefined} & \text{if out of bounds} \end{cases}$$
|
|
||||||
|
|
||||||
#### Algorithm 3: Lookahead MIDI Scheduler
|
|
||||||
|
|
||||||
JavaScript's `setInterval` function lacks the temporal precision required for audio playback. We employ the **Web Audio Lookahead Scheduler** algorithm combined with Ticks $\rightarrow$ Seconds translation.
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
const PPQ = 960; // 960 Pulses Per Quarter note (Standard MIDI resolution)
|
|
||||||
let nextNoteIndex = 0;
|
|
||||||
const scheduleAheadTime = 0.2; // 200ms Lookahead buffer
|
|
||||||
const lookaheadMs = 25; // Polling interval interval block (25ms)
|
|
||||||
|
|
||||||
function ticksToSeconds(ticks, bpm) {
|
|
||||||
const secondsPerQuarterNote = 60.0 / bpm;
|
|
||||||
return (ticks / PPQ) * secondsPerQuarterNote;
|
|
||||||
}
|
|
||||||
|
|
||||||
function scheduler(midiItem, audioCtx, currentPlayheadTime) {
|
|
||||||
// Extract notes mapped within [currentPlayheadTime, currentPlayheadTime + scheduleAheadTime]
|
|
||||||
while (nextNoteIndex < midiItem.notes.length) {
|
|
||||||
const note = midiItem.notes[nextNoteIndex];
|
|
||||||
const noteStartTimeSec = ticksToSeconds(note.startTick, currentBpm);
|
|
||||||
|
|
||||||
if (noteStartTimeSec >= currentPlayheadTime + scheduleAheadTime) {
|
|
||||||
break; // Note start bounds exceed the active Lookahead window
|
|
||||||
}
|
|
||||||
|
|
||||||
if (noteStartTimeSec >= currentPlayheadTime) {
|
|
||||||
// Calculate absolute scheduling time against the AudioContext Clock
|
|
||||||
const audioCtxStartTime = audioCtx.currentTime + (noteStartTimeSec - currentPlayheadTime);
|
|
||||||
const durationSec = ticksToSeconds(note.durationTicks, currentBpm);
|
|
||||||
|
|
||||||
// Fire the VSTi/Synth Engine
|
|
||||||
trackSynthEngine.playNote(note.pitch, note.velocity, audioCtxStartTime, durationSec);
|
|
||||||
}
|
|
||||||
nextNoteIndex++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Algorithm 4: Grid Snapping & Quantization (Piano Roll)
|
|
||||||
|
|
||||||
When adding or dragging a MIDI Note in the Piano Roll Tab, the $X$ coordinate of the mouse cursor must snap to the nearest rhythmic grid boundary (1/4, 1/8, 1/16, 1/32 Note).
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
function snapTickToGrid(rawTick, gridFraction, ppq) {
|
|
||||||
// gridFraction: 0.25 (1/4 note), 0.125 (1/8 note), 0.0625 (1/16 note)
|
|
||||||
const ticksPerGridStep = ppq * (gridFraction * 4);
|
|
||||||
|
|
||||||
// Snap rounding formula targeting the nearest grid boundary
|
|
||||||
const snappedTick = Math.round(rawTick / ticksPerGridStep) * ticksPerGridStep;
|
|
||||||
return Math.max(0, snappedTick);
|
|
||||||
}
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 5. Performance Optimization
|
|
||||||
|
|
||||||
* **Virtual Rendering for Piano Roll, Audio Sample Editor & Main Session:**
|
|
||||||
* Never render the entire array of MIDI Notes or total Audio Waveforms simultaneously into the HTML DOM.
|
|
||||||
* Mandatory use of HTML5 Canvas 2D / WebGL paired with **Virtual Viewport Rendering** (only drawing Notes/Samples situated within the active Viewport Rect boundary).
|
|
||||||
|
|
||||||
|
|
||||||
* **Audio Bouncing / Freezing (For Heavy Sections):**
|
|
||||||
* If a Sub-Session houses too many Tracks and VSTi plugins, causing CPU bottlenecks during Main Session playback:
|
|
||||||
* Enable the **"Freeze Section"** action: The Python backend processes the request, rendering that entire Sub-Session block into a single temporary Audio WAV file (Bounce to Disk).
|
|
||||||
* The Main Session then only processes one discrete Audio file instead of simultaneously calculating dozens of child tracks.
|
|
||||||
|
|
||||||
|
|
||||||
* **Immutable State & Undo/Redo Engine:**
|
|
||||||
* Project State management is handled via the Redux/Zustand pattern model.
|
|
||||||
* Every add/edit/delete operation applied to Notes on the Piano Roll or edits made to Audio Clips generates an Action that pushes to the `UndoStack`, supporting seamless `Ctrl + Z` shortcuts across every Sub-tab context.
|
|
||||||
@@ -0,0 +1,426 @@
|
|||||||
|
# TECHNICAL SPECIFICATION: CLIENT-SIDE REAL-TIME RECORDING ENGINE
|
||||||
|
|
||||||
|
## Browser-Based Microphone & Hardware MIDI Keyboard Recording Module
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. System Overview
|
||||||
|
|
||||||
|
The Client-side recording module enables the DAW to capture live audio signals directly from Microphone/Line-in interfaces (via the Web MediaDevices API) and keypress events from Hardware MIDI Keyboards/Controllers (via the Web MIDI API) in real time. The module operates with low latency and includes hardware latency compensation.
|
||||||
|
|
||||||
|
```text
|
||||||
|
+-----------------------------------------------------------------------------------+
|
||||||
|
| CLIENT BROWSER |
|
||||||
|
| |
|
||||||
|
| +-------------------------+ +-----------------------------+ |
|
||||||
|
| | Hardware MIDI Keyboard | | Live Microphone / Line-In | |
|
||||||
|
| +------------+------------+ +--------------+--------------+ |
|
||||||
|
| | | |
|
||||||
|
| v (Web MIDI API) v (getUserMedia) |
|
||||||
|
| +------------+------------+ +--------------+--------------+ |
|
||||||
|
| | Web MIDI Input Handler | | MediaStreamAudioSourceNode | |
|
||||||
|
| +------------+------------+ +--------------+--------------+ |
|
||||||
|
| | | |
|
||||||
|
| +------------------+ | |
|
||||||
|
| | | v |
|
||||||
|
| v v +--------------+--------------+ |
|
||||||
|
| +------------+-----+ +---------+-----------+ | Track Input Gain Node | |
|
||||||
|
| | Event Clock / | | WebAudio Virtual | +--------------+--------------+ |
|
||||||
|
| | Latency Engine | | Synth Engine | | |
|
||||||
|
| +------------+-----+ +---------+-----------+ +--------+--------+ |
|
||||||
|
| | | | | |
|
||||||
|
| v v v v |
|
||||||
|
| +------------+-----+ (Live Sound) +-------+-------+ +-------+------+ |
|
||||||
|
| | Recorded MIDI | | AudioWorklet | | Monitoring | |
|
||||||
|
| | Buffer | | Ring-Buffer | | Switch | |
|
||||||
|
| +------------+-----+ | Recorder | +-------+------+ |
|
||||||
|
| | +-------+-------+ | |
|
||||||
|
| v | v |
|
||||||
|
| [ Timeline MIDI ] v [ Master Mix ] |
|
||||||
|
| [ Item Creation ] +-------+-------+ |
|
||||||
|
| | Float32 PCM | |
|
||||||
|
| | Audio Buffer | |
|
||||||
|
| +-------+-------+ |
|
||||||
|
| | |
|
||||||
|
| v |
|
||||||
|
| [ Timeline Audio] |
|
||||||
|
| [ Item Creation ] |
|
||||||
|
+-----------------------------------------------------------------------------------+
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Hardware I/O & API Contracts
|
||||||
|
|
||||||
|
### 2.1 MediaDevices (Microphone Capture)
|
||||||
|
|
||||||
|
**Permission Request:** Uses `navigator.mediaDevices.getUserMedia` configured to disable automatic browser processing DSP algorithms to capture pure, unprocessed audio signals:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
const audioConstraints = {
|
||||||
|
audio: {
|
||||||
|
deviceId: selectedDeviceId ? { exact: selectedDeviceId } : undefined,
|
||||||
|
echoCancellation: false, // Disables echo cancellation to prevent instrument sound distortion
|
||||||
|
noiseSuppression: false, // Disables automatic noise suppression to preserve full frequency range
|
||||||
|
autoGainControl: false, // Disables Automatic Gain Control (AGC)
|
||||||
|
latency: 0 // Requests minimal latency from OS audio driver
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 Web MIDI API Integration
|
||||||
|
|
||||||
|
**Device Enumeration & Listener Assignment:**
|
||||||
|
|
||||||
|
* Uses `navigator.requestMIDIAccess({ sysex: false })` to scan for USB-connected keyboard devices.
|
||||||
|
* **Timestamp Precision:** Obtains event timestamps from `MIDIMessageEvent.timeStamp` (as a `DOMHighResTimeStamp` in microseconds) and synchronizes them with `AudioContext.currentTime`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Recording Lifecycle & State Machine
|
||||||
|
|
||||||
|
```text
|
||||||
|
[IDLE] ───► (User Arms Track) ───► [ARMED] ───► (Press Rec + Play) ───► [COUNT-IN / PRE-ROLL]
|
||||||
|
|
|
||||||
|
[STOP & COMMIT] ◄─── (Press Stop) ◄─── [RECORDING IN PROGRESS] ◄───────────────+
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
* **Arming Phase (Record Enable):**
|
||||||
|
* The user selects an input source and activates the Arm (R) button on the target track.
|
||||||
|
* Initializes the input level meter (VU Meter Canvas) to display input volume levels in real time.
|
||||||
|
|
||||||
|
|
||||||
|
* **Pre-Roll / Count-In Phase:**
|
||||||
|
* Transport triggers the metronome count-in (e.g., 1 Bar = 4 beats). The metronome plays click sounds based on project BPM.
|
||||||
|
* The engine does not write data to the Timeline yet, but begins reading the input buffer to prepare memory buffers.
|
||||||
|
|
||||||
|
|
||||||
|
* **Recording Phase:**
|
||||||
|
* Once the transport passes the Start Bar boundary, incoming MIDI key events or PCM Float32 audio samples are written into the active recording buffer memory.
|
||||||
|
* Canvas UI displays real-time visual feedback, rendering waveforms or MIDI note blocks dynamically.
|
||||||
|
|
||||||
|
|
||||||
|
* **Stop & Commit Phase:**
|
||||||
|
* Pressing Stop halts the recording process.
|
||||||
|
* Converts temporary memory buffers into a structured `MIDIItem` or `AudioItem`.
|
||||||
|
* Inserts the new Item onto the target track within the Main Session or Section tab.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Data Structures
|
||||||
|
|
||||||
|
### 4.1 Live MIDI Event Buffer Element Schema
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"pitch": { "type": "integer", "minimum": 0, "maximum": 127 },
|
||||||
|
"start_beat": { "type": "number", "description": "Start position in beats on the timeline" },
|
||||||
|
"duration_beats": { "type": "number", "description": "Keypress duration in beats" },
|
||||||
|
"velocity": { "type": "number", "minimum": 0.0, "maximum": 1.0 },
|
||||||
|
"channel": { "type": "integer", "default": 0 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.2 Recording Track Input Configuration State
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"track_id": "track_midi_01",
|
||||||
|
"is_armed": true,
|
||||||
|
"monitoring_enabled": true,
|
||||||
|
"input_source": {
|
||||||
|
"device_type": "MIDI_KEYBOARD",
|
||||||
|
"device_id": "midi_input_usb_keyboard_0",
|
||||||
|
"channel": 1
|
||||||
|
},
|
||||||
|
"input_gain_db": 0.0,
|
||||||
|
"latency_offset_ms": 12.5
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Core Algorithms & Latency Compensation
|
||||||
|
|
||||||
|
### 5.1 Algorithm 1: Hardware Latency Compensation Formula
|
||||||
|
|
||||||
|
When recording, the physical moment a key is pressed or sound enters the microphone is inherently delayed relative to speaker output due to input buffers ($L_{\text{input}}$) and output buffers ($L_{\text{output}}$).
|
||||||
|
|
||||||
|
#### Mathematical Formulation
|
||||||
|
|
||||||
|
Let:
|
||||||
|
|
||||||
|
* $T_{\text{audio\_ctx}}$ = Current timestamp in seconds on the `AudioContext` clock (`audioCtx.currentTime`).
|
||||||
|
* $T_{\text{rec\_start}}$ = Recording start timestamp in seconds.
|
||||||
|
* $\text{BPM}$ = Song tempo (Beats Per Minute).
|
||||||
|
* $\text{TS}_{\text{num}}$ = Time Signature Numerator (beats per bar).
|
||||||
|
* $\text{Bar}_{\text{start}}$ = Target timeline start bar for recording.
|
||||||
|
* $L_{\text{comp}}$ = Total hardware latency offset ($L_{\text{input}} + L_{\text{output}} + L_{\text{user\_offset}}$) in seconds.
|
||||||
|
|
||||||
|
**Actual Elapsed Audio Time ($T_{\text{elapsed}}$):**
|
||||||
|
|
||||||
|
$$T_{\text{elapsed}} = \max\left(0, T_{\text{audio\_ctx}} - T_{\text{rec\_start}} - L_{\text{comp}}\right)$$
|
||||||
|
|
||||||
|
**Audio Time to Beat Conversion ($\text{Beat}_{\text{current}}$):**
|
||||||
|
|
||||||
|
$$\text{SecondsPerBeat} = \frac{60.0}{\text{BPM}}$$
|
||||||
|
|
||||||
|
$$\text{Beat}_{\text{current}} = \frac{T_{\text{elapsed}}}{\text{SecondsPerBeat}} + \left(\text{Bar}_{\text{start}} \times \text{TS}_{\text{num}}\right)$$
|
||||||
|
|
||||||
|
**Timeline Placement Mapping:**
|
||||||
|
|
||||||
|
$$\text{StartBeat}_{\text{item}} = \text{Beat}_{\text{current}}$$
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5.2 Algorithm 2: AudioWorklet PCM Ring-Buffer Processor
|
||||||
|
|
||||||
|
To prevent audio glitches or missing PCM frames when the browser's main thread is processing heavy UI renders, microphone recording runs inside an `AudioWorkletProcessor`:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// public/processors/pcm-recorder-processor.js
|
||||||
|
class PCMRecorderProcessor extends AudioWorkletProcessor {
|
||||||
|
constructor() {
|
||||||
|
super();
|
||||||
|
this.bufferSize = 4096;
|
||||||
|
this.buffer = new Float32Array(this.bufferSize);
|
||||||
|
this.bufferIndex = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
process(inputs, outputs, parameters) {
|
||||||
|
const input = inputs[0];
|
||||||
|
if (input && input.length > 0) {
|
||||||
|
const inputChannel = input[0]; // Mono Channel 0
|
||||||
|
|
||||||
|
for (let i = 0; i < inputChannel.length; i++) {
|
||||||
|
this.buffer[this.bufferIndex++] = inputChannel[i];
|
||||||
|
|
||||||
|
// When Ring-Buffer fills, send Float32Array to Main Thread
|
||||||
|
if (this.bufferIndex >= this.bufferSize) {
|
||||||
|
this.port.postMessage({
|
||||||
|
type: 'PCM_DATA',
|
||||||
|
buffer: this.buffer.slice(0, this.bufferSize)
|
||||||
|
});
|
||||||
|
this.bufferIndex = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true; // Keep worklet active
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
registerProcessor('pcm-recorder-processor', PCMRecorderProcessor);
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5.3 Algorithm 3: Client MIDIRecorder Class Implementation
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
class ClientMIDIRecorder {
|
||||||
|
constructor(audioContext, bpm = 120, timeSigNumerator = 4) {
|
||||||
|
this.audioCtx = audioContext;
|
||||||
|
this.bpm = bpm;
|
||||||
|
this.timeSigNum = timeSigNumerator;
|
||||||
|
this.isRecording = false;
|
||||||
|
|
||||||
|
this.activeNotes = new Map(); // Store pitch -> { noteId, startBeat, velocity }
|
||||||
|
this.recordedNotes = [];
|
||||||
|
this.recStartAudioTime = 0.0;
|
||||||
|
this.recStartBar = 0.0;
|
||||||
|
|
||||||
|
// Compute round-trip browser latency
|
||||||
|
this.latencyCompSec = (this.audioCtx.baseLatency || 0) + (this.audioCtx.outputLatency || 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
start(startBar = 0.0) {
|
||||||
|
this.isRecording = true;
|
||||||
|
this.recordedNotes = [];
|
||||||
|
this.activeNotes.clear();
|
||||||
|
this.recStartBar = startBar;
|
||||||
|
this.recStartAudioTime = this.audioCtx.currentTime;
|
||||||
|
|
||||||
|
this.bindMIDIInputs();
|
||||||
|
}
|
||||||
|
|
||||||
|
bindMIDIInputs() {
|
||||||
|
if (navigator.requestMIDIAccess) {
|
||||||
|
navigator.requestMIDIAccess().then(midiAccess => {
|
||||||
|
for (let input of midiAccess.inputs.values()) {
|
||||||
|
input.onmidimessage = (event) => this.handleMIDIMessage(event);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
handleMIDIMessage(event) {
|
||||||
|
if (!this.isRecording) return;
|
||||||
|
|
||||||
|
const [status, pitch, velocity] = event.data;
|
||||||
|
const command = status >> 4;
|
||||||
|
|
||||||
|
// Apply latency compensation formula
|
||||||
|
const currentTimeSec = Math.max(0, this.audioCtx.currentTime - this.recStartAudioTime - this.latencyCompSec);
|
||||||
|
const secondsPerBeat = 60.0 / this.bpm;
|
||||||
|
const currentBeat = (currentTimeSec / secondsPerBeat) + (this.recStartBar * this.timeSigNum);
|
||||||
|
|
||||||
|
// Command 0x9: Note On
|
||||||
|
if (command === 0x9 && velocity > 0) {
|
||||||
|
const noteId = `rec_${Date.now()}_${pitch}`;
|
||||||
|
this.activeNotes.set(pitch, {
|
||||||
|
id: noteId,
|
||||||
|
pitch: pitch,
|
||||||
|
start_beat: currentBeat,
|
||||||
|
velocity: velocity / 127.0
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Command 0x8: Note Off (or Note On with velocity = 0)
|
||||||
|
else if (command === 0x8 || (command === 0x9 && velocity === 0)) {
|
||||||
|
if (this.activeNotes.has(pitch)) {
|
||||||
|
const note = this.activeNotes.get(pitch);
|
||||||
|
const durationBeats = Math.max(0.125, currentBeat - note.start_beat); // Min 1/32 note
|
||||||
|
|
||||||
|
this.recordedNotes.push({
|
||||||
|
id: note.id,
|
||||||
|
pitch: note.pitch,
|
||||||
|
start_beat: note.start_beat,
|
||||||
|
duration_beats: durationBeats,
|
||||||
|
velocity: note.velocity,
|
||||||
|
pan: 0.0
|
||||||
|
});
|
||||||
|
|
||||||
|
this.activeNotes.delete(pitch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stop() {
|
||||||
|
this.isRecording = false;
|
||||||
|
|
||||||
|
// Flush remaining active keypresses when stop is triggered
|
||||||
|
const currentTimeSec = Math.max(0, this.audioCtx.currentTime - this.recStartAudioTime - this.latencyCompSec);
|
||||||
|
const currentBeat = (currentTimeSec / (60.0 / this.bpm)) + (this.recStartBar * this.timeSigNum);
|
||||||
|
|
||||||
|
for (let [pitch, note] of this.activeNotes.entries()) {
|
||||||
|
this.recordedNotes.push({
|
||||||
|
id: note.id,
|
||||||
|
pitch: note.pitch,
|
||||||
|
start_beat: note.start_beat,
|
||||||
|
duration_beats: Math.max(0.25, currentBeat - note.start_beat),
|
||||||
|
velocity: note.velocity,
|
||||||
|
pan: 0.0
|
||||||
|
});
|
||||||
|
}
|
||||||
|
this.activeNotes.clear();
|
||||||
|
return this.recordedNotes;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5.4 Algorithm 4: Client AudioRecorder & AudioBuffer Splicing Class Implementation
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
class ClientAudioRecorder {
|
||||||
|
constructor(audioContext) {
|
||||||
|
this.audioCtx = audioContext;
|
||||||
|
this.mediaStream = null;
|
||||||
|
this.sourceNode = null;
|
||||||
|
this.workletNode = null;
|
||||||
|
this.pcmChunks = [];
|
||||||
|
this.isRecording = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async initializeInput(deviceId = null) {
|
||||||
|
const constraints = {
|
||||||
|
audio: {
|
||||||
|
deviceId: deviceId ? { exact: deviceId } : undefined,
|
||||||
|
echoCancellation: false,
|
||||||
|
noiseSuppression: false,
|
||||||
|
autoGainControl: false
|
||||||
|
}
|
||||||
|
};
|
||||||
|
this.mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
|
||||||
|
this.sourceNode = this.audioCtx.createMediaStreamSource(this.mediaStream);
|
||||||
|
}
|
||||||
|
|
||||||
|
async start(destinationTrackGainNode, enableMonitoring = true) {
|
||||||
|
this.pcmChunks = [];
|
||||||
|
this.isRecording = true;
|
||||||
|
|
||||||
|
// Load Worklet Processor Module
|
||||||
|
await this.audioCtx.audioWorklet.addModule('/processors/pcm-recorder-processor.js');
|
||||||
|
this.workletNode = new AudioWorkletNode(this.audioCtx, 'pcm-recorder-processor');
|
||||||
|
|
||||||
|
// Receive PCM data streams from AudioWorklet
|
||||||
|
this.workletNode.port.onmessage = (event) => {
|
||||||
|
if (this.isRecording && event.data.type === 'PCM_DATA') {
|
||||||
|
this.pcmChunks.push(new Float32Array(event.data.buffer));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Route Audio Nodes
|
||||||
|
this.sourceNode.connect(this.workletNode);
|
||||||
|
|
||||||
|
// Enable Live Input Monitoring if requested
|
||||||
|
if (enableMonitoring) {
|
||||||
|
this.sourceNode.connect(destinationTrackGainNode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async stop() {
|
||||||
|
this.isRecording = false;
|
||||||
|
|
||||||
|
if (this.sourceNode && this.workletNode) {
|
||||||
|
this.sourceNode.disconnect(this.workletNode);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Concatenate PCM Float32Array chunks into a single AudioBuffer
|
||||||
|
const totalSamples = this.pcmChunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||||
|
if (totalSamples === 0) return null;
|
||||||
|
|
||||||
|
const audioBuffer = this.audioCtx.createBuffer(1, totalSamples, this.audioCtx.sampleRate);
|
||||||
|
const channelData = audioBuffer.getChannelData(0);
|
||||||
|
|
||||||
|
let offset = 0;
|
||||||
|
for (const chunk of this.pcmChunks) {
|
||||||
|
channelData.set(chunk, offset);
|
||||||
|
offset += chunk.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
return audioBuffer; // Return compiled AudioBuffer for timeline insertion
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. UI Components & User Interactions
|
||||||
|
|
||||||
|
* **Track Header Arming Controls:**
|
||||||
|
* **[R] Button (Arm Track):** Highlights red when armed for recording on the target track.
|
||||||
|
* **[I] Button (Input Monitor):** Toggles live monitoring for incoming Microphone or Synth audio during performance.
|
||||||
|
* **Input Selector Dropdown:** Allows selection of available Microphone devices or USB Hardware MIDI Keyboards.
|
||||||
|
|
||||||
|
|
||||||
|
* **Real-time VU Meter Component:**
|
||||||
|
* Displays input signal gain level from $-60\text{ dB}$ to $0\text{ dB}$. Displays red clipping indicators when signal levels exceed $0\text{ dBFS}$.
|
||||||
|
|
||||||
|
|
||||||
|
* **Live Waveform & MIDI Preview Rendering:**
|
||||||
|
* **Microphone Recording:** The canvas UI renders incoming waveform signals progressing along the Playhead position in real time.
|
||||||
|
* **MIDI Performance:** Rectangular note blocks (green/orange) appear at note-on trigger events and extend until key release (note-off).
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
# DIAGNOSTIC REPORT: WHY MIDI SIGNAL IS RECEIVED BUT NOT RECORDED / RENDERED
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Executive Summary & Root Cause Analysis
|
||||||
|
|
||||||
|
Based on the DAW UI screenshot provided, the system is successfully receiving MIDI hardware signals (as indicated by the active VU meter on Track 01 set to `MIDIIN2 (SE49)`), but no MIDI data is being written or displayed on the timeline.
|
||||||
|
|
||||||
|
This issue occurs due to four architectural and state-management gaps in the current implementation.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Detailed Root Causes
|
||||||
|
|
||||||
|
### Root Cause 1: Global Transport Record vs. Track Arm Disconnect
|
||||||
|
|
||||||
|
* **Observed State:** Track 01 has its individual Arm `[R]` button active (red indicator ON). However, the Global Transport Record button (red circle on the top toolbar) is inactive/stopped at time position `0:01.951`.
|
||||||
|
* **Technical Issue:** Arming a track only enables Live Monitoring (routing MIDI input to the virtual synth engine for real-time audio playback). Recording MIDI into timeline buffers requires both **Track Arm = `true**` AND **Transport Engine State = `RECORDING**`.
|
||||||
|
|
||||||
|
```text
|
||||||
|
[ Track Armed ] + [ Transport STOPPED ] --> Live Monitoring ONLY (VU meter lights up, no recording)
|
||||||
|
[ Track Armed ] + [ Transport RECORDING ] --> Live Monitoring + Event Buffer Write + Canvas Redraw
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
### Root Cause 2: Gate Condition in `handleMIDIMessage`
|
||||||
|
|
||||||
|
In the client recording engine (`ClientMIDIRecorder`), incoming MIDI events trigger live synth audio, but note recording is gated behind a transport flag:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
handleMIDIMessage(event) {
|
||||||
|
// BUG: If global transport is not in RECORD mode, execution stops here.
|
||||||
|
// Synth gets triggered elsewhere, but recordedNotes array remains empty.
|
||||||
|
if (!this.isRecording) return;
|
||||||
|
|
||||||
|
const [status, pitch, velocity] = event.data;
|
||||||
|
// ... logic to write to activeNotes and recordedNotes
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
### Root Cause 3: Absence of Real-Time Canvas Redraw Loop
|
||||||
|
|
||||||
|
For notes to render dynamically inside the MIDI Item clip as keys are pressed:
|
||||||
|
|
||||||
|
* The UI Canvas must run a `requestAnimationFrame` render loop while `isRecording === true`.
|
||||||
|
* The renderer must query the `activeNotes` Map (currently held keys) in addition to finalized `recordedNotes`.
|
||||||
|
* If the UI only renders on static session updates (e.g., when clicking or stopping transport), live notes will not appear on screen during playback.
|
||||||
|
|
||||||
|
### Root Cause 4: Track Target ID Unbound to Input Stream
|
||||||
|
|
||||||
|
If multiple tracks exist, `ClientMIDIRecorder` must know which `track_id` is currently armed and matched to device `MIDIIN2 (SE49)`. If events arrive without a target track context, they cannot be routed into the target `MIDIItem.source_data.notes` array.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Technical Solutions & Code Adjustments
|
||||||
|
|
||||||
|
### Step 1: Ensure Dual-Stage Recording State Verification
|
||||||
|
|
||||||
|
Update the transport control logic so pressing **Record + Play** on the top toolbar initializes active record buffers on all armed tracks:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// Transport Controller
|
||||||
|
function startTransportRecording() {
|
||||||
|
const armedTracks = session.tracks.filter(t => t.is_armed);
|
||||||
|
|
||||||
|
if (armedTracks.length === 0) {
|
||||||
|
console.warn("No tracks armed for recording.");
|
||||||
|
startPlaybackOnly();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Activate global transport record state
|
||||||
|
transport.isRecording = true;
|
||||||
|
transport.isPlaying = true;
|
||||||
|
|
||||||
|
// Initialize temporary recording items on each armed track
|
||||||
|
armedTracks.forEach(track => {
|
||||||
|
const newRecordingItem = {
|
||||||
|
id: `rec_item_${Date.now()}`,
|
||||||
|
type: "MIDI_ITEM",
|
||||||
|
start_bar: transport.currentBar,
|
||||||
|
duration_bars: 0.1, // Expands dynamically during recording
|
||||||
|
clip_start_offset_bars: 0.0,
|
||||||
|
source_data: { total_buffer_bars: 8.0, notes: [] }
|
||||||
|
};
|
||||||
|
|
||||||
|
track.activeRecordingItem = newRecordingItem;
|
||||||
|
midiRecorder.start(track.id, transport.currentBar);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Start UI animation loop for live waveform/note preview
|
||||||
|
requestAnimationFrame(renderLiveRecordingUI);
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 2: Live MIDI Note Binding & Duration Expansion
|
||||||
|
|
||||||
|
Update `ClientMIDIRecorder` to feed both the active buffer and the active recording clip:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
handleMIDIMessage(event) {
|
||||||
|
const [status, pitch, velocity] = event.data;
|
||||||
|
const command = status >> 4;
|
||||||
|
|
||||||
|
// 1. Always trigger Live Audio Preview (VU Meter + Synth Node)
|
||||||
|
this.triggerSynthPreview(pitch, velocity);
|
||||||
|
|
||||||
|
// 2. Gate recording buffer write behind global transport record state
|
||||||
|
if (!transport.isRecording || !this.targetTrack) return;
|
||||||
|
|
||||||
|
const currentBeat = this.calculateLatencyCompensatedBeat();
|
||||||
|
|
||||||
|
// Command 0x9: Note On
|
||||||
|
if (command === 0x9 && velocity > 0) {
|
||||||
|
const liveNote = {
|
||||||
|
id: `note_${Date.now()}_${pitch}`,
|
||||||
|
pitch: pitch,
|
||||||
|
start_beat: currentBeat,
|
||||||
|
duration_beats: 0.25, // Default initial length until Note Off
|
||||||
|
velocity: velocity / 127.0
|
||||||
|
};
|
||||||
|
|
||||||
|
this.activeNotes.set(pitch, liveNote);
|
||||||
|
this.targetTrack.activeRecordingItem.source_data.notes.push(liveNote);
|
||||||
|
}
|
||||||
|
// Command 0x8: Note Off
|
||||||
|
else if (command === 0x8 || (command === 0x9 && velocity === 0)) {
|
||||||
|
if (this.activeNotes.has(pitch)) {
|
||||||
|
const note = this.activeNotes.get(pitch);
|
||||||
|
note.duration_beats = Math.max(0.125, currentBeat - note.start_beat);
|
||||||
|
this.activeNotes.delete(pitch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 3: Real-Time UI Canvas Render Loop
|
||||||
|
|
||||||
|
Add real-time item length expansion and live note drawing on the main canvas during recording:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
function renderLiveRecordingUI() {
|
||||||
|
if (!transport.isRecording) return;
|
||||||
|
|
||||||
|
const currentBar = transport.getCurrentBarPosition();
|
||||||
|
|
||||||
|
session.tracks.forEach(track => {
|
||||||
|
if (track.is_armed && track.activeRecordingItem) {
|
||||||
|
const item = track.activeRecordingItem;
|
||||||
|
|
||||||
|
// Expand item duration on timeline as playhead moves forward
|
||||||
|
item.duration_bars = Math.max(0.5, currentBar - item.start_bar);
|
||||||
|
|
||||||
|
// Draw item bounding box and active/completed MIDI note rectangles
|
||||||
|
drawTimelineItem(trackCanvasCtx, item);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
requestAnimationFrame(renderLiveRecordingUI);
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Checklist to Fix in Your Application
|
||||||
|
|
||||||
|
* [ ] Check if clicking top toolbar **Record + Play** sets `transport.isRecording = true`.
|
||||||
|
* [ ] Verify that Track 01 generates a temporary `activeRecordingItem` on record start.
|
||||||
|
* [ ] Confirm `requestAnimationFrame` is re-rendering the canvas continuously while transport is moving.
|
||||||
|
* [ ] Ensure incoming MIDI events on `MIDIIN2 (SE49)` push notes into Track 01's item note array rather than just playing the synth.
|
||||||
@@ -0,0 +1,237 @@
|
|||||||
|
Here is the clean, nicely formatted Markdown version of the technical specification document:
|
||||||
|
|
||||||
|
# TECHNICAL INSTALLATION & INTEGRATION GUIDE FOR SOUNDFONT / VSTI IN DAW
|
||||||
|
|
||||||
|
This document provides a detailed technical architecture model for integrating SoundFonts, WebAssembly Plugins (Client), and Native VSTi/AU (Server). It clearly delineates components pre-installed by the Developer (Coder) versus those open for User uploads and additions.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Architectural Distribution Overview (Developer vs. User)
|
||||||
|
|
||||||
|
| Plugin / Asset Category | Processing Location | Installed By | Storage & Management Method | Security & Safety Profile |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| **Default SoundFont (`.sf2`)** | Client (Wasm) | Coder | Static Assets hosted on Web Server / CDN | Extremely High |
|
||||||
|
| **User Custom SoundFont (`.sf2`)** | Client (Wasm) | User | Browser `IndexedDB` or User Cloud Storage | Extremely High (Runs inside Wasm Sandbox) |
|
||||||
|
| **WebAssembly Synths (WAMs)** | Client (JS/Wasm) | Coder | Bundled within Frontend Source Code | Extremely High |
|
||||||
|
| **Core Server VSTi (Vital, Surge...)** | Server (Python) | Coder | System Directory inside Docker/Linux Container | High (Controlled binary footprint) |
|
||||||
|
| **User Custom VST3 / Preset** | Server (Python) | User (Restricted) | Stores `.vst3` files or `.fxp`/`.json` on Container | High Security Risk (Requires Sandboxing) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Client-Side Integration Tech (Browser / WebAssembly)
|
||||||
|
|
||||||
|
The Client-Side handles zero-latency real-time composition and audio previews.
|
||||||
|
|
||||||
|
### 2.1 Coder Pre-bundled Assets
|
||||||
|
|
||||||
|
* **Static SoundFont Hosting:**
|
||||||
|
* The developer places standard `.sf2` files (such as `GeneralUser_GS.sf2`) into the `public/soundfonts/` directory or hosts them via CDN.
|
||||||
|
* Upon application startup, default SoundFonts are queried via REST API:
|
||||||
|
```http
|
||||||
|
GET /api/v1/assets/default-soundfonts
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{ "id": "sf_generaluser", "name": "GeneralUser GS v1.471", "size_mb": 31.2, "url": "/soundfonts/GeneralUser.sf2" },
|
||||||
|
{ "id": "sf_sso", "name": "Sonatina Symphonic Orchestra", "size_mb": 95.0, "url": "/soundfonts/SSO.sf2" }
|
||||||
|
]
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
* **FluidSynth WebAssembly Engine Integration:**
|
||||||
|
* Compiles FluidSynth C/C++ code into WebAssembly (`fluidsynth.wasm` + `fluidsynth.js`) using Emscripten.
|
||||||
|
* Alternatively, leverages open JavaScript wrappers such as `@soundfont/player` or `SpessaSynth`.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### 2.2 Allowing User Custom SoundFont (`.sf2`) Uploads
|
||||||
|
|
||||||
|
Delivers a flexible user experience without overloading server storage:
|
||||||
|
|
||||||
|
* **Upload Mechanism & Local Cache (`IndexedDB`):**
|
||||||
|
* Users drag and drop `.sf2` files directly into the DAW interface.
|
||||||
|
* JavaScript reads the file as an `ArrayBuffer` via the `FileReader` API.
|
||||||
|
* The file persists directly within the browser's local `IndexedDB` cache for immediate reuse across sessions without re-uploading to the server.
|
||||||
|
|
||||||
|
|
||||||
|
* **Dynamic Injection into WebAssembly Memory:**
|
||||||
|
```javascript
|
||||||
|
// Client-side JavaScript snippet
|
||||||
|
async function loadUserSoundFont(fileBuffer) {
|
||||||
|
const uint8Array = new Uint8Array(fileBuffer);
|
||||||
|
// Write buffer straight into Emscripten FluidSynth Virtual File System (MEMFS)
|
||||||
|
Module.FS.writeFile('/user_font.sf2', uint8Array);
|
||||||
|
|
||||||
|
// Call Wasm C-function to load bank
|
||||||
|
const sfont_id = Module._fluid_synth_sfload(synthInstance, '/user_font.sf2', 1);
|
||||||
|
console.log(`User SoundFont loaded successfully with ID: ${sfont_id}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Server-Side Integration Tech (Python Backend Engine)
|
||||||
|
|
||||||
|
The Server-Side executes high-resolution offline WAV rendering when an operator triggers the Export / Bounce workflow.
|
||||||
|
|
||||||
|
### 3.1 Server Environment Installed by Coder
|
||||||
|
|
||||||
|
The developer configures the Server environment (or Docker Container) with pre-installed Native C++ libraries and Python utilities.
|
||||||
|
|
||||||
|
1. **Server Base `Dockerfile` Configuration:**
|
||||||
|
```dockerfile
|
||||||
|
FROM python:3.10-slim
|
||||||
|
|
||||||
|
# Install Linux audio libraries
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
fluidsynth \
|
||||||
|
libfluidsynth-dev \
|
||||||
|
libasound2-dev \
|
||||||
|
libjack-jackd2-dev \
|
||||||
|
build-essential \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Initialize directories for Native VST3 and system SoundFonts
|
||||||
|
RUN mkdir -p /opt/daw_engine/vst3 \
|
||||||
|
&& mkdir -p /opt/daw_engine/soundfonts
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
2. **Pre-installing Native VST3 Plugins:**
|
||||||
|
Places 64-bit Linux `.vst3` binary builds of open-source synths inside `/opt/daw_engine/vst3/`:
|
||||||
|
* `/opt/daw_engine/vst3/Vital.vst3`
|
||||||
|
* `/opt/daw_engine/vst3/Surge XT.vst3`
|
||||||
|
* `/opt/daw_engine/vst3/Dexed.vst3`
|
||||||
|
|
||||||
|
|
||||||
|
3. **Python Backend Integration via Spotify `pedalboard`:**
|
||||||
|
```python
|
||||||
|
# render_engine/vst_loader.py
|
||||||
|
import os
|
||||||
|
from pedalboard import VST3Plugin, Pedalboard
|
||||||
|
|
||||||
|
class PluginManager:
|
||||||
|
def __init__(self, vst_dir="/opt/daw_engine/vst3"):
|
||||||
|
self.vst_dir = vst_dir
|
||||||
|
self.available_plugins = self._scan_plugins()
|
||||||
|
|
||||||
|
def _scan_plugins(self):
|
||||||
|
plugins = {}
|
||||||
|
for root, dirs, files in os.walk(self.vst_dir):
|
||||||
|
for file in files:
|
||||||
|
if file.endswith(".vst3") or file.endswith(".so"):
|
||||||
|
plugin_path = os.path.join(root, file)
|
||||||
|
plugin_name = os.path.splitext(file)[0]
|
||||||
|
plugins[plugin_name] = plugin_path
|
||||||
|
return plugins
|
||||||
|
|
||||||
|
def load_vst(self, plugin_name: str, preset_data: dict = None) -> VST3Plugin:
|
||||||
|
if plugin_name not in self.available_plugins:
|
||||||
|
raise FileNotFoundError(f"VST3 Plugin '{plugin_name}' not found on server.")
|
||||||
|
|
||||||
|
path = self.available_plugins[plugin_name]
|
||||||
|
vst_instance = VST3Plugin(path)
|
||||||
|
|
||||||
|
# Inject parameters if provided
|
||||||
|
if preset_data:
|
||||||
|
for param_name, param_value in preset_data.items():
|
||||||
|
setattr(vst_instance, param_name, param_value)
|
||||||
|
|
||||||
|
return vst_instance
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### 3.2 Handling User Custom Plugins / Presets
|
||||||
|
|
||||||
|
#### Option 1: User Presets / Patches Uploads (**RECOMMENDED - Safe**)
|
||||||
|
|
||||||
|
* **Implementation:** The backend locks native VST3 installations to common open engines (Vital, Dexed, Surge XT). Users upload lightweight preset patches like `.vitalbank`, `.syx` (DX7 patches), `.fxp`, or JSON parameter states.
|
||||||
|
* **Workflow:**
|
||||||
|
1. User selects the Vital Synth on the Client UI.
|
||||||
|
2. User clicks "Import Preset" $\rightarrow$ Uploads a `.vital` file or JSON parameter bundle.
|
||||||
|
3. Server parses JSON parameters and injects them directly into the VST3 instance via `pedalboard` during render execution.
|
||||||
|
|
||||||
|
|
||||||
|
* **Benefits:** Absolutely safe, minimal footprint, zero security vulnerabilities to the host infrastructure.
|
||||||
|
|
||||||
|
#### Option 2: User Native Binary VST3 Uploads (**HIGH RISK - Requires Isolation**)
|
||||||
|
|
||||||
|
* **Risk:** A `.vst3` file contains executable machine code (`.so` Shared Object on Linux). Accepting arbitrary uploads grants 100% vector exposure to Remote Code Execution (RCE) attacks.
|
||||||
|
* **Technical Mitigation (If Mandatory):**
|
||||||
|
* **Sandboxing Isolation:** Every user Export/Render request runs inside an isolated, short-lived container (Ephemeral Docker / Firejail / gVisor) stripped of `root` privileges and completely isolated from external internet interfaces.
|
||||||
|
* **Time-To-Live (TTL):** User `.vst3` binaries persist inside temporary directories `/tmp/user_sessions/{user_id}/` and purge automatically upon render job completion.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. API Specification for SoundFonts & Plugins
|
||||||
|
|
||||||
|
### 4.1 OpenAPI Endpoint Spec for Frontend
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
/api/v1/plugins/available:
|
||||||
|
get:
|
||||||
|
summary: Query available VSTi engines and SoundFont resources on the Server
|
||||||
|
responses:
|
||||||
|
200:
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
example:
|
||||||
|
vst_instruments:
|
||||||
|
- id: "vst_vital"
|
||||||
|
name: "Vital Wavetable Synth"
|
||||||
|
type: "VST3"
|
||||||
|
has_native_support: true
|
||||||
|
- id: "vst_dexed"
|
||||||
|
name: "Dexed FM Synth"
|
||||||
|
type: "VST3"
|
||||||
|
has_native_support: true
|
||||||
|
soundfonts:
|
||||||
|
- id: "sf_generaluser"
|
||||||
|
name: "GeneralUser GS"
|
||||||
|
file: "GeneralUser.sf2"
|
||||||
|
|
||||||
|
/api/v1/projects/render:
|
||||||
|
post:
|
||||||
|
summary: Trigger offline DAW Project rendering to WAV on the Server
|
||||||
|
requestBody:
|
||||||
|
required: true
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
$ref: '#/components/schemas/ProjectSchema'
|
||||||
|
responses:
|
||||||
|
200:
|
||||||
|
description: Returns the URL pointing to the rendered WAV file
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Development Team Best Practices Summary
|
||||||
|
|
||||||
|
* **SoundFont (`.sf2`):**
|
||||||
|
* **For Users:** Encourage unrestricted local uploads on the Client (Browser). Store assets in `IndexedDB` to ensure optimal real-time performance without straining server resources.
|
||||||
|
* **For Developers:** Supply 1–2 default General MIDI (GM) SoundFont banks (`GeneralUser_GS.sf2`) bundled on both Client and Server.
|
||||||
|
|
||||||
|
|
||||||
|
* **VSTi Instruments:**
|
||||||
|
* **For Developers:** Pre-install top open-source Linux-native synths on the Server (Vital, Surge XT, Dexed, OB-Xd).
|
||||||
|
* **For Users:** Do **not** allow direct `.vst3` binary uploads to the production server. Instead, permit users to upload Presets / Patches / JSON parameters for the supported synth models. This guarantees 100% security while saving storage and network bandwidth.
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
# Kế hoạch nâng cấp chức năng Piano Roll, quản lý tab và cơ chế Lưu (Save) cho Section/MIDI items
|
||||||
|
|
||||||
|
Tài liệu này đề xuất các chỉnh sửa lớn cho hệ thống **Piano Roll** và **Section Tabs** để tăng trải nghiệm soạn nhạc (DAW workflow):
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Các yêu cầu chức năng cần bổ sung
|
||||||
|
|
||||||
|
### 1. Quản lý Tab & Tên Tab
|
||||||
|
- **Gán tên tab**: Khi mở Piano Roll, gán tên tab hiển thị là `Piano Roll: [Tên MIDI Item]` để khớp chính xác.
|
||||||
|
- **Không mất dữ liệu khi mở Section Tab**: Khi mở thêm Section tab mới, trạng thái và các nốt nhạc đang sửa đổi trong tab Piano Roll phải được giữ nguyên trong React state (`subTabs`), không bị xoá hay reset.
|
||||||
|
- **Tạo tab con cho Section Tab**: Nếu mở MIDI item thuộc một Section-tab, hệ thống sẽ tạo một tab Piano Roll tương ứng (nhận `parent_tab_id` là Section tab đó) để quản lý độc lập.
|
||||||
|
- **Mở lại tab cũ**: Khi kích hoạt chỉnh sửa MIDI item đã có tab Piano Roll mở sẵn, chuyển đổi `activeTab` về tab đó mà không tạo thêm tab mới trùng lặp.
|
||||||
|
|
||||||
|
### 2. Cơ chế Lưu (Save) thủ công
|
||||||
|
- **Lưu nốt MIDI**: Thay vì tự động cập nhật nốt nhạc thời gian thực về track gốc làm nặng ứng dụng, các chỉnh sửa nốt nhạc trong Piano Roll sẽ lưu cục bộ trong tab. Khi nhấn nút **"Lưu"** (Save) trên Piano Roll tab, nốt nhạc mới được cập nhật vào MIDI item cha.
|
||||||
|
- **Lưu Section Tab**: Bổ sung nút **"Lưu Section"** trên thanh công cụ của Section Tab. Khi bấm, tên và thời lượng (duration) của Section Tab sẽ cập nhật ngược lại vào block Section item ở **Main Session**.
|
||||||
|
|
||||||
|
### 3. Cải tiến tương tác Chuột trong Piano Roll
|
||||||
|
- **Ctrl + Click**: Vẽ nhanh 1 nốt nhạc mới tại tọa độ click (áp dụng snap).
|
||||||
|
- **Nhấp & Kéo (Click & Drag) ở khoảng trống**: Quét khung chọn (marquee selection) để chọn nhiều nốt cùng lúc.
|
||||||
|
- **Kéo cạnh phải nốt**: Thay đổi thời lượng nốt nhạc (Resize).
|
||||||
|
- **Kéo phần thân nốt**: Di chuyển nốt nhạc theo cao độ (pitch) và thời gian (beat). Nếu có nhiều nốt đang được chọn, di chuyển toàn bộ các nốt được chọn cùng nhau (multi-note move).
|
||||||
|
- **Shift + Cuộn chuột (Scroll)**:
|
||||||
|
- Nếu cuộn chuột trên 1 nốt nhạc: Tăng/giảm velocity của nốt đó (khoảng cách 0.05, giới hạn từ 0.1 đến 1.0).
|
||||||
|
- Nếu cuộn chuột ngoài khoảng trống khi có các nốt được chọn: Tăng/giảm velocity của toàn bộ các nốt đang được chọn.
|
||||||
|
- **Click chuột phải**: Xóa nhanh note
|
||||||
|
|
||||||
|
### 4. Thêm Thanh thước nhịp (Bar Ruler)
|
||||||
|
- Bổ sung thanh thước nhịp nằm phía trên lưới Piano Roll, hiển thị nhịp số bắt đầu từ `Bar 0`, `Bar 1`, `Bar 2`,...
|
||||||
|
- Thanh thước nhịp sẽ tự động cuộn ngang đồng bộ với lưới Piano Roll khi người dùng cuộn chuột ngang.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Đề xuất thay đổi mã nguồn
|
||||||
|
|
||||||
|
### [app.jsx](file:///home/locpham/SonicForgeStudio/app/static/js/app.jsx)
|
||||||
|
|
||||||
|
#### A. Cấu trúc lại `handleUpdateMidiNotes` và thêm `handleSaveMidiNotes`
|
||||||
|
- Tách biệt cập nhật cục bộ và lưu trữ về MIDI item gốc.
|
||||||
|
|
||||||
|
#### B. Thêm nút "Lưu Section" và "Lưu MIDI Notes"
|
||||||
|
- Hiển thị nút **Lưu Section** trên thanh công cụ khi `activeTab` là Section Tab.
|
||||||
|
- Hiển thị nút **Lưu** màu xanh lá trên thanh điều khiển của `PianoRollTabEditor`.
|
||||||
|
|
||||||
|
#### C. Viết lại `PianoRollTabEditor` hỗ trợ sự kiện chuột
|
||||||
|
- Quản lý trạng thái `selectedNoteIds` và `selectionMarquee`.
|
||||||
|
- Thêm sự kiện `wheel` hỗ trợ `Shift+Scroll` thay đổi velocity.
|
||||||
|
- Vẽ khung chọn `selectionMarquee` trên canvas.
|
||||||
|
- Thêm component / thẻ div Bar Ruler phía trên canvas và đồng bộ hóa cuộn (`onScroll` cập nhật `rulerScrollRef.current.scrollLeft`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Kế hoạch xác minh
|
||||||
|
|
||||||
|
### Kiểm tra tự động
|
||||||
|
- Chạy `PYTHONPATH=. pytest` để đảm bảo logic lưu trữ phía Python không bị ảnh hưởng.
|
||||||
|
|
||||||
|
### Kiểm tra thủ công
|
||||||
|
1. **Kiểm tra tab & lưu trữ**:
|
||||||
|
- Bật Section Tab, tạo một MIDI item. Double-click để mở Piano Roll của nó.
|
||||||
|
- Sửa đổi nốt trong Piano Roll, chuyển qua lại giữa Main Session, Section Tab và Piano Roll. Xác minh nốt không bị mất.
|
||||||
|
- Bấm **Lưu** trong Piano Roll -> Nhìn ngoài timeline thấy MIDI item cập nhật nốt.
|
||||||
|
- Bấm **Lưu Section** -> Nhìn ngoài Main Session thấy Section item cập nhật tên/độ dài.
|
||||||
|
2. **Kiểm tra thao tác chuột**:
|
||||||
|
- Kiểm tra quét chuột chọn nhiều nốt, kéo di chuyển cả cụm.
|
||||||
|
- Kiểm tra `Ctrl+Click` vẽ nốt.
|
||||||
|
- Kiểm tra `Shift+Scroll` thay đổi độ đậm nhạt (velocity) của nốt.
|
||||||
|
- Kiểm tra thanh Bar Ruler hiển thị `Bar 0, Bar 1...` cuộn mượt mà.
|
||||||
@@ -10,3 +10,5 @@ scipy>=1.10.0
|
|||||||
soundfile>=0.12.1
|
soundfile>=0.12.1
|
||||||
jinja2>=3.1.2
|
jinja2>=3.1.2
|
||||||
httpx>=0.24.0
|
httpx>=0.24.0
|
||||||
|
jsonschema>=4.18.0
|
||||||
|
pedalboard>=0.8.0
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
import os
|
||||||
|
import json
|
||||||
|
import pytest
|
||||||
|
from app.core.render_engine import PythonRenderEngine
|
||||||
|
|
||||||
|
def test_render_engine_init():
|
||||||
|
engine = PythonRenderEngine()
|
||||||
|
assert engine.sample_rate == 44100
|
||||||
|
|
||||||
|
def test_bars_to_samples():
|
||||||
|
engine = PythonRenderEngine()
|
||||||
|
samples = engine.bars_to_samples(4.0, 120.0, 4)
|
||||||
|
assert samples == 352800
|
||||||
|
|
||||||
|
def test_render_session_container_empty():
|
||||||
|
engine = PythonRenderEngine()
|
||||||
|
session = {"tracks": []}
|
||||||
|
buf = engine.render_session_container(session, {}, 120.0, 4, 1000)
|
||||||
|
assert buf.shape == (2, 1000)
|
||||||
|
assert (buf == 0.0).all()
|
||||||
Reference in New Issue
Block a user