feat: embed native VST GUI in mastering + fx rack panels (bridge --fx-gui frame capture + input relay)

Linux parts (0L1-4L1 per TASKS_LINUX_WINDOWS.md):
- native_bridge: FxGuiServer.cpp HTTP server (Winsock) — /frame BitBlt +
  PrintWindow fallback -> JPEG (libjpeg-turbo q70), /input PostMessage relay
  (mouse/wheel/key, coords scaled), /close, /ping; RenderFxJob fxGuiSetup +
  run_fx_gui_server (offscreen); main.cpp --fx-gui dispatch; vcpkg +
  CMakeLists WIN32-only + JPEG.
- api: open_fx_gui embed flag spawn --fx-gui, parse SF_FXGUI_PORT -> embed_url,
  fx-gui/close endpoint; 4 unit tests.
- ui: VstGuiEmbed component (frame refresh 100ms, ping 2s, input relay,
  scale coords) shared by MasteringModal + FXRackModal; swap graph<->GUI,
  cleanup on unmount, dead-bridge toast fallback.
- docs: TASKS_LINUX_WINDOWS.md, WALKTHROUGH_WINDOWS.md, PLAN status.
Windows tasks 0W1-4W1 remain (build + E2E real VST).
This commit is contained in:
2026-08-17 22:37:55 +07:00
parent 38f4cbd8b5
commit c382afe2a6
13 changed files with 3098 additions and 2051 deletions
+3 -1
View File
@@ -59,6 +59,7 @@ add_executable(daw_vst_bridge
src/main.cpp
src/RenderJob.cpp
src/RenderFxJob.cpp
src/FxGuiServer.cpp
src/NativeInstrumentEngine.cpp
src/SharedMemoryIPC.cpp
src/Vst3Instrument.cpp
@@ -89,7 +90,8 @@ if(VST3_SDK_TARGET)
endif()
endif()
if(WIN32)
target_link_libraries(daw_vst_bridge PRIVATE ${FLUIDSYNTH_LIBRARY} ${SFIZZ_LIBRARY})
find_package(JPEG REQUIRED) # libjpeg-turbo (vcpkg) — FxGuiServer.cpp JPEG encode
target_link_libraries(daw_vst_bridge PRIVATE ${FLUIDSYNTH_LIBRARY} ${SFIZZ_LIBRARY} JPEG::JPEG)
# Required Windows libs
target_link_libraries(daw_vst_bridge PRIVATE winmm imm32)
else()
+9
View File
@@ -44,4 +44,13 @@ int run_scan_dir(const std::string& dirPath);
// editor attach failure.
int run_open_fx_gui(const std::string& jobPath);
// Embedded GUI server mode (Phase 0 of PLAN_MASTERBUS_VST_GUI_EMBED.md):
// `daw_vst_bridge --fx-gui <job.json>` — same plugin load + editor attach as
// --open-fx-gui but the host window is parked off-screen and a tiny HTTP
// server (FxGuiServer.cpp) serves JPEG frames (GET /frame) and relays pointer/
// key input (POST /input) so the web UI can embed the native GUI in a panel.
// Prints `SF_FXGUI_PORT=<port>` to stdout. Windows only. Exit codes same as
// --open-fx-gui.
int run_fx_gui_server(const std::string& jobPath);
#endif // RENDER_FX_JOB_H
+318
View File
@@ -0,0 +1,318 @@
// native_bridge/src/FxGuiServer.cpp
// Embedded VST GUI server (Phase 0 of PLAN_MASTERBUS_VST_GUI_EMBED.md):
// `daw_vst_bridge --fx-gui <job.json>` loads ONE VST3 effect plugin, attaches
// its native editor to an off-screen host window, and serves the window as
// JPEG frames over a tiny HTTP server so the web frontend can embed it in a
// panel (frame-capture + input relay, VNC-style). Windows only.
//
// Routes:
// GET /ping -> {"ok":true}
// GET /frame -> image/jpeg (BitBlt, fallback PrintWindow)
// POST /input -> {type:"mousedown|mousemove|mouseup|wheel|keydown|keyup",
// x,y,button,deltaY,keyCode} -> PostMessage to HWND
// POST /close -> detach + exit 0
//
// The HTTP server runs on a worker thread; the calling (STA) thread runs the
// message pump — VST3 editors need COM apartment + pump on the same thread.
// The listen port is random and printed to stdout as `SF_FXGUI_PORT=<port>`.
#ifdef _WIN32
#include <winsock2.h>
#include <windows.h>
#include <ws2tcpip.h>
#pragma comment(lib, "ws2_32.lib")
#include <jpeglib.h>
#include <atomic>
#include <cstdio>
#include <cstring>
#include <functional>
#include <iostream>
#include <string>
#include <thread>
#include <vector>
// sheredom/json.h (vendored with the VST3 SDK) — same include as RenderFxJob.cpp.
#include "vst3sdk/public.sdk/source/vst/moduleinfo/json.h"
namespace {
// --- tiny JSON helpers (same API as RenderFxJob.cpp) ------------------------
const json_value_s* memberValue(const json_object_s* o, const char* key) {
const json_object_element_s* m = member(o, key);
return m ? m->value : nullptr;
}
std::string memberString(const json_object_s* o, const char* key, const std::string& def) {
const json_value_s* v = memberValue(o, key);
if (v && v->type == json_type_string) {
const auto* s = static_cast<const json_string_s*>(v->payload);
return std::string(s->string, s->string_size);
}
return def;
}
int64_t memberInt(const json_object_s* o, const char* key, int64_t def) {
const json_value_s* v = memberValue(o, key);
if (v && v->type == json_type_number)
return (int64_t)std::atof(static_cast<const json_number_s*>(v->payload)->number);
return def;
}
// --- HTTP plumbing -----------------------------------------------------------
struct HttpReq {
std::string method;
std::string path;
std::string body;
};
// Read one request (headers up to \r\n\r\n + Content-Length body). Returns
// false on socket error or malformed request line.
bool readRequest(SOCKET c, HttpReq& out) {
std::string raw;
char buf[4096];
while (raw.find("\r\n\r\n") == std::string::npos) {
if (raw.size() > 16384) return false;
int n = recv(c, buf, (int)sizeof(buf), 0);
if (n <= 0) return false;
raw.append(buf, (size_t)n);
}
const size_t headEnd = raw.find("\r\n\r\n");
const std::string head = raw.substr(0, headEnd);
const size_t sp1 = head.find(' ');
const size_t sp2 = head.find(' ', sp1 + 1);
if (sp1 == std::string::npos || sp2 == std::string::npos) return false;
out.method = head.substr(0, sp1);
out.path = head.substr(sp1 + 1, sp2 - sp1 - 1);
long contentLength = 0;
size_t pos = 0;
while ((pos = head.find("\r\n", pos)) != std::string::npos) {
const size_t lineEnd = head.find("\r\n", pos + 2);
const std::string line = head.substr(pos + 2, lineEnd == std::string::npos ? std::string::npos : lineEnd - pos - 2);
pos = lineEnd == std::string::npos ? std::string::npos : lineEnd;
if (line.rfind("Content-Length:", 0) == 0 || line.rfind("content-length:", 0) == 0) {
contentLength = std::atol(line.c_str() + 15);
break;
}
}
if (contentLength > 0) {
while (raw.size() < headEnd + 4 + (size_t)contentLength) {
int n = recv(c, buf, (int)sizeof(buf), 0);
if (n <= 0) return false;
raw.append(buf, (size_t)n);
}
out.body = raw.substr(headEnd + 4, (size_t)contentLength);
}
return true;
}
void sendResponse(SOCKET c, int code, const char* contentType,
const void* data, size_t len) {
char hdr[512];
std::snprintf(hdr, sizeof(hdr),
"HTTP/1.1 %d %s\r\nContent-Type: %s\r\n"
"Content-Length: %zu\r\nAccess-Control-Allow-Origin: *\r\n"
"Connection: close\r\n\r\n",
code, code == 200 ? "OK" : "Error", contentType, len);
send(c, hdr, (int)std::strlen(hdr), 0);
if (len > 0) send(c, (const char*)data, (int)len, 0);
}
// --- frame capture -----------------------------------------------------------
// BitBlt the window client rect; if the result is all-black (plugin uses a
// layered/DirectX surface GDI cannot see), fall back to PrintWindow with
// PW_RENDERFULLCONTENT. Returns JPEG bytes in `out`.
bool captureHwndToJpeg(HWND hwnd, std::vector<unsigned char>& out, int quality) {
RECT rc;
if (!GetClientRect(hwnd, &rc)) return false;
const int w = rc.right - rc.left;
const int h = rc.bottom - rc.top;
if (w <= 0 || h <= 0) return false;
HDC win = GetDC(hwnd);
HDC mem = CreateCompatibleDC(win);
HBITMAP bmp = CreateCompatibleBitmap(win, w, h);
HGDIOBJ oldBmp = SelectObject(mem, bmp);
std::vector<unsigned char> px((size_t)w * h * 4);
auto grab = [&]() -> bool {
BITMAPINFO bi = {};
bi.bmiHeader.biSize = sizeof(BITMAPINFOHEADER);
bi.bmiHeader.biWidth = w;
bi.bmiHeader.biHeight = -h; // top-down
bi.bmiHeader.biPlanes = 1;
bi.bmiHeader.biBitCount = 32;
bi.bmiHeader.biCompression = BI_RGB;
return GetDIBits(mem, bmp, 0, h, px.data(), &bi, DIB_RGB_COLORS) == h;
};
auto isAllBlack = [&]() -> bool {
// Sample a coarse grid of pixels; all zero => invisible to GDI.
const int step = 8;
for (int y = 0; y < h; y += step)
for (int x = 0; x < w; x += step) {
const size_t i = ((size_t)y * w + x) * 4;
if (px[i] || px[i + 1] || px[i + 2]) return false;
}
return true;
};
bool ok = BitBlt(mem, 0, 0, w, h, win, 0, 0, SRCCOPY | CAPTUREBLT) != 0 && grab();
if (ok && isAllBlack()) {
ok = PrintWindow(hwnd, mem, PW_RENDERFULLCONTENT) != 0 && grab();
}
SelectObject(mem, oldBmp);
DeleteObject(bmp);
DeleteDC(mem);
ReleaseDC(hwnd, win);
if (!ok) return false;
// BGRA (GDI) -> RGB, then JPEG.
std::vector<unsigned char> rgb((size_t)w * h * 3);
for (int i = 0; i < w * h; ++i) {
rgb[(size_t)i * 3 + 0] = px[(size_t)i * 4 + 2];
rgb[(size_t)i * 3 + 1] = px[(size_t)i * 4 + 1];
rgb[(size_t)i * 3 + 2] = px[(size_t)i * 4 + 0];
}
jpeg_compress_struct cinfo = {};
jpeg_error_mgr jerr;
cinfo.err = jpeg_std_error(&jerr);
jpeg_create_compress(&cinfo);
unsigned char* memBuf = nullptr;
unsigned long memSize = 0;
jpeg_mem_dest(&cinfo, &memBuf, &memSize);
cinfo.image_width = (JDIMENSION)w;
cinfo.image_height = (JDIMENSION)h;
cinfo.input_components = 3;
cinfo.in_color_space = JCS_RGB;
jpeg_set_defaults(&cinfo);
jpeg_set_quality(&cinfo, quality, TRUE);
jpeg_start_compress(&cinfo, TRUE);
JSAMPROW row = rgb.data();
while (cinfo.next_scanline < cinfo.image_height)
jpeg_write_scanlines(&cinfo, &row, 1);
jpeg_finish_compress(&cinfo);
if (memBuf && memSize > 0) out.assign(memBuf, memBuf + memSize);
std::free(memBuf);
jpeg_destroy_compress(&cinfo);
return !out.empty();
}
// --- input relay -------------------------------------------------------------
void relayInput(HWND hwnd, const json_object_s* o) {
const std::string type = memberString(o, "type", "");
const int x = (int)memberInt(o, "x", 0);
const int y = (int)memberInt(o, "y", 0);
const int button = (int)memberInt(o, "button", 0);
const int deltaY = (int)memberInt(o, "deltaY", 0);
const WPARAM keyCode = (WPARAM)memberInt(o, "keyCode", 0);
const LPARAM lp = MAKELPARAM(x, y);
if (type == "mousemove") {
PostMessageA(hwnd, WM_MOUSEMOVE, 0, lp);
} else if (type == "mousedown") {
UINT m = button == 2 ? WM_RBUTTONDOWN : button == 1 ? WM_MBUTTONDOWN : WM_LBUTTONDOWN;
PostMessageA(hwnd, m, MAKEWPARAM(0, button == 2 ? MK_RBUTTON : button == 1 ? MK_MBUTTON : MK_LBUTTON), lp);
PostMessageA(hwnd, WM_MOUSEMOVE, 0, lp); // plugins tracking absolute pos
} else if (type == "mouseup") {
UINT m = button == 2 ? WM_RBUTTONUP : button == 1 ? WM_MBUTTONUP : WM_LBUTTONUP;
PostMessageA(hwnd, m, 0, lp);
} else if (type == "wheel") {
PostMessageA(hwnd, WM_MOUSEWHEEL, MAKEWPARAM(0, (short)(-(int64_t)deltaY * 120)), lp);
} else if (type == "keydown") {
SetFocus(hwnd);
PostMessageA(hwnd, WM_KEYDOWN, keyCode, 0);
} else if (type == "keyup") {
PostMessageA(hwnd, WM_KEYUP, keyCode, 0);
}
}
// --- HTTP server loop (worker thread) ---------------------------------------
void runHttpServer(HWND hwnd, std::atomic<bool>& stop) {
WSADATA wsa;
if (WSAStartup(MAKEWORD(2, 2), &wsa) != 0) { std::cerr << "[FxGuiServer] WSAStartup failed" << std::endl; return; }
SOCKET lsock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (lsock == INVALID_SOCKET) { WSACleanup(); return; }
sockaddr_in addr = {};
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
addr.sin_port = 0; // random free port
if (bind(lsock, (sockaddr*)&addr, sizeof(addr)) != 0 || listen(lsock, 4) != 0) {
closesocket(lsock);
WSACleanup();
std::cerr << "[FxGuiServer] bind/listen failed" << std::endl;
return;
}
sockaddr_in bound = {};
int blen = sizeof(bound);
getsockname(lsock, (sockaddr*)&bound, &blen);
std::cout << "SF_FXGUI_PORT=" << ntohs(bound.sin_port) << std::endl;
std::cout.flush();
while (!stop.load()) {
fd_set rf;
FD_ZERO(&rf);
FD_SET(lsock, &rf);
timeval tv = { 0, 100000 }; // 100ms poll so /close can break out
int sel = select(0, &rf, nullptr, nullptr, &tv);
if (sel <= 0) continue;
SOCKET c = accept(lsock, nullptr, nullptr);
if (c == INVALID_SOCKET) continue;
HttpReq req;
if (!readRequest(c, req)) { closesocket(c); continue; }
if (req.path == "/ping") {
sendResponse(c, 200, "application/json", "{\"ok\":true}", 11);
} else if (req.path == "/frame") {
std::vector<unsigned char> jpg;
if (captureHwndToJpeg(hwnd, jpg, 70))
sendResponse(c, 200, "image/jpeg", jpg.data(), jpg.size());
else
sendResponse(c, 500, "application/json", "{\"error\":\"capture failed\"}", 25);
} else if (req.path == "/input") {
json_parse_result_s pres = {};
json_value_s* root = json_parse_ex(req.body.data(), req.body.size(),
json_parse_flags_default, nullptr, nullptr, &pres);
if (root && root->type == json_type_object) {
relayInput(hwnd, static_cast<const json_object_s*>(root->payload));
std::free(root);
sendResponse(c, 200, "application/json", "{\"ok\":true}", 11);
} else {
if (root) std::free(root);
sendResponse(c, 400, "application/json", "{\"error\":\"bad json\"}", 20);
}
} else if (req.path == "/close") {
sendResponse(c, 200, "application/json", "{\"ok\":true}", 11);
stop.store(true);
PostMessageA(hwnd, WM_CLOSE); // pump exits -> process returns
} else {
sendResponse(c, 404, "application/json", "{\"error\":\"not found\"}", 22);
}
closesocket(c);
}
closesocket(lsock);
WSACleanup();
}
} // namespace
// Runs the embedded GUI server: starts the HTTP worker thread, then pumps
// messages on this (STA) thread until the host window is destroyed (user
// closed it or POST /close). Returns 0.
int fxGuiServerLoop(HWND hwnd) {
std::atomic<bool> stop{false};
std::thread http(runHttpServer, hwnd, std::ref(stop));
MSG msg;
while (IsWindow(hwnd)) {
BOOL r = GetMessageA(&msg, nullptr, 0, 0);
if (r <= 0) break;
TranslateMessage(&msg);
DispatchMessageA(&msg);
}
stop.store(true);
if (http.joinable()) http.join();
return 0;
}
#endif // _WIN32
+53 -13
View File
@@ -931,12 +931,11 @@ static void fxGuiDetachSafe(void* fx) {
#endif
}
int run_open_fx_gui(const std::string& jobPath) {
// The editor window and its message pump live on this (STA) thread for the
// whole GUI session — VST3 editors need the COM apartment + pump together.
CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
struct ComGuard { ~ComGuard() { CoUninitialize(); } } comGuard;
// Shared setup for both GUI modes: parse job, load plugin, create host window
// (offscreen for embedded capture), attach editor (SEH-guarded). Returns 0 ok,
// 1 job/argument error, 2 plugin load or editor attach failure.
static int fxGuiSetup(const std::string& jobPath, Vst3Fx* fx, HWND* hwndOut,
bool offscreen, LONG_PTR userData) {
// 1. Parse job { path, name }.
std::ifstream jf(jobPath, std::ios::binary);
if (!jf) { std::cerr << "[FxGui] cannot read job file: " << jobPath << std::endl; return 1; }
@@ -954,31 +953,47 @@ int run_open_fx_gui(const std::string& jobPath) {
if (path.empty()) { std::cerr << "[FxGui] job missing path" << std::endl; return 1; }
// 2. Load the plugin (fixed 44.1k/512 — GUI only, no audio processed here).
Vst3Fx fx;
if (!fx.load(path, 44100.0, 512)) { std::cerr << "[FxGui] plugin load failed: " << path << std::endl; return 2; }
if (!fx->load(path, 44100.0, 512)) { std::cerr << "[FxGui] plugin load failed: " << path << std::endl; return 2; }
// 3. Host window.
// 3. Host window. Embedded mode parks it far off-screen (never steals
// focus; still visible to GDI so BitBlt captures real pixels).
WNDCLASSA wc = {};
wc.lpfnWndProc = FxGuiWndProc;
wc.hInstance = GetModuleHandleA(nullptr);
wc.lpszClassName = kFxGuiWndClass;
RegisterClassA(&wc);
FxGuiCtx ctx;
const int x = offscreen ? -20000 : CW_USEDEFAULT;
const int y = offscreen ? -20000 : CW_USEDEFAULT;
HWND hwnd = CreateWindowExA(0, kFxGuiWndClass,
(name.empty() ? "VST FX" : name).c_str(),
WS_OVERLAPPEDWINDOW | WS_VISIBLE | WS_CLIPCHILDREN | WS_CLIPSIBLINGS,
CW_USEDEFAULT, CW_USEDEFAULT, 800, 600,
x, y, 800, 600,
nullptr, nullptr, GetModuleHandleA(nullptr), nullptr);
if (!hwnd) { std::cerr << "[FxGui] create window failed" << std::endl; return 1; }
SetWindowLongPtrA(hwnd, GWLP_USERDATA, (LONG_PTR)&ctx);
SetWindowLongPtrA(hwnd, GWLP_USERDATA, userData);
// 4. Attach the plugin editor (SEH-guarded — plugin GUI code may crash).
if (!fxGuiAttachSafe(&fx, hwnd)) {
if (!fxGuiAttachSafe(fx, hwnd)) {
std::cerr << "[FxGui] editor attach failed for " << path << std::endl;
DestroyWindow(hwnd);
return 2;
}
std::cout << "[FxGui] editor open: " << (name.empty() ? path : name) << std::endl;
*hwndOut = hwnd;
return 0;
}
int run_open_fx_gui(const std::string& jobPath) {
// The editor window and its message pump live on this (STA) thread for the
// whole GUI session — VST3 editors need the COM apartment + pump together.
CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
struct ComGuard { ~ComGuard() { CoUninitialize(); } } comGuard;
Vst3Fx fx;
HWND hwnd = nullptr;
FxGuiCtx ctx;
const int rc = fxGuiSetup(jobPath, &fx, &hwnd, /*offscreen=*/false, (LONG_PTR)&ctx);
if (rc) return rc;
// 5. Pump messages until the window is destroyed.
MSG msg;
@@ -995,6 +1010,25 @@ int run_open_fx_gui(const std::string& jobPath) {
return 0;
}
// Embedded GUI server mode: same setup but off-screen, then hand the message
// pump + HTTP capture/input to FxGuiServer.cpp (see PLAN_MASTERBUS_VST_GUI_EMBED.md).
extern int fxGuiServerLoop(HWND hwnd);
int run_fx_gui_server(const std::string& jobPath) {
CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
struct ComGuard { ~ComGuard() { CoUninitialize(); } } comGuard;
Vst3Fx fx;
HWND hwnd = nullptr;
const int rc = fxGuiSetup(jobPath, &fx, &hwnd, /*offscreen=*/true, 0);
if (rc) return rc;
const int srv = fxGuiServerLoop(hwnd);
fxGuiDetachSafe(&fx);
std::cout << "[FxGuiServer] closed" << std::endl;
return srv;
}
#else // !_WIN32
int run_open_fx_gui(const std::string& jobPath) {
@@ -1003,6 +1037,12 @@ int run_open_fx_gui(const std::string& jobPath) {
return 1;
}
int run_fx_gui_server(const std::string& jobPath) {
(void)jobPath;
std::cerr << "[FxGui] GUI mode chỉ hỗ trợ Windows" << std::endl;
return 1;
}
#endif // _WIN32
// --- --scan: classify VST3 modules as instrument/effect ----------------------
+8
View File
@@ -646,6 +646,14 @@ int main(int argc, char* argv[]) {
}
if (!fxGuiJob.empty()) return run_open_fx_gui(fxGuiJob);
// --fx-gui <job.json>: embedded GUI server (frame capture + input relay).
// See PLAN_MASTERBUS_VST_GUI_EMBED.md Phase 0.
std::string fxGuiServerJob;
for (int i = 1; i + 1 < argc; ++i) {
if (std::strcmp(argv[i], "--fx-gui") == 0) fxGuiServerJob = argv[i + 1];
}
if (!fxGuiServerJob.empty()) return run_fx_gui_server(fxGuiServerJob);
// 1. Shared memory name: argv --shm <name> | env SF_SHM_NAME | default
std::string shmName = "SonicForge_DAW_IPC";
for (int i = 1; i + 1 < argc; ++i) {
+1
View File
@@ -3,6 +3,7 @@
"version-string": "1.0.0",
"dependencies": [
"fluidsynth",
"libjpeg-turbo",
"pkgconf"
]
}