diff --git a/docker/backend/package-lock.json b/docker/backend/package-lock.json index b894ad7..f1dd3df 100644 --- a/docker/backend/package-lock.json +++ b/docker/backend/package-lock.json @@ -10,7 +10,8 @@ "dependencies": { "better-sqlite3": "^12.11.1", "fastify": "^5.12.5", - "nodemailer": "^7.0.13" + "nodemailer": "^7.0.13", + "onnxruntime-node": "^1.30.0" }, "devDependencies": { "@types/better-sqlite3": "^9.6.0", @@ -615,6 +616,15 @@ "integrity": "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==", "license": "MIT" }, + "node_modules/adm-zip": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.1.tgz", + "integrity": "sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==", + "license": "MIT", + "engines": { + "node": ">=14.0" + } + }, "node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", @@ -814,6 +824,40 @@ "node": ">=4.0.0" } }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -841,6 +885,24 @@ "once": "^1.4.0" } }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esbuild": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", @@ -883,6 +945,18 @@ "@esbuild/win32-x64": "0.28.2" } }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", @@ -1042,6 +1116,61 @@ "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", "license": "MIT" }, + "node_modules/global-agent": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-4.1.3.tgz", + "integrity": "sha512-KUJEViiuFT3I97t+GYMikLPJS2Lfo/S2F+DQuBWzuzaMPnvt5yyZePzArx36fBzpGTxZjIpDbXLeySLgh+k76g==", + "license": "BSD-3-Clause", + "dependencies": { + "globalthis": "^1.0.2", + "matcher": "^4.0.0", + "semver": "^7.3.5", + "serialize-error": "^8.1.0" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "license": "MIT", + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -1145,6 +1274,21 @@ ], "license": "MIT" }, + "node_modules/matcher": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-4.0.0.tgz", + "integrity": "sha512-S6x5wmcDmsDRRU/c2dkccDwQPXoFczc5+HpQ2lON8pnvHlnvHAHj5WlLVvw6n6vNyHuVugYrFohYxbS+pvFpKQ==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/mimic-response": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", @@ -1199,6 +1343,15 @@ "node": ">=6.0.0" } }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -1217,6 +1370,29 @@ "wrappy": "1" } }, + "node_modules/onnxruntime-common": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.30.0.tgz", + "integrity": "sha512-7fdVWjAID1dVhH/G8qK3APARunV4VkBFoCQAP7qp4Wkab0mrorvmc+sqiT+mKXOzDqdjN5j+/Z9nb4gzNPWcyA==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.30.0.tgz", + "integrity": "sha512-twhs1C2C/BFkz1yc5OY0KIU2GUq6DURO7hD4bx5Q2Qy3nAMJwRXW8xU3NVczE29VA9lolLOYepoD8fjTGOfIqw==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.6.0", + "global-agent": "^4.1.3", + "onnxruntime-common": "1.30.0" + } + }, "node_modules/pino": { "version": "10.3.1", "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", @@ -1464,6 +1640,21 @@ "node": ">=10" } }, + "node_modules/serialize-error": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", + "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/set-cookie-parser": { "version": "2.7.2", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", @@ -1637,6 +1828,18 @@ "node": "*" } }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", diff --git a/docker/backend/package.json b/docker/backend/package.json index c4496ec..a6e331b 100644 --- a/docker/backend/package.json +++ b/docker/backend/package.json @@ -14,7 +14,8 @@ "dependencies": { "better-sqlite3": "^12.11.1", "fastify": "^5.12.5", - "nodemailer": "^7.0.13" + "nodemailer": "^7.0.13", + "onnxruntime-node": "^1.30.0" }, "devDependencies": { "@types/better-sqlite3": "^9.6.0", diff --git a/docker/backend/src/inpaintService.ts b/docker/backend/src/inpaintService.ts new file mode 100644 index 0000000..062282c --- /dev/null +++ b/docker/backend/src/inpaintService.ts @@ -0,0 +1,74 @@ +import * as ort from 'onnxruntime-node'; +import { existsSync } from 'node:fs'; + +const MODEL_PATH = process.env.INPAINT_MODEL_PATH || '/models/migan_pipeline_v2.onnx'; + +let sessionPromise: Promise | null = null; + +export async function getInpaintSession(): Promise { + if (!sessionPromise) { + sessionPromise = (async () => { + const candidates = [ + MODEL_PATH, + '/home/locpham/RecipesCam/docker/frontend/public/models/migan_pipeline_v2.onnx', + '../frontend/public/models/migan_pipeline_v2.onnx', + './models/migan_pipeline_v2.onnx', + ]; + let foundPath: string | null = null; + for (const p of candidates) { + if (existsSync(p)) { + foundPath = p; + break; + } + } + if (!foundPath) { + console.warn('[inpaintService] Model file not found in candidates:', candidates); + return null; + } + + console.log('[inpaintService] Loading ONNX model from:', foundPath); + const t0 = Date.now(); + const session = await ort.InferenceSession.create(foundPath, { + executionProviders: ['cpu'], + graphOptimizationLevel: 'all', + }); + console.log('[inpaintService] Model loaded in ' + (Date.now() - t0) + 'ms'); + return session; + })().catch((err) => { + console.error('[inpaintService] Failed to load ONNX model:', err); + sessionPromise = null; + return null; + }); + } + return sessionPromise; +} + +export async function runServerInpaint( + srcBytes: Uint8Array, + maskBytes: Uint8Array, + w: number, + h: number +): Promise { + const session = await getInpaintSession(); + if (!session) return null; + + const in0 = new ort.Tensor('uint8', srcBytes, [1, 3, h, w]); + const in1 = new ort.Tensor('uint8', maskBytes, [1, 1, h, w]); + + const out = await session.run({ + [session.inputNames[0]]: in0, + [session.inputNames[1]]: in1, + }); + + const tensor = out[session.outputNames[0]]; + const data = tensor.data as Uint8Array; + const patch = new Uint8Array(4 * w * h); + const plane = w * h; + for (let i = 0; i < plane; i++) { + patch[i * 4] = data[i]; + patch[i * 4 + 1] = data[plane + i]; + patch[i * 4 + 2] = data[2 * plane + i]; + patch[i * 4 + 3] = 255; + } + return patch; +} diff --git a/docker/backend/src/server.ts b/docker/backend/src/server.ts index 4d982b5..ee895af 100644 --- a/docker/backend/src/server.ts +++ b/docker/backend/src/server.ts @@ -1,3 +1,4 @@ +import { runServerInpaint } from './inpaintService'; import { recipeFile } from './recipeFile'; import { sendVerificationMail } from './mailer'; import { placeName } from './place'; @@ -425,6 +426,43 @@ async function lookupGeo(ip: string): Promise { // ---- routes ---- app.get('/api/health', async () => ({ ok: true })); +// AI Inpainting endpoint (Hybrid Option 2) +app.post('/api/inpaint', { bodyLimit: 10 * 1024 * 1024 }, async (req, reply) => { + const b = bodyOf(req) as { + src?: string; + mask?: string; + w?: number; + h?: number; + }; + if (!b || !b.src || !b.mask || typeof b.w !== 'number' || typeof b.h !== 'number') { + return reply.status(400).send({ ok: false, error: 'Missing src, mask, w, or h' }); + } + + const { w, h } = b; + const srcBuf = Buffer.from(b.src, 'base64'); + const maskBuf = Buffer.from(b.mask, 'base64'); + + if (srcBuf.length !== 3 * w * h || maskBuf.length !== w * h) { + return reply.status(400).send({ ok: false, error: 'Dimension mismatch' }); + } + + try { + const patch = await runServerInpaint(new Uint8Array(srcBuf), new Uint8Array(maskBuf), w, h); + if (!patch) { + return reply.status(503).send({ ok: false, error: 'Inpaint model unavailable' }); + } + return reply.send({ + ok: true, + patch: Buffer.from(patch).toString('base64'), + w, + h, + }); + } catch (err: any) { + req.log.error(err, 'Inpaint inference error'); + return reply.status(500).send({ ok: false, error: String(err?.message || err) }); + } +}); + // Native file manager launcher (Open in window explorer) app.post('/api/open-explorer', async (req, reply) => { const b = bodyOf(req); diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 667412d..26726ed 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -24,9 +24,12 @@ services: SMTP_PASS: ${SMTP_PASS:-} SMTP_FROM: ${SMTP_FROM:-} SMTP_SECURE: ${SMTP_SECURE:-} + INPAINT_MODEL_PATH: /models/migan_pipeline_v2.onnx volumes: # SQLite (WAL) lives on the host so a rebuild never loses accounts. - ./data:/data + # Models directory mounted read-only for server-side AI inpainting + - ./frontend/public/models:/models:ro # Static SPA + /api reverse proxy. "api" is the DNS name nginx.conf proxies to, # so this service name is load-bearing.