feat(heal): add server-side AI inpainting endpoint via ONNX Runtime

2026-10-16 20:23:24 +07:00
parent a5031c76d5
commit 8aa59bcfb2
5 changed files with 321 additions and 2 deletions
@@ -10,7 +10,8 @@
      "dependencies": {
        "better-sqlite3": "^12.11.1",
        "fastify": "^5.12.5",
-        "nodemailer": "^7.0.13"
+        "nodemailer": "^7.0.13",
+        "onnxruntime-node": "^1.30.0"
      },
      "devDependencies": {
        "@types/better-sqlite3": "^9.6.0",
@@ -615,6 +616,15 @@
      "integrity": "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==",
      "license": "MIT"
    },
+    "node_modules/adm-zip": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.1.tgz",
+      "integrity": "sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=14.0"
+      }
+    },
    "node_modules/ajv": {
      "version": "8.20.0",
      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz",
@@ -814,6 +824,40 @@
        "node": ">=4.0.0"
      }
    },
+    "node_modules/define-data-property": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz",
+      "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==",
+      "license": "MIT",
+      "dependencies": {
+        "es-define-property": "^1.0.0",
+        "es-errors": "^1.3.0",
+        "gopd": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/define-properties": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz",
+      "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==",
+      "license": "MIT",
+      "dependencies": {
+        "define-data-property": "^1.0.1",
+        "has-property-descriptors": "^1.0.0",
+        "object-keys": "^1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
    "node_modules/dequal": {
      "version": "2.0.3",
      "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
@@ -841,6 +885,24 @@
        "once": "^1.4.0"
      }
    },
+    "node_modules/es-define-property": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
+      "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-errors": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+      "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
    "node_modules/esbuild": {
      "version": "0.28.2",
      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz",
@@ -883,6 +945,18 @@
        "@esbuild/win32-x64": "0.28.2"
      }
    },
+    "node_modules/escape-string-regexp": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
+      "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
    "node_modules/expand-template": {
      "version": "2.0.3",
      "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
@@ -1042,6 +1116,61 @@
      "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",
      "license": "MIT"
    },
+    "node_modules/global-agent": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-4.1.3.tgz",
+      "integrity": "sha512-KUJEViiuFT3I97t+GYMikLPJS2Lfo/S2F+DQuBWzuzaMPnvt5yyZePzArx36fBzpGTxZjIpDbXLeySLgh+k76g==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "globalthis": "^1.0.2",
+        "matcher": "^4.0.0",
+        "semver": "^7.3.5",
+        "serialize-error": "^8.1.0"
+      },
+      "engines": {
+        "node": ">=10.0"
+      }
+    },
+    "node_modules/globalthis": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz",
+      "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==",
+      "license": "MIT",
+      "dependencies": {
+        "define-properties": "^1.2.1",
+        "gopd": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/gopd": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
+      "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/has-property-descriptors": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz",
+      "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==",
+      "license": "MIT",
+      "dependencies": {
+        "es-define-property": "^1.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
    "node_modules/ieee754": {
      "version": "1.2.1",
      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
@@ -1145,6 +1274,21 @@
      ],
      "license": "MIT"
    },
+    "node_modules/matcher": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/matcher/-/matcher-4.0.0.tgz",
+      "integrity": "sha512-S6x5wmcDmsDRRU/c2dkccDwQPXoFczc5+HpQ2lON8pnvHlnvHAHj5WlLVvw6n6vNyHuVugYrFohYxbS+pvFpKQ==",
+      "license": "MIT",
+      "dependencies": {
+        "escape-string-regexp": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
    "node_modules/mimic-response": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
@@ -1199,6 +1343,15 @@
        "node": ">=6.0.0"
      }
    },
+    "node_modules/object-keys": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
+      "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
    "node_modules/on-exit-leak-free": {
      "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
@@ -1217,6 +1370,29 @@
        "wrappy": "1"
      }
    },
+    "node_modules/onnxruntime-common": {
+      "version": "1.30.0",
+      "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.30.0.tgz",
+      "integrity": "sha512-7fdVWjAID1dVhH/G8qK3APARunV4VkBFoCQAP7qp4Wkab0mrorvmc+sqiT+mKXOzDqdjN5j+/Z9nb4gzNPWcyA==",
+      "license": "MIT"
+    },
+    "node_modules/onnxruntime-node": {
+      "version": "1.30.0",
+      "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.30.0.tgz",
+      "integrity": "sha512-twhs1C2C/BFkz1yc5OY0KIU2GUq6DURO7hD4bx5Q2Qy3nAMJwRXW8xU3NVczE29VA9lolLOYepoD8fjTGOfIqw==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "os": [
+        "win32",
+        "darwin",
+        "linux"
+      ],
+      "dependencies": {
+        "adm-zip": "^0.6.0",
+        "global-agent": "^4.1.3",
+        "onnxruntime-common": "1.30.0"
+      }
+    },
    "node_modules/pino": {
      "version": "10.3.1",
      "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz",
@@ -1464,6 +1640,21 @@
        "node": ">=10"
      }
    },
+    "node_modules/serialize-error": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz",
+      "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==",
+      "license": "MIT",
+      "dependencies": {
+        "type-fest": "^0.20.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
    "node_modules/set-cookie-parser": {
      "version": "2.7.2",
      "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz",
@@ -1637,6 +1828,18 @@
        "node": "*"
      }
    },
+    "node_modules/type-fest": {
+      "version": "0.20.2",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
+      "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
+      "license": "(MIT OR CC0-1.0)",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
    "node_modules/typescript": {
      "version": "5.9.3",
      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
@@ -14,7 +14,8 @@
  "dependencies": {
    "better-sqlite3": "^12.11.1",
    "fastify": "^5.12.5",
-    "nodemailer": "^7.0.13"
+    "nodemailer": "^7.0.13",
+    "onnxruntime-node": "^1.30.0"
  },
  "devDependencies": {
    "@types/better-sqlite3": "^9.6.0",
@@ -0,0 +1,74 @@
+import * as ort from 'onnxruntime-node';
+import { existsSync } from 'node:fs';
+
+const MODEL_PATH = process.env.INPAINT_MODEL_PATH || '/models/migan_pipeline_v2.onnx';
+
+let sessionPromise: Promise<ort.InferenceSession | null> | null = null;
+
+export async function getInpaintSession(): Promise<ort.InferenceSession | null> {
+  if (!sessionPromise) {
+    sessionPromise = (async () => {
+      const candidates = [
+        MODEL_PATH,
+        '/home/locpham/RecipesCam/docker/frontend/public/models/migan_pipeline_v2.onnx',
+        '../frontend/public/models/migan_pipeline_v2.onnx',
+        './models/migan_pipeline_v2.onnx',
+      ];
+      let foundPath: string | null = null;
+      for (const p of candidates) {
+        if (existsSync(p)) {
+          foundPath = p;
+          break;
+        }
+      }
+      if (!foundPath) {
+        console.warn('[inpaintService] Model file not found in candidates:', candidates);
+        return null;
+      }
+
+      console.log('[inpaintService] Loading ONNX model from:', foundPath);
+      const t0 = Date.now();
+      const session = await ort.InferenceSession.create(foundPath, {
+        executionProviders: ['cpu'],
+        graphOptimizationLevel: 'all',
+      });
+      console.log('[inpaintService] Model loaded in ' + (Date.now() - t0) + 'ms');
+      return session;
+    })().catch((err) => {
+      console.error('[inpaintService] Failed to load ONNX model:', err);
+      sessionPromise = null;
+      return null;
+    });
+  }
+  return sessionPromise;
+}
+
+export async function runServerInpaint(
+  srcBytes: Uint8Array,
+  maskBytes: Uint8Array,
+  w: number,
+  h: number
+): Promise<Uint8Array | null> {
+  const session = await getInpaintSession();
+  if (!session) return null;
+
+  const in0 = new ort.Tensor('uint8', srcBytes, [1, 3, h, w]);
+  const in1 = new ort.Tensor('uint8', maskBytes, [1, 1, h, w]);
+
+  const out = await session.run({
+    [session.inputNames[0]]: in0,
+    [session.inputNames[1]]: in1,
+  });
+
+  const tensor = out[session.outputNames[0]];
+  const data = tensor.data as Uint8Array;
+  const patch = new Uint8Array(4 * w * h);
+  const plane = w * h;
+  for (let i = 0; i < plane; i++) {
+    patch[i * 4] = data[i];
+    patch[i * 4 + 1] = data[plane + i];
+    patch[i * 4 + 2] = data[2 * plane + i];
+    patch[i * 4 + 3] = 255;
+  }
+  return patch;
+}
@@ -1,3 +1,4 @@
+import { runServerInpaint } from './inpaintService';
 import { recipeFile } from './recipeFile';
 import { sendVerificationMail } from './mailer';
 import { placeName } from './place';
@@ -425,6 +426,43 @@ async function lookupGeo(ip: string): Promise<GeoPlace | null> {
 // ---- routes ----
 app.get('/api/health', async () => ({ ok: true }));

+// AI Inpainting endpoint (Hybrid Option 2)
+app.post('/api/inpaint', { bodyLimit: 10 * 1024 * 1024 }, async (req, reply) => {
+  const b = bodyOf(req) as {
+    src?: string;
+    mask?: string;
+    w?: number;
+    h?: number;
+  };
+  if (!b || !b.src || !b.mask || typeof b.w !== 'number' || typeof b.h !== 'number') {
+    return reply.status(400).send({ ok: false, error: 'Missing src, mask, w, or h' });
+  }
+
+  const { w, h } = b;
+  const srcBuf = Buffer.from(b.src, 'base64');
+  const maskBuf = Buffer.from(b.mask, 'base64');
+
+  if (srcBuf.length !== 3 * w * h || maskBuf.length !== w * h) {
+    return reply.status(400).send({ ok: false, error: 'Dimension mismatch' });
+  }
+
+  try {
+    const patch = await runServerInpaint(new Uint8Array(srcBuf), new Uint8Array(maskBuf), w, h);
+    if (!patch) {
+      return reply.status(503).send({ ok: false, error: 'Inpaint model unavailable' });
+    }
+    return reply.send({
+      ok: true,
+      patch: Buffer.from(patch).toString('base64'),
+      w,
+      h,
+    });
+  } catch (err: any) {
+    req.log.error(err, 'Inpaint inference error');
+    return reply.status(500).send({ ok: false, error: String(err?.message || err) });
+  }
+});
+
 // Native file manager launcher (Open in window explorer)
 app.post('/api/open-explorer', async (req, reply) => {
  const b = bodyOf(req);