feat(admin): back the data up, and put it back, from the admin tool
A new BACKUP tab downloads the deployment's whole state — the SQLite file and both media folders, photos included — as one .tar.gz, and takes the same file back. That one artefact therefore does both jobs: the operator's backup and the data package that moves an install onto another box. The database is snapshotted through SQLite's own backup rather than copied, because the file is written to while the archive streams; the media folders are tarred straight off the volume, so no second copy of them is made. A restore replaces the data on disk and then exits — the container's restart policy brings the API back on the restored files, which is the only moment the open handle can be dropped. The state being replaced is tarred aside first, and the archive is checked for `..` entries before anything is unpacked. The API authenticates that route before it reads a byte, and nginx lets that one path past the body cap which holds everywhere else.
This commit is contained in:
@@ -2,10 +2,13 @@ import { recipeFile } from './recipeFile';
|
||||
import { sendVerificationMail } from './mailer';
|
||||
import { placeName } from './place';
|
||||
import Fastify, { type FastifyReply, type FastifyRequest } from 'fastify';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import { readFileSync, unlinkSync, writeFileSync } from 'node:fs';
|
||||
import { basename } from 'node:path';
|
||||
import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { basename, join } from 'node:path';
|
||||
import {
|
||||
DATA_DIR,
|
||||
MAX_PHOTO_BYTES,
|
||||
MAX_PHOTOS_PER_USER,
|
||||
MAX_RECIPE_BYTES,
|
||||
@@ -19,6 +22,7 @@ import {
|
||||
createRecipe,
|
||||
createSession,
|
||||
createUser,
|
||||
db,
|
||||
deleteAllPhotos,
|
||||
deletePhoto,
|
||||
deletePhotoOf,
|
||||
@@ -130,6 +134,14 @@ app.addContentTypeParser(['image/jpeg', 'image/png', 'image/webp'], { parseAs: '
|
||||
done(null, body);
|
||||
});
|
||||
|
||||
// A restore body is a whole data dir, so it is the one body far past the
|
||||
// instance limit. Buffered, not streamed: `tar` needs the archive to be
|
||||
// seekable, and the route is admin-gated (see gateAdmin) before a byte is read.
|
||||
const RESTORE_LIMIT = 4 * 1024 * 1024 * 1024;
|
||||
app.addContentTypeParser('application/gzip', { parseAs: 'buffer', bodyLimit: RESTORE_LIMIT }, (_req, body, done) => {
|
||||
done(null, body);
|
||||
});
|
||||
|
||||
// ---- rate limiting --------------------------------------------------------
|
||||
// Fixed window keyed on what the caller is trying to abuse — an email, or a
|
||||
// user id — rather than an address: the API sits behind two proxies, so a
|
||||
@@ -1085,6 +1097,118 @@ app.patch<{ Params: { id: string } }>('/api/admin/photos/:id', async (req, reply
|
||||
return reply.status(200).send({ id, slots: set });
|
||||
});
|
||||
|
||||
// ---- backup / restore -----------------------------------------------------
|
||||
// The deployment's whole state is DATA_DIR: one SQLite file plus the two media
|
||||
// folders. The archive is a plain tar.gz of exactly those three, which makes
|
||||
// one artefact serve both jobs — the operator's backup, and the data package
|
||||
// that moves an install onto another box (the restore route below takes the
|
||||
// very same file).
|
||||
const BACKUP_PATHS = ['recipescam.db', 'uploads', 'avatars'];
|
||||
|
||||
// Sortable and filename-safe, so two backups in one day never collide.
|
||||
const stamp = () => new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
||||
|
||||
// Authenticate before the body is read. Every other route can afford to parse
|
||||
// first and refuse after; the restore route cannot, because a rejected caller
|
||||
// would already have cost the process gigabytes of memory.
|
||||
function gateAdmin(req: FastifyRequest, reply: FastifyReply): boolean {
|
||||
const user = admin(req);
|
||||
if ('status' in user) {
|
||||
reply.status(user.status).send({ error: user.status === 401 ? 'unauthorized' : 'forbidden' });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Admin only: download the data dir. The database is written to while the
|
||||
// archive streams, so it is snapshotted through SQLite's own backup rather than
|
||||
// copied — a plain cp could catch a half-committed page. The media folders are
|
||||
// append-only and are tarred straight off the volume, so no second copy of them
|
||||
// is ever made.
|
||||
app.get('/api/admin/backup', async (req, reply) => {
|
||||
const user = admin(req);
|
||||
if ('status' in user) return reply.status(user.status).send({ error: user.status === 401 ? 'unauthorized' : 'forbidden' });
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'rc-backup-'));
|
||||
try {
|
||||
await db.backup(join(tmp, 'recipescam.db'));
|
||||
} catch (err) {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
req.log.error(err);
|
||||
return reply.status(500).send({ error: 'could not snapshot the database' });
|
||||
}
|
||||
const tar = spawn('tar', ['-czf', '-', '-C', tmp, 'recipescam.db', '-C', DATA_DIR, 'uploads', 'avatars']);
|
||||
tar.on('close', () => rmSync(tmp, { recursive: true, force: true }));
|
||||
tar.on('error', (err) => {
|
||||
req.log.error(err);
|
||||
tar.stdout.destroy();
|
||||
});
|
||||
tar.stderr.on('data', (chunk) => req.log.error(`backup tar: ${String(chunk).trim()}`));
|
||||
reply.header('content-type', 'application/gzip');
|
||||
reply.header('content-disposition', `attachment; filename="recipescam-backup-${stamp()}.tar.gz"`);
|
||||
return reply.send(tar.stdout);
|
||||
});
|
||||
|
||||
// Admin only: put a backup back. The body is the archive itself (see the gzip
|
||||
// parser above), and it replaces the data on disk — so the current state is
|
||||
// tarred aside first, and the process then exits: `restart: unless-stopped`
|
||||
// brings it back up on the restored files, which is the only moment the open
|
||||
// SQLite handle can be dropped safely.
|
||||
app.post(
|
||||
'/api/admin/restore',
|
||||
{
|
||||
bodyLimit: RESTORE_LIMIT,
|
||||
onRequest: (req, reply, done) => {
|
||||
if (gateAdmin(req, reply)) done();
|
||||
},
|
||||
},
|
||||
async (req, reply) => {
|
||||
const body = req.body as unknown;
|
||||
if (!Buffer.isBuffer(body) || body.length === 0) return reply.status(400).send({ error: 'empty body' });
|
||||
// Unpacked beside the data it is about to replace, not in /tmp: the media
|
||||
// folders are hundreds of megabytes, and this keeps the swap on one
|
||||
// filesystem. The name is dotted so it can never show up as a media folder.
|
||||
const tmp = join(DATA_DIR, `.restore-${Date.now()}`);
|
||||
try {
|
||||
mkdirSync(tmp, { recursive: true });
|
||||
const archive = join(tmp, 'in.tar.gz');
|
||||
writeFileSync(archive, body);
|
||||
// An archive is untrusted input even when an admin sent it: a `..` entry
|
||||
// would write anywhere in the container. The listing is checked before
|
||||
// anything is unpacked, and tar itself refuses absolute names.
|
||||
const listed = spawnSync('tar', ['-tzf', archive], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
|
||||
if (listed.status !== 0) return reply.status(400).send({ error: 'not a readable .tar.gz' });
|
||||
const unsafe = listed.stdout.split('\n').some((name) => name.startsWith('/') || name.split('/').includes('..'));
|
||||
if (unsafe) return reply.status(400).send({ error: 'archive has unsafe paths' });
|
||||
const unpacked = spawnSync('tar', ['-xzf', archive, '-C', tmp, '--no-same-owner'], {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
});
|
||||
if (unpacked.status !== 0 || !existsSync(join(tmp, 'recipescam.db')))
|
||||
return reply.status(400).send({ error: 'archive has no recipescam.db' });
|
||||
// Close before the swap: a checkpoint on close is what makes the safety
|
||||
// copy below a complete database rather than one missing its WAL.
|
||||
db.close();
|
||||
spawnSync('tar', ['-czf', join(DATA_DIR, `.pre-restore-${stamp()}.tar.gz`), '-C', DATA_DIR, ...BACKUP_PATHS], {
|
||||
stdio: 'ignore',
|
||||
});
|
||||
for (const dir of ['uploads', 'avatars']) {
|
||||
const from = join(tmp, dir);
|
||||
rmSync(join(DATA_DIR, dir), { recursive: true, force: true });
|
||||
mkdirSync(join(DATA_DIR, dir), { recursive: true });
|
||||
if (existsSync(from)) cpSync(from, join(DATA_DIR, dir), { recursive: true });
|
||||
}
|
||||
cpSync(join(tmp, 'recipescam.db'), join(DATA_DIR, 'recipescam.db'));
|
||||
// Whatever the outgoing database left behind must not be replayed onto the
|
||||
// incoming one.
|
||||
for (const suffix of ['-wal', '-shm']) rmSync(join(DATA_DIR, `recipescam.db${suffix}`), { force: true });
|
||||
setTimeout(() => process.exit(0), 250);
|
||||
return reply.status(200).send({ ok: true, restarting: true });
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
app
|
||||
.listen({ port: PORT, host: HOST })
|
||||
.catch((err) => {
|
||||
|
||||
Reference in New Issue
Block a user