From 7e16ddc4c63e33a3450d3b73cffcb99a7b69ec5d Mon Sep 17 00:00:00 2001 From: Julien Date: Mon, 7 Sep 2026 10:52:34 +0200 Subject: [PATCH] Add configuration export and restore --- README.md | 6 +- src/configuration-export.test.ts | 50 ++++++++++ src/configuration-export.ts | 123 ++++++++++++++++++++++++ src/configuration-restore.test.ts | 154 ++++++++++++++++++++++++++++++ src/configuration-restore.ts | 154 ++++++++++++++++++++++++++++++ src/db.test.ts | 34 +++++++ src/db.ts | 25 +++++ src/executor.test.ts | 64 ++++++++++++- src/executor.ts | 115 +++++++++++++++------- src/schemas.test.ts | 7 ++ src/schemas.ts | 11 ++- src/server.ts | 6 ++ web/src/main.tsx | 83 +++++++++++----- web/src/styles.css | 1 + 14 files changed, 767 insertions(+), 66 deletions(-) create mode 100644 src/configuration-export.test.ts create mode 100644 src/configuration-export.ts create mode 100644 src/configuration-restore.test.ts create mode 100644 src/configuration-restore.ts create mode 100644 src/db.test.ts diff --git a/README.md b/README.md index e60f7ce..8078ac8 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Supported operations: - Execute commands with zero or more declared outputs, allowing preparation and cleanup steps. - Create compressed PostgreSQL or MySQL dumps using tools installed on the remote host. - Archive a remote directory as `tar.gz`. +- Export restore-ready Backup Manager configuration with encrypted credential envelopes. - Run jobs manually or with timezone-aware cron schedules. - Retain a configured number of successful runs and notify by webhook or SMTP. @@ -49,6 +50,7 @@ Every job targets one SSH host and runs its steps sequentially over one pinned c - **Remote command:** runs an executable with explicit arguments on the SSH host. It can collect generated host files or archives. - **PostgreSQL/MySQL dump:** creates a compressed dump using an encrypted password associated with that step. - **Directory archive:** archives an absolute host path. +- **Backup Manager configuration:** creates a local, integrity-protected JSON artifact containing hosts, jobs, schedules, and notification settings. Credential values remain encrypted and restoration requires the original `MASTER_KEY`; the key itself is never exported. Command arguments are entered one per line and are passed as distinct shell-quoted values; shell pipelines and redirection are not interpreted. Add an explicit script on the remote system when more complex command logic is required. @@ -86,4 +88,6 @@ Run all checks in that order with `npm run check`. - `backups///` contains completed artifacts. - `backups/.staging/` contains in-progress downloads and is cleaned when a run fails. -Back up both the data and artifact volumes together. Restore workflows are not included in this release. +Configuration exports omit run history, artifact metadata, artifact files, the administrator password, runtime paths, and `MASTER_KEY`. Store the exported JSON and the key separately. For a complete storage-level backup, back up both the data and artifact volumes together. + +Restore a configuration export from **Settings > Restore configuration**. Restores require the original `MASTER_KEY`, reject modified exports, update records with matching stable identities, add missing records, and leave unrelated local configuration and run history untouched. Numeric IDs and host references are remapped when necessary. A backup from another manager can initialize an empty installation but cannot merge into a populated installation. Restore is blocked while any job is queued or running, and conflicting names owned by unrelated local records must be resolved first. diff --git a/src/configuration-export.test.ts b/src/configuration-export.test.ts new file mode 100644 index 0000000..8c3633c --- /dev/null +++ b/src/configuration-export.test.ts @@ -0,0 +1,50 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { configurationExportSchema, createConfigurationExport } from './configuration-export.js'; +import { encryptJson } from './crypto.js'; +import { openDatabase } from './db.js'; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) fs.rmSync(directory, { recursive: true, force: true }); +}); + +describe('configuration export', () => { + it('exports restore-relevant configuration without operational history', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'backup-manager-export-')); + temporaryDirectories.push(directory); + const db = openDatabase(directory); + const key = Buffer.alloc(32, 7); + const hostSecret = encryptJson({ password: 'host-plaintext' }, key); + const jobSecret = encryptJson({ database: 'database-plaintext' }, key); + const settingsSecret = encryptJson({ smtp: { password: 'smtp-plaintext' } }, key); + db.prepare('INSERT INTO hosts (id, name, hostname, port, username, fingerprint, auth_type, secret) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') + .run(7, 'server', 'server.local', 22, 'backup', `SHA256:${'A'.repeat(43)}`, 'password', hostSecret); + db.prepare('INSERT INTO jobs (id, name, host_id, config, secret, schedule, timezone, enabled, retention_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)') + .run(11, 'manager', 7, JSON.stringify({ steps: [{ id: 'config', name: 'Manager configuration', type: 'managerConfig', continueOnError: false, outputName: 'manager-config' }] }), jobSecret, '0 2 * * *', 'UTC', 1, 5); + db.prepare('INSERT INTO settings (key, value) VALUES (?, ?)').run('notifications', settingsSecret); + db.prepare("INSERT INTO runs (job_id, status, trigger) VALUES (?, 'succeeded', 'manual')").run(11); + + const exported = createConfigurationExport(db, key, new Date('2026-09-05T12:00:00Z')); + db.close(); + + expect(exported).toMatchObject({ + format: 'backup-script-manager/configuration', version: 1, exportedAt: '2026-09-05T12:00:00.000Z', + hosts: [{ id: 7, authType: 'password', secret: hostSecret }], + jobs: [{ id: 11, hostId: 7, secret: jobSecret, retentionCount: 5 }], + settings: [{ key: 'notifications', value: settingsSecret }], + }); + const serialized = JSON.stringify(exported); + expect(serialized).not.toContain('runs'); + expect(serialized).not.toContain('artifacts'); + expect(serialized).not.toContain('host-plaintext'); + expect(serialized).not.toContain('database-plaintext'); + expect(serialized).not.toContain('smtp-plaintext'); + expect(exported.jobs[0]).not.toHaveProperty('nextRunAt'); + expect(configurationExportSchema.safeParse({ ...exported, version: 2 }).success).toBe(false); + expect(configurationExportSchema.safeParse({ ...exported, unexpected: true }).success).toBe(false); + }); +}); diff --git a/src/configuration-export.ts b/src/configuration-export.ts new file mode 100644 index 0000000..84d90ae --- /dev/null +++ b/src/configuration-export.ts @@ -0,0 +1,123 @@ +import { createHmac, timingSafeEqual } from 'node:crypto'; +import type Database from 'better-sqlite3'; +import { CronExpressionParser } from 'cron-parser'; +import { z } from 'zod'; +import type { HostRecord, JobRecord } from './db.js'; +import { jobConfigSchema, normalizeJobConfig } from './schemas.js'; + +const exportedHostSchema = z.object({ + id: z.number().int().positive(), + stableId: z.string().regex(/^[a-f0-9]{32}$/), + name: z.string().trim().min(1).max(100), + hostname: z.string().trim().min(1).max(253), + port: z.number().int().min(1).max(65535), + username: z.string().trim().min(1).max(100), + fingerprint: z.string().regex(/^SHA256:[A-Za-z0-9+/]{43}=?$/, 'Expected an SHA256 SSH fingerprint'), + authType: z.enum(['password', 'privateKey']), + secret: z.string().min(1).max(2 * 1024 * 1024), +}).strict(); + +const exportedJobSchema = z.object({ + id: z.number().int().positive(), + stableId: z.string().regex(/^[a-f0-9]{32}$/), + name: z.string().trim().min(1).max(100), + hostId: z.number().int().positive(), + config: jobConfigSchema, + secret: z.string().min(1).max(2 * 1024 * 1024).nullable(), + schedule: z.string().trim().min(1).nullable(), + timezone: z.string().trim().min(1).max(100), + enabled: z.boolean(), + retentionCount: z.number().int().min(1).max(1000), +}).strict().superRefine((job, context) => { + if (!job.schedule) return; + try { + CronExpressionParser.parse(job.schedule, { tz: job.timezone }); + } catch { + context.addIssue({ code: 'custom', path: ['schedule'], message: 'Invalid cron expression or timezone' }); + } +}); + +const configurationExportFields = { + format: z.literal('backup-script-manager/configuration'), + version: z.literal(1), + sourceInstanceId: z.uuid(), + exportedAt: z.iso.datetime(), + hosts: z.array(exportedHostSchema).max(10_000), + jobs: z.array(exportedJobSchema).max(10_000), + settings: z.array(z.object({ key: z.string().min(1).max(100).regex(/^[a-zA-Z0-9._-]+$/), value: z.string().min(1).max(2 * 1024 * 1024) }).strict()).max(1_000), +}; + +export const configurationExportSchema = z.object({ + ...configurationExportFields, + integrity: z.string().regex(/^[A-Za-z0-9_-]{43}$/), +}).strict().superRefine((configuration, context) => { + checkUnique(configuration.hosts.map((host) => host.id), ['hosts'], 'Host IDs must be unique', context); + checkUnique(configuration.hosts.map((host) => host.stableId), ['hosts'], 'Host stable IDs must be unique', context); + checkUnique(configuration.hosts.map((host) => host.name), ['hosts'], 'Host names must be unique', context); + checkUnique(configuration.jobs.map((job) => job.id), ['jobs'], 'Job IDs must be unique', context); + checkUnique(configuration.jobs.map((job) => job.stableId), ['jobs'], 'Job stable IDs must be unique', context); + checkUnique(configuration.jobs.map((job) => job.name), ['jobs'], 'Job names must be unique', context); + checkUnique(configuration.settings.map((setting) => setting.key), ['settings'], 'Setting keys must be unique', context); + const hostIds = new Set(configuration.hosts.map((host) => host.id)); + for (const [index, job] of configuration.jobs.entries()) { + if (!hostIds.has(job.hostId)) context.addIssue({ code: 'custom', path: ['jobs', index, 'hostId'], message: 'Job references a host that is not in the export' }); + } +}); + +export type ConfigurationExport = z.infer; +type UnsignedConfigurationExport = Omit; + +export function createConfigurationExport(db: Database.Database, masterKey: Buffer, exportedAt = new Date()): ConfigurationExport { + return db.transaction(() => { + const hostRows = db.prepare('SELECT * FROM hosts ORDER BY id').all() as HostRecord[]; + const jobRows = db.prepare('SELECT * FROM jobs ORDER BY id').all() as JobRecord[]; + const settings = db.prepare('SELECT key, value FROM settings ORDER BY key').all() as Array<{ key: string; value: string }>; + const metadata = db.prepare("SELECT value FROM metadata WHERE key = 'instance_id'").get() as { value: string }; + const unsigned = { + format: 'backup-script-manager/configuration' as const, + version: 1 as const, + sourceInstanceId: metadata.value, + exportedAt: exportedAt.toISOString(), + hosts: hostRows.map((host) => ({ + id: host.id, + stableId: host.stable_id, + name: host.name, + hostname: host.hostname, + port: host.port, + username: host.username, + fingerprint: host.fingerprint, + authType: host.auth_type, + secret: host.secret, + })), + jobs: jobRows.map((job) => ({ + id: job.id, + stableId: job.stable_id, + name: job.name, + hostId: job.host_id, + config: normalizeJobConfig(JSON.parse(job.config), String(job.id)), + secret: job.secret, + schedule: job.schedule, + timezone: job.timezone, + enabled: Boolean(job.enabled), + retentionCount: job.retention_count, + })), + settings, + }; + return { ...unsigned, integrity: configurationIntegrity(unsigned, masterKey) }; + })(); +} + +export function verifyConfigurationExportIntegrity(configuration: ConfigurationExport, masterKey: Buffer): boolean { + const { integrity, ...unsigned } = configuration; + const expected = Buffer.from(configurationIntegrity(unsigned, masterKey), 'base64url'); + const actual = Buffer.from(integrity, 'base64url'); + return actual.length === expected.length && timingSafeEqual(actual, expected); +} + +function checkUnique(values: Array, path: PropertyKey[], message: string, context: z.RefinementCtx): void { + if (new Set(values).size !== values.length) context.addIssue({ code: 'custom', path, message }); +} + +function configurationIntegrity(configuration: UnsignedConfigurationExport, masterKey: Buffer): string { + return createHmac('sha256', masterKey).update('backup-script-manager/configuration/v1\0').update(JSON.stringify(configuration)).digest('base64url'); +} diff --git a/src/configuration-restore.test.ts b/src/configuration-restore.test.ts new file mode 100644 index 0000000..f3febaf --- /dev/null +++ b/src/configuration-restore.test.ts @@ -0,0 +1,154 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createConfigurationExport, type ConfigurationExport } from './configuration-export.js'; +import { ConfigurationRestoreError, restoreConfiguration } from './configuration-restore.js'; +import { encryptJson } from './crypto.js'; +import { openDatabase } from './db.js'; + +const temporaryDirectories: string[] = []; +const masterKey = Buffer.alloc(32, 9); +const nextRunAt = '2030-01-01T02:00:00.000Z'; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) fs.rmSync(directory, { recursive: true, force: true }); +}); + +describe('configuration restore', () => { + it('updates matching records, adds missing records, and preserves unrelated configuration and history', () => { + const configuration = sourceConfiguration(); + const db = targetDatabase(configuration); + + const result = restoreConfiguration(db, configuration, masterKey, () => nextRunAt); + + expect(result).toEqual({ hosts: { added: 0, updated: 1 }, jobs: { added: 0, updated: 1 }, settings: { added: 0, updated: 1 } }); + expect(db.prepare('SELECT name, hostname, secret FROM hosts WHERE id = 1').get()).toEqual({ name: 'restored-host', hostname: 'restored.local', secret: configuration.hosts[0]!.secret }); + expect(db.prepare('SELECT name, schedule, next_run_at AS nextRunAt, secret FROM jobs WHERE id = 1').get()).toEqual({ name: 'restored-job', schedule: '0 2 * * *', nextRunAt, secret: configuration.jobs[0]!.secret }); + expect(db.prepare('SELECT COUNT(*) AS count FROM runs WHERE job_id = 1').get()).toEqual({ count: 1 }); + expect(db.prepare('SELECT name FROM hosts WHERE id = 2').get()).toEqual({ name: 'unrelated-host' }); + expect(db.prepare('SELECT name FROM jobs WHERE id = 2').get()).toEqual({ name: 'unrelated-job' }); + db.close(); + }); + + it('rejects a backup encrypted with another key before changing records', () => { + const configuration = sourceConfiguration(); + const db = targetDatabase(configuration); + expect(() => restoreConfiguration(db, configuration, Buffer.alloc(32, 3), () => nextRunAt)).toThrow('MASTER_KEY'); + expect(db.prepare('SELECT name FROM hosts WHERE id = 1').get()).toEqual({ name: 'current-host' }); + db.close(); + }); + + it('rejects restore while a run is active', () => { + const configuration = sourceConfiguration(); + const db = targetDatabase(configuration); + db.prepare("INSERT INTO runs (job_id, status, trigger) VALUES (2, 'queued', 'manual')").run(); + try { + restoreConfiguration(db, configuration, masterKey, () => nextRunAt); + expect.unreachable('Restore should have been rejected'); + } catch (error) { + expect(error).toBeInstanceOf(ConfigurationRestoreError); + expect((error as ConfigurationRestoreError).statusCode).toBe(409); + } + db.close(); + }); + + it('rolls back all changes when a database write fails', () => { + const configuration = sourceConfiguration(); + const db = targetDatabase(configuration); + db.exec("CREATE TRIGGER reject_restore BEFORE UPDATE ON jobs BEGIN SELECT RAISE(FAIL, 'restore failure'); END"); + + expect(() => restoreConfiguration(db, configuration, masterKey, () => nextRunAt)).toThrow('restore failure'); + + expect(db.prepare('SELECT name FROM hosts WHERE id = 1').get()).toEqual({ name: 'current-host' }); + expect(db.prepare('SELECT name FROM jobs WHERE id = 1').get()).toEqual({ name: 'current-job' }); + db.close(); + }); + + it('rejects modified configuration even when its encrypted values are intact', () => { + const configuration = sourceConfiguration(); + const db = targetDatabase(configuration); + configuration.jobs[0]!.name = 'tampered-job'; + expect(() => restoreConfiguration(db, configuration, masterKey, () => nextRunAt)).toThrow('integrity verification failed'); + expect(db.prepare('SELECT name FROM jobs WHERE id = 1').get()).toEqual({ name: 'current-job' }); + db.close(); + }); + + it('rejects a foreign-instance backup when local configuration exists', () => { + const configuration = sourceConfiguration(); + const db = targetDatabase(); + expect(() => restoreConfiguration(db, configuration, masterKey, () => nextRunAt)).toThrow('different Backup Manager instance'); + db.close(); + }); + + it('rejects names owned by unrelated records in the same manager', () => { + const configuration = sourceConfiguration('unrelated-host'); + const db = targetDatabase(configuration); + expect(() => restoreConfiguration(db, configuration, masterKey, () => nextRunAt)).toThrow('belongs to an unrelated local record'); + expect(db.prepare('SELECT name FROM hosts WHERE id = 1').get()).toEqual({ name: 'current-host' }); + db.close(); + }); + + it('adopts the source identity when restoring into an empty manager', () => { + const configuration = sourceConfiguration(); + const db = newDatabase('empty-target'); + + const result = restoreConfiguration(db, configuration, masterKey, () => nextRunAt); + + expect(result.hosts).toEqual({ added: 1, updated: 0 }); + expect(db.prepare("SELECT value FROM metadata WHERE key = 'instance_id'").get()).toEqual({ value: configuration.sourceInstanceId }); + expect(db.prepare('SELECT name FROM jobs WHERE id = 1').get()).toEqual({ name: 'restored-job' }); + db.close(); + }); + + it('does not overwrite reused numeric IDs or attach their history to restored jobs', () => { + const configuration = sourceConfiguration(); + const db = targetDatabase(configuration, false); + + const result = restoreConfiguration(db, configuration, masterKey, () => nextRunAt); + + expect(result.jobs).toEqual({ added: 1, updated: 0 }); + expect(db.prepare('SELECT name FROM jobs WHERE id = 1').get()).toEqual({ name: 'current-job' }); + expect(db.prepare('SELECT jobs.name FROM runs JOIN jobs ON jobs.id = runs.job_id WHERE runs.job_id = 1').get()).toEqual({ name: 'current-job' }); + expect(db.prepare("SELECT jobs.name, hosts.name AS hostName FROM jobs JOIN hosts ON hosts.id = jobs.host_id WHERE jobs.stable_id = ?").get(configuration.jobs[0]!.stableId)).toEqual({ name: 'restored-job', hostName: 'restored-host' }); + db.close(); + }); +}); + +function sourceConfiguration(hostName = 'restored-host'): ConfigurationExport { + const db = newDatabase('source'); + db.prepare('INSERT INTO hosts (id, name, hostname, port, username, fingerprint, auth_type, secret) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') + .run(1, hostName, 'restored.local', 2222, 'restore', `SHA256:${'A'.repeat(43)}`, 'password', encryptJson({ password: 'restored-password' }, masterKey)); + const config = { steps: [{ id: 'database', name: 'Database', type: 'postgres', continueOnError: false, database: 'app', username: 'postgres', databaseHost: 'localhost', databasePort: 5432, outputName: 'app', timeoutSeconds: 300 }] }; + db.prepare('INSERT INTO jobs (id, name, host_id, config, secret, schedule, timezone, enabled, retention_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)') + .run(1, 'restored-job', 1, JSON.stringify(config), encryptJson({ database: 'restored-database-password' }, masterKey), '0 2 * * *', 'UTC', 1, 5); + db.prepare('INSERT INTO settings (key, value) VALUES (?, ?)').run('notifications', encryptJson({ notifySuccess: false, notifyFailure: true }, masterKey)); + const configuration = createConfigurationExport(db, masterKey, new Date('2026-09-07T12:00:00Z')); + db.close(); + return configuration; +} + +function targetDatabase(configuration?: ConfigurationExport, preserveStableIds = true) { + const db = newDatabase('target'); + if (configuration) db.prepare("UPDATE metadata SET value = ? WHERE key = 'instance_id'").run(configuration.sourceInstanceId); + db.prepare('INSERT INTO hosts (id, name, hostname, port, username, fingerprint, auth_type, secret) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') + .run(1, 'current-host', 'current.local', 22, 'backup', `SHA256:${'B'.repeat(43)}`, 'password', encryptJson({ password: 'current' }, masterKey)); + db.prepare('INSERT INTO hosts (id, name, hostname, port, username, fingerprint, auth_type, secret) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') + .run(2, 'unrelated-host', 'unrelated.local', 22, 'backup', `SHA256:${'C'.repeat(43)}`, 'password', encryptJson({ password: 'unrelated' }, masterKey)); + const config = JSON.stringify({ steps: [{ id: 'files', name: 'Files', type: 'directory', continueOnError: false, path: '/srv/files', outputName: 'files', timeoutSeconds: 300 }] }); + db.prepare('INSERT INTO jobs (id, name, host_id, config, timezone, retention_count) VALUES (?, ?, ?, ?, ?, ?)').run(1, 'current-job', 1, config, 'UTC', 10); + db.prepare('INSERT INTO jobs (id, name, host_id, config, timezone, retention_count) VALUES (?, ?, ?, ?, ?, ?)').run(2, 'unrelated-job', 2, config, 'UTC', 10); + if (configuration && preserveStableIds) { + db.prepare('UPDATE hosts SET stable_id = ? WHERE id = 1').run(configuration.hosts[0]!.stableId); + db.prepare('UPDATE jobs SET stable_id = ? WHERE id = 1').run(configuration.jobs[0]!.stableId); + } + db.prepare("INSERT INTO runs (job_id, status, trigger) VALUES (1, 'succeeded', 'manual')").run(); + db.prepare('INSERT INTO settings (key, value) VALUES (?, ?)').run('notifications', encryptJson({ notifySuccess: true, notifyFailure: true }, masterKey)); + return db; +} + +function newDatabase(label: string) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), `backup-manager-restore-${label}-`)); + temporaryDirectories.push(directory); + return openDatabase(directory); +} diff --git a/src/configuration-restore.ts b/src/configuration-restore.ts new file mode 100644 index 0000000..0ecb922 --- /dev/null +++ b/src/configuration-restore.ts @@ -0,0 +1,154 @@ +import { randomUUID } from 'node:crypto'; +import type Database from 'better-sqlite3'; +import { z } from 'zod'; +import { configurationExportSchema, type ConfigurationExport, verifyConfigurationExportIntegrity } from './configuration-export.js'; +import { decryptJson } from './crypto.js'; +import { normalizeStepSecrets } from './job-credentials.js'; + +const hostPasswordSchema = z.object({ password: z.string().min(1) }).strict(); +const hostPrivateKeySchema = z.object({ privateKey: z.string().min(1), passphrase: z.string().optional() }).strict(); +const stepSecretsSchema = z.record(z.string(), z.string()); +const notificationSettingsSchema = z.object({ + webhookUrl: z.url().optional(), + notifySuccess: z.boolean(), + notifyFailure: z.boolean(), + smtp: z.object({ + host: z.string().min(1), port: z.number().int().min(1).max(65535), secure: z.boolean(), + username: z.string().optional(), password: z.string().optional(), from: z.email(), to: z.email(), + }).optional(), +}).strict(); + +export interface RestoreResult { + hosts: { added: number; updated: number }; + jobs: { added: number; updated: number }; + settings: { added: number; updated: number }; +} + +export class ConfigurationRestoreError extends Error { + constructor(message: string, readonly statusCode = 400) { + super(message); + } +} + +export function restoreConfiguration(db: Database.Database, value: unknown, masterKey: Buffer, calculateNextRun: (schedule: string | null, timezone: string) => string | null): RestoreResult { + const configuration = configurationExportSchema.parse(value); + if (db.prepare("SELECT id FROM runs WHERE status IN ('queued', 'running') LIMIT 1").get()) throw new ConfigurationRestoreError('Cannot restore configuration while a job is queued or running', 409); + if (!verifyConfigurationExportIntegrity(configuration, masterKey)) throw new ConfigurationRestoreError('Configuration integrity verification failed; the backup may be modified or use a different MASTER_KEY'); + validateSourceInstance(db, configuration); + validateSecrets(configuration, masterKey); + validateNameConflicts(db, configuration); + + const existingHosts = identityMap(db, 'hosts'); + const existingJobs = identityMap(db, 'jobs'); + const existingSettingKeys = new Set((db.prepare('SELECT key FROM settings').all() as Array<{ key: string }>).map((row) => row.key)); + const result: RestoreResult = { + hosts: countChanges(configuration.hosts.map((host) => host.stableId), new Set(existingHosts.keys())), + jobs: countChanges(configuration.jobs.map((job) => job.stableId), new Set(existingJobs.keys())), + settings: countChanges(configuration.settings.map((setting) => setting.key), existingSettingKeys), + }; + + db.transaction(() => { + const localInstance = db.prepare("SELECT value FROM metadata WHERE key = 'instance_id'").get() as { value: string }; + if (localInstance.value !== configuration.sourceInstanceId) db.prepare("UPDATE metadata SET value = ? WHERE key = 'instance_id'").run(configuration.sourceInstanceId); + const temporaryHostName = db.prepare('UPDATE hosts SET name = ? WHERE id = ?'); + const temporaryJobName = db.prepare('UPDATE jobs SET name = ? WHERE id = ?'); + for (const host of configuration.hosts) { + const existing = existingHosts.get(host.stableId); + if (existing) temporaryHostName.run(`__restore_host_${randomUUID()}`, existing.id); + } + for (const job of configuration.jobs) { + const existing = existingJobs.get(job.stableId); + if (existing) temporaryJobName.run(`__restore_job_${randomUUID()}`, existing.id); + } + + const updateHost = db.prepare('UPDATE hosts SET stable_id = ?, name = ?, hostname = ?, port = ?, username = ?, fingerprint = ?, auth_type = ?, secret = ? WHERE id = ?'); + const insertHostWithId = db.prepare('INSERT INTO hosts (id, stable_id, name, hostname, port, username, fingerprint, auth_type, secret) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)'); + const insertHost = db.prepare('INSERT INTO hosts (stable_id, name, hostname, port, username, fingerprint, auth_type, secret) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'); + const hostIdMap = new Map(); + for (const host of configuration.hosts) { + const existing = existingHosts.get(host.stableId); + if (existing) { + updateHost.run(host.stableId, host.name, host.hostname, host.port, host.username, host.fingerprint, host.authType, host.secret, existing.id); + hostIdMap.set(host.id, existing.id); + } else if (db.prepare('SELECT 1 FROM hosts WHERE id = ?').get(host.id)) { + const inserted = insertHost.run(host.stableId, host.name, host.hostname, host.port, host.username, host.fingerprint, host.authType, host.secret); + hostIdMap.set(host.id, Number(inserted.lastInsertRowid)); + } else { + insertHostWithId.run(host.id, host.stableId, host.name, host.hostname, host.port, host.username, host.fingerprint, host.authType, host.secret); + hostIdMap.set(host.id, host.id); + } + } + + const updateJob = db.prepare('UPDATE jobs SET stable_id = ?, name = ?, host_id = ?, config = ?, secret = ?, schedule = ?, timezone = ?, enabled = ?, retention_count = ?, next_run_at = ? WHERE id = ?'); + const insertJobWithId = db.prepare('INSERT INTO jobs (id, stable_id, name, host_id, config, secret, schedule, timezone, enabled, retention_count, next_run_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); + const insertJob = db.prepare('INSERT INTO jobs (stable_id, name, host_id, config, secret, schedule, timezone, enabled, retention_count, next_run_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); + for (const job of configuration.jobs) { + const existing = existingJobs.get(job.stableId); + const hostId = hostIdMap.get(job.hostId)!; + const values = [job.stableId, job.name, hostId, JSON.stringify(job.config), job.secret, job.schedule, job.timezone, Number(job.enabled), job.retentionCount, calculateNextRun(job.schedule, job.timezone)] as const; + if (existing) updateJob.run(...values, existing.id); + else if (db.prepare('SELECT 1 FROM jobs WHERE id = ?').get(job.id)) insertJob.run(...values); + else insertJobWithId.run(job.id, ...values); + } + + const upsertSetting = db.prepare('INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value'); + for (const setting of configuration.settings) upsertSetting.run(setting.key, setting.value); + })(); + + return result; +} + +function validateSourceInstance(db: Database.Database, configuration: ConfigurationExport): void { + const local = db.prepare("SELECT value FROM metadata WHERE key = 'instance_id'").get() as { value: string }; + if (local.value === configuration.sourceInstanceId) return; + const populated = ['hosts', 'jobs', 'settings'].some((table) => (db.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get() as unknown) !== undefined); + if (populated) throw new ConfigurationRestoreError('This backup belongs to a different Backup Manager instance; restore it into an empty manager', 409); +} + +function validateSecrets(configuration: ConfigurationExport, masterKey: Buffer): void { + try { + for (const host of configuration.hosts) { + const secret = decryptJson(host.secret, masterKey); + (host.authType === 'password' ? hostPasswordSchema : hostPrivateKeySchema).parse(secret); + } + for (const job of configuration.jobs) { + const decrypted = job.secret ? stepSecretsSchema.parse(decryptJson(job.secret, masterKey)) : {}; + const secrets = normalizeStepSecrets(decrypted, job.config); + for (const step of job.config.steps) { + if ((step.type === 'postgres' || step.type === 'mysql') && !secrets[step.id]) throw new Error('Missing database password'); + } + } + for (const setting of configuration.settings) { + const decrypted = decryptJson(setting.value, masterKey); + if (setting.key === 'notifications') notificationSettingsSchema.parse(decrypted); + } + } catch { + throw new ConfigurationRestoreError('Configuration credentials cannot be decrypted and validated with the current MASTER_KEY'); + } +} + +function validateNameConflicts(db: Database.Database, configuration: ConfigurationExport): void { + const importedHostIds = new Set(configuration.hosts.map((host) => host.stableId)); + const importedJobIds = new Set(configuration.jobs.map((job) => job.stableId)); + for (const host of configuration.hosts) { + const conflict = db.prepare('SELECT stable_id AS stableId FROM hosts WHERE name = ?').get(host.name) as { stableId: string } | undefined; + if (conflict && conflict.stableId !== host.stableId && !importedHostIds.has(conflict.stableId)) throw new ConfigurationRestoreError(`Host name "${host.name}" belongs to an unrelated local record`, 409); + } + for (const job of configuration.jobs) { + const conflict = db.prepare('SELECT stable_id AS stableId FROM jobs WHERE name = ?').get(job.name) as { stableId: string } | undefined; + if (conflict && conflict.stableId !== job.stableId && !importedJobIds.has(conflict.stableId)) throw new ConfigurationRestoreError(`Job name "${job.name}" belongs to an unrelated local record`, 409); + } +} + +function identityMap(db: Database.Database, table: 'hosts' | 'jobs'): Map { + const rows = db.prepare(`SELECT id, stable_id AS stableId FROM ${table}`).all() as Array<{ id: number; stableId: string }>; + return new Map(rows.map((row) => [row.stableId, { id: row.id }])); +} + +function countChanges(values: T[], existing: Set): { added: number; updated: number } { + return values.reduce((count, value) => { + if (existing.has(value)) count.updated += 1; + else count.added += 1; + return count; + }, { added: 0, updated: 0 }); +} diff --git a/src/db.test.ts b/src/db.test.ts new file mode 100644 index 0000000..ecced44 --- /dev/null +++ b/src/db.test.ts @@ -0,0 +1,34 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import Database from 'better-sqlite3'; +import { afterEach, describe, expect, it } from 'vitest'; +import { openDatabase } from './db.js'; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) fs.rmSync(directory, { recursive: true, force: true }); +}); + +describe('database migrations', () => { + it('assigns stable identities to existing and newly inserted records', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'backup-manager-migration-')); + temporaryDirectories.push(directory); + const legacy = new Database(path.join(directory, 'app.db')); + legacy.exec(` + CREATE TABLE hosts (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, hostname TEXT NOT NULL, port INTEGER NOT NULL, username TEXT NOT NULL, fingerprint TEXT NOT NULL, auth_type TEXT NOT NULL, secret TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); + CREATE TABLE jobs (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, host_id INTEGER NOT NULL REFERENCES hosts(id) ON DELETE RESTRICT, config TEXT NOT NULL, secret TEXT, schedule TEXT, timezone TEXT NOT NULL DEFAULT 'UTC', enabled INTEGER NOT NULL DEFAULT 1, retention_count INTEGER NOT NULL DEFAULT 10, next_run_at TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); + INSERT INTO hosts (name, hostname, port, username, fingerprint, auth_type, secret) VALUES ('legacy', 'legacy.local', 22, 'backup', 'SHA256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA', 'password', 'secret'); + INSERT INTO jobs (name, host_id, config) VALUES ('legacy-job', 1, '{}'); + `); + legacy.close(); + + const db = openDatabase(directory); + expect((db.prepare('SELECT stable_id AS stableId FROM hosts WHERE id = 1').get() as { stableId: string }).stableId).toMatch(/^[a-f0-9]{32}$/); + expect((db.prepare('SELECT stable_id AS stableId FROM jobs WHERE id = 1').get() as { stableId: string }).stableId).toMatch(/^[a-f0-9]{32}$/); + const id = Number(db.prepare('INSERT INTO hosts (name, hostname, port, username, fingerprint, auth_type, secret) VALUES (?, ?, ?, ?, ?, ?, ?)').run('new', 'new.local', 22, 'backup', `SHA256:${'B'.repeat(43)}`, 'password', 'secret').lastInsertRowid); + expect((db.prepare('SELECT stable_id AS stableId FROM hosts WHERE id = ?').get(id) as { stableId: string }).stableId).toMatch(/^[a-f0-9]{32}$/); + db.close(); + }); +}); diff --git a/src/db.ts b/src/db.ts index 7d518a5..e249546 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,9 +1,11 @@ +import { randomUUID } from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import Database from 'better-sqlite3'; export interface HostRecord { id: number; + stable_id: string; name: string; hostname: string; port: number; @@ -16,6 +18,7 @@ export interface HostRecord { export interface JobRecord { id: number; + stable_id: string; name: string; host_id: number; config: string; @@ -49,11 +52,13 @@ export function openDatabase(dataDir: string): Database.Database { CREATE TABLE IF NOT EXISTS hosts ( id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, hostname TEXT NOT NULL, port INTEGER NOT NULL, username TEXT NOT NULL, fingerprint TEXT NOT NULL, auth_type TEXT NOT NULL, secret TEXT NOT NULL, + stable_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE IF NOT EXISTS jobs ( id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, host_id INTEGER NOT NULL REFERENCES hosts(id) ON DELETE RESTRICT, config TEXT NOT NULL, secret TEXT, schedule TEXT, timezone TEXT NOT NULL DEFAULT 'UTC', enabled INTEGER NOT NULL DEFAULT 1, + stable_id TEXT, retention_count INTEGER NOT NULL DEFAULT 10, next_run_at TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE IF NOT EXISTS runs ( @@ -67,8 +72,28 @@ export function openDatabase(dataDir: string): Database.Database { created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL); CREATE INDEX IF NOT EXISTS runs_job_id ON runs(job_id, id DESC); `); + ensureColumn(db, 'hosts', 'stable_id TEXT'); + ensureColumn(db, 'jobs', 'stable_id TEXT'); + db.exec(` + UPDATE hosts SET stable_id = lower(hex(randomblob(16))) WHERE stable_id IS NULL; + UPDATE jobs SET stable_id = lower(hex(randomblob(16))) WHERE stable_id IS NULL; + CREATE UNIQUE INDEX IF NOT EXISTS hosts_stable_id ON hosts(stable_id); + CREATE UNIQUE INDEX IF NOT EXISTS jobs_stable_id ON jobs(stable_id); + CREATE TRIGGER IF NOT EXISTS hosts_assign_stable_id AFTER INSERT ON hosts WHEN NEW.stable_id IS NULL + BEGIN UPDATE hosts SET stable_id = lower(hex(randomblob(16))) WHERE id = NEW.id; END; + CREATE TRIGGER IF NOT EXISTS jobs_assign_stable_id AFTER INSERT ON jobs WHEN NEW.stable_id IS NULL + BEGIN UPDATE jobs SET stable_id = lower(hex(randomblob(16))) WHERE id = NEW.id; END; + `); + db.prepare("INSERT OR IGNORE INTO metadata (key, value) VALUES ('instance_id', ?)").run(randomUUID()); db.prepare("UPDATE runs SET status = 'failed', finished_at = CURRENT_TIMESTAMP, error = 'Interrupted by application restart' WHERE status IN ('queued', 'running')").run(); return db; } + +function ensureColumn(db: Database.Database, table: 'hosts' | 'jobs', definition: string): void { + const column = definition.split(' ')[0]!; + const columns = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>; + if (!columns.some((entry) => entry.name === column)) db.exec(`ALTER TABLE ${table} ADD COLUMN ${definition}`); +} diff --git a/src/executor.test.ts b/src/executor.test.ts index 4be9d88..09e3c3e 100644 --- a/src/executor.test.ts +++ b/src/executor.test.ts @@ -1,5 +1,16 @@ -import { describe, expect, it, vi } from 'vitest'; -import { executeSequentialSteps } from './executor.js'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { AppConfig } from './config.js'; +import { openDatabase, type RunRecord } from './db.js'; +import { BackupExecutor, executeSequentialSteps } from './executor.js'; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) fs.rmSync(directory, { recursive: true, force: true }); +}); describe('step execution policy', () => { it('continues after tolerated failures and records a warning', async () => { @@ -23,3 +34,52 @@ describe('step execution policy', () => { expect(visited).toEqual(['first']); }); }); + +describe('local configuration execution', () => { + it('publishes a protected configuration artifact without opening SSH', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'backup-manager-local-step-')); + temporaryDirectories.push(directory); + const backupDir = path.join(directory, 'backups'); + fs.mkdirSync(backupDir); + const db = openDatabase(path.join(directory, 'data')); + db.prepare('INSERT INTO hosts (id, name, hostname, port, username, fingerprint, auth_type, secret) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') + .run(1, 'unused-host', 'unreachable.invalid', 22, 'backup', `SHA256:${'A'.repeat(43)}`, 'password', 'not-an-encrypted-envelope'); + db.prepare('INSERT INTO jobs (id, name, host_id, config, timezone, retention_count) VALUES (?, ?, ?, ?, ?, ?)') + .run(1, 'manager-config', 1, JSON.stringify({ steps: [{ id: 'config', name: 'Manager configuration', type: 'managerConfig', continueOnError: false, outputName: 'manager-config' }] }), 'UTC', 3); + const runId = Number(db.prepare("INSERT INTO runs (job_id, status, trigger) VALUES (1, 'queued', 'manual')").run().lastInsertRowid); + const run = db.prepare('SELECT * FROM runs WHERE id = ?').get(runId) as RunRecord; + const config: AppConfig = { port: 0, dataDir: path.join(directory, 'data'), backupDir, adminPassword: 'not-exported', masterKey: Buffer.alloc(32), secureCookie: false }; + + await new BackupExecutor(db, config).execute(run); + + const artifact = db.prepare('SELECT name, path FROM artifacts WHERE run_id = ?').get(runId) as { name: string; path: string }; + const artifactPath = path.join(backupDir, artifact.path); + expect(db.prepare('SELECT status FROM runs WHERE id = ?').get(runId)).toEqual({ status: 'succeeded' }); + expect(artifact.name).toBe('manager-config.json'); + expect(JSON.parse(fs.readFileSync(artifactPath, 'utf8'))).toMatchObject({ format: 'backup-script-manager/configuration', version: 1 }); + expect(fs.statSync(artifactPath).mode & 0o777).toBe(0o600); + db.close(); + }); + + it('removes published files when artifact metadata cannot be committed', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'backup-manager-local-failure-')); + temporaryDirectories.push(directory); + const backupDir = path.join(directory, 'backups'); + fs.mkdirSync(backupDir); + const db = openDatabase(path.join(directory, 'data')); + db.prepare('INSERT INTO hosts (id, name, hostname, port, username, fingerprint, auth_type, secret) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') + .run(1, 'unused-host', 'unreachable.invalid', 22, 'backup', `SHA256:${'A'.repeat(43)}`, 'password', 'not-an-encrypted-envelope'); + db.prepare('INSERT INTO jobs (id, name, host_id, config, timezone, retention_count) VALUES (?, ?, ?, ?, ?, ?)') + .run(1, 'manager-config', 1, JSON.stringify({ steps: [{ id: 'config', name: 'Manager configuration', type: 'managerConfig', continueOnError: false, outputName: 'manager-config' }] }), 'UTC', 3); + const runId = Number(db.prepare("INSERT INTO runs (job_id, status, trigger) VALUES (1, 'queued', 'manual')").run().lastInsertRowid); + const run = db.prepare('SELECT * FROM runs WHERE id = ?').get(runId) as RunRecord; + db.exec("CREATE TRIGGER reject_artifact BEFORE INSERT ON artifacts BEGIN SELECT RAISE(FAIL, 'metadata failure'); END"); + const config: AppConfig = { port: 0, dataDir: path.join(directory, 'data'), backupDir, adminPassword: 'not-exported', masterKey: Buffer.alloc(32), secureCookie: false }; + + await expect(new BackupExecutor(db, config).execute(run)).rejects.toThrow('metadata failure'); + + expect(fs.existsSync(path.join(backupDir, '1', String(runId)))).toBe(false); + expect(db.prepare('SELECT status FROM runs WHERE id = ?').get(runId)).toEqual({ status: 'failed' }); + db.close(); + }); +}); diff --git a/src/executor.ts b/src/executor.ts index 8db818a..bfd583a 100644 --- a/src/executor.ts +++ b/src/executor.ts @@ -2,18 +2,18 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import type Database from 'better-sqlite3'; -import type { Client } from 'ssh2'; +import type { Client, SFTPWrapper } from 'ssh2'; import type { AppConfig } from './config.js'; +import { createConfigurationExport } from './configuration-export.js'; import { decryptJson } from './crypto.js'; import type { HostRecord, JobRecord, RunRecord } from './db.js'; import { normalizeStepSecrets } from './job-credentials.js'; import { normalizeJobConfig, type JobConfig, type JobStep } from './schemas.js'; import { connect, download, exec, getSftp, shellQuote, uploadText, type HostSecret } from './ssh.js'; -interface RemoteArtifact { - remotePath: string; - name: string; -} +type PendingArtifact = + | { source: 'remote'; remotePath: string; name: string } + | { source: 'local'; localPath: string; name: string }; export class BackupExecutor { constructor(private db: Database.Database, private appConfig: AppConfig) {} @@ -24,27 +24,42 @@ export class BackupExecutor { const host = this.db.prepare('SELECT * FROM hosts WHERE id = ?').get(job.host_id) as HostRecord | undefined; if (!host) throw new Error('Host no longer exists'); const jobConfig = normalizeJobConfig(JSON.parse(job.config), String(job.id)); - const hostSecret = decryptJson(host.secret, this.appConfig.masterKey); + const needsRemote = requiresRemoteConnection(jobConfig); + const hostSecret = needsRemote ? decryptJson(host.secret, this.appConfig.masterKey) : {}; const stepSecrets = this.readStepSecrets(job, jobConfig); const redactions = [hostSecret.password, hostSecret.passphrase, ...Object.values(stepSecrets)].filter((value): value is string => Boolean(value)); + const staging = path.join(this.appConfig.backupDir, '.staging', String(run.id)); this.updateRun(run.id, "status = 'running', started_at = CURRENT_TIMESTAMP"); - this.log(run.id, `Connecting to ${host.name} (${host.hostname}:${host.port})`); let client: Client | undefined; let remoteDir: string | undefined; + const ensureRemote = async (): Promise<{ client: Client; remoteDir: string }> => { + if (client && remoteDir) return { client, remoteDir }; + this.log(run.id, `Connecting to ${host.name} (${host.hostname}:${host.port})`); + const connected = await connect(host, hostSecret); + try { + const created = await exec(connected, `umask 077 && mktemp -d ${shellQuote(`/tmp/backup-manager-${run.id}-XXXXXX`)}`, 30); + const createdDir = created.stdout.trim(); + if (!new RegExp(`^/tmp/backup-manager-${run.id}-[A-Za-z0-9]+$`).test(createdDir)) throw new Error('Remote host returned an invalid staging path'); + client = connected; + remoteDir = createdDir; + return { client, remoteDir }; + } catch (error) { + connected.end(); + throw error; + } + }; try { - client = await connect(host, hostSecret); - const created = await exec(client, `umask 077 && mktemp -d ${shellQuote(`/tmp/backup-manager-${run.id}-XXXXXX`)}`, 30); - remoteDir = created.stdout.trim(); - if (!new RegExp(`^/tmp/backup-manager-${run.id}-[A-Za-z0-9]+$`).test(remoteDir)) throw new Error('Remote host returned an invalid staging path'); - const { artifacts, warningCount } = await this.runSteps(client, jobConfig, stepSecrets, remoteDir, (message) => this.log(run.id, redact(message, redactions))); - await this.collectArtifacts(client, job, run, artifacts); + fs.rmSync(staging, { recursive: true, force: true }); + fs.mkdirSync(staging, { recursive: true, mode: 0o700 }); + const { artifacts, warningCount } = await this.runSteps(jobConfig, stepSecrets, staging, ensureRemote, (message) => this.log(run.id, redact(message, redactions))); + await this.collectArtifacts(client, job, run, artifacts, staging); const status = warningCount ? 'succeeded_with_warnings' : 'succeeded'; this.updateRun(run.id, 'status = ?, finished_at = CURRENT_TIMESTAMP, error = NULL', status); this.log(run.id, `Completed with ${artifacts.length} artifact(s)${warningCount ? ` and ${warningCount} warning(s)` : ''}`); this.applyRetention(job); } catch (error) { - fs.rmSync(path.join(this.appConfig.backupDir, '.staging', String(run.id)), { recursive: true, force: true }); + fs.rmSync(staging, { recursive: true, force: true }); const message = redact(error instanceof Error ? error.message : String(error), redactions); this.updateRun(run.id, "status = 'failed', finished_at = CURRENT_TIMESTAMP, error = ?", message); this.log(run.id, `Failed: ${message}`); @@ -57,15 +72,30 @@ export class BackupExecutor { } } - private async runSteps(client: Client, config: JobConfig, secrets: Record, remoteDir: string, log: (message: string) => void): Promise<{ artifacts: RemoteArtifact[]; warningCount: number }> { + private async runSteps(config: JobConfig, secrets: Record, staging: string, ensureRemote: () => Promise<{ client: Client; remoteDir: string }>, log: (message: string) => void): Promise<{ artifacts: PendingArtifact[]; warningCount: number }> { const result = await executeSequentialSteps(config.steps, async (step, index) => { log(`Step ${index + 1}/${config.steps.length}: ${step.name}`); - return this.runStep(client, step, secrets[step.id], remoteDir, index, log); + if (step.type === 'managerConfig') return this.runStep(undefined, step, secrets[step.id], undefined, staging, index, log); + const remote = await ensureRemote(); + return this.runStep(remote.client, step, secrets[step.id], remote.remoteDir, staging, index, log); }, (error) => log(`Warning: ${error instanceof Error ? error.message : String(error)}`)); return { artifacts: result.values.flat(), warningCount: result.warningCount }; } - private async runStep(client: Client, config: JobStep, password: string | undefined, remoteDir: string, stepIndex: number, log: (message: string) => void): Promise { + private async runStep(client: Client | undefined, config: JobStep, password: string | undefined, remoteDir: string | undefined, staging: string, stepIndex: number, log: (message: string) => void): Promise { + if (config.type === 'managerConfig') { + const name = config.outputName.endsWith('.json') ? config.outputName : `${config.outputName}.json`; + const localPath = path.join(staging, name); + log('Exporting Backup Manager configuration'); + try { + fs.writeFileSync(localPath, `${JSON.stringify(createConfigurationExport(this.db, this.appConfig.masterKey), null, 2)}\n`, { encoding: 'utf8', mode: 0o600, flag: 'wx' }); + } catch (error) { + fs.rmSync(localPath, { force: true }); + throw error; + } + return [{ source: 'local', localPath, name }]; + } + if (!client || !remoteDir) throw new Error('Remote connection is unavailable'); switch (config.type) { case 'dockerCommand': { log(`Running Docker command in ${config.container}`); @@ -75,7 +105,7 @@ export class BackupExecutor { if (result.stdout.trim()) log(result.stdout.trim()); if (result.stderr.trim()) log(result.stderr.trim()); - const artifacts: RemoteArtifact[] = []; + const artifacts: PendingArtifact[] = []; for (const [index, output] of config.outputs.entries()) { const copiedPath = `${remoteDir}/step-${stepIndex}-output-${index}`; await exec(client, `docker cp ${shellQuote(`${config.container}:${output.path}`)} ${shellQuote(copiedPath)}`, config.timeoutSeconds); @@ -83,10 +113,10 @@ export class BackupExecutor { const archiveName = output.name.endsWith('.tar.gz') ? output.name : `${output.name}.tar.gz`; const archivePath = `${remoteDir}/${archiveName}`; await exec(client, `tar -czf ${shellQuote(archivePath)} -C ${shellQuote(remoteDir)} -- ${shellQuote(`step-${stepIndex}-output-${index}`)}`, config.timeoutSeconds); - artifacts.push({ remotePath: archivePath, name: archiveName }); + artifacts.push({ source: 'remote', remotePath: archivePath, name: archiveName }); } else { await exec(client, `test -f ${shellQuote(copiedPath)}`, 30); - artifacts.push({ remotePath: copiedPath, name: output.name }); + artifacts.push({ source: 'remote', remotePath: copiedPath, name: output.name }); } } return artifacts; @@ -98,17 +128,17 @@ export class BackupExecutor { const result = await exec(client, command, config.timeoutSeconds); if (result.stdout.trim()) log(result.stdout.trim()); if (result.stderr.trim()) log(result.stderr.trim()); - const artifacts: RemoteArtifact[] = []; + const artifacts: PendingArtifact[] = []; for (const [index, output] of config.outputs.entries()) { if (output.archive) { const archiveName = output.name.endsWith('.tar.gz') ? output.name : `${output.name}.tar.gz`; const archivePath = `${remoteDir}/${archiveName}`; await exec(client, `tar -czf ${shellQuote(archivePath)} -C ${shellQuote(path.posix.dirname(output.path))} -- ${shellQuote(path.posix.basename(output.path))}`, config.timeoutSeconds); - artifacts.push({ remotePath: archivePath, name: archiveName }); + artifacts.push({ source: 'remote', remotePath: archivePath, name: archiveName }); } else { const copiedPath = `${remoteDir}/step-${stepIndex}-output-${index}`; await exec(client, `test -f ${shellQuote(output.path)} && cp -- ${shellQuote(output.path)} ${shellQuote(copiedPath)}`, config.timeoutSeconds); - artifacts.push({ remotePath: copiedPath, name: output.name }); + artifacts.push({ source: 'remote', remotePath: copiedPath, name: output.name }); } } return artifacts; @@ -118,7 +148,7 @@ export class BackupExecutor { const name = config.outputName.endsWith('.tar.gz') ? config.outputName : `${config.outputName}.tar.gz`; const target = `${remoteDir}/${name}`; await exec(client, `tar -czf ${shellQuote(target)} -C ${shellQuote(path.posix.dirname(config.path))} -- ${shellQuote(path.posix.basename(config.path))}`, config.timeoutSeconds); - return [{ remotePath: target, name }]; + return [{ source: 'remote', remotePath: target, name }]; } case 'postgres': { if (!password) throw new Error('PostgreSQL password is missing'); @@ -129,7 +159,7 @@ export class BackupExecutor { const raw = `${remoteDir}/dump.sql`; const command = `PGPASSFILE=${shellQuote(`${remoteDir}/pgpass`)} pg_dump --host=${shellQuote(config.databaseHost)} --port=${shellQuote(String(config.databasePort))} --username=${shellQuote(config.username)} --no-password --dbname=${shellQuote(config.database)} > ${shellQuote(raw)} && gzip -c ${shellQuote(raw)} > ${shellQuote(target)} && rm ${shellQuote(raw)}`; await exec(client, command, config.timeoutSeconds); - return [{ remotePath: target, name }]; + return [{ source: 'remote', remotePath: target, name }]; } case 'mysql': { if (!password) throw new Error('MySQL password is missing'); @@ -140,23 +170,27 @@ export class BackupExecutor { const raw = `${remoteDir}/dump.sql`; const command = `mysqldump --defaults-extra-file=${shellQuote(`${remoteDir}/my.cnf`)} --host=${shellQuote(config.databaseHost)} --port=${shellQuote(String(config.databasePort))} --user=${shellQuote(config.username)} --databases ${shellQuote(config.database)} > ${shellQuote(raw)} && gzip -c ${shellQuote(raw)} > ${shellQuote(target)} && rm ${shellQuote(raw)}`; await exec(client, command, config.timeoutSeconds); - return [{ remotePath: target, name }]; + return [{ source: 'remote', remotePath: target, name }]; } } } - private async collectArtifacts(client: Client, job: JobRecord, run: RunRecord, artifacts: RemoteArtifact[]): Promise { - const staging = path.join(this.appConfig.backupDir, '.staging', String(run.id)); + private async collectArtifacts(client: Client | undefined, job: JobRecord, run: RunRecord, artifacts: PendingArtifact[], staging: string): Promise { const finalDir = path.join(this.appConfig.backupDir, String(job.id), String(run.id)); - fs.rmSync(staging, { recursive: true, force: true }); - fs.mkdirSync(staging, { recursive: true, mode: 0o700 }); - const sftp = await getSftp(client); const records: Array<{ name: string; relativePath: string; size: number; checksum: string }> = []; + let sftp: SFTPWrapper | undefined; try { for (const artifact of artifacts) { - this.log(run.id, `Downloading ${artifact.name}`); - const localPath = path.join(staging, artifact.name); - await download(sftp, artifact.remotePath, localPath); + let localPath: string; + if (artifact.source === 'remote') { + if (!client) throw new Error('Remote connection is unavailable'); + sftp ??= await getSftp(client); + this.log(run.id, `Downloading ${artifact.name}`); + localPath = path.join(staging, artifact.name); + await download(sftp, artifact.remotePath, localPath); + } else { + localPath = artifact.localPath; + } const { size, checksum } = await hashFile(localPath); records.push({ name: artifact.name, @@ -166,12 +200,17 @@ export class BackupExecutor { }); } } finally { - sftp.end(); + sftp?.end(); } fs.mkdirSync(path.dirname(finalDir), { recursive: true, mode: 0o700 }); fs.renameSync(staging, finalDir); const insert = this.db.prepare('INSERT INTO artifacts (run_id, name, path, size, checksum) VALUES (?, ?, ?, ?, ?)'); - this.db.transaction(() => records.forEach((record) => insert.run(run.id, record.name, record.relativePath, record.size, record.checksum)))(); + try { + this.db.transaction(() => records.forEach((record) => insert.run(run.id, record.name, record.relativePath, record.size, record.checksum)))(); + } catch (error) { + fs.rmSync(finalDir, { recursive: true, force: true }); + throw error; + } } private applyRetention(job: JobRecord): void { @@ -202,6 +241,10 @@ export class BackupExecutor { } } +export function requiresRemoteConnection(config: JobConfig): boolean { + return config.steps.some((step) => step.type !== 'managerConfig'); +} + async function hashFile(filePath: string): Promise<{ size: number; checksum: string }> { const hash = createHash('sha256'); let size = 0; diff --git a/src/schemas.test.ts b/src/schemas.test.ts index 15e4fe3..6721fdf 100644 --- a/src/schemas.test.ts +++ b/src/schemas.test.ts @@ -38,6 +38,13 @@ describe('job validation', () => { expect(jobInputSchema.parse(input).config.steps).toHaveLength(2); }); + it('accepts a Backup Manager configuration export as an artifact-producing step', () => { + const input = { ...dockerJob, config: { steps: [ + { id: 'manager-config', name: 'Manager configuration', continueOnError: false, type: 'managerConfig', outputName: 'backup-manager-config' }, + ] } }; + expect(jobInputSchema.parse(input).config.steps[0]).toMatchObject({ type: 'managerConfig', outputName: 'backup-manager-config' }); + }); + it('rejects invalid cron expressions', () => { expect(jobInputSchema.safeParse({ ...dockerJob, schedule: 'not a cron' }).success).toBe(false); }); diff --git a/src/schemas.ts b/src/schemas.ts index 2369277..5661ea4 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -96,7 +96,13 @@ export const mysqlStepSchema = z.object({ timeoutSeconds: z.number().int().min(1).max(86400).default(3600), }); -export const jobStepSchema = z.discriminatedUnion('type', [dockerStepSchema, remoteCommandStepSchema, directoryStepSchema, postgresStepSchema, mysqlStepSchema]); +export const managerConfigStepSchema = z.object({ + ...stepFields, + type: z.literal('managerConfig'), + outputName: safeName, +}); + +export const jobStepSchema = z.discriminatedUnion('type', [dockerStepSchema, remoteCommandStepSchema, directoryStepSchema, postgresStepSchema, mysqlStepSchema, managerConfigStepSchema]); export type JobStep = z.infer; export const jobConfigSchema = z.object({ @@ -171,11 +177,12 @@ export type JobInput = z.infer; export type JobUpdateInput = z.infer; function finalArtifactNames(step: JobStep): string[] { + if (step.type === 'managerConfig') return [step.outputName.endsWith('.json') ? step.outputName : `${step.outputName}.json`]; if (step.type === 'directory') return [step.outputName.endsWith('.tar.gz') ? step.outputName : `${step.outputName}.tar.gz`]; if (step.type === 'postgres' || step.type === 'mysql') return [step.outputName.endsWith('.sql.gz') ? step.outputName : `${step.outputName}.sql.gz`]; return step.outputs.map((output) => output.archive && !output.name.endsWith('.tar.gz') ? `${output.name}.tar.gz` : output.name); } function defaultStepName(type: string): string { - return { dockerCommand: 'Docker command', directory: 'Directory archive', postgres: 'PostgreSQL dump', mysql: 'MySQL dump' }[type] ?? 'Backup step'; + return { dockerCommand: 'Docker command', directory: 'Directory archive', postgres: 'PostgreSQL dump', mysql: 'MySQL dump', managerConfig: 'Backup Manager configuration' }[type] ?? 'Backup step'; } diff --git a/src/server.ts b/src/server.ts index fbf535e..4d4f4b4 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5,6 +5,7 @@ import fastifyStatic from '@fastify/static'; import Fastify from 'fastify'; import { z } from 'zod'; import { loadConfig } from './config.js'; +import { restoreConfiguration } from './configuration-restore.js'; import { createSession, decryptJson, encryptJson, secureEqual, verifySession } from './crypto.js'; import { openDatabase, type HostRecord, type JobRecord } from './db.js'; import { resolveUpdatedHostSecret } from './host-credentials.js'; @@ -227,6 +228,11 @@ app.put('/api/settings/notifications', async (request) => { return { saved: true }; }); +app.put('/api/settings/configuration', { bodyLimit: 10 * 1024 * 1024 }, async (request) => ({ + restored: true, + ...restoreConfiguration(db, request.body, config.masterKey, nextRun), +})); + app.setErrorHandler((error: Error & { code?: string; statusCode?: number }, _request, reply) => { if (error instanceof z.ZodError) return reply.code(400).send({ error: 'Validation failed', details: z.flattenError(error).fieldErrors }); if (error.code === 'SQLITE_CONSTRAINT_UNIQUE') return reply.code(409).send({ error: 'A record with that name already exists' }); diff --git a/web/src/main.tsx b/web/src/main.tsx index 951fccd..9a9ba90 100644 --- a/web/src/main.tsx +++ b/web/src/main.tsx @@ -5,10 +5,10 @@ import './styles.css'; type Tab = 'overview' | 'hosts' | 'jobs' | 'settings'; type Host = { id: number; name: string; hostname: string; port: number; username: string; fingerprint: string; authType: 'password' | 'privateKey' }; -type StepType = 'dockerCommand' | 'remoteCommand' | 'directory' | 'postgres' | 'mysql'; +type StepType = 'dockerCommand' | 'remoteCommand' | 'directory' | 'postgres' | 'mysql' | 'managerConfig'; type StepOutput = { path: string; name: string; archive: boolean }; type PersistedStep = { - id: string; name: string; type: StepType; continueOnError: boolean; timeoutSeconds: number; + id: string; name: string; type: StepType; continueOnError: boolean; timeoutSeconds?: number; executable?: string; arguments?: string[]; workingDirectory?: string; container?: string; user?: string; outputs?: StepOutput[]; path?: string; outputName?: string; database?: string; username?: string; databaseHost?: string; databasePort?: number; }; @@ -189,6 +189,7 @@ function Jobs() { const outputs = step.collectOutput ? [{ path: step.outputPath, name: step.outputName, archive: step.archive }, ...step.additionalOutputs] : []; if (step.type === 'dockerCommand') return { ...base, container: step.container, executable: step.executable, arguments: parseArguments(step.arguments), user: step.dockerUser || undefined, workingDirectory: step.workingDirectory || undefined, outputs }; if (step.type === 'remoteCommand') return { ...base, executable: step.executable, arguments: parseArguments(step.arguments), workingDirectory: step.workingDirectory || undefined, outputs }; + if (step.type === 'managerConfig') return { ...base, outputName: step.outputName }; if (step.type === 'directory') return { ...base, path: step.directoryPath, outputName: step.outputName }; if (step.databasePassword) stepSecrets[step.id] = step.databasePassword; return { ...base, database: step.database, username: step.databaseUsername, databaseHost: step.databaseHost, databasePort: Number(step.databasePort), outputName: step.outputName }; @@ -218,7 +219,7 @@ function Jobs() { {hosts.length === 0 ? :
{steps.map((step, index) => setSteps((current) => current.filter((item) => item.id !== id))} key={step.id} />)}{steps.length === 0 &&
Select a type below to add the first step.
}
-
+
Job policy
{message &&

{message}

}
{editingJob && }
} @@ -233,9 +234,10 @@ function StepEditor({ step, index, count, update, move, remove }: { step: StepDr {step.type === 'dockerCommand' && <>
} {step.type === 'remoteCommand' && <>} {command && <>