diff --git a/Cargo.lock b/Cargo.lock index dd6f6c6ca43..ca14d4cf68d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8069,15 +8069,19 @@ dependencies = [ "serde_json", "serde_with", "slab", + "sled", "spacetimedb-auth", "spacetimedb-client-api-messages", "spacetimedb-codegen", + "spacetimedb-commitlog", "spacetimedb-data-structures", "spacetimedb-fs-utils", "spacetimedb-lib", "spacetimedb-paths", "spacetimedb-primitives", "spacetimedb-schema", + "spacetimedb-snapshot", + "spacetimedb-table", "syntect", "tabled", "tar", @@ -8141,6 +8145,7 @@ dependencies = [ "spacetimedb-core", "spacetimedb-data-structures", "spacetimedb-datastore", + "spacetimedb-fs-utils", "spacetimedb-lib", "spacetimedb-paths", "spacetimedb-schema", @@ -8543,6 +8548,7 @@ dependencies = [ "tempdir", "thiserror 1.0.69", "tokio", + "windows-sys 0.59.0", "zstd-framed", ] @@ -8918,6 +8924,7 @@ dependencies = [ "futures", "hostname", "http", + "humantime", "log", "netstat2", "once_cell", @@ -8933,6 +8940,7 @@ dependencies = [ "spacetimedb-client-api-messages", "spacetimedb-core", "spacetimedb-datastore", + "spacetimedb-fs-utils", "spacetimedb-lib", "spacetimedb-paths", "spacetimedb-pg", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index bb71fb8b6e5..bf83b5d0fa8 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -24,12 +24,15 @@ bench = false spacetimedb-auth.workspace = true spacetimedb-client-api-messages.workspace = true spacetimedb-codegen.workspace = true +spacetimedb-commitlog.workspace = true spacetimedb-data-structures.workspace = true spacetimedb-fs-utils.workspace = true spacetimedb-lib.workspace = true spacetimedb-paths.workspace = true spacetimedb-primitives.workspace = true spacetimedb-schema.workspace = true +spacetimedb-snapshot.workspace = true +spacetimedb-table.workspace = true anyhow.workspace = true base64.workspace = true @@ -61,6 +64,7 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value", "preserve_order", "arbitrary_precision"] } serde_with = { workspace = true, features = ["chrono_0_4"] } slab.workspace = true +sled.workspace = true syntect.workspace = true tabled.workspace = true tar.workspace = true diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index def9a0928cb..3e9b1b41f03 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -21,6 +21,7 @@ pub use tasks::build; pub fn get_subcommands() -> Vec { vec![ + backup::cli(), publish::cli(), delete::cli(), logs::cli(), @@ -52,6 +53,7 @@ pub async fn exec_subcommand( args: &ArgMatches, ) -> anyhow::Result { match cmd { + "backup" => backup::exec(config, paths, args).await, "call" => call::exec(config, args).await, "describe" => describe::exec(config, args).await, "dev" => dev::exec(config, args).await, diff --git a/crates/cli/src/subcommands/backup.rs b/crates/cli/src/subcommands/backup.rs new file mode 100644 index 00000000000..0b2bb985c8b --- /dev/null +++ b/crates/cli/src/subcommands/backup.rs @@ -0,0 +1,2230 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::common_args; +use crate::config::Config; +use crate::subcommands::db_arg_resolution::{load_config_db_targets, resolve_database_arg}; +use crate::util::{add_auth_header_opt, database_identity, get_auth_header, ResponseExt}; +use anyhow::Context; +use clap::{Arg, ArgMatches, Command}; +use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize}; +use spacetimedb_commitlog::{commits, committed_meta, segment, DEFAULT_LOG_FORMAT_VERSION}; +use spacetimedb_fs_utils::{copy_dir_all, copy_file_sync, create_dir_all_sync, sync_dir}; +use spacetimedb_lib::{ + bsatn, de::Deserialize as BsatnDeserialize, hash_bytes, ser::Serialize as BsatnSerialize, Hash, Identity, +}; +use spacetimedb_paths::{ + server::{CommitLogDir, ServerDataDir, SnapshotsPath}, + FromPathUnchecked, SpacetimePaths, +}; +use spacetimedb_snapshot::SnapshotRepository; +use spacetimedb_table::page_pool::PagePool; + +static RESTORE_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +pub fn cli() -> Command { + Command::new("backup") + .about("Create server-side database backups") + .subcommand_required(true) + .subcommand( + Command::new("create") + .about("Create a hot backup of a running database") + .arg( + Arg::new("database") + .long("database") + .required(false) + .help("The name or identity of the database to back up"), + ) + .arg( + Arg::new("output_dir") + .long("output-dir") + .value_name("ROOT_RELATIVE_OUTPUT_DIR") + .required(true) + .value_parser(clap::value_parser!(PathBuf)) + .help("Directory relative to the server's configured hot-backup root; it must be empty"), + ) + .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database")) + .arg(common_args::anonymous()) + .arg( + Arg::new("no_config") + .long("no-config") + .action(clap::ArgAction::SetTrue) + .help("Ignore spacetime.json configuration"), + ), + ) + .subcommand( + Command::new("restore") + .about("Restore a hot backup into an offline server data directory") + .arg( + Arg::new("input_dir") + .long("input-dir") + .value_name("BACKUP_DIR") + .required(true) + .value_parser(clap::value_parser!(PathBuf)) + .help("Directory containing manifest.json, snapshots/, and clog/"), + ) + .arg( + Arg::new("data_dir") + .long("data-dir") + .value_name("SERVER_DATA_DIR") + .value_parser(clap::value_parser!(ServerDataDir)) + .help("Offline server data directory whose matching replica will be restored; defaults to the CLI root data-dir"), + ) + .arg( + Arg::new("force") + .long("force") + .action(clap::ArgAction::SetTrue) + .help("Overwrite the existing target replica directory"), + ), + ) +} + +pub async fn exec(config: Config, paths: &SpacetimePaths, args: &ArgMatches) -> Result<(), anyhow::Error> { + let (cmd, subcommand_args) = args.subcommand().expect("Subcommand required"); + match cmd { + "create" => exec_create(config, subcommand_args).await, + "restore" => exec_restore(paths, subcommand_args), + unknown => Err(anyhow::anyhow!("Invalid subcommand: {unknown}")), + } +} + +#[derive(SerdeSerialize)] +struct BackupRequest { + server_output_dir: PathBuf, +} + +#[derive(Debug, SerdeDeserialize)] +struct BackupManifest { + version: u32, + database_identity: String, + replica_id: u64, + output_dir: PathBuf, + snapshot_offset: u64, + durable_offset: u64, + snapshot_ms: u64, + copy_ms: u64, + total_ms: u64, + bytes: u64, +} + +async fn exec_create(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { + let server_from_cli = args.get_one::("server").map(|s| s.as_ref()); + let no_config = args.get_flag("no_config"); + let anon_identity = args.get_flag("anon_identity"); + let database_arg = args.get_one::("database").map(|s| s.as_str()); + let config_targets = load_config_db_targets(no_config)?; + let resolved = resolve_database_arg( + database_arg, + config_targets.as_deref(), + "spacetime backup create --database --output-dir [--no-config]", + )?; + let server = server_from_cli.or(resolved.server.as_deref()); + + let identity = database_identity(&config, &resolved.database, server).await?; + let host_url = config.get_host_url(server)?; + let auth_header = get_auth_header(&mut config, anon_identity, server, true).await?; + let server_output_dir = args.get_one::("output_dir").unwrap().clone(); + + let mut builder = reqwest::Client::new() + .post(format!("{host_url}/v1/database/{identity}/backup")) + .json(&BackupRequest { server_output_dir }); + builder = add_auth_header_opt(builder, &auth_header); + let manifest: BackupManifest = builder.send().await?.json_or_error().await?; + + println!("Backup written on server to {}", manifest.output_dir.display()); + println!("snapshot_offset: {}", manifest.snapshot_offset); + println!("durable_offset: {}", manifest.durable_offset); + println!("bytes: {}", manifest.bytes); + println!("snapshot_ms: {}", manifest.snapshot_ms); + println!("copy_ms: {}", manifest.copy_ms); + println!("total_ms: {}", manifest.total_ms); + Ok(()) +} + +fn exec_restore(paths: &SpacetimePaths, args: &ArgMatches) -> Result<(), anyhow::Error> { + let input_dir = args.get_one::("input_dir").unwrap(); + let data_dir = args.get_one::("data_dir").unwrap_or(&paths.data_dir); + let force = args.get_flag("force"); + + let manifest = restore_backup(input_dir, data_dir, force)?; + + println!( + "Restored database {} replica {} into {}", + manifest.database_identity, + manifest.replica_id, + data_dir.replica(manifest.replica_id).display() + ); + println!("snapshot_offset: {}", manifest.snapshot_offset); + println!("durable_offset: {}", manifest.durable_offset); + Ok(()) +} + +fn restore_backup(input_dir: &Path, data_dir: &ServerDataDir, force: bool) -> anyhow::Result { + let manifest = read_backup_manifest(input_dir)?; + validate_backup(input_dir, &manifest)?; + let backup_log_format_version = read_backup_max_log_format_version(input_dir)?; + validate_target_log_format_version(data_dir, backup_log_format_version)?; + create_dir_all_sync(&data_dir.0)?; + + // The data-dir lock catches the common footgun; per-replica online restore needs a real restore service. + let _pid_file = data_dir + .pid_file() + .context("target data-dir must be offline before restore")?; + restore_backup_inner(input_dir, data_dir, force, manifest) +} + +#[derive(SerdeDeserialize)] +struct RestoreTargetConfig { + #[serde(default)] + commitlog: RestoreTargetCommitlogConfig, +} + +#[derive(SerdeDeserialize)] +struct RestoreTargetCommitlogConfig { + #[serde(default = "default_log_format_version", rename = "log-format-version")] + log_format_version: u8, +} + +impl Default for RestoreTargetCommitlogConfig { + fn default() -> Self { + Self { + log_format_version: DEFAULT_LOG_FORMAT_VERSION, + } + } +} + +const fn default_log_format_version() -> u8 { + DEFAULT_LOG_FORMAT_VERSION +} + +fn read_backup_max_log_format_version(input_dir: &Path) -> anyhow::Result { + let clog_dir = CommitLogDir::from_path_unchecked(input_dir.join("clog")); + anyhow::ensure!( + clog_dir.0.is_dir(), + "backup clog directory is missing: {}", + clog_dir.display() + ); + let initial_segment = clog_dir.segment(0); + anyhow::ensure!( + initial_segment.0.is_file(), + "backup commitlog segment is missing: {}", + initial_segment.display() + ); + + let mut max_log_format_version: Option = None; + for entry in std::fs::read_dir(&clog_dir.0).with_context(|| format!("reading {}", clog_dir.display()))? { + let entry = entry?; + let file_name = entry.file_name(); + let file_name = file_name.to_string_lossy(); + if !file_name.ends_with(".stdb.log") { + continue; + } + anyhow::ensure!( + is_commitlog_segment_name(&file_name) && entry.file_type()?.is_file(), + "invalid backup commitlog segment: {}", + entry.path().display() + ); + let path = entry.path(); + let file = std::fs::File::open(&path) + .with_context(|| format!("opening backup commitlog segment header {}", path.display()))?; + let header = segment::Header::decode(file) + .with_context(|| format!("reading backup commitlog segment header {}", path.display()))?; + max_log_format_version = Some( + max_log_format_version + .unwrap_or_default() + .max(header.log_format_version), + ); + } + max_log_format_version.with_context(|| { + format!( + "backup clog directory contains no segment files: {}", + clog_dir.display() + ) + }) +} + +fn validate_target_log_format_version(data_dir: &ServerDataDir, backup_log_format_version: u8) -> anyhow::Result<()> { + let config_path = data_dir.0.join("config.toml"); + let target_log_format_version = if config_path.exists() { + let config = std::fs::read_to_string(&config_path) + .with_context(|| format!("reading target server config {}", config_path.display()))?; + toml::from_str::(&config) + .with_context(|| format!("parsing target server config {}", config_path.display()))? + .commitlog + .log_format_version + } else { + DEFAULT_LOG_FORMAT_VERSION + }; + anyhow::ensure!( + target_log_format_version >= backup_log_format_version, + "target commitlog log format version {target_log_format_version} is lower than backup commitlog requires version {backup_log_format_version}; update [commitlog].log-format-version in {} before restoring", + config_path.display() + ); + Ok(()) +} + +fn restore_backup_inner( + input_dir: &Path, + data_dir: &ServerDataDir, + force: bool, + manifest: BackupManifest, +) -> anyhow::Result { + let replica_dir = data_dir.replica(manifest.replica_id); + let restore_temp_id = RESTORE_TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); + let tmp_dir = replica_dir.0.with_file_name(format!( + "{}.restore_tmp_{}_{}", + manifest.replica_id, + std::process::id(), + restore_temp_id + )); + let old_tmp_dir = replica_dir.0.with_file_name(format!( + "{}.restore_old_{}_{}", + manifest.replica_id, + std::process::id(), + restore_temp_id + )); + + anyhow::ensure!( + force || !replica_dir.0.exists(), + "target replica directory already exists: {}; pass --force to overwrite it", + replica_dir.display() + ); + anyhow::ensure!( + !tmp_dir.exists(), + "temporary restore directory already exists: {}", + tmp_dir.display() + ); + if force { + anyhow::ensure!( + !old_tmp_dir.exists(), + "temporary old replica directory already exists: {}", + old_tmp_dir.display() + ); + } + + if let Some(parent) = tmp_dir.parent() { + create_dir_all_sync(parent)?; + } + + let mut staged_server_state = stage_missing_server_state(input_dir, data_dir, restore_temp_id, &manifest)?; + let validation = (|| -> anyhow::Result<()> { + validate_target_control_db(input_dir, data_dir, &manifest)?; + anyhow::ensure!( + !(force && replica_dir.0.exists() && !staged_server_state.is_empty()), + "cannot restore over existing replica {} while target data-dir is missing server state; restore into an empty data-dir or complete the target server state first", + replica_dir.display() + ); + Ok(()) + })(); + if let Err(error) = validation { + return Err(staged_server_state.cleanup().attach(error)); + } + + let mut old_tmp_moved = false; + let mut restore_old_tmp_on_error = false; + let mut restored_replica_moved = false; + let mut committed_server_state = None; + let res = (|| -> anyhow::Result<()> { + create_dir_all_sync(&tmp_dir)?; + copy_dir_all(input_dir.join("snapshots"), tmp_dir.join("snapshots"))?; + copy_dir_all(input_dir.join("clog"), tmp_dir.join("clog"))?; + create_dir_all_sync(&tmp_dir.join("module_logs"))?; + sync_parent_dir(&tmp_dir)?; + + committed_server_state = Some(staged_server_state.commit()?); + + if force && replica_dir.0.exists() { + std::fs::rename(&replica_dir.0, &old_tmp_dir) + .with_context(|| format!("moving existing target replica directory {}", replica_dir.display()))?; + old_tmp_moved = true; + restore_old_tmp_on_error = true; + sync_parent_dir(&replica_dir.0)?; + } + std::fs::rename(&tmp_dir, &replica_dir.0) + .with_context(|| format!("moving restored replica into {}", replica_dir.display()))?; + restored_replica_moved = true; + sync_parent_dir(&replica_dir.0)?; + restore_old_tmp_on_error = false; + + if let Some(committed_server_state) = &mut committed_server_state { + committed_server_state.keep(); + } + + if old_tmp_moved { + if let Err(err) = std::fs::remove_dir_all(&old_tmp_dir) { + tracing::warn!( + "restore completed, but failed to remove old replica directory {}: {err}", + old_tmp_dir.display() + ); + } else { + let _ = sync_parent_dir(&old_tmp_dir); + } + old_tmp_moved = false; + } + Ok(()) + })(); + + if let Err(error) = res { + let mut rollback = rollback_failed_replica_restore( + &tmp_dir, + &replica_dir.0, + &old_tmp_dir, + restored_replica_moved, + restore_old_tmp_on_error && old_tmp_moved, + ); + if let Some(committed_server_state) = &mut committed_server_state { + rollback.extend(committed_server_state.rollback()); + } + return Err(rollback.attach(error)); + } + + Ok(manifest) +} + +#[derive(Debug, Default)] +struct RollbackFailures { + errors: Vec, + recovery_paths: Vec, +} + +impl RollbackFailures { + fn capture(&mut self, result: anyhow::Result<()>, recovery_paths: &[&Path]) -> bool { + match result { + Ok(()) => true, + Err(error) => { + self.errors.push(format!("{error:#}")); + for path in recovery_paths { + let path = (*path).to_path_buf(); + if !self.recovery_paths.contains(&path) { + self.recovery_paths.push(path); + } + } + false + } + } + } + + fn extend(&mut self, other: Self) { + self.errors.extend(other.errors); + for path in other.recovery_paths { + if !self.recovery_paths.contains(&path) { + self.recovery_paths.push(path); + } + } + } + + fn is_empty(&self) -> bool { + self.errors.is_empty() + } + + fn summary(&self) -> String { + let paths = self + .recovery_paths + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(", "); + format!( + "restore rollback was incomplete; possible leftover/recoverable paths: {paths}; rollback errors: {}", + self.errors.join("; ") + ) + } + + fn attach(self, error: anyhow::Error) -> anyhow::Error { + if self.is_empty() { + error + } else { + error.context(self.summary()) + } + } + + fn report_unattached(&self) { + if !self.is_empty() { + tracing::error!("{}", self.summary()); + } + } +} + +fn rollback_failed_replica_restore( + tmp_dir: &Path, + replica_dir: &Path, + old_tmp_dir: &Path, + restored_replica_moved: bool, + restore_old_tmp: bool, +) -> RollbackFailures { + let mut failures = RollbackFailures::default(); + + if tmp_dir.exists() { + failures.capture(rollback_remove_dir_all(tmp_dir), &[tmp_dir]); + failures.capture(rollback_sync_parent_dir(tmp_dir), &[tmp_dir]); + } + if restored_replica_moved && replica_dir.exists() { + failures.capture(rollback_remove_dir_all(replica_dir), &[replica_dir, old_tmp_dir]); + failures.capture(rollback_sync_parent_dir(replica_dir), &[replica_dir, old_tmp_dir]); + } + if restore_old_tmp { + if !old_tmp_dir.exists() { + if !replica_dir.exists() { + failures.capture( + Err(anyhow::anyhow!( + "recoverable old replica directory disappeared during rollback: {}", + old_tmp_dir.display() + )), + &[replica_dir, old_tmp_dir], + ); + } + } else if replica_dir.exists() { + failures.capture( + Err(anyhow::anyhow!( + "could not restore old replica because rollback target still exists: {}", + replica_dir.display() + )), + &[replica_dir, old_tmp_dir], + ); + } else { + let renamed = failures.capture(rollback_rename(old_tmp_dir, replica_dir), &[replica_dir, old_tmp_dir]); + if renamed { + failures.capture(rollback_sync_parent_dir(replica_dir), &[replica_dir, old_tmp_dir]); + } + } + } + + failures +} + +fn rollback_remove_dir_all(path: &Path) -> anyhow::Result<()> { + #[cfg(test)] + maybe_fail_restore_rollback_operation(path, "remove")?; + std::fs::remove_dir_all(path).with_context(|| format!("removing directory {} during rollback", path.display())) +} + +fn rollback_remove_file(path: &Path) -> anyhow::Result<()> { + #[cfg(test)] + maybe_fail_restore_rollback_operation(path, "remove")?; + std::fs::remove_file(path).with_context(|| format!("removing file {} during rollback", path.display())) +} + +fn rollback_rename(from: &Path, to: &Path) -> anyhow::Result<()> { + #[cfg(test)] + maybe_fail_restore_rollback_operation(from, "rename")?; + std::fs::rename(from, to).with_context(|| { + format!( + "restoring recoverable path {} to {} during rollback", + from.display(), + to.display() + ) + }) +} + +fn rollback_sync_parent_dir(path: &Path) -> anyhow::Result<()> { + if let Some(parent) = path.parent().filter(|parent| !parent.as_os_str().is_empty()) { + #[cfg(test)] + maybe_fail_restore_rollback_operation(path, "sync-parent")?; + sync_dir(parent).with_context(|| format!("syncing parent directory {} during rollback", parent.display()))?; + } + Ok(()) +} + +#[cfg(test)] +fn maybe_fail_restore_rollback_operation(path: &Path, operation: &str) -> anyhow::Result<()> { + let Some(parent) = path.parent().filter(|parent| !parent.as_os_str().is_empty()) else { + return Ok(()); + }; + let marker = parent.join(format!(".fail-next-restore-rollback-{operation}")); + anyhow::ensure!( + !marker.exists(), + "injected rollback {operation} failure for {}", + path.display() + ); + Ok(()) +} + +fn sync_parent_dir(path: &Path) -> anyhow::Result<()> { + if let Some(parent) = path.parent().filter(|parent| !parent.as_os_str().is_empty()) { + #[cfg(test)] + maybe_fail_sync_parent_dir(parent)?; + sync_dir(parent).with_context(|| format!("syncing parent directory {}", parent.display()))?; + } + Ok(()) +} + +#[cfg(test)] +fn maybe_fail_sync_parent_dir(parent: &Path) -> anyhow::Result<()> { + let marker = parent.join(".fail-next-restore-sync-parent"); + if !marker.exists() { + return Ok(()); + } + let remaining = std::fs::read_to_string(&marker) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if remaining == 0 { + anyhow::bail!("injected sync_parent_dir failure for {}", parent.display()); + } + std::fs::write(marker, (remaining - 1).to_string())?; + Ok(()) +} + +fn read_backup_manifest(input_dir: &Path) -> anyhow::Result { + let manifest_path = input_dir.join("manifest.json"); + let bytes = std::fs::read(&manifest_path) + .with_context(|| format!("reading backup manifest {}", manifest_path.display()))?; + serde_json::from_slice(&bytes).with_context(|| format!("parsing backup manifest {}", manifest_path.display())) +} + +fn validate_backup(input_dir: &Path, manifest: &BackupManifest) -> anyhow::Result<()> { + anyhow::ensure!( + manifest.version == 1, + "unsupported backup manifest version {}", + manifest.version + ); + anyhow::ensure!( + manifest.durable_offset == manifest.snapshot_offset, + "backup durable_offset {} does not match snapshot_offset {}", + manifest.durable_offset, + manifest.snapshot_offset + ); + let database_identity: Identity = manifest + .database_identity + .parse() + .with_context(|| format!("parsing backup database identity {}", manifest.database_identity))?; + + let snapshots = SnapshotsPath::from_path_unchecked(input_dir.join("snapshots")); + let snapshot_dir = snapshots.snapshot_dir(manifest.snapshot_offset); + anyhow::ensure!( + snapshot_dir.0.is_dir(), + "backup snapshot directory is missing: {}", + snapshot_dir.display() + ); + let snapshot_file = snapshot_dir.snapshot_file(manifest.snapshot_offset); + anyhow::ensure!( + snapshot_file.0.is_file(), + "backup snapshot file is missing: {}", + snapshot_file.display() + ); + let snapshot_repo = + SnapshotRepository::open(snapshots, database_identity, manifest.replica_id).with_context(|| { + format!( + "opening backup snapshots directory {}", + input_dir.join("snapshots").display() + ) + })?; + let snapshot = snapshot_repo + .read_snapshot(manifest.snapshot_offset, &PagePool::new(None)) + .with_context(|| format!("reading backup snapshot {}", snapshot_file.display()))?; + anyhow::ensure!( + snapshot.database_identity == database_identity, + "backup snapshot database identity {} does not match manifest {}", + snapshot.database_identity, + database_identity + ); + anyhow::ensure!( + snapshot.replica_id == manifest.replica_id, + "backup snapshot replica_id {} does not match manifest {}", + snapshot.replica_id, + manifest.replica_id + ); + anyhow::ensure!( + snapshot.tx_offset == manifest.snapshot_offset, + "backup snapshot tx_offset {} does not match manifest {}", + snapshot.tx_offset, + manifest.snapshot_offset + ); + + let clog_dir = CommitLogDir::from_path_unchecked(input_dir.join("clog")); + anyhow::ensure!( + clog_dir.0.is_dir(), + "backup clog directory is missing: {}", + clog_dir.display() + ); + let initial_segment = clog_dir.segment(0); + anyhow::ensure!( + initial_segment.0.is_file(), + "backup commitlog segment is missing: {}", + initial_segment.display() + ); + let mut segment_count = 0u64; + for entry in std::fs::read_dir(&clog_dir.0).with_context(|| format!("reading {}", clog_dir.display()))? { + let entry = entry?; + let file_name = entry.file_name(); + let file_name = file_name.to_string_lossy(); + if file_name.ends_with(".stdb.log") { + anyhow::ensure!( + is_commitlog_segment_name(&file_name) && entry.file_type()?.is_file(), + "invalid backup commitlog segment: {}", + entry.path().display() + ); + segment_count += 1; + } + } + anyhow::ensure!( + segment_count > 0, + "backup clog directory contains no segment files: {}", + clog_dir.display() + ); + let commitlog_meta = committed_meta(clog_dir.clone()) + .with_context(|| format!("reading backup commitlog metadata {}", clog_dir.display()))?; + let max_committed_offset = commitlog_meta.and_then(|meta| meta.metadata().tx_range.end.checked_sub(1)); + anyhow::ensure!( + max_committed_offset == Some(manifest.snapshot_offset), + "backup commitlog max committed offset {:?} does not match manifest snapshot_offset {}", + max_committed_offset, + manifest.snapshot_offset + ); + for commit in commits(clog_dir).with_context(|| { + format!( + "opening backup commitlog for traversal {}", + input_dir.join("clog").display() + ) + })? { + commit.with_context(|| format!("reading backup commitlog {}", input_dir.join("clog").display()))?; + } + Ok(()) +} + +fn is_commitlog_segment_name(file_name: &str) -> bool { + let Some(offset) = file_name.strip_suffix(".stdb.log") else { + return false; + }; + offset.len() == 20 && offset.bytes().all(|byte| byte.is_ascii_digit()) +} + +fn validate_target_control_db( + input_dir: &Path, + data_dir: &ServerDataDir, + manifest: &BackupManifest, +) -> anyhow::Result<()> { + let control_db_dir = data_dir.0.join("control-db"); + if !control_db_dir.exists() { + return Ok(()); + } + let database_identity: Identity = manifest + .database_identity + .parse() + .with_context(|| format!("parsing backup database identity {}", manifest.database_identity))?; + let database = validate_control_db_records( + &control_db_dir, + database_identity, + manifest.replica_id, + ControlDbValidationScope::ExistingTarget, + ) + .with_context(|| { + format!( + "target data-dir already has {}; restore into an empty data-dir or a data-dir whose control-db contains the backed up database from {}", + control_db_dir.display(), + input_dir.display() + ) + })?; + let backup_control_db_dir = input_dir.join("server").join("control-db"); + let backup_database = validate_control_db_records( + &backup_control_db_dir, + database_identity, + manifest.replica_id, + ControlDbValidationScope::ScopedBackup, + ) + .with_context(|| format!("validating backup control-db {}", backup_control_db_dir.display()))?; + validate_control_db_metadata_match(&backup_database, &database)?; + let backup_program_bytes_dir = input_dir.join("server").join("program-bytes"); + validate_program_bytes(&backup_program_bytes_dir, backup_database.initial_program) + .with_context(|| format!("validating backup program-bytes {}", backup_program_bytes_dir.display()))?; + let program_bytes_dir = data_dir.0.join("program-bytes"); + if program_bytes_dir.exists() { + validate_program_bytes(&program_bytes_dir, database.initial_program) + .with_context(|| format!("validating target program-bytes {}", program_bytes_dir.display()))?; + } + Ok(()) +} + +fn validate_control_db_metadata_match(backup: &ControlDbDatabase, target: &ControlDbDatabase) -> anyhow::Result<()> { + anyhow::ensure!( + backup.database_identity == target.database_identity, + "backup control-db database identity {} does not match target {}", + backup.database_identity, + target.database_identity + ); + anyhow::ensure!( + backup.owner_identity == target.owner_identity, + "backup control-db owner identity {} does not match target {}", + backup.owner_identity, + target.owner_identity + ); + anyhow::ensure!( + backup.host_type == target.host_type, + "backup control-db host_type {:?} does not match target {:?}", + backup.host_type, + target.host_type + ); + anyhow::ensure!( + backup.initial_program == target.initial_program, + "backup control-db initial_program {} does not match target {}", + backup.initial_program, + target.initial_program + ); + Ok(()) +} + +#[derive(Debug, Clone, Copy)] +enum ControlDbValidationScope { + ScopedBackup, + ExistingTarget, +} + +#[allow(dead_code)] +#[derive(Debug, Clone, BsatnDeserialize, BsatnSerialize)] +struct ControlDbDatabase { + id: u64, + database_identity: Identity, + owner_identity: Identity, + host_type: ControlDbHostType, + initial_program: Hash, +} + +#[allow(dead_code)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, BsatnDeserialize, BsatnSerialize)] +#[repr(i32)] +enum ControlDbHostType { + Wasm = 0, + Js = 1, +} + +#[allow(dead_code)] +#[derive(BsatnDeserialize, BsatnSerialize)] +struct ControlDbReplica { + id: u64, + database_id: u64, + node_id: u64, + leader: bool, +} + +fn validate_control_db_records( + control_db_dir: &Path, + database_identity: Identity, + replica_id: u64, + scope: ControlDbValidationScope, +) -> anyhow::Result { + anyhow::ensure!( + control_db_dir.is_dir(), + "control-db path is not a directory: {}", + control_db_dir.display() + ); + let db = sled::Config::default() + .path(control_db_dir) + .flush_every_ms(Some(50)) + .mode(sled::Mode::HighThroughput) + .open() + .with_context(|| format!("opening control-db {}", control_db_dir.display()))?; + + let database_by_identity = open_existing_control_db_tree(&db, "database_by_identity")?; + let database_key = database_identity.to_be_byte_array(); + let Some(database_record) = database_by_identity + .get(database_key) + .with_context(|| format!("reading database identity {database_identity} from control-db"))? + else { + anyhow::bail!("control-db is missing database identity {database_identity}"); + }; + let database_from_identity: ControlDbDatabase = bsatn::from_slice(&database_record) + .with_context(|| format!("decoding database identity {database_identity} in control-db"))?; + anyhow::ensure!( + database_from_identity.database_identity == database_identity, + "control-db database_by_identity record identity {} does not match key {}", + database_from_identity.database_identity, + database_identity + ); + + let databases = open_existing_control_db_tree(&db, "database")?; + let mut database_count = 0usize; + for item in databases.iter() { + let (key, value) = item.with_context(|| format!("reading database records in {}", control_db_dir.display()))?; + database_count += 1; + let key_id = control_db_u64_key(&key, "database id")?; + let database: ControlDbDatabase = + bsatn::from_slice(&value).with_context(|| format!("decoding database {key_id} in control-db"))?; + anyhow::ensure!( + database.id == key_id, + "control-db database record id {} does not match key {}", + database.id, + key_id + ); + } + let database_id = database_from_identity.id; + let Some(database_record_by_id) = databases + .get(database_id.to_be_bytes()) + .with_context(|| format!("reading database {database_id} from control-db"))? + else { + anyhow::bail!("control-db database table is missing database identity {database_identity}"); + }; + anyhow::ensure!( + database_record_by_id == database_record, + "control-db database_by_identity record for {database_identity} does not match database table record {database_id}" + ); + + let mut database_by_identity_count = 0usize; + for item in database_by_identity.iter() { + let (key, value) = + item.with_context(|| format!("reading database_by_identity records in {}", control_db_dir.display()))?; + database_by_identity_count += 1; + let key_bytes: [u8; 32] = key + .as_ref() + .try_into() + .with_context(|| format!("invalid control-db database identity key length {}", key.len()))?; + let key_identity = Identity::from_be_byte_array(key_bytes); + let database: ControlDbDatabase = bsatn::from_slice(&value) + .with_context(|| format!("decoding database identity {key_identity} in control-db"))?; + anyhow::ensure!( + database.database_identity == key_identity, + "control-db database_by_identity record identity {} does not match key {}", + database.database_identity, + key_identity + ); + } + + let replicas = open_existing_control_db_tree(&db, "replica")?; + let replica_key = replica_id.to_be_bytes(); + let mut replica_count = 0usize; + let mut matching_replica = None; + for item in replicas.iter() { + let (key, value) = item.with_context(|| format!("reading replica records in {}", control_db_dir.display()))?; + replica_count += 1; + let key_id = control_db_u64_key(&key, "replica id")?; + let replica: ControlDbReplica = + bsatn::from_slice(&value).with_context(|| format!("decoding replica {key_id} in control-db"))?; + anyhow::ensure!( + replica.id == key_id, + "control-db replica record id {} does not match key {}", + replica.id, + key_id + ); + if key.as_ref() == replica_key.as_slice() { + matching_replica = Some(replica); + } + } + let Some(replica) = matching_replica else { + anyhow::bail!("control-db is missing replica {replica_id}"); + }; + anyhow::ensure!( + replica.id == replica_id, + "control-db replica record id {} does not match key {}", + replica.id, + replica_id + ); + anyhow::ensure!( + replica.database_id == database_id, + "control-db replica {} belongs to database {}, not {}", + replica_id, + replica.database_id, + database_id + ); + + if matches!(scope, ControlDbValidationScope::ScopedBackup) { + anyhow::ensure!( + database_by_identity_count == 1, + "backup control-db must contain exactly one database_by_identity record, found {database_by_identity_count}" + ); + anyhow::ensure!( + database_count == 1, + "backup control-db must contain exactly one database record, found {database_count}" + ); + anyhow::ensure!( + replica_count == 1, + "backup control-db must contain exactly one replica record, found {replica_count}" + ); + } + Ok(database_from_identity) +} + +fn validate_program_bytes(program_bytes_dir: &Path, program_hash: Hash) -> anyhow::Result<()> { + anyhow::ensure!( + program_bytes_dir.is_dir(), + "program-bytes path is not a directory: {}", + program_bytes_dir.display() + ); + let relative_path = program_bytes_relative_path(&program_hash); + let path = program_bytes_dir.join(relative_path); + let program_bytes = std::fs::read(&path) + .with_context(|| format!("reading program bytes {} referenced by control-db", path.display()))?; + let actual_hash = hash_bytes(&program_bytes); + anyhow::ensure!( + actual_hash == program_hash, + "program bytes hash mismatch at {}: expected {}, got {}", + path.display(), + program_hash, + actual_hash + ); + Ok(()) +} + +fn program_bytes_relative_path(program_hash: &Hash) -> PathBuf { + let hex = program_hash.to_hex(); + let (prefix, suffix) = hex.split_at(2); + PathBuf::from(prefix).join(suffix) +} + +fn control_db_u64_key(key: &[u8], label: &str) -> anyhow::Result { + let bytes: [u8; 8] = key + .try_into() + .with_context(|| format!("invalid control-db {label} key length {}", key.len()))?; + Ok(u64::from_be_bytes(bytes)) +} + +fn open_existing_control_db_tree(db: &sled::Db, tree_name: &str) -> anyhow::Result { + anyhow::ensure!( + db.tree_names().iter().any(|name| name.as_ref() == tree_name.as_bytes()), + "control-db is missing `{tree_name}` tree" + ); + db.open_tree(tree_name) + .with_context(|| format!("opening control-db `{tree_name}` tree")) +} + +#[derive(Debug)] +struct StagedServerStateEntry { + final_path: PathBuf, + staged_path: PathBuf, +} + +#[derive(Debug)] +struct StagedServerState { + entries: Vec, + cleanup_on_drop: bool, +} + +impl StagedServerState { + fn new() -> Self { + Self { + entries: Vec::new(), + cleanup_on_drop: true, + } + } + + fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn cleanup(&mut self) -> RollbackFailures { + if !self.cleanup_on_drop { + return RollbackFailures::default(); + } + self.cleanup_on_drop = false; + let mut failures = RollbackFailures::default(); + for entry in &self.entries { + if !entry.staged_path.exists() { + continue; + } + if entry.staged_path.is_dir() { + failures.capture(rollback_remove_dir_all(&entry.staged_path), &[&entry.staged_path]) + } else { + failures.capture(rollback_remove_file(&entry.staged_path), &[&entry.staged_path]) + }; + failures.capture(rollback_sync_parent_dir(&entry.staged_path), &[&entry.staged_path]); + } + failures + } + + fn commit(mut self) -> anyhow::Result { + let mut committed = CommittedServerState::default(); + let res = (|| -> anyhow::Result<()> { + for entry in &self.entries { + anyhow::ensure!( + !entry.final_path.exists(), + "target data-dir server state appeared during restore: {}", + entry.final_path.display() + ); + std::fs::rename(&entry.staged_path, &entry.final_path).with_context(|| { + format!( + "moving staged server state {} to {}", + entry.staged_path.display(), + entry.final_path.display() + ) + })?; + committed.paths.push(entry.final_path.clone()); + sync_parent_dir(&entry.final_path)?; + } + Ok(()) + })(); + if let Err(error) = res { + let mut rollback = self.cleanup(); + rollback.extend(committed.rollback()); + return Err(rollback.attach(error)); + } + self.cleanup_on_drop = false; + Ok(committed) + } +} + +impl Drop for StagedServerState { + fn drop(&mut self) { + self.cleanup().report_unattached(); + } +} + +#[derive(Debug, Default)] +struct CommittedServerState { + paths: Vec, + keep: bool, +} + +impl CommittedServerState { + fn keep(&mut self) { + self.keep = true; + } + + fn rollback(&mut self) -> RollbackFailures { + if self.keep { + return RollbackFailures::default(); + } + self.keep = true; + let mut failures = RollbackFailures::default(); + for path in self.paths.iter().rev() { + if !path.exists() { + continue; + } + if path.is_dir() { + failures.capture(rollback_remove_dir_all(path), &[path]) + } else { + failures.capture(rollback_remove_file(path), &[path]) + }; + failures.capture(rollback_sync_parent_dir(path), &[path]); + } + failures + } +} + +impl Drop for CommittedServerState { + fn drop(&mut self) { + if !self.keep { + self.rollback().report_unattached(); + } + } +} + +fn stage_missing_server_state( + input_dir: &Path, + data_dir: &ServerDataDir, + restore_temp_id: u64, + manifest: &BackupManifest, +) -> anyhow::Result { + let server_dir = input_dir.join("server"); + let needs_required_dirs = ["control-db", "program-bytes"] + .into_iter() + .any(|required_dir| !data_dir.0.join(required_dir).exists()); + if !needs_required_dirs && !server_dir.exists() { + return Ok(StagedServerState::new()); + } + + let mut entries = Vec::new(); + for required_dir in ["control-db", "program-bytes"] { + let dst = data_dir.0.join(required_dir); + if dst.exists() { + continue; + } + let src = server_dir.join(required_dir); + anyhow::ensure!( + src.is_dir(), + "target data-dir is missing {}; backup is missing server state {}", + dst.display(), + src.display() + ); + let staged = staged_server_state_path(&dst, restore_temp_id); + anyhow::ensure!( + !staged.exists(), + "temporary server state path already exists: {}", + staged.display() + ); + entries.push((src, dst, staged, true)); + } + for file in ["config.toml", "metadata.toml"] { + let dst = data_dir.0.join(file); + if dst.exists() { + continue; + } + let src = server_dir.join(file); + if !needs_required_dirs && !src.exists() { + continue; + } + anyhow::ensure!( + src.is_file(), + "target data-dir is missing {}; backup is missing server state {}", + dst.display(), + src.display() + ); + let staged = staged_server_state_path(&dst, restore_temp_id); + anyhow::ensure!( + !staged.exists(), + "temporary server state path already exists: {}", + staged.display() + ); + entries.push((src, dst, staged, false)); + } + + let mut staged = StagedServerState::new(); + let res = (|| -> anyhow::Result<()> { + for (src, final_path, staged_path, is_dir) in entries { + copy_staged_server_state_entry(&mut staged, &src, final_path, staged_path, is_dir)?; + } + let control_db_entry = staged + .entries + .iter() + .find(|entry| entry.final_path.ends_with("control-db")); + let program_bytes_entry = staged + .entries + .iter() + .find(|entry| entry.final_path.ends_with("program-bytes")); + if control_db_entry.is_some() || program_bytes_entry.is_some() { + let database_identity: Identity = manifest + .database_identity + .parse() + .with_context(|| format!("parsing backup database identity {}", manifest.database_identity))?; + let control_db_path = control_db_entry + .map(|entry| entry.staged_path.clone()) + .unwrap_or_else(|| data_dir.0.join("control-db")); + let scope = if control_db_entry.is_some() { + ControlDbValidationScope::ScopedBackup + } else { + ControlDbValidationScope::ExistingTarget + }; + let database = validate_control_db_records(&control_db_path, database_identity, manifest.replica_id, scope) + .with_context(|| format!("validating control-db {}", control_db_path.display()))?; + let program_bytes_dir = program_bytes_entry + .map(|entry| entry.staged_path.clone()) + .unwrap_or_else(|| data_dir.0.join("program-bytes")); + validate_program_bytes(&program_bytes_dir, database.initial_program) + .with_context(|| format!("validating program-bytes {}", program_bytes_dir.display()))?; + } + Ok(()) + })(); + if let Err(error) = res { + return Err(staged.cleanup().attach(error)); + } + Ok(staged) +} + +fn copy_staged_server_state_entry( + staged: &mut StagedServerState, + src: &Path, + final_path: PathBuf, + staged_path: PathBuf, + is_dir: bool, +) -> anyhow::Result<()> { + staged.entries.push(StagedServerStateEntry { + final_path, + staged_path, + }); + let entry = staged.entries.last().expect("staged entry was just pushed"); + let res = if is_dir { + copy_dir_all(src, &entry.staged_path) + .with_context(|| format!("copying {} to {}", src.display(), entry.staged_path.display())) + } else { + copy_file_sync(src, &entry.staged_path) + .with_context(|| format!("copying {} to {}", src.display(), entry.staged_path.display())) + .map(|_| ()) + }; + if let Err(error) = res { + return Err(staged.cleanup().attach(error)); + } + Ok(()) +} + +fn staged_server_state_path(path: &Path, restore_temp_id: u64) -> PathBuf { + let suffix = format!("restore_tmp_{}_{}", std::process::id(), restore_temp_id); + if let Some(file_name) = path.file_name() { + path.with_file_name(format!("{}.{}", file_name.to_string_lossy(), suffix)) + } else { + path.with_extension(suffix) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb_commitlog::{payload::Txdata, Commitlog}; + use spacetimedb_lib::ProductValue; + use spacetimedb_paths::FromPathUnchecked; + use spacetimedb_table::{blob_store::HashMapBlobStore, table::Table}; + + #[test] + fn backup_restore_copies_replica_into_existing_data_dir() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let manifest = restore_backup(backup.path(), &data_dir, false)?; + + assert_eq!(manifest.replica_id, 7); + assert!(data + .path() + .join("replicas/7/snapshots/00000000000000000042.snapshot_dir/00000000000000000042.snapshot_bsatn") + .is_file()); + assert!(data + .path() + .join("replicas/7/clog/00000000000000000000.stdb.log") + .is_file()); + assert!(data.path().join("replicas/7/module_logs").is_dir()); + assert!(!data.path().join("spacetime.pid").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_v1_backup_for_v0_target_before_mutation() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let config = "[commitlog]\nlog-format-version = 0\n"; + std::fs::write(data.path().join("config.toml"), config)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + let rendered = format!("{err:#}"); + + assert!(rendered.contains("target commitlog log format version 0")); + assert!(rendered.contains("backup commitlog requires version 1")); + assert_eq!(std::fs::read_to_string(data.path().join("config.toml"))?, config); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_requires_force_for_existing_replica() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + std::fs::create_dir_all(data.path().join("replicas/7"))?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + assert!(err.to_string().contains("--force")); + assert!(data.path().join("replicas/7").is_dir()); + Ok(()) + } + + #[test] + fn backup_restore_requires_force_before_copying_missing_server_state() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + std::fs::create_dir_all(data.path().join("replicas/7"))?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + assert!(err.to_string().contains("--force")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + assert!(!data.path().join("config.toml").exists()); + assert!(!data.path().join("metadata.toml").exists()); + Ok(()) + } + + #[test] + fn backup_restore_force_replaces_existing_replica_without_leaving_old_tmp() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + restore_backup(backup.path(), &data_dir, true)?; + + assert!(!replica_dir.join("old-marker").exists()); + assert!(replica_dir + .join("snapshots/00000000000000000042.snapshot_dir/00000000000000000042.snapshot_bsatn") + .is_file()); + assert!(replica_dir.join("clog/00000000000000000000.stdb.log").is_file()); + assert!(replica_dir.join("module_logs").is_dir()); + + for entry in std::fs::read_dir(data.path().join("replicas"))? { + let entry = entry?; + assert!(!entry.file_name().to_string_lossy().contains(".restore_old_")); + } + Ok(()) + } + + #[test] + fn backup_restore_force_restores_old_replica_when_sync_after_old_rename_fails() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + std::fs::write(data.path().join("replicas/.fail-next-restore-sync-parent"), b"1")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + + assert!(err.to_string().contains("injected sync_parent_dir failure")); + assert_eq!(std::fs::read(replica_dir.join("old-marker"))?, b"old"); + assert!(!replica_dir.join("snapshots").exists()); + assert_no_restore_temps(data.path())?; + Ok(()) + } + + #[test] + fn backup_restore_force_restores_old_replica_when_sync_after_new_rename_fails() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + std::fs::write(data.path().join("replicas/.fail-next-restore-sync-parent"), b"2")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + + assert!(err.to_string().contains("injected sync_parent_dir failure")); + assert_eq!(std::fs::read(replica_dir.join("old-marker"))?, b"old"); + assert!(!replica_dir.join("snapshots").exists()); + assert_no_restore_temps(data.path())?; + Ok(()) + } + + #[test] + fn backup_restore_reports_rollback_remove_failure_with_recovery_paths() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + let replicas_dir = data.path().join("replicas"); + std::fs::write(replicas_dir.join(".fail-next-restore-sync-parent"), b"2")?; + std::fs::write(replicas_dir.join(".fail-next-restore-rollback-remove"), b"fail")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + let rendered = format!("{err:#}"); + let recoverable_old = find_restore_old_dir(&replicas_dir)?; + + assert!(rendered.contains("restore rollback was incomplete")); + assert!(rendered.contains("injected rollback remove failure")); + assert!(rendered.contains("injected sync_parent_dir failure")); + assert!(rendered.contains("possible leftover/recoverable paths")); + assert!(rendered.contains(&replica_dir.display().to_string())); + assert!(rendered.contains(&recoverable_old.display().to_string())); + assert!(replica_dir.join("snapshots").is_dir()); + assert_eq!(std::fs::read(recoverable_old.join("old-marker"))?, b"old"); + Ok(()) + } + + #[test] + fn backup_restore_reports_rollback_rename_failure_with_recoverable_old_replica() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + let replicas_dir = data.path().join("replicas"); + std::fs::write(replicas_dir.join(".fail-next-restore-sync-parent"), b"2")?; + std::fs::write(replicas_dir.join(".fail-next-restore-rollback-rename"), b"fail")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + let rendered = format!("{err:#}"); + let recoverable_old = find_restore_old_dir(&replicas_dir)?; + + assert!(rendered.contains("restore rollback was incomplete")); + assert!(rendered.contains("injected rollback rename failure")); + assert!(rendered.contains("injected sync_parent_dir failure")); + assert!(rendered.contains(&recoverable_old.display().to_string())); + assert!(!replica_dir.exists()); + assert_eq!(std::fs::read(recoverable_old.join("old-marker"))?, b"old"); + Ok(()) + } + + #[test] + fn backup_restore_reports_rollback_fsync_failure_and_preserves_original_error_chain() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + let replicas_dir = data.path().join("replicas"); + std::fs::write(replicas_dir.join(".fail-next-restore-sync-parent"), b"2")?; + std::fs::write(replicas_dir.join(".fail-next-restore-rollback-sync-parent"), b"fail")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + let rendered = format!("{err:#}"); + + assert!(rendered.contains("restore rollback was incomplete")); + assert!(rendered.contains("injected rollback sync-parent failure")); + assert!(rendered.contains("injected sync_parent_dir failure")); + assert!(rendered.contains(&replica_dir.display().to_string())); + assert_eq!(std::fs::read(replica_dir.join("old-marker"))?, b"old"); + assert_no_restore_temps(data.path())?; + Ok(()) + } + + #[test] + fn backup_restore_rolls_back_staged_server_state_when_replica_commit_sync_fails() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + std::fs::create_dir_all(data.path().join("replicas"))?; + std::fs::write(data.path().join("replicas/.fail-next-restore-sync-parent"), b"1")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("injected sync_parent_dir failure")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + assert!(!data.path().join("config.toml").exists()); + assert!(!data.path().join("metadata.toml").exists()); + assert!(!data.path().join("replicas/7").exists()); + assert_no_restore_temps(data.path())?; + Ok(()) + } + + #[test] + fn backup_restore_reports_committed_server_state_rollback_failure() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + std::fs::create_dir_all(data.path().join("replicas"))?; + std::fs::write(data.path().join("replicas/.fail-next-restore-sync-parent"), b"1")?; + std::fs::write(data.path().join(".fail-next-restore-rollback-remove"), b"fail")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + let rendered = format!("{err:#}"); + let metadata_path = data.path().join("metadata.toml"); + + assert!(rendered.contains("restore rollback was incomplete")); + assert!(rendered.contains("injected rollback remove failure")); + assert!(rendered.contains("injected sync_parent_dir failure")); + assert!(rendered.contains(&metadata_path.display().to_string())); + assert!(metadata_path.is_file()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[cfg(windows)] + #[test] + fn backup_restore_force_commits_when_old_replica_cleanup_fails() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + let old_marker = replica_dir.join("old-marker"); + std::fs::write(&old_marker, b"old")?; + let mut permissions = std::fs::metadata(&old_marker)?.permissions(); + permissions.set_readonly(true); + std::fs::set_permissions(&old_marker, permissions)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + restore_backup(backup.path(), &data_dir, true)?; + restore_backup(backup.path(), &data_dir, true)?; + + assert!(replica_dir + .join("snapshots/00000000000000000042.snapshot_dir/00000000000000000042.snapshot_bsatn") + .is_file()); + + for entry in std::fs::read_dir(data.path().join("replicas"))? { + let entry = entry?; + if !entry.file_name().to_string_lossy().contains(".restore_old_") { + continue; + } + clear_readonly_recursively(&entry.path())?; + std::fs::remove_dir_all(entry.path())?; + } + Ok(()) + } + + #[test] + fn backup_restore_copies_server_state_into_empty_data_dir() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + restore_backup(backup.path(), &data_dir, false)?; + + assert!(data.path().join("control-db").is_dir()); + assert!(data + .path() + .join("program-bytes") + .join(program_bytes_relative_path(&test_program_hash())) + .is_file()); + assert_eq!(std::fs::read_to_string(data.path().join("config.toml"))?, "config"); + assert_eq!(std::fs::read_to_string(data.path().join("metadata.toml"))?, "metadata"); + Ok(()) + } + + #[test] + fn backup_restore_copies_program_bytes_when_target_has_control_db() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_test_control_db(&data.path().join("control-db"), test_database_identity()?, 7, false)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + restore_backup(backup.path(), &data_dir, false)?; + + assert!(data.path().join("control-db").is_dir()); + assert!(data + .path() + .join("program-bytes") + .join(program_bytes_relative_path(&test_program_hash())) + .is_file()); + assert!(data.path().join("replicas/7").is_dir()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_missing_program_bytes_referenced_by_control_db() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + std::fs::remove_file( + backup + .path() + .join("server/program-bytes") + .join(program_bytes_relative_path(&test_program_hash())), + )?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("referenced by control-db")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_program_bytes_hash_mismatch() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + std::fs::write( + backup + .path() + .join("server/program-bytes") + .join(program_bytes_relative_path(&test_program_hash())), + b"not the program bytes", + )?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("program bytes hash mismatch")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_existing_target_missing_program_bytes_referenced_by_control_db() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_test_control_db(&data.path().join("control-db"), test_database_identity()?, 7, false)?; + std::fs::create_dir_all(data.path().join("program-bytes"))?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("referenced by control-db")); + assert!(data.path().join("control-db").is_dir()); + assert!(data.path().join("program-bytes").is_dir()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_existing_target_program_bytes_hash_mismatch() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_test_control_db(&data.path().join("control-db"), test_database_identity()?, 7, false)?; + let program_path = data + .path() + .join("program-bytes") + .join(program_bytes_relative_path(&test_program_hash())); + std::fs::create_dir_all(program_path.parent().context("program bytes path has no parent")?)?; + std::fs::write(&program_path, b"not the program bytes")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("program bytes hash mismatch")); + assert!(data.path().join("control-db").is_dir()); + assert!(program_path.is_file()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_existing_target_control_db_metadata_mismatch() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + let target_program_bytes = b"different program"; + let target_program_hash = hash_bytes(target_program_bytes); + make_test_control_db_with_database( + &data.path().join("control-db"), + ControlDbDatabase { + id: 1, + database_identity: test_database_identity()?, + owner_identity: Identity::ZERO, + host_type: ControlDbHostType::Wasm, + initial_program: target_program_hash, + }, + 7, + false, + )?; + let target_program_path = data + .path() + .join("program-bytes") + .join(program_bytes_relative_path(&target_program_hash)); + std::fs::create_dir_all( + target_program_path + .parent() + .context("program bytes path has no parent")?, + )?; + std::fs::write(&target_program_path, target_program_bytes)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("initial_program")); + assert!(data.path().join("control-db").is_dir()); + assert!(target_program_path.is_file()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_errors_when_empty_data_dir_lacks_server_state() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + assert!(err.to_string().contains("backup is missing server state")); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_missing_snapshot_file_before_copying_server_state() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + std::fs::remove_file( + backup + .path() + .join("snapshots/00000000000000000042.snapshot_dir/00000000000000000042.snapshot_bsatn"), + )?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("backup snapshot file is missing")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_corrupt_snapshot_before_copying_server_state() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + std::fs::write( + backup + .path() + .join("snapshots/00000000000000000042.snapshot_dir/00000000000000000042.snapshot_bsatn"), + b"not a snapshot", + )?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("reading backup snapshot")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_clog_without_segment_file_before_copying_server_state() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + std::fs::remove_file(backup.path().join("clog/00000000000000000000.stdb.log"))?; + std::fs::write(backup.path().join("clog/not-a-segment"), b"log")?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("backup commitlog segment is missing")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_invalid_commitlog_segment_name() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + std::fs::write(backup.path().join("clog/bad.stdb.log"), b"log")?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("invalid backup commitlog segment")); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_does_not_leave_partial_server_state_when_server_state_is_incomplete() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_test_control_db( + &backup.path().join("server/control-db"), + test_database_identity()?, + 7, + false, + )?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("backup is missing server state")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + assert!(!data.path().join("config.toml").exists()); + assert!(!data.path().join("metadata.toml").exists()); + assert!(!data.path().join("replicas/7").exists()); + for entry in std::fs::read_dir(data.path())? { + let entry = entry?; + assert!(!entry.file_name().to_string_lossy().contains(".restore_tmp_")); + } + Ok(()) + } + + #[test] + fn backup_restore_rejects_unscoped_backup_control_db() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_test_control_db( + &backup.path().join("server/control-db"), + test_database_identity()?, + 7, + true, + )?; + write_test_program_bytes(&backup.path().join("server/program-bytes"))?; + std::fs::write(backup.path().join("server/config.toml"), b"config")?; + std::fs::write(backup.path().join("server/metadata.toml"), b"metadata")?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("backup control-db must contain exactly one database")); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_control_db_database_record_identity_mismatch() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + let manifest_identity = test_database_identity()?; + let other_identity: Identity = "c300000000000000000000000000000000000000000000000000000000000000".parse()?; + std::fs::create_dir_all(backup.path().join("server/control-db"))?; + let db = sled::Config::default() + .path(backup.path().join("server/control-db")) + .flush_every_ms(Some(50)) + .mode(sled::Mode::HighThroughput) + .open()?; + let database = ControlDbDatabase { + id: 1, + database_identity: other_identity, + owner_identity: Identity::ZERO, + host_type: ControlDbHostType::Wasm, + initial_program: Hash::ZERO, + }; + let database_record = sled::IVec::from(encode_stored_control_db_database(&database)?); + db.open_tree("database_by_identity")? + .insert(manifest_identity.to_be_byte_array(), database_record.clone())?; + db.open_tree("database")?.insert(1u64.to_be_bytes(), database_record)?; + db.open_tree("replica")?.insert( + 7u64.to_be_bytes(), + bsatn::to_vec(&ControlDbReplica { + id: 7, + database_id: 1, + node_id: 0, + leader: true, + })?, + )?; + db.flush()?; + drop(db); + write_test_program_bytes(&backup.path().join("server/program-bytes"))?; + std::fs::write(backup.path().join("server/config.toml"), b"config")?; + std::fs::write(backup.path().join("server/metadata.toml"), b"metadata")?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("database_by_identity record identity")); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_existing_control_db_without_database_metadata() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + std::fs::create_dir_all(data.path().join("control-db"))?; + std::fs::create_dir_all(data.path().join("program-bytes"))?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("target data-dir already has")); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_staged_server_state_cleanup_removes_current_entry_after_copy_error() -> anyhow::Result<()> { + let data = tempfile::tempdir()?; + let final_path = data.path().join("control-db"); + let staged_path = staged_server_state_path(&final_path, 99); + let mut staged = StagedServerState::new(); + + let err = copy_staged_server_state_entry( + &mut staged, + &data.path().join("missing-control-db"), + final_path, + staged_path.clone(), + true, + ) + .unwrap_err(); + + assert!(err.to_string().contains("copying")); + assert_eq!(staged.entries.len(), 1); + assert!(!staged_path.exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_force_existing_replica_when_server_state_is_missing() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + + assert!(err.to_string().contains("cannot restore over existing replica")); + assert!(replica_dir.is_dir()); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + Ok(()) + } + + fn make_target_data_dir(path: &Path, replica_id: u64) -> anyhow::Result<()> { + make_test_control_db(&path.join("control-db"), test_database_identity()?, replica_id, false)?; + write_test_program_bytes(&path.join("program-bytes"))?; + std::fs::write(path.join("config.toml"), b"[commitlog]\nlog-format-version = 1\n")?; + std::fs::write(path.join("metadata.toml"), b"metadata")?; + Ok(()) + } + + fn make_backup_server_state(path: &Path) -> anyhow::Result<()> { + make_test_control_db(&path.join("server/control-db"), test_database_identity()?, 7, false)?; + write_test_program_bytes(&path.join("server/program-bytes"))?; + std::fs::write(path.join("server/config.toml"), b"config")?; + std::fs::write(path.join("server/metadata.toml"), b"metadata")?; + Ok(()) + } + + fn test_program_bytes() -> &'static [u8] { + b"program" + } + + fn test_program_hash() -> Hash { + hash_bytes(test_program_bytes()) + } + + fn write_test_program_bytes(program_bytes_dir: &Path) -> anyhow::Result<()> { + let program_hash = test_program_hash(); + let path = program_bytes_dir.join(program_bytes_relative_path(&program_hash)); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, test_program_bytes())?; + Ok(()) + } + + fn assert_no_restore_temps(data_dir: &Path) -> anyhow::Result<()> { + for entry in std::fs::read_dir(data_dir.join("replicas"))? { + let entry = entry?; + let file_name = entry.file_name(); + let file_name = file_name.to_string_lossy(); + assert!( + !file_name.contains(".restore_tmp_"), + "unexpected restore tmp {file_name}" + ); + assert!( + !file_name.contains(".restore_old_"), + "unexpected restore old tmp {file_name}" + ); + } + Ok(()) + } + + fn find_restore_old_dir(replicas_dir: &Path) -> anyhow::Result { + std::fs::read_dir(replicas_dir)? + .filter_map(Result::ok) + .find(|entry| entry.file_name().to_string_lossy().contains(".restore_old_")) + .map(|entry| entry.path()) + .context("expected recoverable restore-old directory") + } + + fn make_test_control_db( + path: &Path, + database_identity: Identity, + replica_id: u64, + include_extra_database: bool, + ) -> anyhow::Result<()> { + make_test_control_db_with_database( + path, + ControlDbDatabase { + id: 1, + database_identity, + owner_identity: Identity::ZERO, + host_type: ControlDbHostType::Wasm, + initial_program: test_program_hash(), + }, + replica_id, + include_extra_database, + ) + } + + fn make_test_control_db_with_database( + path: &Path, + database: ControlDbDatabase, + replica_id: u64, + include_extra_database: bool, + ) -> anyhow::Result<()> { + std::fs::create_dir_all(path)?; + let db = sled::Config::default() + .path(path) + .flush_every_ms(Some(50)) + .mode(sled::Mode::HighThroughput) + .open()?; + let database_identity = database.database_identity; + let database_id = database.id; + let database_record = sled::IVec::from(encode_stored_control_db_database(&database)?); + db.open_tree("database_by_identity")? + .insert(database_identity.to_be_byte_array(), database_record.clone())?; + db.open_tree("database")? + .insert(database_id.to_be_bytes(), database_record.clone())?; + db.open_tree("replica")?.insert( + replica_id.to_be_bytes(), + bsatn::to_vec(&ControlDbReplica { + id: replica_id, + database_id, + node_id: 0, + leader: true, + })?, + )?; + if include_extra_database { + let extra_identity: Identity = + "c300000000000000000000000000000000000000000000000000000000000000".parse()?; + let extra_database = ControlDbDatabase { + id: 2, + database_identity: extra_identity, + owner_identity: Identity::ZERO, + host_type: ControlDbHostType::Wasm, + initial_program: test_program_hash(), + }; + let extra_database_record = sled::IVec::from(encode_stored_control_db_database(&extra_database)?); + db.open_tree("database_by_identity")? + .insert(extra_identity.to_be_byte_array(), extra_database_record.clone())?; + db.open_tree("database")? + .insert(2u64.to_be_bytes(), extra_database_record)?; + db.open_tree("replica")?.insert( + (replica_id + 1).to_be_bytes(), + bsatn::to_vec(&ControlDbReplica { + id: replica_id + 1, + database_id: 2, + node_id: 0, + leader: true, + })?, + )?; + } + db.flush()?; + drop(db); + Ok(()) + } + + #[derive(BsatnSerialize)] + struct StoredControlDbDatabaseFixture { + id: u64, + database_identity: Identity, + owner_identity: Identity, + host_type: ControlDbHostType, + initial_program: Hash, + } + + fn encode_stored_control_db_database(database: &ControlDbDatabase) -> anyhow::Result> { + bsatn::to_vec(&StoredControlDbDatabaseFixture { + id: database.id, + database_identity: database.database_identity, + owner_identity: database.owner_identity, + host_type: database.host_type, + initial_program: database.initial_program, + }) + .map_err(Into::into) + } + + fn test_database_identity() -> anyhow::Result { + Ok("c200000000000000000000000000000000000000000000000000000000000000".parse()?) + } + + #[cfg(windows)] + #[allow(clippy::permissions_set_readonly_false)] + fn clear_readonly_recursively(path: &Path) -> anyhow::Result<()> { + let metadata = std::fs::symlink_metadata(path)?; + let mut permissions = metadata.permissions(); + if permissions.readonly() { + permissions.set_readonly(false); + std::fs::set_permissions(path, permissions)?; + } + if metadata.is_dir() { + for entry in std::fs::read_dir(path)? { + clear_readonly_recursively(&entry?.path())?; + } + } + Ok(()) + } + + fn make_backup_dir(path: &Path, replica_id: u64, offset: u64) -> anyhow::Result<()> { + let database_identity: Identity = test_database_identity()?; + + let snapshots_path = SnapshotsPath::from_path_unchecked(path.join("snapshots")); + std::fs::create_dir_all(&snapshots_path.0)?; + let snapshot_repo = SnapshotRepository::open(snapshots_path, database_identity, replica_id)?; + let blobs = HashMapBlobStore::default(); + snapshot_repo + .create_snapshot(std::iter::empty::<&mut Table>(), &blobs, offset)? + .sync_all()?; + + let clog_dir = path.join("clog"); + std::fs::create_dir_all(&clog_dir)?; + let clog = Commitlog::>::open( + CommitLogDir::from_path_unchecked(&clog_dir), + Default::default(), + None, + )?; + for tx_offset in 0..=offset { + clog.commit([( + tx_offset, + Txdata { + inputs: None, + outputs: None, + mutations: None, + }, + )])?; + } + clog.flush_and_sync()?; + + let manifest = serde_json::json!({ + "version": 1, + "database_identity": database_identity, + "replica_id": replica_id, + "snapshot_offset": offset, + "durable_offset": offset, + "output_dir": path, + "snapshot_ms": 1, + "copy_ms": 2, + "total_ms": 3, + "bytes": 4 + }); + std::fs::write(path.join("manifest.json"), serde_json::to_vec_pretty(&manifest)?)?; + Ok(()) + } +} diff --git a/crates/cli/src/subcommands/mod.rs b/crates/cli/src/subcommands/mod.rs index ef51b66f055..4a7d13a6ca8 100644 --- a/crates/cli/src/subcommands/mod.rs +++ b/crates/cli/src/subcommands/mod.rs @@ -1,3 +1,4 @@ +pub mod backup; pub mod build; pub mod call; pub mod db_arg_resolution; diff --git a/crates/client-api/Cargo.toml b/crates/client-api/Cargo.toml index afbb5a2a340..e34ffb0c0ed 100644 --- a/crates/client-api/Cargo.toml +++ b/crates/client-api/Cargo.toml @@ -11,6 +11,7 @@ spacetimedb-client-api-messages.workspace = true spacetimedb-core.workspace = true spacetimedb-data-structures.workspace = true spacetimedb-datastore.workspace = true +spacetimedb-fs-utils.workspace = true spacetimedb-lib = { workspace = true, features = ["serde"] } spacetimedb-paths.workspace = true spacetimedb-schema.workspace = true diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 7fd6f238128..cc9012efd39 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -1,6 +1,7 @@ use std::fmt; use std::future::Future; use std::num::NonZeroU8; +use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::anyhow; @@ -11,7 +12,10 @@ use http::StatusCode; use spacetimedb::client::ClientActorIndex; use spacetimedb::energy::{EnergyBalance, EnergyQuanta}; -use spacetimedb::host::{HostController, MigratePlanResult, ModuleHost, NoSuchModule, UpdateDatabaseResult}; +use spacetimedb::host::{ + HostController, HotBackupInProgress, HotBackupManifest, MigratePlanResult, ModuleHost, NoSuchModule, + UpdateDatabaseResult, +}; use spacetimedb::identity::{AuthCtx, Identity}; use spacetimedb::messages::control_db::{Database, HostType, Node, Replica}; use spacetimedb::sql; @@ -59,6 +63,17 @@ pub trait NodeDelegate: Send + Sync { /// /// The [`Host`] is spawned implicitly if not already running. async fn leader(&self, database_id: u64) -> Result; + async fn create_hot_backup(&self, leader: Host, output_dir: PathBuf) -> anyhow::Result { + leader.create_hot_backup(output_dir).await + } + /// Root directory under which HTTP-initiated hot backups may be written. + /// + /// Requested backup output directories are resolved against and confined + /// to this directory. Returning `None` (the default) disables the HTTP + /// backup endpoints entirely. + fn hot_backup_root(&self) -> Option { + None + } fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir; } @@ -107,6 +122,20 @@ impl Host { self.host_controller.get_module_host(self.replica_id).await } + pub async fn create_hot_backup(&self, output_dir: impl AsRef) -> anyhow::Result { + let _ = output_dir; + anyhow::bail!("hot backup requires the node delegate to export control-db and finalize the backup manifest") + } + + pub async fn create_hot_backup_without_control_db( + &self, + output_dir: impl AsRef, + ) -> anyhow::Result { + self.host_controller + .create_hot_backup_without_control_db(self.replica_id, output_dir) + .await + } + /// Wait for the module host to become available, retrying with backoff. /// /// This is useful for routes like `/schema` that may be called while the @@ -496,6 +525,14 @@ impl NodeDelegate for Arc { (**self).leader(database_id).await } + async fn create_hot_backup(&self, leader: Host, output_dir: PathBuf) -> anyhow::Result { + (**self).create_hot_backup(leader, output_dir).await + } + + fn hot_backup_root(&self) -> Option { + (**self).hot_backup_root() + } + fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir { (**self).module_logs_dir(replica_id) } @@ -569,6 +606,7 @@ pub enum Action { DeleteDatabase, RenameDatabase, ViewModuleLogs, + CreateHotBackup, } impl fmt::Display for Action { @@ -587,6 +625,7 @@ impl fmt::Display for Action { Self::DeleteDatabase => f.write_str("delete database"), Self::RenameDatabase => f.write_str("rename database"), Self::ViewModuleLogs => f.write_str("view module logs"), + Self::CreateHotBackup => f.write_str("create hot backup"), } } } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 862597d289c..2992ec70128 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -1,6 +1,7 @@ use std::borrow::Cow; use std::future::Future; use std::num::NonZeroU8; +use std::path::{Component, Path as StdPath, PathBuf}; use std::str::FromStr; use std::time::Duration; use std::{env, io}; @@ -42,6 +43,7 @@ use spacetimedb_client_api_messages::name::{ self, DatabaseName, DomainName, MigrationPolicy, PrePublishAutoMigrateResult, PrePublishManualMigrateResult, PrePublishResult, PrettyPrintStyle, PublishOp, PublishResult, }; +use spacetimedb_fs_utils::normalize_absolute_path; use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::{http as st_http, ConnectionId}; @@ -676,6 +678,86 @@ fn mime_ndjson() -> mime::Mime { "application/x-ndjson".parse().unwrap() } +#[derive(Deserialize)] +pub struct BackupParams { + name_or_identity: NameOrIdentity, +} + +#[derive(Deserialize)] +pub struct BackupRequest { + #[serde(alias = "output_dir")] + server_output_dir: PathBuf, +} + +fn bad_backup_request(message: impl Into) -> ErrorResponse { + (StatusCode::BAD_REQUEST, message.into()).into() +} + +fn resolve_hot_backup_output_dir( + hot_backup_root: Option, + requested: &StdPath, +) -> axum::response::Result { + let root = hot_backup_root + .ok_or_else(|| ErrorResponse::from((StatusCode::NOT_FOUND, "hot backup endpoint is disabled")))?; + + if requested.as_os_str().is_empty() { + return Err(bad_backup_request("backup output path must not be empty")); + } + if requested.is_absolute() { + return Err(bad_backup_request( + "backup output path must be relative to the configured hot backup root", + )); + } + if requested + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(bad_backup_request( + "backup output path must contain only normal relative path components", + )); + } + + let root = normalize_absolute_path(&root).map_err(log_and_500)?; + let output_dir = normalize_absolute_path(&root.join(requested)).map_err(log_and_500)?; + if !output_dir.starts_with(&root) { + return Err(bad_backup_request( + "backup output path must remain within the configured hot backup root", + )); + } + + Ok(output_dir) +} + +pub async fn backup( + State(worker_ctx): State, + Path(BackupParams { name_or_identity }): Path, + Extension(auth): Extension, + axum::Json(BackupRequest { server_output_dir }): axum::Json, +) -> axum::response::Result +where + S: ControlStateDelegate + NodeDelegate + Authorization, +{ + let server_output_dir = resolve_hot_backup_output_dir(worker_ctx.hot_backup_root(), &server_output_dir)?; + let database_identity = name_or_identity.resolve(&worker_ctx).await?; + let database = worker_ctx_find_database(&worker_ctx, &database_identity) + .await? + .ok_or(NO_SUCH_DATABASE)?; + worker_ctx + .authorize_action( + auth.claims.identity, + database.database_identity, + Action::CreateHotBackup, + ) + .await?; + + let leader = worker_ctx.leader(database.id).await.map_err(Into::into)?; + let manifest = worker_ctx + .create_hot_backup(leader, server_output_dir) + .await + .map_err(log_and_500)?; + Ok(axum::Json(manifest)) +} + pub(crate) async fn worker_ctx_find_database( worker_ctx: &(impl ControlStateDelegate + ?Sized), database_identity: &Identity, @@ -1561,6 +1643,8 @@ pub struct DatabaseRoutes { pub schema_get: MethodRouter, /// GET: /database/:name_or_identity/logs pub logs_get: MethodRouter, + /// POST: /database/:name_or_identity/backup + pub backup_post: MethodRouter, /// POST: /database/:name_or_identity/sql pub sql_post: MethodRouter, /// POST: /database/:name_or_identity/pre-publish @@ -1600,6 +1684,7 @@ where call_reducer_procedure_post: post(call::), schema_get: get(schema::), logs_get: get(logs::), + backup_post: post(backup::), sql_post: post(sql::), pre_publish: post(pre_publish::), db_reset: put(reset::), @@ -1630,6 +1715,7 @@ where .route("/call/:reducer", self.call_reducer_procedure_post) .route("/schema", self.schema_get) .route("/logs", self.logs_get) + .route("/backup", self.backup_post) .route("/sql", self.sql_post) .route("/unstable/timestamp", self.timestamp_get) .route("/pre_publish", self.pre_publish) @@ -1669,13 +1755,14 @@ where #[cfg(test)] mod tests { use super::*; - use crate::auth::JwtAuthProvider; + use crate::auth::{JwtAuthProvider, SpacetimeCreds}; use crate::routes::subscribe::{HasWebSocketOptions, WebSocketOptions}; use crate::{ Action, Authorization, ControlStateReadAccess, ControlStateWriteAccess, MaybeMisdirected, Unauthorized, }; use async_trait::async_trait; use axum::body::Body; + use axum::response::IntoResponse; use http::Request; use spacetimedb::auth::identity::{JwtError, JwtErrorKind, SpacetimeIdentityClaims}; use spacetimedb::auth::token_validation::{TokenSigner, TokenValidationError, TokenValidator}; @@ -1689,6 +1776,7 @@ mod tests { use spacetimedb_paths::server::ModuleLogsDir; use spacetimedb_paths::FromPathUnchecked; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; + use std::path::Path as StdPath; use tower::util::ServiceExt; #[derive(Clone, Default)] struct DummyValidator; @@ -1732,6 +1820,16 @@ mod tests { jwt: DummyJwtProvider, client_actor_index: std::sync::Arc, module_logs_dir: ModuleLogsDir, + database: Option, + hot_backup_root: Option, + leader_calls: std::sync::Arc, + authorize_action_result: DummyAuthorizeActionResult, + } + + #[derive(Clone, Copy)] + enum DummyAuthorizeActionResult { + Deny, + InternalError, } impl DummyState { @@ -1742,8 +1840,31 @@ mod tests { }, client_actor_index: std::sync::Arc::new(ClientActorIndex::new()), module_logs_dir: ModuleLogsDir::from_path_unchecked(std::env::temp_dir()), + database: None, + hot_backup_root: None, + leader_calls: std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)), + authorize_action_result: DummyAuthorizeActionResult::InternalError, } } + + fn with_database(mut self, database: Database) -> Self { + self.database = Some(database); + self + } + + fn with_hot_backup_root(mut self, root: PathBuf) -> Self { + self.hot_backup_root = Some(root); + self + } + + fn with_authorize_action_result(mut self, result: DummyAuthorizeActionResult) -> Self { + self.authorize_action_result = result; + self + } + + fn leader_calls(&self) -> usize { + self.leader_calls.load(std::sync::atomic::Ordering::SeqCst) + } } impl HasWebSocketOptions for DummyState { @@ -1770,9 +1891,14 @@ mod tests { } async fn leader(&self, _database_id: u64) -> Result { + self.leader_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); Err(DummyLeaderError) } + fn hot_backup_root(&self) -> Option { + self.hot_backup_root.clone() + } + fn module_logs_dir(&self, _replica_id: u64) -> ModuleLogsDir { self.module_logs_dir.clone() } @@ -1814,7 +1940,10 @@ mod tests { Ok(None) } async fn get_database_by_identity(&self, _database_identity: &Identity) -> anyhow::Result> { - Ok(None) + Ok(self + .database + .clone() + .filter(|database| database.database_identity == *_database_identity)) } async fn get_databases(&self) -> anyhow::Result> { Ok(Vec::new()) @@ -1920,11 +2049,21 @@ mod tests { impl Authorization for DummyState { async fn authorize_action( &self, - _subject: Identity, - _database: Identity, - _action: Action, + subject: Identity, + database: Identity, + action: Action, ) -> Result<(), Unauthorized> { - Err(Unauthorized::InternalError(anyhow::anyhow!("unused"))) + match self.authorize_action_result { + DummyAuthorizeActionResult::Deny => Err(Unauthorized::Unauthorized { + subject, + action, + database: Some(database), + source: None, + }), + DummyAuthorizeActionResult::InternalError => { + Err(Unauthorized::InternalError(anyhow::anyhow!("unused"))) + } + } } async fn authorize_sql(&self, _subject: Identity, _database: Identity) -> Result { @@ -1932,6 +2071,109 @@ mod tests { } } + fn err_status(result: axum::response::Result) -> StatusCode { + axum::response::Result::<()>::Err(result.unwrap_err()) + .into_response() + .status() + } + + fn test_auth(identity: Identity) -> SpacetimeAuth { + SpacetimeAuth { + creds: SpacetimeCreds::from_signed_token("unused.header.signature".to_owned()), + claims: spacetimedb::auth::identity::SpacetimeIdentityClaims { + identity, + subject: "subject".into(), + issuer: "issuer".into(), + audience: [].into(), + extra: None, + iat: std::time::SystemTime::now(), + exp: None, + }, + jwt_payload: "{}".into(), + } + } + + #[test] + fn backup_output_dir_requires_configured_root() { + assert_eq!( + err_status(resolve_hot_backup_output_dir(None, StdPath::new("backup-1"))), + StatusCode::NOT_FOUND + ); + } + + #[test] + fn backup_output_dir_resolves_relative_path_under_root() { + let root = tempfile::tempdir().unwrap(); + + let output = + resolve_hot_backup_output_dir(Some(root.path().to_path_buf()), StdPath::new("db/backup-1")).unwrap(); + + assert_eq!(output, root.path().canonicalize().unwrap().join("db/backup-1")); + } + + #[test] + fn backup_output_dir_rejects_absolute_path() { + let root = tempfile::tempdir().unwrap(); + + assert_eq!( + err_status(resolve_hot_backup_output_dir( + Some(root.path().to_path_buf()), + root.path() + )), + StatusCode::BAD_REQUEST + ); + } + + #[test] + fn backup_output_dir_rejects_parent_components() { + let root = tempfile::tempdir().unwrap(); + + assert_eq!( + err_status(resolve_hot_backup_output_dir( + Some(root.path().to_path_buf()), + StdPath::new("../escape") + )), + StatusCode::BAD_REQUEST + ); + } + + #[tokio::test] + async fn backup_authorizes_before_starting_leader() { + let root = tempfile::tempdir().unwrap(); + let subject = Identity::from_claims("issuer", "owner"); + let database_identity = Identity::from_claims("issuer", "database"); + let state = DummyState::new() + .with_hot_backup_root(root.path().to_path_buf()) + .with_database(Database { + id: 1, + database_identity, + owner_identity: subject, + host_type: HostType::Wasm, + initial_program: spacetimedb_lib::Hash::ZERO, + bootstrap_generation: 0, + }) + .with_authorize_action_result(DummyAuthorizeActionResult::Deny); + + let result = backup( + State(state.clone()), + Path(BackupParams { + name_or_identity: NameOrIdentity::Identity(database_identity.into()), + }), + Extension(test_auth(subject)), + axum::Json(BackupRequest { + server_output_dir: PathBuf::from("manual/backup-1"), + }), + ) + .await; + + let response = match result { + Ok(response) => response.into_response(), + Err(err) => axum::response::Result::<()>::Err(err).into_response(), + }; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(state.leader_calls(), 0); + } + /// Tests that requests to user-defined routes under `/database/:name-or-identity/routes` /// bypass the usual SpacetimeDB auth middleware, /// and accept requests with `Authorization` headers that SpacetimeDB would treat as malformed. diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 2f9c232bb0b..029988b47f7 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -1,3 +1,4 @@ +use super::hot_backup::{self, HotBackupInProgress}; use super::module_host::{DurableOffset, EventStatus, InitDatabaseResult, ModuleHost, ModuleInfo, NoSuchModule}; use super::scheduler::SchedulerStarter; use super::v8::V8HeapMetrics; @@ -46,6 +47,7 @@ use spacetimedb_schema::def::{ModuleDef, RawModuleDefVersion}; use spacetimedb_table::page_pool::PagePool; use std::future::Future; use std::ops::Deref; +use std::path::Path; use std::sync::Arc; use std::time::Duration; use tokio::sync::{watch, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock as AsyncRwLock}; @@ -690,6 +692,24 @@ impl HostController { .ok_or(NoSuchModule) } + /// Create a hot backup for `replica_id`, leaving scoped server state for + /// the caller to export before finalizing the manifest. + pub async fn create_hot_backup_without_control_db( + &self, + replica_id: u64, + output_dir: impl AsRef, + ) -> anyhow::Result { + let module = self.get_module_host(replica_id).await?; + hot_backup::create_hot_backup_without_control_db( + module.relational_db(), + &self.data_dir.replica(replica_id), + Some(&self.data_dir), + replica_id, + output_dir, + ) + .await + } + /// Subscribe to updates of the [`ModuleHost`] identified by `replica_id`, /// or return an error if it is not registered with the controller. /// diff --git a/crates/core/src/host/hot_backup.rs b/crates/core/src/host/hot_backup.rs new file mode 100644 index 00000000000..8c679d53ce9 --- /dev/null +++ b/crates/core/src/host/hot_backup.rs @@ -0,0 +1,1090 @@ +use crate::db::relational_db::{open_snapshot_repo, RelationalDB, TxOffset}; +use crate::util::asyncify; +use anyhow::{anyhow, Context}; +use parking_lot::Mutex; +use spacetimedb_commitlog::repo::Repo; +use spacetimedb_commitlog::{self as commitlog}; +use spacetimedb_fs_utils::{ + atomic_write_bytes, copy_dir_all, copy_file_sync, create_dir_all_sync, dir_size, normalize_absolute_path, sync_dir, +}; +use spacetimedb_lib::Identity; +use spacetimedb_paths::server::{CommitLogDir, ReplicaDir, ServerDataDir}; +use spacetimedb_paths::FromPathUnchecked; +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::fs::OpenOptions; +use std::io::{ErrorKind, Write as _}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +#[derive(Debug, serde::Serialize)] +pub struct HotBackupManifest { + pub version: u32, + pub database_identity: Identity, + pub replica_id: u64, + pub snapshot_offset: TxOffset, + pub durable_offset: TxOffset, + pub output_dir: PathBuf, + pub snapshot_ms: u64, + pub copy_ms: u64, + pub total_ms: u64, + pub bytes: u64, +} + +impl HotBackupManifest { + /// Convert a [`Duration`] to whole milliseconds, saturating at `u64::MAX`. + /// + /// Used to fill in the `*_ms` timing fields of the manifest. + pub fn elapsed_ms(duration: Duration) -> u64 { + duration.as_millis().try_into().unwrap_or(u64::MAX) + } +} + +/// A hot backup whose output directory is still being finalized. +/// +/// Dropping this value releases the in-process guard for its output directory. +/// Call [`Self::finalize_with_blocking`] to add caller-owned state and write the +/// final `manifest.json` while the guard is still held. +pub struct HotBackupInProgress { + manifest: HotBackupManifest, + _dir_guard: BackupDirGuard, +} + +impl HotBackupInProgress { + /// The manifest that will be written when this backup is finalized. + pub fn manifest(&self) -> &HotBackupManifest { + &self.manifest + } + + fn into_manifest(self) -> HotBackupManifest { + self.manifest + } + + /// Run caller-provided blocking finalization work, then write + /// `manifest.json` while still holding the backup output directory guard. + /// + /// The guard is moved into the blocking task with the manifest. If the + /// awaiting future is cancelled, the blocking task continues to own the + /// guard until the caller's work and final manifest write have returned. + pub async fn finalize_with_blocking(self, f: F) -> anyhow::Result + where + F: FnOnce(&mut HotBackupManifest) -> anyhow::Result<()> + Send + 'static, + { + asyncify(move || { + let mut backup = self; + f(&mut backup.manifest)?; + write_hot_backup_manifest_sync(&backup.manifest.output_dir, &backup.manifest)?; + Ok(backup.into_manifest()) + }) + .await + } +} + +/// Export a point-in-time hot backup into `output_dir`. +#[cfg(test)] +async fn create_hot_backup( + db: &RelationalDB, + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + replica_id: u64, + output_dir: impl AsRef, +) -> anyhow::Result { + let backup = create_hot_backup_inner(db, replica_dir, server_data_dir, replica_id, output_dir, true).await?; + Ok(backup.into_manifest()) +} + +/// Export a point-in-time hot backup into `output_dir`, leaving scoped server +/// state for the caller to provide. +/// +/// Unlike a finalized hot backup, this does **not** compute +/// [`HotBackupManifest::bytes`] (it is left at 0) and does **not** write +/// `manifest.json`: the caller is expected to add server state that must be +/// exported with broader context, such as `server/control-db`, and then finalize +/// the manifest itself. This both avoids traversing the backup tree twice and +/// makes `manifest.json` a reliable marker of a fully-written backup. +pub(super) async fn create_hot_backup_without_control_db( + db: &RelationalDB, + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + replica_id: u64, + output_dir: impl AsRef, +) -> anyhow::Result { + create_hot_backup_inner(db, replica_dir, server_data_dir, replica_id, output_dir, false).await +} + +async fn create_hot_backup_inner( + db: &RelationalDB, + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + replica_id: u64, + output_dir: impl AsRef, + copy_control_db: bool, +) -> anyhow::Result { + let total_start = Instant::now(); + let output_dir = ensure_backup_path(replica_dir, server_data_dir, output_dir.as_ref())?; + // Guard against two concurrent backups interleaving writes into the same + // directory after both passed the `ensure_empty_dir` check. + let dir_guard = BackupDirGuard::acquire(&output_dir)?; + anyhow::ensure!( + !(copy_control_db && server_data_dir.is_some()), + "copying live server/control-db is not supported by hot backup; export control-db separately and finalize the manifest after the export" + ); + + let snapshot_start = Instant::now(); + let snapshot_offset = db.request_hot_backup_snapshot().await?; + let snapshot_ms = HotBackupManifest::elapsed_ms(snapshot_start.elapsed()); + let durable_offset = snapshot_offset; + + let copy_start = Instant::now(); + let snapshot_repo = open_snapshot_repo(replica_dir.snapshots(), db.database_identity(), replica_id)?; + let src_snapshot_dir = snapshot_repo.snapshot_dir_path(snapshot_offset); + let dst_snapshots = output_dir.join("snapshots"); + let dst_snapshot_dir = dst_snapshots.join( + src_snapshot_dir + .0 + .file_name() + .context("snapshot directory has no file name")?, + ); + // `request_hot_backup_snapshot` returns only after the snapshot worker has + // fsynced the snapshot to disk (see `SnapshotWorker::ensure_snapshot_at_least`), + // so both paths must already exist; a missing path is a bug, not a timing issue. + anyhow::ensure!( + src_snapshot_dir.0.is_dir(), + "snapshot directory does not exist: {}", + src_snapshot_dir.display() + ); + let src_snapshot_file = src_snapshot_dir.snapshot_file(snapshot_offset); + anyhow::ensure!( + src_snapshot_file.0.is_file(), + "snapshot file does not exist: {}", + src_snapshot_file.display() + ); + let snapshot_file_name: OsString = src_snapshot_file + .0 + .file_name() + .context("snapshot file has no file name")? + .to_owned(); + + asyncify({ + let output_dir = output_dir.clone(); + let src_snapshot_dir = src_snapshot_dir.clone(); + let snapshot_file_name = snapshot_file_name.clone(); + let dir_guard = dir_guard.clone(); + move || -> anyhow::Result<()> { + ensure_empty_dir(&output_dir, &dir_guard)?; + copy_dir_all_retry(&src_snapshot_dir.0, &dst_snapshot_dir, &snapshot_file_name)?; + Ok(()) + } + }) + .await?; + + if let Some(server_data_dir) = server_data_dir { + copy_server_state( + server_data_dir, + &output_dir, + copy_control_db, + commitlog::DEFAULT_LOG_FORMAT_VERSION, + dir_guard.clone(), + ) + .await?; + } + copy_commitlog_range( + replica_dir.commit_log(), + output_dir.join("clog"), + snapshot_offset, + dir_guard.clone(), + ) + .await?; + let copy_ms = HotBackupManifest::elapsed_ms(copy_start.elapsed()); + + let mut manifest = HotBackupManifest { + version: 1, + database_identity: db.database_identity(), + replica_id, + snapshot_offset, + durable_offset, + output_dir: output_dir.clone(), + snapshot_ms, + copy_ms, + total_ms: HotBackupManifest::elapsed_ms(total_start.elapsed()), + bytes: 0, + }; + // When the caller provides `server/control-db` itself, leave `bytes` + // and `manifest.json` to it, so the backup tree is only traversed once + // and the manifest is written exactly once, after the backup is complete. + if copy_control_db { + manifest.bytes = asyncify({ + let output_dir = output_dir.clone(); + let dir_guard = dir_guard.clone(); + move || { + let _dir_guard = dir_guard; + dir_size(&output_dir) + } + }) + .await?; + write_hot_backup_manifest_guarded(&output_dir, &manifest, dir_guard.clone()).await?; + } + Ok(HotBackupInProgress { + manifest, + _dir_guard: dir_guard, + }) +} + +async fn copy_commitlog_range( + src: CommitLogDir, + dst: PathBuf, + through: TxOffset, + dir_guard: BackupDirGuard, +) -> anyhow::Result<()> { + asyncify(move || -> anyhow::Result<()> { + let _dir_guard = dir_guard; + copy_commitlog_range_sync(src, &dst, through)?; + sync_dir(&dst).with_context(|| format!("syncing backup commitlog dir {}", dst.display()))?; + if let Some(parent) = dst.parent() { + sync_dir(parent).with_context(|| format!("syncing backup commitlog parent {}", parent.display()))?; + } + Ok(()) + }) + .await?; + Ok(()) +} + +fn copy_commitlog_range_sync(src: CommitLogDir, dst: &Path, through: TxOffset) -> anyhow::Result<()> { + let src_repo = commitlog::repo::Fs::new(src, None)?; + let dst_repo = commitlog::repo::Fs::new(CommitLogDir::from_path_unchecked(dst), None)?; + let mut copied_any_commit = false; + let mut next_tx_offset = None; + for segment_offset in src_repo.existing_offsets()? { + let reader = + commitlog::repo::open_segment_reader(&src_repo, commitlog::DEFAULT_LOG_FORMAT_VERSION, segment_offset) + .with_context(|| format!("opening source commitlog segment {segment_offset}"))?; + let mut writer = None; + for commit in reader.commits() { + let commit = commit.with_context(|| format!("reading source commitlog segment {segment_offset}"))?; + if commit.min_tx_offset > through { + break; + } + let commit_range = commit.tx_range(); + if let Some(expected_tx_offset) = next_tx_offset { + anyhow::ensure!( + commit.min_tx_offset == expected_tx_offset, + "source commitlog is not contiguous at segment {segment_offset}: expected tx offset {expected_tx_offset}, got {}", + commit.min_tx_offset + ); + } + next_tx_offset = Some(commit_range.end); + let writer = writer.get_or_insert_with(|| { + dst_repo.create_segment( + segment_offset, + commitlog::segment::Header { + log_format_version: commitlog::DEFAULT_LOG_FORMAT_VERSION, + checksum_algorithm: commitlog::Commit::CHECKSUM_ALGORITHM, + }, + ) + }); + let writer = writer + .as_mut() + .map_err(|err| anyhow!("creating target commitlog segment {segment_offset}: {err}"))?; + commitlog::Commit::from(commit) + .write(&mut *writer) + .with_context(|| format!("writing target commitlog segment {segment_offset}"))?; + copied_any_commit = true; + } + if let Some(writer) = writer { + let mut writer = writer.with_context(|| format!("creating target commitlog segment {segment_offset}"))?; + writer + .flush() + .with_context(|| format!("flushing target commitlog segment {segment_offset}"))?; + writer + .sync_data() + .with_context(|| format!("syncing target commitlog segment {segment_offset}"))?; + } + } + anyhow::ensure!( + copied_any_commit, + "source commitlog does not contain commits through snapshot offset {through}" + ); + Ok(()) +} + +async fn write_hot_backup_manifest_guarded( + output_dir: &Path, + manifest: &HotBackupManifest, + dir_guard: BackupDirGuard, +) -> anyhow::Result<()> { + let output_dir = output_dir.to_path_buf(); + let json = serde_json::to_vec_pretty(manifest)?; + asyncify(move || { + let _dir_guard = dir_guard; + write_hot_backup_manifest_bytes(&output_dir, &json) + }) + .await?; + Ok(()) +} + +fn write_hot_backup_manifest_sync(output_dir: &Path, manifest: &HotBackupManifest) -> anyhow::Result<()> { + let json = serde_json::to_vec_pretty(manifest)?; + write_hot_backup_manifest_bytes(output_dir, &json) +} + +fn write_hot_backup_manifest_bytes(output_dir: &Path, json: &[u8]) -> anyhow::Result<()> { + atomic_write_bytes(&output_dir.join("manifest.json"), json) + .with_context(|| format!("writing hot backup manifest in {}", output_dir.display())) +} + +const BACKUP_DIR_CLAIM_MARKER: &str = ".spacetimedb-hot-backup-in-progress"; + +/// Serializes concurrent hot backups into overlapping output directories in +/// this process and claims the exact output directory across processes. +/// +/// Two concurrent backups into the same (empty) directory would both pass the +/// `ensure_empty_dir` check and interleave their writes, producing a corrupt +/// backup. The process-wide map rejects overlapping paths locally, while a +/// hidden `create_new` marker atomically claims the exact path on the +/// filesystem. Both claims are released when the last guard clone drops. +struct BackupDirGuard(PathBuf); + +static ACTIVE_BACKUP_DIRS: Mutex> = Mutex::new(BTreeMap::new()); + +impl BackupDirGuard { + fn acquire(output_dir: &Path) -> anyhow::Result { + let mut active = ACTIVE_BACKUP_DIRS.lock(); + if let Some(overlapping) = active + .keys() + .find(|active_dir| backup_paths_overlap(output_dir, active_dir)) + { + anyhow::bail!( + "backup output path {} overlaps with in-progress backup {}", + output_dir.display(), + overlapping.display() + ); + } + + ensure_no_claimed_ancestor(output_dir)?; + create_dir_all_sync(output_dir) + .with_context(|| format!("creating backup output directory {}", output_dir.display()))?; + let marker_path = Self::marker_path_for(output_dir); + let marker = match OpenOptions::new().write(true).create_new(true).open(&marker_path) { + Ok(marker) => marker, + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + anyhow::bail!( + "backup output directory {} is already claimed by marker {}", + output_dir.display(), + marker_path.display() + ) + } + Err(err) => { + return Err(err).with_context(|| format!("claiming backup output directory {}", output_dir.display())) + } + }; + let claim_result = marker + .sync_all() + .map_err(anyhow::Error::from) + .and_then(|()| ensure_no_claimed_ancestor(output_dir)) + .and_then(|()| sync_dir(output_dir).map_err(anyhow::Error::from)); + drop(marker); + if let Err(err) = claim_result { + let _ = std::fs::remove_file(&marker_path); + let _ = sync_dir(output_dir); + return Err(err).with_context(|| format!("persisting backup output claim {}", marker_path.display())); + } + + active.insert(output_dir.to_path_buf(), 1); + Ok(Self(output_dir.to_path_buf())) + } + + fn marker_path_for(output_dir: &Path) -> PathBuf { + output_dir.join(BACKUP_DIR_CLAIM_MARKER) + } + + fn marker_path(&self) -> PathBuf { + Self::marker_path_for(&self.0) + } +} + +fn ensure_no_claimed_ancestor(output_dir: &Path) -> anyhow::Result<()> { + for ancestor in output_dir.ancestors().skip(1) { + let marker_path = BackupDirGuard::marker_path_for(ancestor); + anyhow::ensure!( + !marker_path + .try_exists() + .with_context(|| format!("checking backup output claim {}", marker_path.display()))?, + "backup output directory {} is inside claimed backup directory {}", + output_dir.display(), + ancestor.display() + ); + } + Ok(()) +} + +impl Clone for BackupDirGuard { + fn clone(&self) -> Self { + let mut active = ACTIVE_BACKUP_DIRS.lock(); + let count = active + .get_mut(&self.0) + .expect("cloning active backup directory guard after it was released"); + *count = count + .checked_add(1) + .expect("active backup directory guard refcount overflow"); + Self(self.0.clone()) + } +} + +#[cfg(windows)] +fn backup_path_components(path: &Path) -> Vec { + path.components() + .map(|component| component.as_os_str().to_string_lossy().to_ascii_lowercase()) + .collect() +} + +#[cfg(windows)] +fn backup_path_starts_with(path: &Path, base: &Path) -> bool { + let path = backup_path_components(path); + let base = backup_path_components(base); + base.len() <= path.len() && path.iter().zip(&base).all(|(path, base)| path == base) +} + +#[cfg(not(windows))] +fn backup_path_starts_with(path: &Path, base: &Path) -> bool { + path.starts_with(base) +} + +fn backup_paths_overlap(a: &Path, b: &Path) -> bool { + backup_path_starts_with(a, b) || backup_path_starts_with(b, a) +} + +impl Drop for BackupDirGuard { + fn drop(&mut self) { + let mut active = ACTIVE_BACKUP_DIRS.lock(); + let Some(count) = active.get_mut(&self.0) else { + panic!("dropping backup directory guard after it was released"); + }; + if *count == 1 { + let _ = std::fs::remove_file(self.marker_path()); + let _ = sync_dir(&self.0); + active.remove(&self.0); + } else { + *count -= 1; + } + } +} + +fn ensure_empty_dir(path: &Path, dir_guard: &BackupDirGuard) -> anyhow::Result<()> { + anyhow::ensure!( + path == dir_guard.0.as_path(), + "backup output guard does not own directory: {}", + path.display() + ); + anyhow::ensure!( + path.is_dir(), + "backup output path is not a directory: {}", + path.display() + ); + + let marker_path = dir_guard.marker_path(); + let mut found_marker = false; + for entry in path.read_dir()? { + let entry = entry?; + anyhow::ensure!( + entry.path() == marker_path, + "backup output directory must be empty: {}", + path.display() + ); + let file_type = entry.file_type()?; + let metadata = entry.metadata()?; + anyhow::ensure!( + file_type.is_file() && metadata.len() == 0, + "backup output claim marker is invalid: {}", + marker_path.display() + ); + found_marker = true; + } + anyhow::ensure!( + found_marker, + "backup output claim marker is missing: {}", + marker_path.display() + ); + Ok(()) +} + +/// Validate `output_dir` as a hot backup destination and return its normalized +/// form. +/// +/// The path must be absolute and, after resolving symlinks in its existing +/// ancestors and rejecting `..` components (see +/// [`spacetimedb_fs_utils::normalize_absolute_path`]), must not point inside +/// the server data directory or the replica directory. All subsequent backup +/// I/O must use the returned path, so the checks cannot be bypassed via +/// symlinks or path traversal. +fn ensure_backup_path( + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + output_dir: &Path, +) -> anyhow::Result { + anyhow::ensure!( + output_dir.is_absolute(), + "backup output directory must be an absolute server path: {}", + output_dir.display() + ); + let output_dir = normalize_absolute_path(output_dir).context("normalizing the backup output directory path")?; + if let Some(server_data_dir) = server_data_dir { + let data_dir = normalize_absolute_path(&server_data_dir.0).unwrap_or_else(|_| server_data_dir.0.to_path_buf()); + anyhow::ensure!( + !output_dir.starts_with(&data_dir), + "backup output directory must not be inside the server data directory: {}", + output_dir.display() + ); + } + let replica_dir_normalized = + normalize_absolute_path(&replica_dir.0).unwrap_or_else(|_| replica_dir.0.to_path_buf()); + anyhow::ensure!( + !output_dir.starts_with(&replica_dir_normalized), + "backup output directory must not be inside the replica directory: {}", + output_dir.display() + ); + Ok(output_dir) +} + +async fn copy_server_state( + data_dir: &ServerDataDir, + output_dir: &Path, + copy_control_db: bool, + output_log_format_version: u8, + dir_guard: BackupDirGuard, +) -> anyhow::Result<()> { + let data_dir = data_dir.clone(); + let output_dir = output_dir.to_path_buf(); + asyncify(move || -> anyhow::Result<()> { + let _dir_guard = dir_guard; + assert!(!copy_control_db); + let server_dir = output_dir.join("server"); + create_dir_all_sync(&server_dir)?; + + copy_server_config( + &data_dir.config_toml().0, + &server_dir.join("config.toml"), + output_log_format_version, + )?; + copy_required_file(&data_dir.metadata_toml().0, &server_dir.join("metadata.toml"))?; + sync_dir(&server_dir)?; + Ok(()) + }) + .await?; + Ok(()) +} + +fn copy_server_config(src: &Path, dst: &Path, output_log_format_version: u8) -> anyhow::Result<()> { + anyhow::ensure!(src.is_file(), "server state file is missing: {}", src.display()); + let source = std::fs::read_to_string(src).with_context(|| format!("reading {}", src.display()))?; + let mut config: toml::Table = + toml::from_str(&source).with_context(|| format!("parsing server config {}", src.display()))?; + let commitlog = config + .entry("commitlog".to_owned()) + .or_insert_with(|| toml::Value::Table(Default::default())) + .as_table_mut() + .with_context(|| format!("server config [commitlog] is not a table in {}", src.display()))?; + commitlog.insert( + "log-format-version".to_owned(), + toml::Value::Integer(i64::from(output_log_format_version)), + ); + + let serialized = toml::to_string_pretty(&config).context("serializing backup server config")?; + atomic_write_bytes(dst, serialized.as_bytes()) + .with_context(|| format!("writing backup server config {}", dst.display()))?; + Ok(()) +} + +fn copy_required_file(src: &Path, dst: &Path) -> anyhow::Result<()> { + anyhow::ensure!(src.is_file(), "server state file is missing: {}", src.display()); + if let Some(parent) = dst.parent() { + create_dir_all_sync(parent)?; + } + copy_file_sync(src, dst).with_context(|| format!("copying {} to {}", src.display(), dst.display()))?; + Ok(()) +} + +fn copy_dir_all_retry(src: &Path, dst: &Path, required_file_name: &OsString) -> anyhow::Result<()> { + const RETRY_LIMIT: u32 = 3_000; + const RETRY_SLEEP: Duration = Duration::from_millis(10); + + let mut last_err = None; + for _ in 0..RETRY_LIMIT { + match copy_snapshot_dir(src, dst, required_file_name) { + Ok(()) => return Ok(()), + Err(err) => { + last_err = Some(err); + std::thread::sleep(RETRY_SLEEP); + } + } + } + let err = last_err.expect("copy_dir_all_retry loop must run at least once"); + Err(err).with_context(|| format!("copying snapshot {} to {}", src.display(), dst.display())) +} + +fn copy_snapshot_dir(src: &Path, dst: &Path, required_file_name: &OsString) -> anyhow::Result<()> { + create_dir_all_sync(dst)?; + let src_required = src.join(required_file_name); + let dst_required = dst.join(required_file_name); + copy_file_sync(&src_required, &dst_required) + .with_context(|| format!("copying {} to {}", src_required.display(), dst_required.display()))?; + + for entry in std::fs::read_dir(src).with_context(|| format!("reading {}", src.display()))? { + let entry = entry?; + if entry.file_name() == *required_file_name { + continue; + } + let ty = entry.file_type()?; + let dst = dst.join(entry.file_name()); + if ty.is_dir() { + copy_dir_all(entry.path(), dst)?; + } else { + copy_file_sync(&entry.path(), &dst)?; + } + } + anyhow::ensure!( + dst_required.is_file(), + "copying snapshot {} to {} did not copy {}", + src.display(), + dst.display(), + required_file_name.to_string_lossy() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::disallowed_macros)] + + use super::*; + use crate::db::relational_db::tests_utils::{begin_mut_tx, TestDB}; + use crate::db::relational_db::Txdata; + use spacetimedb_commitlog::Commitlog; + use spacetimedb_lib::db::raw_def::v9::{RawModuleDefV9Builder, RawTableDefBuilder}; + use spacetimedb_paths::server::{CommitLogDir, SnapshotsPath}; + use spacetimedb_paths::FromPathUnchecked; + use spacetimedb_primitives::TableId; + use spacetimedb_sats::raw_identifier::RawIdentifier; + use spacetimedb_sats::{AlgebraicType, ProductType}; + use spacetimedb_schema::def::ModuleDef; + use spacetimedb_schema::schema::{Schema as _, TableSchema}; + use spacetimedb_table::page_pool::PagePool; + + const HOT_BACKUP_TEST_TIMEOUT: Duration = Duration::from_secs(60); + + fn my_table(col_type: AlgebraicType) -> TableSchema { + table("MyTable", ProductType::from([("my_col", col_type)]), |builder| builder) + } + + fn table( + name: &str, + columns: ProductType, + f: impl FnOnce(RawTableDefBuilder<'_>) -> RawTableDefBuilder, + ) -> TableSchema { + let mut builder = RawModuleDefV9Builder::new(); + f(builder.build_table_with_new_type(RawIdentifier::new(name), columns, true)); + let raw = builder.finish(); + let def: ModuleDef = raw.try_into().expect("table validation failed"); + let table = def.table(name).expect("table not found"); + TableSchema::from_module_def(&def, table, (), TableId::SENTINEL) + } + + fn add_table(stdb: &RelationalDB) -> anyhow::Result<()> { + let mut tx = begin_mut_tx(stdb); + stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; + stdb.commit_tx(tx)?; + Ok(()) + } + + fn write_test_server_state(data_dir: &Path, config: &str) -> std::io::Result<()> { + std::fs::write(data_dir.join("config.toml"), config)?; + std::fs::write(data_dir.join("metadata.toml"), b"metadata")?; + Ok(()) + } + + #[test] + fn hot_backup_create_writes_manifest_snapshot_and_commitlog() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + add_table(&stdb)?; + + let backup_dir = tempfile::tempdir()?; + let manifest = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + create_hot_backup(&stdb, stdb.path().unwrap(), None, 0, backup_dir.path()), + ) + .await + })??; + + assert!(backup_dir.path().join("manifest.json").is_file()); + assert!(manifest.bytes > 0); + assert_eq!(manifest.durable_offset, manifest.snapshot_offset); + let manifest_json: serde_json::Value = + serde_json::from_slice(&std::fs::read(backup_dir.path().join("manifest.json"))?)?; + assert_eq!( + manifest_json["snapshot_offset"].as_u64(), + Some(manifest.snapshot_offset) + ); + assert_eq!(manifest_json["durable_offset"].as_u64(), Some(manifest.snapshot_offset)); + let repo = open_snapshot_repo( + SnapshotsPath::from_path_unchecked(backup_dir.path().join("snapshots")), + stdb.database_identity(), + 0, + )?; + let snapshot = repo.read_snapshot(manifest.snapshot_offset, &PagePool::new_for_test())?; + assert_eq!(snapshot.database_identity, manifest.database_identity); + assert_eq!(snapshot.replica_id, manifest.replica_id); + assert_eq!(snapshot.tx_offset, manifest.snapshot_offset); + let clog = Commitlog::::open( + CommitLogDir::from_path_unchecked(backup_dir.path().join("clog")), + Default::default(), + None, + )?; + assert_eq!(clog.max_committed_offset(), Some(manifest.snapshot_offset)); + + Ok(()) + } + + #[test] + fn hot_backup_create_without_control_db_rewrites_v0_server_config() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + add_table(&stdb)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + let source_config = r#"[commitlog] +log-format-version = 0 +max-segment-size = 4096 + +[logs] +directives = ["info"] +"#; + write_test_server_state(data.path(), source_config)?; + std::fs::create_dir_all(data.path().join("control-db"))?; + std::fs::write(data.path().join("control-db/control"), b"control")?; + + let backup_dir = tempfile::tempdir()?; + let backup = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + create_hot_backup_without_control_db( + &stdb, + stdb.path().unwrap(), + Some(&data_dir), + 0, + backup_dir.path(), + ), + ) + .await + })??; + + assert_eq!(backup.manifest().bytes, 0); + assert!(!backup_dir.path().join("manifest.json").exists()); + assert_eq!(std::fs::read_to_string(data.path().join("config.toml"))?, source_config); + let backup_config: toml::Value = + toml::from_str(&std::fs::read_to_string(backup_dir.path().join("server/config.toml"))?)?; + assert_eq!( + backup_config["commitlog"]["log-format-version"].as_integer(), + Some(i64::from(commitlog::DEFAULT_LOG_FORMAT_VERSION)) + ); + assert_eq!(backup_config["commitlog"]["max-segment-size"].as_integer(), Some(4096)); + assert_eq!(backup_config["logs"]["directives"][0].as_str(), Some("info")); + assert_eq!( + std::fs::read_to_string(backup_dir.path().join("server/metadata.toml"))?, + "metadata" + ); + assert!(!backup_dir.path().join("server/control-db").exists()); + assert!(!backup_dir.path().join("server/program-bytes").exists()); + + let output_commitlog = + commitlog::repo::Fs::new(CommitLogDir::from_path_unchecked(backup_dir.path().join("clog")), None)?; + let segment_offset = output_commitlog + .existing_offsets()? + .into_iter() + .next() + .context("backup commitlog has no segments")?; + let segment = output_commitlog.open_segment_reader(segment_offset)?; + let header = commitlog::segment::Header::decode(segment)?; + assert_eq!(header.log_format_version, commitlog::DEFAULT_LOG_FORMAT_VERSION); + + Ok(()) + } + + #[test] + fn hot_backup_without_control_db_keeps_output_guard_until_manifest_finalization() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + add_table(&stdb)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + write_test_server_state(data.path(), "[commitlog]\nlog-format-version = 1\n")?; + + let backup_dir = tempfile::tempdir()?; + let backup = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + create_hot_backup_without_control_db( + &stdb, + stdb.path().unwrap(), + Some(&data_dir), + 0, + backup_dir.path(), + ), + ) + .await + })??; + let output_dir = backup.manifest().output_dir.clone(); + + let err = match BackupDirGuard::acquire(&output_dir.join("nested")) { + Ok(_) => panic!("overlapping backup directory guard was accepted"), + Err(err) => err, + }; + assert!(err.to_string().contains("overlaps")); + + drop(backup); + let guard = BackupDirGuard::acquire(&output_dir.join("nested"))?; + drop(guard); + + Ok(()) + } + + #[test] + fn hot_backup_finalize_with_blocking_keeps_output_guard_after_cancellation() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + add_table(&stdb)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + write_test_server_state(data.path(), "[commitlog]\nlog-format-version = 1\n")?; + + let backup_dir = tempfile::tempdir()?; + let backup = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + create_hot_backup_without_control_db( + &stdb, + stdb.path().unwrap(), + Some(&data_dir), + 0, + backup_dir.path(), + ), + ) + .await + })??; + let output_dir = backup.manifest().output_dir.clone(); + + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let (finish_tx, finish_rx) = std::sync::mpsc::channel(); + let handle = stdb.runtime().unwrap().spawn(async move { + backup + .finalize_with_blocking(move |_| { + started_tx + .send(()) + .map_err(|_| anyhow!("finalize start receiver dropped"))?; + finish_rx + .recv() + .context("waiting for test to release hot backup finalize")?; + Ok(()) + }) + .await + }); + + started_rx + .recv_timeout(HOT_BACKUP_TEST_TIMEOUT) + .context("hot backup finalize did not start")?; + handle.abort(); + + let err = match BackupDirGuard::acquire(&output_dir.join("nested")) { + Ok(_) => panic!("overlapping backup directory guard was accepted after cancellation"), + Err(err) => err, + }; + assert!(err.to_string().contains("overlaps")); + + finish_tx + .send(()) + .map_err(|_| anyhow!("finalize blocking task finished before release signal"))?; + let deadline = Instant::now() + HOT_BACKUP_TEST_TIMEOUT; + loop { + match BackupDirGuard::acquire(&output_dir.join("nested")) { + Ok(guard) => { + drop(guard); + break; + } + Err(err) if err.to_string().contains("overlaps") && Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(err) => return Err(err), + } + } + assert!(output_dir.join("manifest.json").is_file()); + + Ok(()) + } + + #[test] + fn hot_backup_rejects_raw_copy_of_live_control_db() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + add_table(&stdb)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + write_test_server_state(data.path(), "[commitlog]\nlog-format-version = 1\n")?; + std::fs::create_dir_all(data.path().join("control-db"))?; + std::fs::write(data.path().join("control-db/control"), b"control")?; + + let backup_dir = tempfile::tempdir()?; + let result = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + create_hot_backup(&stdb, stdb.path().unwrap(), Some(&data_dir), 0, backup_dir.path()), + ) + .await + })?; + let err = result.unwrap_err(); + + assert!(err.to_string().contains("control-db")); + assert!(!backup_dir.path().join("manifest.json").exists()); + Ok(()) + } + + #[test] + fn hot_backup_rejects_unsafe_output_dirs() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + let replica_dir = stdb.path().unwrap(); + + let err = stdb + .runtime() + .unwrap() + .block_on(create_hot_backup( + &stdb, + replica_dir, + None, + 0, + PathBuf::from("relative"), + )) + .unwrap_err(); + assert!(err.to_string().contains("absolute server path")); + + let err = stdb + .runtime() + .unwrap() + .block_on(create_hot_backup( + &stdb, + replica_dir, + None, + 0, + replica_dir.0.join("backup"), + )) + .unwrap_err(); + assert!(err.to_string().contains("replica directory")); + + Ok(()) + } + + #[test] + fn hot_backup_dir_guard_rejects_parent_child_overlap() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let parent = temp.path().join("backup"); + let child = parent.join("nested"); + + let parent_guard = BackupDirGuard::acquire(&parent)?; + let Err(err) = BackupDirGuard::acquire(&child) else { + panic!("child backup directory overlapped active parent"); + }; + assert!(err.to_string().contains("overlaps")); + drop(parent_guard); + + let child_guard = BackupDirGuard::acquire(&child)?; + let Err(err) = BackupDirGuard::acquire(&parent) else { + panic!("parent backup directory overlapped active child"); + }; + assert!(err.to_string().contains("overlaps")); + drop(child_guard); + + Ok(()) + } + + #[test] + fn hot_backup_dir_guard_clone_keeps_path_active() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let parent = temp.path().join("backup"); + let child = parent.join("nested"); + + let parent_guard = BackupDirGuard::acquire(&parent)?; + let blocking_guard = parent_guard.clone(); + drop(parent_guard); + + let Err(err) = BackupDirGuard::acquire(&child) else { + panic!("child backup directory overlapped cloned active parent"); + }; + assert!(err.to_string().contains("overlaps")); + + drop(blocking_guard); + let child_guard = BackupDirGuard::acquire(&child)?; + drop(child_guard); + + Ok(()) + } + + #[test] + fn hot_backup_dir_guard_rejects_existing_filesystem_claim() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let output_dir = temp.path().join("backup"); + std::fs::create_dir(&output_dir)?; + let marker_path = BackupDirGuard::marker_path_for(&output_dir); + std::fs::write(&marker_path, [])?; + + let Err(err) = BackupDirGuard::acquire(&output_dir) else { + panic!("existing filesystem claim marker was accepted"); + }; + assert!(err.to_string().contains("already claimed")); + assert!(marker_path.is_file()); + + Ok(()) + } + + #[test] + fn hot_backup_dir_guard_rejects_claimed_ancestor() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let parent = temp.path().join("parent"); + std::fs::create_dir(&parent)?; + let marker_path = BackupDirGuard::marker_path_for(&parent); + std::fs::write(&marker_path, [])?; + + let child = parent.join("child"); + let err = match BackupDirGuard::acquire(&child) { + Ok(_) => panic!("backup directory inside a claimed ancestor was accepted"), + Err(err) => err, + }; + + assert!(err.to_string().contains("inside claimed backup directory")); + assert!(!child.exists()); + Ok(()) + } + + #[test] + fn hot_backup_dir_guard_marker_is_only_allowed_entry_and_cleans_up() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let output_dir = temp.path().join("backup"); + let guard = BackupDirGuard::acquire(&output_dir)?; + let marker_path = guard.marker_path(); + + assert!(marker_path.is_file()); + assert_eq!(std::fs::metadata(&marker_path)?.len(), 0); + ensure_empty_dir(&output_dir, &guard)?; + + let other_path = output_dir.join("other"); + std::fs::write(&other_path, b"not empty")?; + let err = ensure_empty_dir(&output_dir, &guard).unwrap_err(); + assert!(err.to_string().contains("must be empty")); + std::fs::remove_file(other_path)?; + + let cloned_guard = guard.clone(); + drop(guard); + assert!(marker_path.is_file()); + drop(cloned_guard); + assert!(!marker_path.exists()); + assert!(output_dir.read_dir()?.next().is_none()); + + Ok(()) + } +} diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 68f6035df80..4ddd2ea0996 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -12,6 +12,7 @@ use spacetimedb_schema::def::ModuleDef; mod disk_storage; mod host_controller; +mod hot_backup; mod module_common; #[allow(clippy::too_many_arguments)] pub mod module_host; @@ -29,6 +30,7 @@ pub use host_controller::{ HostController, HostRuntimeConfig, MigratePlanResult, ModuleHostWithBootstrap, ProcedureCallResult, ProgramStorage, ReducerCallResult, ReducerCallResultWithTxOffset, ReducerOutcome, }; +pub use hot_backup::{HotBackupInProgress, HotBackupManifest}; pub use module_host::{ InitDatabaseResult, ModuleHost, NoSuchModule, ProcedureCallError, ReducerCallError, UpdateDatabaseResult, }; diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index a94b675bf96..20f717e4d70 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -8,7 +8,7 @@ use crate::MetricsRecorderQueue; use anyhow::{anyhow, Context}; use enum_map::EnumMap; use spacetimedb_commitlog::repo::OnNewSegmentFn; -use spacetimedb_commitlog::{self as commitlog, Commitlog, SizeOnDisk}; +use spacetimedb_commitlog::{self as commitlog, Commitlog, CompressionStats, SizeOnDisk}; use spacetimedb_data_structures::map::HashSet; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::error::{DatastoreError, TableError, ViewError}; @@ -321,6 +321,11 @@ impl RelationalDB { } apply_history(&inner, database_identity, history)?; + if durable_tx_offset.is_some() + && let Some(snapshot_worker) = persistence.as_ref().and_then(|p| p.snapshots.as_ref()) + { + snapshot_worker.request_snapshot(); + } let elapsed_time = start_time.elapsed(); ENGINE_METRICS @@ -593,17 +598,24 @@ impl RelationalDB { } } - if let Some((snapshot_repo, durable_tx_offset)) = snapshot_repo.zip(durable_tx_offset) { - // Mark any newer snapshots as invalid, as the history past - // `durable_tx_offset` may have been reset and thus diverge from - // any snapshots taken earlier. - snapshot_repo - .invalidate_newer_snapshots(durable_tx_offset) - .map_err(|e| RestoreSnapshotError::Invalidate { - offset: durable_tx_offset, - source: Box::new(e), + if let Some(snapshot_repo) = snapshot_repo { + // Mark every snapshot without durable history as invalid. Otherwise, + // invalidate only snapshots after the last durable transaction. + let first_invalid_offset = match durable_tx_offset { + None => Some(0), + Some(offset) => offset.checked_add(1), + }; + if let Some(offset) = first_invalid_offset { + snapshot_repo.invalidate_snapshots_at_or_after(offset).map_err(|e| { + RestoreSnapshotError::Invalidate { + offset, + source: Box::new(e), + } })?; + } + } + if let Some((snapshot_repo, durable_tx_offset)) = snapshot_repo.zip(durable_tx_offset) { // Try to restore from any snapshot that was taken within the // range `(min_commitlog_offset + 1)..=durable_tx_offset`. let mut upper_bound = durable_tx_offset; @@ -938,6 +950,24 @@ impl RelationalDB { self.snapshot_worker.as_ref().map(|snap| snap.subscribe()) } + /// Create or reuse a snapshot that covers the current committed state, and wait until it is durable. + pub async fn request_hot_backup_snapshot(&self) -> Result { + let snapshot_worker = self + .snapshot_worker + .as_ref() + .context("snapshots are not enabled for this database")?; + let mut durable_offset = self + .durable_tx_offset() + .context("durability is not enabled for this database")?; + + let tx = self.begin_tx(Workload::Internal); + let (target_offset, _, _) = self.release_tx(tx); + + let snapshot_offset = snapshot_worker.ensure_snapshot_at_least(target_offset).await?; + durable_offset.wait_for(snapshot_offset).await?; + Ok(snapshot_offset) + } + /// Run a fallible function in a transaction. /// /// If the supplied function returns `Ok`, the transaction is automatically @@ -1806,10 +1836,15 @@ pub async fn snapshot_watching_commitlog_compressor( durability: LocalDurability, runtime: Handle, ) { - let mut prev_snapshot_offset = *snapshot_rx.borrow_and_update(); + let snapshot_offset = *snapshot_rx.borrow_and_update(); + let res = + compress_commitlog_segments_older_than_snapshot(snapshot_offset, durability.clone(), runtime.clone()).await; + if let Err(err) = res { + tracing::warn!("failed to compress commitlog segments on startup for snapshot offset {snapshot_offset}: {err}"); + } + while snapshot_rx.changed().await.is_ok() { let snapshot_offset = *snapshot_rx.borrow_and_update(); - let durability = durability.clone(); if let Some(snap_tx) = &mut snap_tx && let Err(err) = snap_tx.try_send(snapshot_offset) @@ -1817,35 +1852,17 @@ pub async fn snapshot_watching_commitlog_compressor( tracing::warn!("failed to send offset {snapshot_offset} after snapshot creation: {err}"); } - let res: io::Result<_> = asyncify(&runtime, move || { - let segment_offsets = durability.existing_segment_offsets()?; - let start_idx = segment_offsets - .binary_search(&prev_snapshot_offset) - // if the snapshot is in the middle of a segment, we want to round down. - // [0, 2].binary_search(1) will return Err(1), so we subtract 1. - .unwrap_or_else(|i| i.saturating_sub(1)); - let segment_offsets = &segment_offsets[start_idx..]; - let end_idx = segment_offsets - .binary_search(&snapshot_offset) - .unwrap_or_else(|i| i.saturating_sub(1)); - // in this case, segment_offsets[end_idx] is the segment that contains the snapshot, - // which we don't want to compress, so an exclusive range is correct. - let segment_offsets = &segment_offsets[..end_idx]; - durability.compress_segments(segment_offsets)?; - let n = segment_offsets.len(); - let last_compressed_segment = if n > 0 { Some(segment_offsets[n - 1]) } else { None }; - Ok(last_compressed_segment) - }) - .await; + let res = + compress_commitlog_segments_older_than_snapshot(snapshot_offset, durability.clone(), runtime.clone()).await; - let last_compressed_segment = match res { - Ok(opt_offset) => opt_offset, + let (last_compressed_segment, stats) = match res { + Ok(res) => res, Err(err) => { tracing::warn!("failed to compress segments: {err}"); continue; } }; - prev_snapshot_offset = snapshot_offset; + log_commitlog_compression_result(snapshot_offset, last_compressed_segment, stats); if let Some((clog_tx, last_compressed_segment)) = clog_tx.as_mut().zip(last_compressed_segment) && let Err(err) = clog_tx.try_send(last_compressed_segment) @@ -1855,6 +1872,46 @@ pub async fn snapshot_watching_commitlog_compressor( } } +async fn compress_commitlog_segments_older_than_snapshot( + snapshot_offset: TxOffset, + durability: LocalDurability, + runtime: Handle, +) -> io::Result<(Option, CompressionStats)> { + asyncify(&runtime, move || { + let segment_offsets = durability.existing_segment_offsets()?; + let segment_offsets = commitlog_segments_older_than_snapshot(&segment_offsets, snapshot_offset); + let last_compressed_segment = segment_offsets.last().copied(); + let stats = durability.compress_segments(segment_offsets)?; + Ok((last_compressed_segment, stats)) + }) + .await +} + +fn commitlog_segments_older_than_snapshot(segment_offsets: &[TxOffset], snapshot_offset: TxOffset) -> &[TxOffset] { + let end_idx = segment_offsets + .binary_search(&snapshot_offset) + // If the snapshot is in the middle of a segment, round down to the segment + // containing it. That segment is still needed for replay from the snapshot. + .unwrap_or_else(|i| i.saturating_sub(1)); + &segment_offsets[..end_idx] +} + +fn log_commitlog_compression_result( + snapshot_offset: TxOffset, + last_compressed_segment: Option, + stats: CompressionStats, +) { + if let Some(last_compressed_segment) = last_compressed_segment { + tracing::info!( + snapshot_offset, + last_compressed_segment, + bytes_read = stats.bytes_read, + bytes_written = stats.bytes_written, + "compressed commitlog segments older than snapshot" + ); + } +} + /// Open a [`SnapshotRepository`] at `db_path/snapshots`, /// configured to store snapshots of the database `database_identity`/`replica_id`. pub fn open_snapshot_repo( @@ -2379,6 +2436,8 @@ mod tests { use std::rc::Rc; use std::time::{Duration, Instant}; + const HOT_BACKUP_TEST_TIMEOUT: Duration = Duration::from_secs(60); + use super::tests_utils::begin_mut_tx; use super::*; use crate::relational_db::tests_utils::{begin_tx, create_view_for_test, insert, make_snapshot, TestDB}; @@ -2409,7 +2468,7 @@ mod tests { #[cfg(unix)] use spacetimedb_snapshot::Snapshot; use spacetimedb_table::read_column::ReadColumn; - use spacetimedb_table::table::RowRef; + use spacetimedb_table::table::{RowRef, Table}; use tempfile::TempDir; use tests::tests_utils::TestHistory; @@ -2492,6 +2551,21 @@ mod tests { Ok(()) } + #[test] + fn commitlog_segments_older_than_snapshot_excludes_segment_containing_snapshot() { + let segment_offsets = [0, 100, 200, 300]; + + assert_eq!(commitlog_segments_older_than_snapshot(&segment_offsets, 0), &[]); + assert_eq!(commitlog_segments_older_than_snapshot(&segment_offsets, 99), &[]); + assert_eq!(commitlog_segments_older_than_snapshot(&segment_offsets, 100), &[0]); + assert_eq!(commitlog_segments_older_than_snapshot(&segment_offsets, 199), &[0]); + assert_eq!(commitlog_segments_older_than_snapshot(&segment_offsets, 200), &[0, 100]); + assert_eq!( + commitlog_segments_older_than_snapshot(&segment_offsets, 350), + &[0, 100, 200] + ); + } + #[test] fn test_open_twice() -> ResultTest<()> { let stdb = TestDB::durable()?; @@ -3808,6 +3882,55 @@ mod tests { Ok(()) } + #[test] + fn hot_backup_snapshot_waits_until_snapshot_is_durable() -> ResultTest<()> { + let stdb = TestDB::durable()?; + + let mut tx = begin_mut_tx(&stdb); + stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; + stdb.commit_tx(tx)?; + + let snapshot_offset = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout(HOT_BACKUP_TEST_TIMEOUT, stdb.request_hot_backup_snapshot()).await + })??; + + assert_eq!(Some(snapshot_offset), stdb.durable_tx_offset().unwrap().last_seen()); + let repo = open_snapshot_repo(stdb.path().unwrap().snapshots(), stdb.database_identity(), 0)?; + assert_eq!(repo.latest_snapshot()?, Some(snapshot_offset)); + + Ok(()) + } + + #[test] + fn hot_backup_concurrent_snapshot_requests_do_not_rewrite_same_offset_snapshot() -> ResultTest<()> { + let stdb = TestDB::durable()?; + + let mut tx = begin_mut_tx(&stdb); + stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; + stdb.commit_tx(tx)?; + + let offsets = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout(HOT_BACKUP_TEST_TIMEOUT, async { + futures::future::join_all((0..8).map(|_| stdb.request_hot_backup_snapshot())) + .await + .into_iter() + .collect::, _>>() + }) + .await + })??; + let snapshot_offset = offsets[0]; + assert!(offsets.iter().all(|offset| *offset == snapshot_offset)); + let repo = open_snapshot_repo(stdb.path().unwrap().snapshots(), stdb.database_identity(), 0)?; + let snapshot_dir = repo.snapshot_dir_path(snapshot_offset); + let sentinel = snapshot_dir.0.join("hot-backup-sentinel"); + std::fs::write(&sentinel, b"keep")?; + + std::thread::sleep(Duration::from_millis(250)); + + assert!(sentinel.is_file(), "same-offset snapshot was rewritten"); + Ok(()) + } + // For test compression into an existing database. // Must supply the path to the database and the identity of the replica using the `ENV`: // - `SNAPSHOT` the path to the database, like `/tmp/db/replicas/.../8/database` @@ -3906,6 +4029,30 @@ mod tests { Ok(()) } + #[test] + fn restore_without_durable_history_invalidates_all_snapshots() -> ResultTest<()> { + let temp = TempDir::with_prefix("snapshot-without-durable-history")?; + let snapshots_path = SnapshotsPath::from_path_unchecked(temp.path()); + snapshots_path.create()?; + let repo = SnapshotRepository::open(snapshots_path, Identity::ZERO, 0)?; + let blobs = spacetimedb_table::blob_store::HashMapBlobStore::default(); + + repo.create_snapshot(std::iter::empty::<&mut Table>(), &blobs, 0)? + .sync_all()?; + assert_eq!(repo.latest_snapshot()?, Some(0)); + + RelationalDB::restore_from_snapshot_or_bootstrap( + Identity::ZERO, + Some(&repo), + None, + 0, + PagePool::new_for_test(), + )?; + + assert_eq!(repo.latest_snapshot()?, None); + Ok(()) + } + #[test] /// Test that we can create a table after replaying a durable database /// without a snapshot. diff --git a/crates/engine/src/snapshot.rs b/crates/engine/src/snapshot.rs index 11b08727b7d..efb7b3713c2 100644 --- a/crates/engine/src/snapshot.rs +++ b/crates/engine/src/snapshot.rs @@ -7,7 +7,10 @@ use std::{ }; use anyhow::Context as _; -use futures::{channel::mpsc, StreamExt as _}; +use futures::{ + channel::{mpsc, oneshot}, + StreamExt as _, +}; use log::{info, warn}; use parking_lot::RwLock; use prometheus::{Histogram, IntGauge}; @@ -127,7 +130,10 @@ impl SnapshotWorker { /// which is likely due to it having panicked. pub fn request_snapshot(&self) { self.request_snapshot - .unbounded_send(Request::TakeSnapshot) + .unbounded_send(Request::TakeSnapshot { + minimum_offset: None, + result_sender: None, + }) .expect("snapshot worker panicked"); } @@ -136,7 +142,10 @@ impl SnapshotWorker { /// Used by the durability to request snapshots on commitlog segment rotation, /// since the durability should continue writing queued TXes even if the snapshot worker panics. pub fn request_snapshot_ignore_closed(&self) { - let _ = self.request_snapshot.unbounded_send(Request::TakeSnapshot); + let _ = self.request_snapshot.unbounded_send(Request::TakeSnapshot { + minimum_offset: None, + result_sender: None, + }); } /// Subscribe to the [TxOffset]s of snapshots created by this worker. @@ -147,6 +156,31 @@ impl SnapshotWorker { pub fn subscribe(&self) -> watch::Receiver { self.snapshot_created.subscribe() } + + /// Request a snapshot if needed, and wait until one exists at or beyond `offset`. + pub async fn ensure_snapshot_at_least(&self, offset: TxOffset) -> anyhow::Result { + if let Some(latest) = self.snapshot_repository.latest_snapshot()? + && latest >= offset + { + return Ok(latest); + } + + let (result_sender, result_receiver) = oneshot::channel(); + self.request_snapshot + .unbounded_send(Request::TakeSnapshot { + minimum_offset: Some(offset), + result_sender: Some(result_sender), + }) + .context("snapshot worker closed before accepting requested snapshot")?; + let snapshot_offset = result_receiver + .await + .context("snapshot worker closed before creating requested snapshot")??; + assert!( + snapshot_offset >= offset, + "snapshot worker returned offset {snapshot_offset}, below requested offset {offset}" + ); + Ok(snapshot_offset) + } } struct SnapshotMetrics { @@ -168,7 +202,10 @@ impl SnapshotMetrics { type WeakDatabaseState = Weak>; enum Request { - TakeSnapshot, + TakeSnapshot { + minimum_offset: Option, + result_sender: Option>>, + }, ReplaceState(SnapshotDatabaseState), } @@ -181,6 +218,15 @@ struct SnapshotWorkerActor { compression: Option, } +fn send_snapshot_request_result( + result_sender: Option>>, + value: anyhow::Result, +) { + if let Some(result_sender) = result_sender { + let _ = result_sender.send(value); + } +} + impl SnapshotWorkerActor { /// Read messages from `snapshot_requests` indefinitely. /// @@ -201,14 +247,46 @@ impl SnapshotWorkerActor { let mut database_state: Option = None; while let Some(req) = self.snapshot_requests.next().await { match req { - Request::TakeSnapshot => { - let res = self - .maybe_take_snapshot(database_state.as_ref()) - .await - .inspect_err(|e| warn!("database={database_identity} SnapshotWorker: {e:#}")); - if let Ok(snapshot_offset) = res { - self.maybe_compress_snapshots(snapshot_offset).await; - self.snapshot_created.send_replace(snapshot_offset); + Request::TakeSnapshot { + minimum_offset, + result_sender, + } => { + if let Some(minimum_offset) = minimum_offset { + match self.snapshot_repo.latest_snapshot() { + Ok(Some(latest)) if latest >= minimum_offset => { + self.snapshot_created.send_replace(latest); + send_snapshot_request_result(result_sender, Ok(latest)); + continue; + } + Ok(_) => {} + Err(err) => { + warn!("database={database_identity} SnapshotWorker: {err:#}"); + send_snapshot_request_result(result_sender, Err(err.into())); + continue; + } + } + } + let res = self.maybe_take_snapshot(database_state.as_ref()).await; + match res { + Ok(snapshot_offset) => { + if let Some(minimum_offset) = minimum_offset { + assert!( + snapshot_offset >= minimum_offset, + "captured snapshot offset {snapshot_offset} below requested offset {minimum_offset}" + ); + } + self.maybe_compress_snapshots(snapshot_offset).await; + // INVARIANT: only signal `snapshot_created` after the snapshot + // is fully durable on disk (`take_snapshot` returns post-fsync). + // Hot backup relies on this to copy the snapshot directory as + // soon as it observes the offset. + self.snapshot_created.send_replace(snapshot_offset); + send_snapshot_request_result(result_sender, Ok(snapshot_offset)); + } + Err(err) => { + warn!("database={database_identity} SnapshotWorker: {err:#}"); + send_snapshot_request_result(result_sender, Err(err)); + } } } Request::ReplaceState(new_state) => { @@ -385,3 +463,49 @@ impl Compressor { } } } + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb_paths::{server::SnapshotsPath, FromPathUnchecked}; + use spacetimedb_snapshot::SnapshotRepository; + + #[test] + fn ensure_snapshot_at_least_returns_snapshot_errors() -> anyhow::Result<()> { + let rt = tokio::runtime::Runtime::new()?; + let temp = tempfile::tempdir()?; + std::fs::create_dir_all(temp.path().join("snapshots"))?; + let repo: Arc = Arc::new(SnapshotRepository::open( + SnapshotsPath::from_path_unchecked(temp.path().join("snapshots")), + Identity::ZERO, + 0, + )?); + let worker = SnapshotWorker::new(repo, Compression::Disabled, Handle::tokio(rt.handle().clone())); + + let err = rt + .block_on(async { tokio::time::timeout(Duration::from_secs(5), worker.ensure_snapshot_at_least(1)).await })? + .unwrap_err(); + + assert!(format!("{err:#}").contains("database state not set")); + Ok(()) + } + + #[test] + fn ensure_snapshot_at_least_returns_error_when_worker_is_closed() -> anyhow::Result<()> { + let rt = tokio::runtime::Runtime::new()?; + let temp = tempfile::tempdir()?; + std::fs::create_dir_all(temp.path().join("snapshots"))?; + let repo: Arc = Arc::new(SnapshotRepository::open( + SnapshotsPath::from_path_unchecked(temp.path().join("snapshots")), + Identity::ZERO, + 0, + )?); + let worker = SnapshotWorker::new(repo, Compression::Disabled, Handle::tokio(rt.handle().clone())); + drop(rt); + + let err = futures::executor::block_on(worker.ensure_snapshot_at_least(1)).unwrap_err(); + + assert!(format!("{err:#}").contains("closed before accepting requested snapshot")); + Ok(()) + } +} diff --git a/crates/fs-utils/Cargo.toml b/crates/fs-utils/Cargo.toml index 2446d6c888b..9827225b670 100644 --- a/crates/fs-utils/Cargo.toml +++ b/crates/fs-utils/Cargo.toml @@ -17,6 +17,9 @@ thiserror.workspace = true tokio.workspace = true zstd-framed.workspace = true +[target.'cfg(windows)'.dependencies] +windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem"] } + [dev-dependencies] tempdir.workspace = true diff --git a/crates/fs-utils/src/lib.rs b/crates/fs-utils/src/lib.rs index c4d1a6ba0c1..261b8ed6bf1 100644 --- a/crates/fs-utils/src/lib.rs +++ b/crates/fs-utils/src/lib.rs @@ -1,5 +1,5 @@ -use std::io::Write; -use std::path::Path; +use std::io::{ErrorKind, Write}; +use std::path::{Component, Path, PathBuf}; pub mod compression; pub mod dir_trie; @@ -27,20 +27,279 @@ pub fn create_parent_dir(file: &Path) -> Result<(), std::io::Error> { } pub fn atomic_write(file_path: &Path, data: String) -> anyhow::Result<()> { + atomic_write_bytes(file_path, data.as_bytes()) +} + +/// Atomically write `data` to `file_path`. +/// +/// The bytes are written to a same-directory temporary file, fsynced, renamed +/// over `file_path`, and followed by a best-effort parent directory sync on +/// platforms that support opening directories as files. +pub fn atomic_write_bytes(file_path: &Path, data: &[u8]) -> anyhow::Result<()> { + const ATOMIC_WRITE_MAX_TEMP_ATTEMPTS: u32 = 1024; + let mut temp_path = file_path.to_path_buf(); - let mut temp_file: std::fs::File; - loop { + for _ in 0..ATOMIC_WRITE_MAX_TEMP_ATTEMPTS { temp_path.set_extension(format!(".tmp{}", rand::random::())); let opened = std::fs::OpenOptions::new() .write(true) .create_new(true) .open(&temp_path); - if let Ok(file) = opened { - temp_file = file; - break; + match opened { + Ok(file) => { + let mut temp_file = file; + let write_result = (|| -> anyhow::Result<()> { + temp_file.write_all(data)?; + temp_file.sync_all()?; + drop(temp_file); + replace_file(&temp_path, file_path)?; + if let Some(parent) = non_empty_parent(file_path) { + sync_dir(parent)?; + } + Ok(()) + })(); + if write_result.is_err() { + let _ = std::fs::remove_file(&temp_path); + } + return write_result; + } + Err(err) if err.kind() == ErrorKind::AlreadyExists => {} + Err(err) => return Err(err.into()), + } + } + anyhow::bail!( + "failed to create a temporary file for atomic write after repeated name collisions: {}", + file_path.display() + ) +} + +#[cfg(windows)] +fn replace_file(source: &Path, destination: &Path) -> std::io::Result<()> { + use std::os::windows::ffi::OsStrExt as _; + use windows_sys::Win32::Storage::FileSystem::{MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH}; + + fn encode_path(path: &Path) -> std::io::Result> { + let mut path: Vec<_> = path.as_os_str().encode_wide().collect(); + if path.contains(&0) { + return Err(std::io::Error::new( + ErrorKind::InvalidInput, + "path contains a null character", + )); + } + path.push(0); + Ok(path) + } + + let source = encode_path(source)?; + let destination = encode_path(destination)?; + let flags = MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH; + + // SAFETY: Both paths are valid, null-terminated UTF-16 strings that live + // for the duration of the call. + if unsafe { MoveFileExW(source.as_ptr(), destination.as_ptr(), flags) } == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(not(windows))] +fn replace_file(source: &Path, destination: &Path) -> std::io::Result<()> { + std::fs::rename(source, destination) +} + +fn non_empty_parent(path: &Path) -> Option<&Path> { + path.parent().filter(|parent| !parent.as_os_str().is_empty()) +} + +/// Best-effort directory sync. +/// +/// On Unix, opening and syncing a directory persists directory-entry updates +/// such as file creation and rename. On platforms where directories cannot be +/// opened this is a no-op, so callers still get durable file contents without +/// losing Windows compatibility. +pub fn sync_dir(path: &Path) -> std::io::Result<()> { + #[cfg(unix)] + { + std::fs::File::open(path)?.sync_all() + } + #[cfg(not(unix))] + { + let _ = path; + Ok(()) + } +} + +/// Recursively create `path` and best-effort sync its parent directory. +pub fn create_dir_all_sync(path: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(path)?; + if let Some(parent) = non_empty_parent(path) { + sync_dir(parent)?; + } + Ok(()) +} + +/// Copy a file and fsync the destination file before returning. +pub fn copy_file_sync(src: &Path, dst: &Path) -> std::io::Result { + if let Some(parent) = non_empty_parent(dst) { + std::fs::create_dir_all(parent)?; + } + let copied = std::fs::copy(src, dst)?; + std::fs::OpenOptions::new().write(true).open(dst)?.sync_all()?; + if let Some(parent) = non_empty_parent(dst) { + sync_dir(parent)?; + } + Ok(copied) +} + +/// Recursively compute the total size in bytes of all files under `path`. +pub fn dir_size(path: &Path) -> std::io::Result { + let mut bytes = 0; + for entry in std::fs::read_dir(path)? { + let entry = entry?; + let meta = entry.metadata()?; + if meta.is_dir() { + bytes += dir_size(&entry.path())?; + } else { + bytes += meta.len(); + } + } + Ok(bytes) +} + +/// Recursively copy the directory `src` into `dst`, creating `dst` if necessary. +pub fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + use anyhow::Context as _; + + let src = src.as_ref(); + let dst = dst.as_ref(); + create_dir_all_sync(dst).with_context(|| format!("creating {}", dst.display()))?; + for entry in std::fs::read_dir(src).with_context(|| format!("reading {}", src.display()))? { + let entry = entry?; + let ty = entry.file_type()?; + let dst = dst.join(entry.file_name()); + if ty.is_dir() { + copy_dir_all(entry.path(), &dst)?; + } else { + copy_file_sync(&entry.path(), &dst) + .with_context(|| format!("copying {} to {}", entry.path().display(), dst.display()))?; } } - temp_file.write_all(data.as_bytes())?; - std::fs::rename(&temp_path, file_path)?; + sync_dir(dst).with_context(|| format!("syncing {}", dst.display()))?; Ok(()) } + +/// Normalize an absolute path that may not exist yet. +/// +/// Canonicalizes the deepest existing ancestor of `path` — resolving any +/// symlinks in it — and re-appends the remaining, not-yet-existing components. +/// +/// Errors if `path` is relative or contains `..` components, as those cannot +/// be resolved reliably for paths that do not exist yet. +pub fn normalize_absolute_path(path: &Path) -> anyhow::Result { + use anyhow::Context as _; + + anyhow::ensure!(path.is_absolute(), "path must be absolute: {}", path.display()); + anyhow::ensure!( + !path.components().any(|c| matches!(c, Component::ParentDir)), + "path must not contain `..` components: {}", + path.display() + ); + + // Split off trailing components until we find an existing ancestor. + // Use `symlink_metadata` so a dangling symlink counts as existing and + // fails the subsequent `canonicalize` instead of being skipped over. + let mut prefix = path.to_path_buf(); + let mut tail: Vec = Vec::new(); + while prefix.symlink_metadata().is_err() { + match (prefix.file_name().map(|n| n.to_owned()), prefix.parent()) { + (Some(name), Some(parent)) if parent != Path::new("") => { + tail.push(name); + prefix = parent.to_path_buf(); + } + _ => anyhow::bail!("path has no existing ancestor: {}", path.display()), + } + } + let mut normalized = prefix + .canonicalize() + .with_context(|| format!("canonicalizing {}", prefix.display()))?; + for name in tail.into_iter().rev() { + normalized.push(name); + } + Ok(normalized) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_absolute_path_rejects_relative_and_parent_components() { + assert!(normalize_absolute_path(Path::new("relative/path")).is_err()); + let dir = tempdir::TempDir::new("normalize").unwrap(); + assert!(normalize_absolute_path(&dir.path().join("a/../b")).is_err()); + } + + #[test] + fn normalize_absolute_path_resolves_nonexistent_suffix() { + let dir = tempdir::TempDir::new("normalize").unwrap(); + let root = dir.path().canonicalize().unwrap(); + let normalized = normalize_absolute_path(&dir.path().join("does/not/exist")).unwrap(); + assert_eq!(normalized, root.join("does/not/exist")); + } + + #[cfg(unix)] + #[test] + fn normalize_absolute_path_resolves_symlinks() { + let dir = tempdir::TempDir::new("normalize").unwrap(); + let target = dir.path().join("target"); + std::fs::create_dir(&target).unwrap(); + let link = dir.path().join("link"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + let normalized = normalize_absolute_path(&link.join("new")).unwrap(); + assert_eq!(normalized, target.canonicalize().unwrap().join("new")); + } + + #[test] + fn atomic_write_bytes_replaces_file_without_leaving_temps() { + let dir = tempdir::TempDir::new("atomic-write").unwrap(); + let path = dir.path().join("manifest.json"); + std::fs::write(&path, b"old").unwrap(); + + atomic_write_bytes(&path, b"new").unwrap(); + + assert_eq!(std::fs::read(&path).unwrap(), b"new"); + let leftover_temp = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(Result::ok) + .any(|entry| entry.file_name().to_string_lossy().contains(".tmp")); + assert!(!leftover_temp, "atomic write left a temp file behind"); + } + + #[test] + fn atomic_write_bytes_returns_error_for_missing_parent() { + let dir = tempdir::TempDir::new("atomic-write").unwrap(); + let path = dir.path().join("missing/manifest.json"); + + let err = atomic_write_bytes(&path, b"new").unwrap_err(); + + let io_err = err + .downcast_ref::() + .expect("expected missing parent to return the original io error"); + assert_eq!(io_err.kind(), std::io::ErrorKind::NotFound); + assert!(!path.exists()); + } + + #[test] + fn copy_dir_all_syncs_nested_file_contents() { + let src = tempdir::TempDir::new("copy-src").unwrap(); + let dst = tempdir::TempDir::new("copy-dst").unwrap(); + std::fs::create_dir_all(src.path().join("nested")).unwrap(); + std::fs::write(src.path().join("nested/file"), b"contents").unwrap(); + + copy_dir_all(src.path(), dst.path().join("out")).unwrap(); + + assert_eq!(std::fs::read(dst.path().join("out/nested/file")).unwrap(), b"contents"); + } +} diff --git a/crates/snapshot/src/lib.rs b/crates/snapshot/src/lib.rs index b04ee20216f..485644a1a69 100644 --- a/crates/snapshot/src/lib.rs +++ b/crates/snapshot/src/lib.rs @@ -860,7 +860,7 @@ impl SnapshotRepository { // failover causes the commitlog to be reset to offset 9. The next // transaction (also offset 10) will trigger snapshot creation, but we'd // mistake the existing snapshot (now invalid) as the previous snapshot. - self.invalidate_newer_snapshots(tx_offset.saturating_sub(1))?; + self.invalidate_snapshots_at_or_after(tx_offset)?; // If a previous snapshot exists in this snapshot repo, // get a handle on its object repo in order to hardlink shared objects into the new snapshot. @@ -1242,9 +1242,23 @@ impl SnapshotRepository { /// /// If this method returns an error, some snapshots may have been invalidated, but not all will have been. pub fn invalidate_newer_snapshots(&self, upper_bound: TxOffset) -> Result<(), SnapshotError> { + let Some(lower_bound) = upper_bound.checked_add(1) else { + return Ok(()); + }; + self.invalidate_snapshots_at_or_after(lower_bound) + } + + /// Mark every snapshot at or after `lower_bound` invalid. + /// + /// Does not invalidate snapshots which are locked. + /// + /// This may overwrite previously-invalidated snapshots. + /// + /// If this method returns an error, some snapshots may have been invalidated, but not all will have been. + pub fn invalidate_snapshots_at_or_after(&self, lower_bound: TxOffset) -> Result<(), SnapshotError> { let newer_snapshots = self .all_snapshots()? - .filter(|tx_offset| *tx_offset > upper_bound) + .filter(|tx_offset| *tx_offset >= lower_bound) // Collect to a vec to avoid iterator invalidation, // as the subsequent `for` loop will mutate the directory. .collect::>(); @@ -1485,6 +1499,19 @@ pub trait SnapshotRepo: Send + Sync { /// Invalidate every snapshot newer than `upper_bound`. fn invalidate_newer_snapshots(&self, upper_bound: TxOffset) -> Result<(), SnapshotError>; + /// Invalidate every snapshot at or after `lower_bound`. + fn invalidate_snapshots_at_or_after(&self, lower_bound: TxOffset) -> Result<(), SnapshotError> { + if let Some(upper_bound) = lower_bound.checked_sub(1) { + return self.invalidate_newer_snapshots(upper_bound); + } + + self.invalidate_newer_snapshots(0)?; + if self.latest_snapshot_older_than(0)?.is_some() { + self.invalidate_snapshot(0)?; + } + Ok(()) + } + /// Invalidate the snapshot at `tx_offset`. fn invalidate_snapshot(&self, tx_offset: TxOffset) -> Result<(), SnapshotError>; } @@ -1527,6 +1554,10 @@ impl SnapshotRepo for SnapshotRepository { SnapshotRepository::invalidate_newer_snapshots(self, upper_bound) } + fn invalidate_snapshots_at_or_after(&self, lower_bound: TxOffset) -> Result<(), SnapshotError> { + SnapshotRepository::invalidate_snapshots_at_or_after(self, lower_bound) + } + fn invalidate_snapshot(&self, tx_offset: TxOffset) -> Result<(), SnapshotError> { SnapshotRepository::invalidate_snapshot(self, tx_offset) } @@ -1574,7 +1605,10 @@ impl FileOrDirPath<'_> { fn sync_all(&self) -> io::Result<()> { match self { #[cfg(target_os = "windows")] - Self::Dir(path) => Ok(()), + Self::Dir(path) => { + let _ = path; + Ok(()) + } #[cfg(not(target_os = "windows"))] Self::Dir(path) => File::open(path) .map_err(|e| { diff --git a/crates/standalone/Cargo.toml b/crates/standalone/Cargo.toml index 180b3a60b4c..39e9ded8f87 100644 --- a/crates/standalone/Cargo.toml +++ b/crates/standalone/Cargo.toml @@ -29,6 +29,7 @@ spacetimedb-client-api-messages.workspace = true spacetimedb-client-api.workspace = true spacetimedb-core.workspace = true spacetimedb-datastore.workspace = true +spacetimedb-fs-utils.workspace = true spacetimedb-lib.workspace = true spacetimedb-paths.workspace = true spacetimedb-pg.workspace = true @@ -43,6 +44,7 @@ dirs.workspace = true futures.workspace = true hostname.workspace = true http.workspace = true +humantime.workspace = true log.workspace = true netstat2.workspace = true openssl.workspace = true diff --git a/crates/standalone/config.toml b/crates/standalone/config.toml index 9eeef5d3535..679591b24ff 100644 --- a/crates/standalone/config.toml +++ b/crates/standalone/config.toml @@ -64,4 +64,16 @@ directives = [ # Size in bytes of the memory buffer holding commit data before flushing to storage. # write-buffer-size = 131072 +# [hot-backup] +# HTTP/CLI-triggered backup output paths are relative to this server-side root. +# Standalone database-owner tokens are not authorized to use this endpoint. +# Omit to disable POST /v1/database/:name/backup. +# root-dir = "/var/backups/stdb" + +# [scheduled-backup] +# database = "mydb" +# output-dir = "/var/backups/stdb" +# interval = "1h" +# keep-last = 24 + # vim: set nowritebackup: << otherwise triggers cargo-watch diff --git a/crates/standalone/src/control_db.rs b/crates/standalone/src/control_db.rs index a637d38afcc..0b7131d53da 100644 --- a/crates/standalone/src/control_db.rs +++ b/crates/standalone/src/control_db.rs @@ -10,6 +10,7 @@ use spacetimedb::messages::control_db::{Database, EnergyBalance, Node, Replica}; use spacetimedb_client_api_messages::name::{ DomainName, DomainParsingError, InsertDomainResult, RegisterTldResult, SetDomainsResult, Tld, TldRef, }; +use spacetimedb_fs_utils::sync_dir; use spacetimedb_lib::bsatn; use spacetimedb_paths::standalone::ControlDbDir; @@ -28,6 +29,9 @@ pub struct ControlDb { pub type Result = core::result::Result; +const CONTROL_ID_COUNTER_TREE: &str = "control_id_counter"; +const CONTROL_ID_COUNTER_KEY: &[u8] = b"next"; + #[derive(thiserror::Error, Debug)] pub enum Error { #[error("collection not found: {0}")] @@ -75,6 +79,103 @@ impl ControlDb { Ok(Self { db }) } + #[cfg(test)] + pub fn export_to_path(&self, path: &ControlDbDir) -> Result<()> { + ensure_empty_control_db_export_dir(path)?; + + // Sled owns live files on Windows; export/import avoids copying locked files. + self.db.flush()?; + let dst = sled::Config::default() + .path(path) + .flush_every_ms(Some(50)) + .mode(sled::Mode::HighThroughput) + .open()?; + dst.import(self.db.export()); + dst.flush()?; + sync_dir(path.as_ref()).with_context(|| format!("syncing control db export dir {}", path.display()))?; + if let Some(parent) = path.0.parent() { + sync_dir(parent).with_context(|| format!("syncing control db export parent {}", parent.display()))?; + } + Ok(()) + } + + pub fn export_database_to_path( + &self, + path: &ControlDbDir, + database_identity: &Identity, + replica_id: u64, + ) -> Result { + ensure_empty_control_db_export_dir(path)?; + let database = self + .get_database_by_identity(database_identity)? + .with_context(|| format!("database {database_identity} not found in control-db"))?; + let replica = self + .get_replica_by_id(replica_id)? + .with_context(|| format!("replica {replica_id} not found in control-db"))?; + if replica.database_id != database.id { + return Err(anyhow::anyhow!( + "replica {} belongs to database {}, not {}", + replica.id, + replica.database_id, + database.id + ) + .into()); + } + + self.db.flush()?; + let dst = sled::Config::default() + .path(path) + .flush_every_ms(Some(50)) + .mode(sled::Mode::HighThroughput) + .open()?; + + let database_buf = sled::IVec::from(compat::Database::from(database.clone()).to_vec()?); + let database_by_identity = dst.open_tree("database_by_identity")?; + database_by_identity.insert(database_identity.to_be_byte_array(), database_buf.clone())?; + database_by_identity.flush()?; + + let databases = dst.open_tree("database")?; + databases.insert(database.id.to_be_bytes(), database_buf)?; + databases.flush()?; + + let replicas = dst.open_tree("replica")?; + replicas.insert(replica.id.to_be_bytes(), bsatn::to_vec(&replica)?)?; + replicas.flush()?; + + let next_control_id = database + .id + .max(replica.id) + .max(replica.node_id) + .checked_add(1) + .context("control db id counter overflow")?; + set_control_id_counter_at_least(&dst, next_control_id)?; + + copy_optional_control_db_tree_entry(&self.db, &dst, "database_locks", database_identity.to_be_byte_array())?; + copy_optional_control_db_tree_entry(&self.db, &dst, "energy_budget", database.owner_identity.to_byte_array())?; + + let domains = self.spacetime_reverse_dns(database_identity)?; + if !domains.is_empty() { + let identity_bytes = database_identity.to_byte_array(); + copy_optional_control_db_tree_entry(&self.db, &dst, "reverse_dns", identity_bytes)?; + for domain in domains { + copy_optional_control_db_tree_entry(&self.db, &dst, "dns", domain.to_lowercase().as_bytes())?; + copy_optional_control_db_tree_entry( + &self.db, + &dst, + "top_level_domains", + domain.tld().to_string().to_lowercase().as_bytes(), + )?; + } + } + + dst.flush()?; + sync_dir(path.as_ref()).with_context(|| format!("syncing control db export dir {}", path.display()))?; + if let Some(parent) = path.0.parent() { + sync_dir(parent).with_context(|| format!("syncing control db export parent {}", parent.display()))?; + } + Ok(database) + } + #[cfg(test)] pub fn at(path: impl AsRef) -> Result { let config = sled::Config::default() @@ -86,6 +187,127 @@ impl ControlDb { } } +fn ensure_empty_control_db_export_dir(path: &ControlDbDir) -> Result<()> { + if path.is_dir() { + if path + .read_dir() + .with_context(|| format!("reading control db export dir {}", path.display()))? + .next() + .is_some() + { + return Err(anyhow::anyhow!("control db export directory must be empty: {}", path.display()).into()); + } + } else { + path.create() + .with_context(|| format!("creating control db export dir {}", path.display()))?; + } + Ok(()) +} + +fn copy_optional_control_db_tree_entry( + src: &sled::Db, + dst: &sled::Db, + tree_name: &str, + key: impl AsRef<[u8]>, +) -> Result<()> { + if !control_db_has_tree(src, tree_name) { + return Ok(()); + } + let src_tree = src.open_tree(tree_name)?; + if let Some(value) = src_tree.get(key.as_ref())? { + let dst_tree = dst.open_tree(tree_name)?; + dst_tree.insert(key.as_ref(), value)?; + dst_tree.flush()?; + } + Ok(()) +} + +fn control_db_has_tree(db: &sled::Db, tree_name: &str) -> bool { + db.tree_names().iter().any(|name| name.as_ref() == tree_name.as_bytes()) +} + +fn decode_control_id(value: &[u8]) -> anyhow::Result { + let bytes: [u8; 8] = value + .try_into() + .with_context(|| format!("invalid control db id counter length: {}", value.len()))?; + Ok(u64::from_be_bytes(bytes)) +} + +fn max_id_key_in_tree(db: &sled::Db, tree_name: &str) -> Result> { + if !control_db_has_tree(db, tree_name) { + return Ok(None); + } + let tree = db.open_tree(tree_name)?; + let mut max_id = None; + for entry in tree.iter() { + let (key, _) = entry?; + let key = key.as_ref(); + let id = decode_control_id(key).with_context(|| format!("invalid id key in control-db `{tree_name}` tree"))?; + max_id = Some(max_id.map_or(id, |max_id: u64| max_id.max(id))); + } + Ok(max_id) +} + +fn next_control_id_seed(db: &sled::Db) -> Result { + let generated_id = db.generate_id()?; + let mut next_id = generated_id; + for tree_name in ["database", "replica", "node"] { + if let Some(id) = max_id_key_in_tree(db, tree_name)? { + next_id = next_id.max(id.checked_add(1).context("control db id counter overflow")?); + } + } + Ok(next_id) +} + +fn set_control_id_counter_at_least(db: &sled::Db, next_id: u64) -> Result<()> { + let counter = db.open_tree(CONTROL_ID_COUNTER_TREE)?; + let result: TransactionResult<(), anyhow::Error> = counter.transaction(|counter_tx| { + let stored_next_id = match counter_tx.get(CONTROL_ID_COUNTER_KEY)? { + Some(value) => decode_control_id(&value).map_err(ConflictableTransactionError::Abort)?, + None => 0, + }; + if stored_next_id < next_id { + counter_tx.insert(CONTROL_ID_COUNTER_KEY, next_id.to_be_bytes().to_vec())?; + counter_tx.flush(); + } + Ok::<_, ConflictableTransactionError>(()) + }); + match result { + Ok(()) => { + counter.flush()?; + Ok(()) + } + Err(TransactionError::Storage(err)) => Err(err.into()), + Err(TransactionError::Abort(err)) => Err(err.into()), + } +} + +fn reserve_control_id(db: &sled::Db) -> Result { + let seed = next_control_id_seed(db)?; + let counter = db.open_tree(CONTROL_ID_COUNTER_TREE)?; + let result: TransactionResult = counter.transaction(|counter_tx| { + let stored_next_id = match counter_tx.get(CONTROL_ID_COUNTER_KEY)? { + Some(value) => decode_control_id(&value).map_err(ConflictableTransactionError::Abort)?, + None => seed, + }; + let id = stored_next_id.max(seed); + let next_id = id + .checked_add(1) + .ok_or_else(|| ConflictableTransactionError::Abort(anyhow::anyhow!("control db id counter overflow")))?; + counter_tx.insert(CONTROL_ID_COUNTER_KEY, next_id.to_be_bytes().to_vec())?; + counter_tx.flush(); + Ok(id) + }); + match result { + Ok(id) => { + counter.flush()?; + Ok(id) + } + Err(TransactionError::Storage(err)) => Err(err.into()), + Err(TransactionError::Abort(err)) => Err(err.into()), + } +} + /// A helper to convert a `sled::IVec` into an `Identity`. /// This expects the identity to be in LITTLE_ENDIAN format. /// This fails if the `sled::IVec` is not 32 bytes long. @@ -366,7 +588,7 @@ impl ControlDb { } pub fn insert_database(&self, mut database: Database) -> Result { - let id = self.db.generate_id()?; + let id = reserve_control_id(&self.db)?; let tree = self.db.open_tree("database_by_identity")?; let key = database.database_identity.to_be_byte_array(); @@ -485,7 +707,7 @@ impl ControlDb { pub fn insert_replica(&self, mut replica: Replica) -> Result { let tree = self.db.open_tree("replica")?; - let id = self.db.generate_id()?; + let id = reserve_control_id(&self.db)?; replica.id = id; let buf = bsatn::to_vec(&replica).unwrap(); @@ -530,7 +752,7 @@ impl ControlDb { pub fn _insert_node(&self, mut node: Node) -> Result { let tree = self.db.open_tree("node")?; - let id = self.db.generate_id()?; + let id = reserve_control_id(&self.db)?; node.id = id; let buf = bsatn::to_vec(&node).unwrap(); diff --git a/crates/standalone/src/control_db/tests.rs b/crates/standalone/src/control_db/tests.rs index 171fcadffc7..ff3d9a4b8d4 100644 --- a/crates/standalone/src/control_db/tests.rs +++ b/crates/standalone/src/control_db/tests.rs @@ -5,6 +5,7 @@ use spacetimedb::messages::control_db::HostType; use spacetimedb_client_api::auth::LOCALHOST; use spacetimedb_lib::error::ResultTest; use spacetimedb_lib::Hash; +use spacetimedb_paths::FromPathUnchecked; use tempfile::TempDir; use super::*; @@ -152,3 +153,139 @@ fn test_decode() -> ResultTest<()> { Ok(()) } + +#[test] +fn test_export_to_path_can_be_opened() -> anyhow::Result<()> { + let src = TempDir::with_prefix("control-db-export-src")?; + let dst = TempDir::with_prefix("control-db-export-dst")?; + let cdb = ControlDb::at(src.path())?; + let database = Database { + id: 0, + database_identity: Identity::ZERO, + owner_identity: *ALICE, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + bootstrap_generation: 0, + }; + let database_id = cdb.insert_database(database)?; + cdb.insert_replica(Replica { + id: 0, + database_id, + node_id: 0, + leader: true, + })?; + + cdb.export_to_path(&ControlDbDir::from_path_unchecked(dst.path()))?; + let exported = ControlDb::at(dst.path())?; + + assert!(exported.get_database_by_id(database_id)?.is_some()); + assert!(exported.get_leader_replica_by_database(database_id).is_some()); + Ok(()) +} + +#[test] +fn test_export_database_to_path_is_scoped_to_one_database() -> anyhow::Result<()> { + let src = TempDir::with_prefix("control-db-export-scoped-src")?; + let dst = TempDir::with_prefix("control-db-export-scoped-dst")?; + let cdb = ControlDb::at(src.path())?; + let first_identity = Identity::from_claims(LOCALHOST, "first"); + let first = Database { + id: 0, + database_identity: first_identity, + owner_identity: *ALICE, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + bootstrap_generation: 0, + }; + let first_database_id = cdb.insert_database(first)?; + let first_replica_id = cdb.insert_replica(Replica { + id: 0, + database_id: first_database_id, + node_id: 0, + leader: true, + })?; + let second_identity = Identity::from_claims(LOCALHOST, "second"); + let second_database_id = cdb.insert_database(Database { + id: 0, + database_identity: second_identity, + owner_identity: *BOB, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + bootstrap_generation: 0, + })?; + cdb.insert_replica(Replica { + id: 0, + database_id: second_database_id, + node_id: 0, + leader: true, + })?; + cdb.set_database_lock(&first_identity, true)?; + cdb.set_database_lock(&second_identity, true)?; + + cdb.export_database_to_path( + &ControlDbDir::from_path_unchecked(dst.path()), + &first_identity, + first_replica_id, + )?; + let exported = ControlDb::at(dst.path())?; + + assert!(exported.get_database_by_identity(&first_identity)?.is_some()); + assert!(exported.get_database_by_identity(&second_identity)?.is_none()); + assert_eq!(exported.get_databases()?.len(), 1); + assert!(exported.get_replica_by_id(first_replica_id)?.is_some()); + assert_eq!(exported.get_replicas()?.len(), 1); + assert!(exported.is_database_locked(&first_identity)?); + assert!(!exported.is_database_locked(&second_identity)?); + Ok(()) +} + +#[test] +fn test_export_database_to_path_advances_control_id_counter() -> anyhow::Result<()> { + let src = TempDir::with_prefix("control-db-export-id-src")?; + let dst = TempDir::with_prefix("control-db-export-id-dst")?; + let cdb = ControlDb::at(src.path())?; + let first_database_id = cdb.insert_database(Database { + id: 0, + database_identity: Identity::ZERO, + owner_identity: *ALICE, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + bootstrap_generation: 0, + })?; + let first_replica_id = cdb.insert_replica(Replica { + id: 0, + database_id: first_database_id, + node_id: 0, + leader: true, + })?; + + cdb.export_database_to_path( + &ControlDbDir::from_path_unchecked(dst.path()), + &Identity::ZERO, + first_replica_id, + )?; + let exported = ControlDb::at(dst.path())?; + let second_identity = Identity::from_claims(LOCALHOST, "after-restore"); + let second_database_id = exported.insert_database(Database { + id: 0, + database_identity: second_identity, + owner_identity: *BOB, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + bootstrap_generation: 0, + })?; + let second_replica_id = exported.insert_replica(Replica { + id: 0, + database_id: second_database_id, + node_id: 0, + leader: true, + })?; + + assert_ne!(second_database_id, first_database_id); + assert_ne!(second_replica_id, first_replica_id); + assert!(exported.get_database_by_identity(&Identity::ZERO)?.is_some()); + assert!(exported.get_replica_by_id(first_replica_id)?.is_some()); + assert_eq!(exported.get_databases()?.len(), 2); + assert_eq!(exported.get_replicas()?.len(), 2); + Ok(()) +} diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index f806f18e995..47b249dac4d 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -23,6 +23,7 @@ use spacetimedb::util::jobs::JobCores; use spacetimedb::worker_metrics::WORKER_METRICS; use spacetimedb_client_api::auth::{self, LOCALHOST}; use spacetimedb_client_api::routes::subscribe::{HasWebSocketOptions, WebSocketOptions}; +pub use spacetimedb_client_api::routes::subscribe::{BIN_PROTOCOL, TEXT_PROTOCOL}; use spacetimedb_client_api::{ControlStateReadAccess, DatabaseResetDef, Host, NodeDelegate}; use spacetimedb_client_api_messages::name::{ DatabaseName, DomainName, InsertDomainResult, RegisterTldResult, SetDomainsResult, Tld, @@ -30,22 +31,27 @@ use spacetimedb_client_api_messages::name::{ use spacetimedb_datastore::db_metrics::data_size::DATA_SIZE_METRICS; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::traits::Program; +use spacetimedb_fs_utils::{atomic_write_bytes, create_dir_all_sync, dir_size, sync_dir}; +use spacetimedb_lib::{hash_bytes, Hash}; use spacetimedb_paths::server::{ModuleLogsDir, PidFile, ServerDataDir}; +use spacetimedb_paths::standalone::ControlDbDir; use spacetimedb_paths::standalone::StandaloneDataDirExt; +use spacetimedb_paths::FromPathUnchecked; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; use spacetimedb_table::page_pool::PagePool; +use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::Duration; - -pub use spacetimedb_client_api::routes::subscribe::{BIN_PROTOCOL, TEXT_PROTOCOL}; +use std::time::{Duration, Instant}; +use tokio::sync::Mutex as AsyncMutex; -#[derive(Clone, Copy)] +#[derive(Clone)] pub struct StandaloneOptions { pub db_config: db::Config, pub durability: DurabilityConfig, pub websocket: WebSocketOptions, pub wasm: WasmConfig, pub v8: V8Config, + pub hot_backup_root: Option, } pub struct StandaloneEnv { @@ -57,6 +63,8 @@ pub struct StandaloneEnv { _pid_file: PidFile, auth_provider: auth::DefaultJwtAuthProvider, websocket_options: WebSocketOptions, + hot_backup_root: Option, + database_lifecycle_lock: Arc>, } impl StandaloneEnv { @@ -110,6 +118,8 @@ impl StandaloneEnv { _pid_file, auth_provider: auth_env, websocket_options: config.websocket, + hot_backup_root: config.hot_backup_root, + database_lifecycle_lock: Arc::new(AsyncMutex::new(())), })) } @@ -191,11 +201,88 @@ impl NodeDelegate for StandaloneEnv { Ok(Host::new(leader.id, self.host_controller.clone())) } + async fn create_hot_backup( + &self, + leader: Host, + output_dir: PathBuf, + ) -> anyhow::Result { + use spacetimedb::host::HotBackupManifest; + + let database_lifecycle_guard = self.database_lifecycle_lock.clone().lock_owned().await; + let total_start = Instant::now(); + // Writes the snapshot, commitlog and generic server state, but leaves + // scoped program bytes, `server/control-db`, `bytes` and `manifest.json` to us. + let backup = leader.create_hot_backup_without_control_db(&output_dir).await?; + // Trust the normalized path used by the engine, not the raw request path. + let output_dir = backup.manifest().output_dir.clone(); + + let export_start = Instant::now(); + let control_db = self.control_db.clone(); + let database_identity = backup.manifest().database_identity; + let replica_id = backup.manifest().replica_id; + let program_bytes_dir = self.data_dir().program_bytes().0; + backup + .finalize_with_blocking(move |manifest| { + let _database_lifecycle_guard = database_lifecycle_guard; + let database = control_db + .export_database_to_path( + &ControlDbDir::from_path_unchecked(output_dir.join("server").join("control-db")), + &database_identity, + replica_id, + ) + .context("exporting scoped server/control-db")?; + copy_program_bytes_to_backup(&program_bytes_dir, &output_dir, database.initial_program)?; + manifest.bytes = dir_size(&output_dir).context("measuring backup size")?; + manifest.copy_ms = manifest + .copy_ms + .saturating_add(HotBackupManifest::elapsed_ms(export_start.elapsed())); + manifest.total_ms = HotBackupManifest::elapsed_ms(total_start.elapsed()); + Ok(()) + }) + .await + } + + fn hot_backup_root(&self) -> Option { + self.hot_backup_root.clone() + } + fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir { self.data_dir().replica(replica_id).module_logs() } } +fn copy_program_bytes_to_backup(program_bytes_dir: &Path, output_dir: &Path, program_hash: Hash) -> anyhow::Result<()> { + let relative_path = program_bytes_relative_path(&program_hash); + let src = program_bytes_dir.join(&relative_path); + let program_bytes = std::fs::read(&src).with_context(|| format!("reading program bytes {}", src.display()))?; + let actual_hash = hash_bytes(&program_bytes); + anyhow::ensure!( + actual_hash == program_hash, + "program bytes hash mismatch at {}: expected {}, got {}", + src.display(), + program_hash, + actual_hash + ); + + let dst = output_dir.join("server").join("program-bytes").join(relative_path); + let program_bytes_root = output_dir.join("server").join("program-bytes"); + if let Some(parent) = dst.parent() { + create_dir_all_sync(parent)?; + } + atomic_write_bytes(&dst, &program_bytes)?; + sync_dir(&program_bytes_root) + .with_context(|| format!("syncing backup program-bytes dir {}", program_bytes_root.display()))?; + let server_dir = output_dir.join("server"); + sync_dir(&server_dir).with_context(|| format!("syncing backup server dir {}", server_dir.display()))?; + Ok(()) +} + +fn program_bytes_relative_path(program_hash: &Hash) -> PathBuf { + let hex = program_hash.to_hex(); + let (prefix, suffix) = hex.split_at(2); + PathBuf::from(prefix).join(suffix) +} + #[async_trait] impl spacetimedb_client_api::ControlStateReadAccess for StandaloneEnv { // Nodes @@ -276,6 +363,7 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { spec: spacetimedb_client_api::DatabaseDef, policy: MigrationPolicy, ) -> anyhow::Result> { + let _database_lifecycle_guard = self.database_lifecycle_lock.lock().await; let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; // standalone does not support replication. @@ -399,6 +487,7 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { } async fn delete_database(&self, _caller_identity: &Identity, database_identity: &Identity) -> anyhow::Result<()> { + let _database_lifecycle_guard = self.database_lifecycle_lock.lock().await; let Some(database) = self.control_db.get_database_by_identity(database_identity)? else { return Ok(()); }; @@ -412,6 +501,7 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { } async fn reset_database(&self, _caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { + let _database_lifecycle_guard = self.database_lifecycle_lock.lock().await; let mut database = self .control_db .get_database_by_identity(&spec.database_identity)? @@ -521,6 +611,14 @@ impl spacetimedb_client_api::Authorization for StandaloneEnv { .await? .with_context(|| format!("database {database} not found")) .with_context(|| format!("Unable to authorize {subject} to perform {action:?})"))?; + if matches!(action, spacetimedb_client_api::Action::CreateHotBackup) { + return Err(spacetimedb_client_api::Unauthorized::Unauthorized { + subject, + action, + database: Some(database.database_identity), + source: Some(anyhow::anyhow!("hot backup is a server-operator action in standalone")), + }); + } if subject == database.owner_identity { return Ok(()); } @@ -643,10 +741,147 @@ mod tests { use super::*; use anyhow::Result; use spacetimedb::db::Storage; + use spacetimedb::messages::control_db::HostType; + use spacetimedb_client_api::Authorization; use spacetimedb_paths::{cli::*, FromPathUnchecked}; use std::fs; use tempfile::TempDir; + #[test] + fn program_bytes_relative_path_matches_disk_storage_layout() { + assert_eq!( + program_bytes_relative_path(&Hash::ZERO), + PathBuf::from("00").join("0".repeat(62)) + ); + } + + #[test] + fn copy_program_bytes_to_backup_uses_disk_storage_layout() -> Result<()> { + let program_bytes_dir = TempDir::new()?; + let output_dir = TempDir::new()?; + let program_bytes = b"test program bytes"; + let program_hash = hash_bytes(program_bytes); + let relative_path = program_bytes_relative_path(&program_hash); + let src = program_bytes_dir.path().join(&relative_path); + fs::create_dir_all(src.parent().unwrap())?; + fs::write(&src, program_bytes)?; + + copy_program_bytes_to_backup(program_bytes_dir.path(), output_dir.path(), program_hash)?; + + assert_eq!( + fs::read( + output_dir + .path() + .join("server") + .join("program-bytes") + .join(relative_path) + )?, + program_bytes + ); + Ok(()) + } + + #[tokio::test] + async fn database_lifecycle_lock_survives_cancelled_blocking_finalizer() -> Result<()> { + let lock = Arc::new(AsyncMutex::new(())); + let guard = lock.clone().lock_owned().await; + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (finish_tx, finish_rx) = std::sync::mpsc::channel(); + + let handle = tokio::spawn(spacetimedb::util::asyncify(move || -> Result<()> { + let _guard = guard; + started_tx + .send(()) + .map_err(|_| anyhow::anyhow!("finalizer start receiver dropped"))?; + finish_rx.recv().context("waiting for test to release finalizer")?; + Ok(()) + })); + + started_rx.await.context("finalizer did not start")?; + handle.abort(); + + assert!( + lock.try_lock().is_err(), + "database lifecycle lock was released after awaiter cancellation" + ); + + finish_tx + .send(()) + .map_err(|_| anyhow::anyhow!("finalizer finished before release signal"))?; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Ok(guard) = lock.try_lock() { + drop(guard); + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .context("database lifecycle lock was not released after finalizer completed")?; + + Ok(()) + } + + #[tokio::test] + async fn standalone_owner_cannot_authorize_hot_backup() -> Result<()> { + let tempdir = TempDir::new()?; + let keys = tempdir.path().join("keys"); + let root = tempdir.path().join("data"); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(root)); + + fs::create_dir(&keys)?; + data_dir.create()?; + + let ca = CertificateAuthority { + jwt_pub_key_path: PubKeyPath(keys.join("public")), + jwt_priv_key_path: PrivKeyPath(keys.join("private")), + }; + ca.get_or_create_keys()?; + let env = StandaloneEnv::init( + StandaloneOptions { + db_config: db::Config { + storage: Storage::Memory, + page_pool_max_size: None, + }, + durability: DurabilityConfig::default(), + websocket: WebSocketOptions::default(), + wasm: WasmConfig::default(), + v8: V8Config::default(), + hot_backup_root: None, + }, + &ca, + data_dir, + JobCores::without_pinned_cores(), + ) + .await?; + + let owner = Identity::from_claims(LOCALHOST, "owner"); + let database_identity = Identity::from_claims(LOCALHOST, "database"); + env.control_db.insert_database(Database { + id: 0, + database_identity, + owner_identity: owner, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + bootstrap_generation: 0, + })?; + + env.authorize_action(owner, database_identity, spacetimedb_client_api::Action::UpdateDatabase) + .await?; + let err = env + .authorize_action( + owner, + database_identity, + spacetimedb_client_api::Action::CreateHotBackup, + ) + .await + .unwrap_err(); + assert!(matches!(err, spacetimedb_client_api::Unauthorized::Unauthorized { .. })); + + Ok(()) + } + #[tokio::test] async fn ensure_init_grabs_lock() -> Result<()> { let tempdir = TempDir::new()?; @@ -676,9 +911,10 @@ mod tests { websocket: WebSocketOptions::default(), wasm: WasmConfig::default(), v8: V8Config::default(), + hot_backup_root: None, }; - let _env = StandaloneEnv::init(config, &ca, data_dir.clone(), JobCores::without_pinned_cores()).await?; + let _env = StandaloneEnv::init(config.clone(), &ca, data_dir.clone(), JobCores::without_pinned_cores()).await?; // Ensure that we have a lock. assert!( StandaloneEnv::init(config, &ca, data_dir.clone(), JobCores::without_pinned_cores()) diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index 01078f7af8d..5be11ded608 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -1,9 +1,14 @@ use netstat2::{get_sockets_info, AddressFamilyFlags, ProtocolFlags, ProtocolSocketInfo, TcpState}; use spacetimedb_client_api::routes::identity::IdentityRoutes; +use spacetimedb_client_api::util::NameOrIdentity; +use spacetimedb_client_api::{ControlStateReadAccess, NodeDelegate}; use spacetimedb_pg::pg_server; use std::io::{self, Write}; use std::net::IpAddr; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::{StandaloneEnv, StandaloneOptions}; use anyhow::Context; @@ -19,9 +24,12 @@ use spacetimedb::worker_metrics; use spacetimedb_client_api::routes::database::DatabaseRoutes; use spacetimedb_client_api::routes::router; use spacetimedb_client_api::routes::subscribe::WebSocketOptions; +use spacetimedb_fs_utils::{create_dir_all_sync, sync_dir}; use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; use spacetimedb_paths::server::{ConfigToml, ServerDataDir}; use tokio::net::TcpListener; +use tokio::sync::watch; +use tokio::task::JoinHandle; pub fn cli() -> clap::Command { clap::Command::new("start") @@ -103,6 +111,26 @@ struct ConfigFile { commitlog: CommitlogConfig, #[serde(default)] websocket: WebSocketOptions, + #[serde(default, alias = "scheduled-backup")] + scheduled_backup: Option, + #[serde(default, alias = "hot-backup")] + hot_backup: HotBackupConfig, +} + +#[derive(Clone, Debug, Default, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +struct HotBackupConfig { + root_dir: Option, +} + +#[derive(Clone, Debug, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +struct ScheduledBackupConfig { + database: NameOrIdentity, + output_dir: PathBuf, + #[serde(deserialize_with = "deserialize_duration")] + interval: Duration, + keep_last: Option, } impl ConfigFile { @@ -111,6 +139,244 @@ impl ConfigFile { } } +fn deserialize_duration<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + #[derive(serde::Deserialize)] + #[serde(untagged)] + enum DurationValue { + String(String), + Seconds(u64), + } + + match ::deserialize(deserializer)? { + DurationValue::String(value) => humantime::parse_duration(&value).map_err(serde::de::Error::custom), + DurationValue::Seconds(value) => Ok(Duration::from_secs(value)), + } +} + +const SCHEDULED_BACKUP_DIR: &str = "scheduled"; +const SCHEDULED_BACKUP_MARKER: &str = ".scheduled-backup"; +const SCHEDULED_BACKUP_CREATE_ATTEMPTS: u32 = 1024; +static SCHEDULED_BACKUP_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +fn backup_dir_name() -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or_default(); + let sequence = SCHEDULED_BACKUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + format!("stdb-{nanos:039}-{:010}-{sequence:020}", std::process::id()) +} + +fn scheduled_backup_database_dir(output_dir: &Path, database_identity: &spacetimedb::Identity) -> PathBuf { + output_dir + .join(SCHEDULED_BACKUP_DIR) + .join(database_identity.to_hex().as_str()) +} + +fn create_scheduled_backup_output(database_dir: &Path) -> anyhow::Result { + create_dir_all_sync(database_dir) + .with_context(|| format!("creating scheduled-backup database dir {}", database_dir.display()))?; + for _ in 0..SCHEDULED_BACKUP_CREATE_ATTEMPTS { + let output_dir = database_dir.join(backup_dir_name()); + match std::fs::create_dir(&output_dir) { + Ok(()) => { + sync_dir(database_dir) + .with_context(|| format!("syncing scheduled-backup database dir {}", database_dir.display()))?; + return Ok(output_dir); + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).with_context(|| format!("creating scheduled-backup output {}", output_dir.display())) + } + } + } + anyhow::bail!( + "failed to reserve a unique scheduled-backup output under {} after {SCHEDULED_BACKUP_CREATE_ATTEMPTS} attempts", + database_dir.display() + ) +} + +fn validate_scheduled_backup_config(config: &ScheduledBackupConfig) -> anyhow::Result<()> { + anyhow::ensure!( + !config.interval.is_zero(), + "scheduled-backup interval must be greater than zero" + ); + anyhow::ensure!( + config.output_dir.is_absolute(), + "scheduled-backup output-dir must be an absolute server path: {}", + config.output_dir.display() + ); + anyhow::ensure!( + !config + .output_dir + .components() + .any(|component| matches!(component, Component::ParentDir)), + "scheduled-backup output-dir must not contain `..` components: {}", + config.output_dir.display() + ); + if let Some(0) = config.keep_last { + anyhow::bail!("scheduled-backup keep-last must be greater than zero"); + } + Ok(()) +} + +/// Remove old scheduler-owned backups for one database. +/// +/// Both `manifest.json` and the scheduler ownership marker must be present. +/// Manual, API-created, and incomplete `stdb-*` directories are preserved and +/// do not occupy a `keep-last` slot. +fn prune_scheduled_backups(database_dir: &Path, keep_last: Option) -> anyhow::Result<()> { + let Some(keep_last) = keep_last else { + return Ok(()); + }; + + let mut complete_owned = Vec::new(); + for entry in std::fs::read_dir(database_dir) + .with_context(|| format!("reading scheduled-backup database dir {}", database_dir.display()))? + { + let entry = entry.with_context(|| format!("reading entry in {}", database_dir.display()))?; + let file_type = entry + .file_type() + .with_context(|| format!("reading file type for {}", entry.path().display()))?; + if !file_type.is_dir() || !entry.file_name().to_string_lossy().starts_with("stdb-") { + continue; + } + + let marker = entry.path().join(SCHEDULED_BACKUP_MARKER); + let has_ownership_marker = marker + .metadata() + .map(|metadata| metadata.is_file() && metadata.len() == 0) + .unwrap_or(false); + if entry.path().join("manifest.json").is_file() && has_ownership_marker { + complete_owned.push(entry); + } + } + + complete_owned.sort_by_key(|entry| entry.file_name()); + let remove_count = complete_owned.len().saturating_sub(keep_last); + for entry in complete_owned.into_iter().take(remove_count) { + std::fs::remove_dir_all(entry.path()) + .with_context(|| format!("removing old backup {}", entry.path().display()))?; + } + if remove_count > 0 { + sync_dir(database_dir) + .with_context(|| format!("syncing scheduled-backup database dir {}", database_dir.display()))?; + } + Ok(()) +} + +fn cleanup_failed_scheduled_backup_output(output_dir: &Path) -> anyhow::Result<()> { + if output_dir.exists() { + std::fs::remove_dir_all(output_dir) + .with_context(|| format!("removing failed scheduled backup {}", output_dir.display()))?; + if let Some(parent) = output_dir.parent() { + sync_dir(parent).with_context(|| format!("syncing scheduled-backup output-dir {}", parent.display()))?; + } + } + Ok(()) +} + +fn write_scheduled_backup_marker(output_dir: &Path) -> anyhow::Result<()> { + let marker_path = output_dir.join(SCHEDULED_BACKUP_MARKER); + let marker = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&marker_path) + .with_context(|| format!("creating scheduled-backup marker {}", marker_path.display()))?; + marker + .sync_all() + .with_context(|| format!("syncing scheduled-backup marker {}", marker_path.display()))?; + sync_dir(output_dir).with_context(|| format!("syncing scheduled-backup output dir {}", output_dir.display()))?; + Ok(()) +} + +async fn run_scheduled_backup(ctx: &Arc, config: &ScheduledBackupConfig) -> anyhow::Result<()> { + validate_scheduled_backup_config(config)?; + let database_identity = config + .database + .try_resolve(ctx) + .await? + .map_err(|name| anyhow::anyhow!("scheduled-backup database `{name}` not found"))?; + let database = ctx + .get_database_by_identity(&database_identity) + .await? + .with_context(|| format!("scheduled-backup database `{}` not found", config.database))?; + let database_dir = scheduled_backup_database_dir(&config.output_dir, &database_identity); + let leader = ctx.leader(database.id).await?; + let create_in = database_dir.clone(); + let output_dir = spacetimedb::util::asyncify(move || create_scheduled_backup_output(&create_in)).await?; + let cleanup_output_dir = output_dir.clone(); + let marker_output_dir = output_dir.clone(); + let backup_result = async { + ctx.create_hot_backup(leader, output_dir).await?; + spacetimedb::util::asyncify(move || write_scheduled_backup_marker(&marker_output_dir)).await + } + .await; + // This path is unique to the current run. Other scheduler, manual, API, + // and incomplete outputs are never cleaned from the failure path. + let cleanup_result = if backup_result.is_err() { + Some(spacetimedb::util::asyncify(move || cleanup_failed_scheduled_backup_output(&cleanup_output_dir)).await) + } else { + None + }; + if let Some(Err(err)) = &cleanup_result { + log::warn!("failed to remove failed scheduled backup output: {err:#}"); + } + // Pruning is blocking filesystem work, so keep it off the async executor. + let keep_last = config.keep_last; + let prune_result = spacetimedb::util::asyncify(move || prune_scheduled_backups(&database_dir, keep_last)).await; + backup_result?; + if let Some(cleanup_result) = cleanup_result { + cleanup_result?; + } + prune_result +} + +struct ScheduledBackupTask { + shutdown: watch::Sender, + handle: JoinHandle<()>, +} + +impl ScheduledBackupTask { + async fn shutdown(self) -> anyhow::Result<()> { + let _ = self.shutdown.send(true); + log::info!("waiting for scheduled backup task to stop"); + self.handle.await.context("waiting for scheduled backup task to stop") + } +} + +fn spawn_scheduled_backup(ctx: Arc, config: ScheduledBackupConfig) -> ScheduledBackupTask { + let (shutdown, mut shutdown_rx) = watch::channel(false); + let handle = tokio::spawn(async move { + log::info!( + "scheduled backup enabled for {} every {:?} into {}", + config.database, + config.interval, + config.output_dir.display() + ); + + loop { + tokio::select! { + _ = shutdown_rx.changed() => break, + _ = tokio::time::sleep(config.interval) => {} + } + if *shutdown_rx.borrow() { + break; + } + if let Err(err) = run_scheduled_backup(&ctx, &config).await { + log::error!("scheduled backup failed: {err:#}"); + } + if *shutdown_rx.borrow() { + break; + } + } + }); + ScheduledBackupTask { shutdown, handle } +} + pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { let listen_addr = args.get_one::("listen_addr").unwrap(); let pg_port = args.get_one::("pg_port"); @@ -181,6 +447,15 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { .context("cannot omit --jwt-{pub,priv}-key-path when those options are not specified in config.toml")?; let data_dir = Arc::new(data_dir.clone()); + let hot_backup_root = if matches!(storage, Storage::Disk) { + config.hot_backup.root_dir.clone() + } else { + None + }; + if config.hot_backup.root_dir.is_some() && matches!(storage, Storage::Memory) { + log::warn!("hot backup disabled for --in-memory server"); + } + let ctx = StandaloneEnv::init( StandaloneOptions { db_config, @@ -190,6 +465,7 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { websocket: config.websocket, wasm: config.common.wasm, v8: config.common.v8, + hot_backup_root, }, &certs, data_dir, @@ -204,6 +480,19 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { ); worker_metrics::spawn_page_pool_stats(listen_addr.clone(), ctx.page_pool().clone()); worker_metrics::spawn_bsatn_rlb_pool_stats(listen_addr.clone(), ctx.bsatn_rlb_pool().clone()); + let backup_task = if matches!(storage, Storage::Disk) { + if let Some(scheduled_backup) = config.scheduled_backup.clone() { + validate_scheduled_backup_config(&scheduled_backup)?; + Some(spawn_scheduled_backup(ctx.clone(), scheduled_backup)) + } else { + None + } + } else { + None + }; + if config.scheduled_backup.is_some() && matches!(storage, Storage::Memory) { + log::warn!("scheduled backup disabled for --in-memory server"); + } let mut db_routes = DatabaseRoutes::default(); db_routes.root_post = db_routes.root_post.layer(DefaultBodyLimit::disable()); db_routes.db_put = db_routes.db_put.layer(DefaultBodyLimit::disable()); @@ -289,6 +578,9 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { }) .await?; } + if let Some(task) = backup_task { + task.shutdown().await?; + } Ok(()) } @@ -507,6 +799,161 @@ mod tests { use super::*; use std::time::Duration; + fn scheduled_backup_config(output_dir: PathBuf, keep_last: Option) -> ScheduledBackupConfig { + ScheduledBackupConfig { + database: NameOrIdentity::Name("mydb".parse().unwrap()), + output_dir, + interval: Duration::from_secs(60), + keep_last, + } + } + + fn create_owned_scheduled_backup(path: &Path) { + std::fs::create_dir(path).unwrap(); + std::fs::write(path.join("manifest.json"), b"{}").unwrap(); + write_scheduled_backup_marker(path).unwrap(); + } + + #[test] + fn scheduled_backup_database_dir_is_scoped_by_identity() { + let identity = spacetimedb::Identity::ONE; + let output_dir = Path::new("/var/backups/stdb"); + + assert_eq!( + scheduled_backup_database_dir(output_dir, &identity), + output_dir.join("scheduled").join(identity.to_hex().as_str()) + ); + } + + #[test] + fn write_scheduled_backup_marker_creates_zero_byte_file() { + let temp = tempfile::tempdir().unwrap(); + let backup = temp.path().join("stdb-0001"); + std::fs::create_dir(&backup).unwrap(); + + write_scheduled_backup_marker(&backup).unwrap(); + + let marker = std::fs::metadata(backup.join(SCHEDULED_BACKUP_MARKER)).unwrap(); + assert!(marker.is_file()); + assert_eq!(marker.len(), 0); + } + + #[test] + fn create_scheduled_backup_output_reserves_an_empty_unique_directory() { + let temp = tempfile::tempdir().unwrap(); + let database_dir = temp.path().join("database"); + + let first = create_scheduled_backup_output(&database_dir).unwrap(); + let second = create_scheduled_backup_output(&database_dir).unwrap(); + + assert_ne!(first, second); + assert!(first.is_dir()); + assert!(second.is_dir()); + assert_eq!(first.read_dir().unwrap().count(), 0); + assert_eq!(second.read_dir().unwrap().count(), 0); + } + + #[test] + fn prune_scheduled_backups_only_removes_complete_owned_backups() { + let temp = tempfile::tempdir().unwrap(); + let old_owned = temp.path().join("stdb-0001"); + let manual = temp.path().join("stdb-0002"); + let incomplete = temp.path().join("stdb-0003"); + let marker_only = temp.path().join("stdb-0004"); + let new_owned = temp.path().join("stdb-0005"); + + create_owned_scheduled_backup(&old_owned); + std::fs::create_dir(&manual).unwrap(); + std::fs::write(manual.join("manifest.json"), b"{}").unwrap(); + std::fs::create_dir(&incomplete).unwrap(); + std::fs::write(incomplete.join("partial"), b"not done").unwrap(); + std::fs::create_dir(&marker_only).unwrap(); + write_scheduled_backup_marker(&marker_only).unwrap(); + create_owned_scheduled_backup(&new_owned); + + prune_scheduled_backups(temp.path(), Some(1)).unwrap(); + + assert!(!old_owned.exists()); + assert!(manual.exists(), "manual stdb-* backup must be preserved"); + assert!(incomplete.exists()); + assert!(marker_only.exists()); + assert!(new_owned.exists()); + } + + #[test] + fn prune_scheduled_backups_is_scoped_to_one_database_dir() { + let temp = tempfile::tempdir().unwrap(); + let database_a = temp.path().join("database-a"); + let database_b = temp.path().join("database-b"); + std::fs::create_dir(&database_a).unwrap(); + std::fs::create_dir(&database_b).unwrap(); + let a_old = database_a.join("stdb-0001"); + let a_new = database_a.join("stdb-0002"); + let b_old = database_b.join("stdb-0001"); + let b_new = database_b.join("stdb-0002"); + create_owned_scheduled_backup(&a_old); + create_owned_scheduled_backup(&a_new); + create_owned_scheduled_backup(&b_old); + create_owned_scheduled_backup(&b_new); + + prune_scheduled_backups(&database_a, Some(1)).unwrap(); + + assert!(!a_old.exists()); + assert!(a_new.exists()); + assert!(b_old.exists()); + assert!(b_new.exists()); + } + + #[test] + fn cleanup_failed_scheduled_backup_output_only_removes_current_output() { + let temp = tempfile::tempdir().unwrap(); + let current = temp.path().join("stdb-current"); + let manual = temp.path().join("stdb-manual"); + std::fs::create_dir(¤t).unwrap(); + std::fs::write(current.join("manifest.json"), b"{}").unwrap(); + std::fs::create_dir(&manual).unwrap(); + std::fs::write(manual.join("manifest.json"), b"{}").unwrap(); + + cleanup_failed_scheduled_backup_output(¤t).unwrap(); + + assert!(!current.exists()); + assert!(manual.exists()); + } + + #[tokio::test] + async fn scheduled_backup_shutdown_waits_for_in_flight_task() { + let (shutdown, _shutdown_rx) = watch::channel(false); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (finish_tx, finish_rx) = tokio::sync::oneshot::channel(); + let handle = tokio::spawn(async move { + started_tx.send(()).unwrap(); + finish_rx.await.unwrap(); + }); + let task = ScheduledBackupTask { shutdown, handle }; + + started_rx.await.unwrap(); + let shutdown = task.shutdown(); + tokio::pin!(shutdown); + tokio::select! { + result = &mut shutdown => panic!("scheduled backup shutdown returned early: {result:?}"), + _ = tokio::time::sleep(Duration::from_millis(10)) => {} + } + finish_tx.send(()).unwrap(); + + shutdown.await.unwrap(); + } + + #[test] + fn scheduled_backup_config_rejects_parent_components() { + let mut path = std::env::temp_dir(); + path.push(".."); + path.push("backups"); + + let err = validate_scheduled_backup_config(&scheduled_backup_config(path, Some(1))).unwrap_err(); + + assert!(err.to_string().contains("must not contain `..`")); + } + #[test] fn options_from_partial_toml() { let toml = r#" @@ -539,6 +986,15 @@ mod tests { offset-index-require-segment-fsync = false preallocate-segments = true write-buffer-size = 131072 + + [hot-backup] + root-dir = "/var/backups/stdb-http" + + [scheduled-backup] + database = "mydb" + output-dir = "/var/backups/stdb" + interval = "1h" + keep-last = 2 "#; let config: ConfigFile = toml::from_str(toml).unwrap(); @@ -572,6 +1028,10 @@ mod tests { config.commitlog.write_buffer_size.map(|val| val.get()), Some(128 * 1024) ); + assert_eq!( + config.hot_backup.root_dir, + Some(PathBuf::from("/var/backups/stdb-http")) + ); assert_eq!( config.websocket, @@ -581,6 +1041,11 @@ mod tests { ..<_>::default() } ); + let scheduled_backup = config.scheduled_backup.unwrap(); + assert_eq!(scheduled_backup.database.to_string(), "mydb"); + assert_eq!(scheduled_backup.output_dir, PathBuf::from("/var/backups/stdb")); + assert_eq!(scheduled_backup.interval, Duration::from_secs(60 * 60)); + assert_eq!(scheduled_backup.keep_last, Some(2)); } #[test] diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index b03b87df9b5..c0d4c2ded43 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -277,6 +277,7 @@ impl CompiledModule { websocket: WebSocketOptions::default(), wasm: Default::default(), v8: Default::default(), + hot_backup_root: None, }, &certs, paths.data_dir.into(), diff --git a/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md b/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md index 62d40384ece..8b38f462975 100644 --- a/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md +++ b/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md @@ -223,7 +223,88 @@ ssh ubuntu@ spacetime publish -s local --bin-path spacetime_module.wasm ` + +###### **Subcommands:** + +* `create` — Create a hot backup of a running database +* `restore` — Restore a hot backup into an offline server data directory + + + +## `spacetime backup create` + +Create a hot backup of a running database + +**Usage:** `spacetime backup create [OPTIONS] --output-dir ` + +###### **Options:** + +* `--database ` — The name or identity of the database to back up +* `--output-dir ` — Directory relative to the server's configured hot-backup root; it must be empty +* `-s`, `--server ` — The nickname, host name or URL of the server hosting the database +* `--anonymous` — Perform this action with an anonymous identity +* `--no-config` — Ignore spacetime.json configuration + + + +## `spacetime backup restore` + +Restore a hot backup into an offline server data directory + +**Usage:** `spacetime backup restore [OPTIONS] --input-dir ` + +###### **Options:** + +* `--input-dir ` — Directory containing manifest.json, snapshots/, and clog/ +* `--data-dir ` — Offline server data directory whose matching replica will be restored; defaults to the CLI root data-dir +* `--force` — Overwrite the existing target replica directory + + + ## `spacetime publish` Create and update a SpacetimeDB database diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md index 6863d642a98..0ca9db1e450 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md @@ -4,7 +4,7 @@ slug: /cli-reference/standalone-config --- -A local database instance (as started by `spacetime start`) can be configured in `{data-dir}/config.toml`, where `{data-dir}` is the database's data directory. This directory is printed when you run `spacetime start`: +A local database instance (as started by `spacetime start`) can be configured in the `config.toml` file inside _data-dir_, where _data-dir_ is the database's data directory. This directory is printed when you run `spacetime start`: 
spacetimedb-standalone version: 1.0.0
 spacetimedb-standalone path: /home/user/.local/share/spacetime/bin/1.0.0/spacetimedb-standalone
@@ -20,6 +20,10 @@ On Linux and macOS, this directory is by default `~/.local/share/spacetime/data`
 
 - [`commitlog`](#commitlog)
 
+- [`hot-backup`](#hot-backup)
+
+- [`scheduled-backup`](#scheduled-backup)
+
 - [`websocket`](#websocket)
 
 ### `certificate-authority`
@@ -101,6 +105,47 @@ If `true`, preallocate disk space for commitlog segments up to `commitlog.max-se
 
 Size in bytes of the memory buffer holding commit data before flushing to storage.
 
+### `hot-backup`
+
+```toml
+[hot-backup]
+root-dir = "/var/backups/stdb"
+```
+
+The `hot-backup` section configures the server-side root for backups created through `spacetime backup create` or the HTTP backup endpoint. These entry points require the server to provide operator authorization; standalone's ordinary database-owner tokens cannot create hot backups through this endpoint. Use scheduled backups for standalone self-hosted servers.
+
+#### `hot-backup.root-dir`
+
+Sets the absolute server path that acts as the root directory for CLI/HTTP-triggered hot backups. Backup creation requests choose an output directory relative to this root. Omit `root-dir` to disable CLI/HTTP-triggered hot backups.
+
+### `scheduled-backup`
+
+```toml
+[scheduled-backup]
+database = "mydb"
+output-dir = "/var/backups/stdb"
+interval = "1h"
+keep-last = 24
+```
+
+The `scheduled-backup` section configures a background task that periodically backs up one local database.
+
+#### `scheduled-backup.database`
+
+Selects the database name or identity to back up.
+
+#### `scheduled-backup.output-dir`
+
+Sets the absolute server path used as the scheduled-backup root. Each backup is written to `output-dir/scheduled/{database-identity}/stdb-*`, where _database-identity_ is the resolved identity even when `scheduled-backup.database` is configured with a name.
+
+#### `scheduled-backup.interval`
+
+Sets how often to create a backup. Values are strings of any format the [`humantime`] crate can parse, such as `"15m"`, `"1h"`, or `"1day"`.
+
+#### `scheduled-backup.keep-last`
+
+Sets the number of complete scheduler-owned backups to retain for this database identity. A backup is eligible for pruning only when its `stdb-*` directory contains both `manifest.json` and the scheduler's zero-byte `.scheduled-backup` ownership marker. Manual, API-created, and incomplete directories are preserved and do not count toward this limit. A failed scheduled run removes only its own output directory. Omit `keep-last` to keep all complete scheduled backups.
+
 ### `websocket`
 
 ```toml