From 75e6c4b1575043c849ecec43d4842f0dd574693e Mon Sep 17 00:00:00 2001 From: kdletters <61648117+kdletters@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:37:59 +0800 Subject: [PATCH 1/8] Add hot backup and restore support --- Cargo.lock | 3 + crates/cli/src/lib.rs | 2 + crates/cli/src/subcommands/backup.rs | 425 ++++++++++++++++++ crates/cli/src/subcommands/mod.rs | 1 + crates/client-api/src/lib.rs | 48 ++ crates/client-api/src/routes/database.rs | 37 ++ crates/engine/Cargo.toml | 5 +- crates/engine/src/relational_db.rs | 487 ++++++++++++++++++++- crates/engine/src/snapshot.rs | 22 + crates/standalone/Cargo.toml | 1 + crates/standalone/config.toml | 6 + crates/standalone/src/control_db.rs | 27 ++ crates/standalone/src/control_db/tests.rs | 28 ++ crates/standalone/src/lib.rs | 48 +- crates/standalone/src/subcommands/start.rs | 140 ++++++ 15 files changed, 1277 insertions(+), 3 deletions(-) create mode 100644 crates/cli/src/subcommands/backup.rs diff --git a/Cargo.lock b/Cargo.lock index dd6f6c6ca43..5629dee8703 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8472,6 +8472,7 @@ dependencies = [ "pretty_assertions", "prometheus", "serde", + "serde_json", "sled", "spacetimedb-commitlog", "spacetimedb-data-structures", @@ -8492,6 +8493,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tokio", + "tokio-util", "tracing", ] @@ -8918,6 +8920,7 @@ dependencies = [ "futures", "hostname", "http", + "humantime", "log", "netstat2", "once_cell", diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index def9a0928cb..3e9b1b41f03 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -21,6 +21,7 @@ pub use tasks::build; pub fn get_subcommands() -> Vec { vec![ + backup::cli(), publish::cli(), delete::cli(), logs::cli(), @@ -52,6 +53,7 @@ pub async fn exec_subcommand( args: &ArgMatches, ) -> anyhow::Result { match cmd { + "backup" => backup::exec(config, paths, args).await, "call" => call::exec(config, args).await, "describe" => describe::exec(config, args).await, "dev" => dev::exec(config, args).await, diff --git a/crates/cli/src/subcommands/backup.rs b/crates/cli/src/subcommands/backup.rs new file mode 100644 index 00000000000..6dbbf0a8454 --- /dev/null +++ b/crates/cli/src/subcommands/backup.rs @@ -0,0 +1,425 @@ +use std::path::{Path, PathBuf}; + +use crate::common_args; +use crate::config::Config; +use crate::subcommands::db_arg_resolution::{load_config_db_targets, resolve_database_arg}; +use crate::util::{add_auth_header_opt, database_identity, get_auth_header, ResponseExt}; +use anyhow::Context; +use clap::{Arg, ArgMatches, Command}; +use serde::{Deserialize, Serialize}; +use spacetimedb_paths::{server::ServerDataDir, SpacetimePaths}; + +pub fn cli() -> Command { + Command::new("backup") + .about("Create server-side database backups") + .subcommand_required(true) + .subcommand( + Command::new("create") + .about("Create a hot backup of a running database") + .arg( + Arg::new("database") + .long("database") + .required(false) + .help("The name or identity of the database to back up"), + ) + .arg( + Arg::new("output_dir") + .long("output-dir") + .value_name("SERVER_OUTPUT_DIR") + .required(true) + .value_parser(clap::value_parser!(PathBuf)) + .help("Directory on the server where the backup will be written; it must be empty"), + ) + .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database")) + .arg(common_args::anonymous()) + .arg( + Arg::new("no_config") + .long("no-config") + .action(clap::ArgAction::SetTrue) + .help("Ignore spacetime.json configuration"), + ), + ) + .subcommand( + Command::new("restore") + .about("Restore a hot backup into an offline server data directory") + .arg( + Arg::new("input_dir") + .long("input-dir") + .value_name("BACKUP_DIR") + .required(true) + .value_parser(clap::value_parser!(PathBuf)) + .help("Directory containing manifest.json, snapshots/, and clog/"), + ) + .arg( + Arg::new("data_dir") + .long("data-dir") + .value_name("SERVER_DATA_DIR") + .value_parser(clap::value_parser!(ServerDataDir)) + .help("Offline server data directory whose matching replica will be restored; defaults to the CLI root data-dir"), + ) + .arg( + Arg::new("force") + .long("force") + .action(clap::ArgAction::SetTrue) + .help("Overwrite the existing target replica directory"), + ), + ) +} + +pub async fn exec(config: Config, paths: &SpacetimePaths, args: &ArgMatches) -> Result<(), anyhow::Error> { + let (cmd, subcommand_args) = args.subcommand().expect("Subcommand required"); + match cmd { + "create" => exec_create(config, subcommand_args).await, + "restore" => exec_restore(paths, subcommand_args), + unknown => Err(anyhow::anyhow!("Invalid subcommand: {unknown}")), + } +} + +#[derive(Serialize)] +struct BackupRequest { + server_output_dir: PathBuf, +} + +#[derive(Debug, Deserialize)] +struct BackupManifest { + version: u32, + database_identity: String, + replica_id: u64, + output_dir: PathBuf, + snapshot_offset: u64, + durable_offset: u64, + snapshot_ms: u64, + copy_ms: u64, + total_ms: u64, + bytes: u64, +} + +async fn exec_create(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { + let server_from_cli = args.get_one::("server").map(|s| s.as_ref()); + let no_config = args.get_flag("no_config"); + let anon_identity = args.get_flag("anon_identity"); + let database_arg = args.get_one::("database").map(|s| s.as_str()); + let config_targets = load_config_db_targets(no_config)?; + let resolved = resolve_database_arg( + database_arg, + config_targets.as_deref(), + "spacetime backup create --database --output-dir [--no-config]", + )?; + let server = server_from_cli.or(resolved.server.as_deref()); + + let identity = database_identity(&config, &resolved.database, server).await?; + let host_url = config.get_host_url(server)?; + let auth_header = get_auth_header(&mut config, anon_identity, server, true).await?; + let server_output_dir = args.get_one::("output_dir").unwrap().clone(); + + let mut builder = reqwest::Client::new() + .post(format!("{host_url}/v1/database/{identity}/backup")) + .json(&BackupRequest { server_output_dir }); + builder = add_auth_header_opt(builder, &auth_header); + let manifest: BackupManifest = builder.send().await?.json_or_error().await?; + + println!("Backup written on server to {}", manifest.output_dir.display()); + println!("snapshot_offset: {}", manifest.snapshot_offset); + println!("durable_offset: {}", manifest.durable_offset); + println!("bytes: {}", manifest.bytes); + println!("snapshot_ms: {}", manifest.snapshot_ms); + println!("copy_ms: {}", manifest.copy_ms); + println!("total_ms: {}", manifest.total_ms); + Ok(()) +} + +fn exec_restore(paths: &SpacetimePaths, args: &ArgMatches) -> Result<(), anyhow::Error> { + let input_dir = args.get_one::("input_dir").unwrap(); + let data_dir = args.get_one::("data_dir").unwrap_or(&paths.data_dir); + let force = args.get_flag("force"); + + let manifest = restore_backup(input_dir, data_dir, force)?; + + println!( + "Restored database {} replica {} into {}", + manifest.database_identity, + manifest.replica_id, + data_dir.replica(manifest.replica_id).display() + ); + println!("snapshot_offset: {}", manifest.snapshot_offset); + println!("durable_offset: {}", manifest.durable_offset); + Ok(()) +} + +fn restore_backup(input_dir: &Path, data_dir: &ServerDataDir, force: bool) -> anyhow::Result { + let manifest = read_backup_manifest(input_dir)?; + validate_backup(input_dir, &manifest)?; + std::fs::create_dir_all(data_dir)?; + + // ponytail: data-dir lock catches the common footgun; per-replica online restore needs a real restore service. + let _pid_file = data_dir + .pid_file() + .context("target data-dir must be offline before restore")?; + copy_missing_server_state(input_dir, data_dir)?; + + let replica_dir = data_dir.replica(manifest.replica_id); + let tmp_dir = replica_dir + .0 + .with_file_name(format!("{}.restore_tmp_{}", manifest.replica_id, std::process::id())); + + anyhow::ensure!( + force || !replica_dir.0.exists(), + "target replica directory already exists: {}; pass --force to overwrite it", + replica_dir.display() + ); + anyhow::ensure!( + !tmp_dir.exists(), + "temporary restore directory already exists: {}", + tmp_dir.display() + ); + + if let Some(parent) = tmp_dir.parent() { + std::fs::create_dir_all(parent)?; + } + + let res = (|| -> anyhow::Result<()> { + std::fs::create_dir_all(&tmp_dir)?; + copy_dir_all(input_dir.join("snapshots"), tmp_dir.join("snapshots"))?; + copy_dir_all(input_dir.join("clog"), tmp_dir.join("clog"))?; + std::fs::create_dir_all(tmp_dir.join("module_logs"))?; + + if replica_dir.0.exists() { + std::fs::remove_dir_all(&replica_dir.0) + .with_context(|| format!("removing existing target replica directory {}", replica_dir.display()))?; + } + std::fs::rename(&tmp_dir, &replica_dir.0) + .with_context(|| format!("moving restored replica into {}", replica_dir.display()))?; + Ok(()) + })(); + + if res.is_err() { + let _ = std::fs::remove_dir_all(&tmp_dir); + } + res?; + + Ok(manifest) +} + +fn read_backup_manifest(input_dir: &Path) -> anyhow::Result { + let manifest_path = input_dir.join("manifest.json"); + let bytes = std::fs::read(&manifest_path) + .with_context(|| format!("reading backup manifest {}", manifest_path.display()))?; + serde_json::from_slice(&bytes).with_context(|| format!("parsing backup manifest {}", manifest_path.display())) +} + +fn validate_backup(input_dir: &Path, manifest: &BackupManifest) -> anyhow::Result<()> { + anyhow::ensure!( + manifest.version == 1, + "unsupported backup manifest version {}", + manifest.version + ); + anyhow::ensure!( + manifest.durable_offset == manifest.snapshot_offset, + "backup durable_offset {} does not match snapshot_offset {}", + manifest.durable_offset, + manifest.snapshot_offset + ); + + let snapshot_dir = input_dir + .join("snapshots") + .join(format!("{:020}.snapshot_dir", manifest.snapshot_offset)); + anyhow::ensure!( + snapshot_dir.is_dir(), + "backup snapshot directory is missing: {}", + snapshot_dir.display() + ); + + let clog_dir = input_dir.join("clog"); + anyhow::ensure!( + clog_dir.is_dir(), + "backup clog directory is missing: {}", + clog_dir.display() + ); + anyhow::ensure!( + clog_dir.read_dir()?.next().is_some(), + "backup clog directory is empty: {}", + clog_dir.display() + ); + Ok(()) +} + +fn copy_missing_server_state(input_dir: &Path, data_dir: &ServerDataDir) -> anyhow::Result<()> { + let server_dir = input_dir.join("server"); + let needs_required_dirs = ["control-db", "program-bytes"] + .into_iter() + .any(|required_dir| !data_dir.0.join(required_dir).exists()); + if !needs_required_dirs && !server_dir.exists() { + return Ok(()); + } + + for required_dir in ["control-db", "program-bytes"] { + let dst = data_dir.0.join(required_dir); + if dst.exists() { + continue; + } + let src = server_dir.join(required_dir); + anyhow::ensure!( + src.is_dir(), + "target data-dir is missing {}; backup is missing server state {}", + dst.display(), + src.display() + ); + copy_dir_all(src, dst)?; + } + for file in ["config.toml", "metadata.toml"] { + let dst = data_dir.0.join(file); + if dst.exists() { + continue; + } + let src = server_dir.join(file); + if !needs_required_dirs && !src.exists() { + continue; + } + anyhow::ensure!( + src.is_file(), + "target data-dir is missing {}; backup is missing server state {}", + dst.display(), + src.display() + ); + std::fs::copy(&src, &dst).with_context(|| format!("copying {} to {}", src.display(), dst.display()))?; + } + Ok(()) +} + +fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + let src = src.as_ref(); + let dst = dst.as_ref(); + std::fs::create_dir_all(dst)?; + for entry in std::fs::read_dir(src).with_context(|| format!("reading {}", src.display()))? { + let entry = entry?; + let ty = entry.file_type()?; + let dst = dst.join(entry.file_name()); + if ty.is_dir() { + copy_dir_all(entry.path(), dst)?; + } else { + std::fs::copy(entry.path(), &dst) + .with_context(|| format!("copying {} to {}", entry.path().display(), dst.display()))?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb_paths::FromPathUnchecked; + + #[test] + fn backup_restore_copies_replica_into_existing_data_dir() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path())?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let manifest = restore_backup(backup.path(), &data_dir, false)?; + + assert_eq!(manifest.replica_id, 7); + assert!(data + .path() + .join("replicas/7/snapshots/00000000000000000042.snapshot_dir/snapshot") + .is_file()); + assert!(data + .path() + .join("replicas/7/clog/00000000000000000000.stdb.log") + .is_file()); + assert!(data.path().join("replicas/7/module_logs").is_dir()); + assert!(!data.path().join("spacetime.pid").exists()); + Ok(()) + } + + #[test] + fn backup_restore_requires_force_for_existing_replica() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path())?; + std::fs::create_dir_all(data.path().join("replicas/7"))?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + assert!(err.to_string().contains("--force")); + assert!(data.path().join("replicas/7").is_dir()); + Ok(()) + } + + #[test] + fn backup_restore_copies_server_state_into_empty_data_dir() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + restore_backup(backup.path(), &data_dir, false)?; + + assert!(data.path().join("control-db/control").is_file()); + assert!(data.path().join("program-bytes/program").is_file()); + assert_eq!(std::fs::read_to_string(data.path().join("config.toml"))?, "config"); + assert_eq!(std::fs::read_to_string(data.path().join("metadata.toml"))?, "metadata"); + Ok(()) + } + + #[test] + fn backup_restore_errors_when_empty_data_dir_lacks_server_state() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + assert!(err.to_string().contains("backup is missing server state")); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + fn make_target_data_dir(path: &Path) -> anyhow::Result<()> { + std::fs::create_dir_all(path.join("control-db"))?; + std::fs::create_dir_all(path.join("program-bytes"))?; + Ok(()) + } + + fn make_backup_server_state(path: &Path) -> anyhow::Result<()> { + std::fs::create_dir_all(path.join("server/control-db"))?; + std::fs::write(path.join("server/control-db/control"), b"control")?; + std::fs::create_dir_all(path.join("server/program-bytes"))?; + std::fs::write(path.join("server/program-bytes/program"), b"program")?; + std::fs::write(path.join("server/config.toml"), b"config")?; + std::fs::write(path.join("server/metadata.toml"), b"metadata")?; + Ok(()) + } + + fn make_backup_dir(path: &Path, replica_id: u64, offset: u64) -> anyhow::Result<()> { + let snapshot_dir = path.join("snapshots").join(format!("{offset:020}.snapshot_dir")); + std::fs::create_dir_all(&snapshot_dir)?; + std::fs::write(snapshot_dir.join("snapshot"), b"snapshot")?; + + let clog_dir = path.join("clog"); + std::fs::create_dir_all(&clog_dir)?; + std::fs::write(clog_dir.join("00000000000000000000.stdb.log"), b"log")?; + + let manifest = serde_json::json!({ + "version": 1, + "database_identity": "c200000000000000000000000000000000000000000000000000000000000000", + "replica_id": replica_id, + "snapshot_offset": offset, + "durable_offset": offset, + "output_dir": path, + "snapshot_ms": 1, + "copy_ms": 2, + "total_ms": 3, + "bytes": 4 + }); + std::fs::write(path.join("manifest.json"), serde_json::to_vec_pretty(&manifest)?)?; + Ok(()) + } +} diff --git a/crates/cli/src/subcommands/mod.rs b/crates/cli/src/subcommands/mod.rs index ef51b66f055..4a7d13a6ca8 100644 --- a/crates/cli/src/subcommands/mod.rs +++ b/crates/cli/src/subcommands/mod.rs @@ -1,3 +1,4 @@ +pub mod backup; pub mod build; pub mod call; pub mod db_arg_resolution; diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 7fd6f238128..93f0287dd9e 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -1,6 +1,7 @@ use std::fmt; use std::future::Future; use std::num::NonZeroU8; +use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::anyhow; @@ -59,6 +60,13 @@ pub trait NodeDelegate: Send + Sync { /// /// The [`Host`] is spawned implicitly if not already running. async fn leader(&self, database_id: u64) -> Result; + async fn create_hot_backup( + &self, + leader: Host, + output_dir: PathBuf, + ) -> anyhow::Result { + leader.create_hot_backup(output_dir).await + } fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir; } @@ -107,6 +115,38 @@ impl Host { self.host_controller.get_module_host(self.replica_id).await } + pub async fn create_hot_backup( + &self, + output_dir: impl AsRef, + ) -> anyhow::Result { + let module = self.module().await?; + module + .relational_db() + .create_hot_backup( + &self.host_controller.data_dir.replica(self.replica_id), + Some(&self.host_controller.data_dir), + self.replica_id, + output_dir, + ) + .await + } + + pub async fn create_hot_backup_without_control_db( + &self, + output_dir: impl AsRef, + ) -> anyhow::Result { + let module = self.module().await?; + module + .relational_db() + .create_hot_backup_without_control_db( + &self.host_controller.data_dir.replica(self.replica_id), + Some(&self.host_controller.data_dir), + self.replica_id, + output_dir, + ) + .await + } + /// Wait for the module host to become available, retrying with backoff. /// /// This is useful for routes like `/schema` that may be called while the @@ -496,6 +536,14 @@ impl NodeDelegate for Arc { (**self).leader(database_id).await } + async fn create_hot_backup( + &self, + leader: Host, + output_dir: PathBuf, + ) -> anyhow::Result { + (**self).create_hot_backup(leader, output_dir).await + } + fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir { (**self).module_logs_dir(replica_id) } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 862597d289c..0e402c9de81 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -1,6 +1,7 @@ use std::borrow::Cow; use std::future::Future; use std::num::NonZeroU8; +use std::path::PathBuf; use std::str::FromStr; use std::time::Duration; use std::{env, io}; @@ -676,6 +677,38 @@ fn mime_ndjson() -> mime::Mime { "application/x-ndjson".parse().unwrap() } +#[derive(Deserialize)] +pub struct BackupParams { + name_or_identity: NameOrIdentity, +} + +#[derive(Deserialize)] +pub struct BackupRequest { + #[serde(alias = "output_dir")] + server_output_dir: PathBuf, +} + +pub async fn backup( + State(worker_ctx): State, + Path(BackupParams { name_or_identity }): Path, + Extension(auth): Extension, + axum::Json(BackupRequest { server_output_dir }): axum::Json, +) -> axum::response::Result +where + S: ControlStateDelegate + NodeDelegate + Authorization, +{ + let (leader, database) = find_leader_and_database(&worker_ctx, name_or_identity).await?; + worker_ctx + .authorize_action(auth.claims.identity, database.database_identity, Action::UpdateDatabase) + .await?; + + let manifest = worker_ctx + .create_hot_backup(leader, server_output_dir) + .await + .map_err(log_and_500)?; + Ok(axum::Json(manifest)) +} + pub(crate) async fn worker_ctx_find_database( worker_ctx: &(impl ControlStateDelegate + ?Sized), database_identity: &Identity, @@ -1561,6 +1594,8 @@ pub struct DatabaseRoutes { pub schema_get: MethodRouter, /// GET: /database/:name_or_identity/logs pub logs_get: MethodRouter, + /// POST: /database/:name_or_identity/backup + pub backup_post: MethodRouter, /// POST: /database/:name_or_identity/sql pub sql_post: MethodRouter, /// POST: /database/:name_or_identity/pre-publish @@ -1600,6 +1635,7 @@ where call_reducer_procedure_post: post(call::), schema_get: get(schema::), logs_get: get(logs::), + backup_post: post(backup::), sql_post: post(sql::), pre_publish: post(pre_publish::), db_reset: put(reset::), @@ -1630,6 +1666,7 @@ where .route("/call/:reducer", self.call_reducer_procedure_post) .route("/schema", self.schema_get) .route("/logs", self.logs_get) + .route("/backup", self.backup_post) .route("/sql", self.sql_post) .route("/unstable/timestamp", self.timestamp_get) .route("/pre_publish", self.pre_publish) diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 740e719b388..dee353c444b 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -24,8 +24,9 @@ once_cell.workspace = true parking_lot.workspace = true prometheus.workspace = true serde.workspace = true +serde_json.workspace = true sled.workspace = true -spacetimedb-commitlog.workspace = true +spacetimedb-commitlog = { workspace = true, features = ["streaming"] } spacetimedb-data-structures.workspace = true spacetimedb-datastore.workspace = true spacetimedb-durability.workspace = true @@ -44,6 +45,8 @@ sqlparser.workspace = true tempfile.workspace = true thiserror.workspace = true tracing.workspace = true +tokio.workspace = true +tokio-util = { workspace = true, features = ["io-util"] } [dev-dependencies] bytes.workspace = true diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index 5ba71da15cb..36171f4b96c 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -42,7 +42,9 @@ use spacetimedb_lib::db::raw_def::v9::{btree, RawModuleDefV9Builder, RawSql}; use spacetimedb_lib::st_var::StVarValue; use spacetimedb_lib::ConnectionId; use spacetimedb_lib::Identity; -use spacetimedb_paths::server::{ReplicaDir, SnapshotsPath}; +use spacetimedb_paths::server::{CommitLogDir, ReplicaDir, ServerDataDir, SnapshotsPath}; +use spacetimedb_paths::standalone::StandaloneDataDirExt; +use spacetimedb_paths::FromPathUnchecked; use spacetimedb_primitives::*; use spacetimedb_runtime::sync::watch; use spacetimedb_runtime::Handle; @@ -62,9 +64,12 @@ use spacetimedb_table::page_pool::PagePool; use spacetimedb_table::table::{RowRef, TableScanIter}; use spacetimedb_table::table_index::IndexKey; use std::borrow::Cow; +use std::ffi::OsString; use std::io; use std::ops::RangeBounds; +use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::{Duration, Instant}; pub use super::persistence::{DiskSizeFn, Durability, Persistence}; pub use super::snapshot::SnapshotWorker; @@ -130,6 +135,20 @@ pub struct RelationalDB { // compiler to find external dependencies. pub const SNAPSHOT_FREQUENCY: u64 = 1_000_000; +#[derive(Debug, serde::Serialize)] +pub struct HotBackupManifest { + pub version: u32, + pub database_identity: Identity, + pub replica_id: u64, + pub snapshot_offset: TxOffset, + pub durable_offset: TxOffset, + pub output_dir: PathBuf, + pub snapshot_ms: u64, + pub copy_ms: u64, + pub total_ms: u64, + pub bytes: u64, +} + impl std::fmt::Debug for RelationalDB { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RelationalDB") @@ -938,6 +957,129 @@ impl RelationalDB { self.snapshot_worker.as_ref().map(|snap| snap.subscribe()) } + /// Create or reuse a snapshot that covers the current committed state, and wait until it is durable. + pub async fn request_hot_backup_snapshot(&self) -> Result { + let snapshot_worker = self + .snapshot_worker + .as_ref() + .context("snapshots are not enabled for this database")?; + let mut durable_offset = self + .durable_tx_offset() + .context("durability is not enabled for this database")?; + + let tx = self.begin_tx(Workload::Internal); + let (target_offset, _, _) = self.release_tx(tx); + + let snapshot_offset = snapshot_worker.ensure_snapshot_at_least(target_offset).await?; + durable_offset.wait_for(snapshot_offset).await?; + Ok(snapshot_offset) + } + + /// Export a point-in-time hot backup into `output_dir`. + pub async fn create_hot_backup( + &self, + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + replica_id: u64, + output_dir: impl AsRef, + ) -> anyhow::Result { + self.create_hot_backup_inner(replica_dir, server_data_dir, replica_id, output_dir, true) + .await + } + + /// Export a point-in-time hot backup into `output_dir`, leaving `server/control-db` + /// for the caller to provide. + pub async fn create_hot_backup_without_control_db( + &self, + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + replica_id: u64, + output_dir: impl AsRef, + ) -> anyhow::Result { + self.create_hot_backup_inner(replica_dir, server_data_dir, replica_id, output_dir, false) + .await + } + + async fn create_hot_backup_inner( + &self, + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + replica_id: u64, + output_dir: impl AsRef, + copy_control_db: bool, + ) -> anyhow::Result { + let total_start = Instant::now(); + let runtime = self + .durability_runtime + .as_ref() + .context("durability runtime is not enabled for this database")? + .clone(); + let output_dir = output_dir.as_ref().to_path_buf(); + ensure_backup_path(replica_dir, server_data_dir, &output_dir)?; + + let snapshot_start = Instant::now(); + let snapshot_offset = self.request_hot_backup_snapshot().await?; + let snapshot_ms = elapsed_ms(snapshot_start.elapsed()); + let durable_offset = snapshot_offset; + + let copy_start = Instant::now(); + let snapshot_repo = open_snapshot_repo(replica_dir.snapshots(), self.database_identity(), replica_id)?; + let src_snapshot_dir = snapshot_repo.snapshot_dir_path(snapshot_offset); + let dst_snapshots = output_dir.join("snapshots"); + let dst_snapshot_dir = dst_snapshots.join( + src_snapshot_dir + .0 + .file_name() + .context("snapshot directory has no file name")?, + ); + wait_for_dir(&src_snapshot_dir.0).await?; + let src_snapshot_file = src_snapshot_dir.snapshot_file(snapshot_offset); + wait_for_file(&src_snapshot_file.0).await?; + let snapshot_file_name: OsString = src_snapshot_file + .0 + .file_name() + .context("snapshot file has no file name")? + .to_owned(); + + asyncify(&runtime, { + let output_dir = output_dir.clone(); + let src_snapshot_dir = src_snapshot_dir.clone(); + let snapshot_file_name = snapshot_file_name.clone(); + move || -> anyhow::Result<()> { + ensure_empty_dir(&output_dir)?; + copy_dir_all_retry(&src_snapshot_dir.0, &dst_snapshot_dir, &snapshot_file_name)?; + Ok(()) + } + }) + .await?; + + if let Some(server_data_dir) = server_data_dir { + copy_server_state(&runtime, server_data_dir, &output_dir, copy_control_db).await?; + } + copy_commitlog_range(replica_dir.commit_log(), output_dir.join("clog"), snapshot_offset).await?; + let copy_ms = elapsed_ms(copy_start.elapsed()); + + let mut manifest = HotBackupManifest { + version: 1, + database_identity: self.database_identity(), + replica_id, + snapshot_offset, + durable_offset, + output_dir: output_dir.clone(), + snapshot_ms, + copy_ms, + total_ms: elapsed_ms(total_start.elapsed()), + bytes: 0, + }; + manifest.bytes = asyncify(&runtime, { + let output_dir = output_dir.clone(); + move || dir_size(&output_dir) + }) + .await?; + write_hot_backup_manifest(&runtime, &output_dir, &manifest).await?; + Ok(manifest) + } + /// Run a fallible function in a transaction. /// /// If the supplied function returns `Ok`, the transaction is automatically @@ -1843,6 +1985,220 @@ pub fn open_snapshot_repo( .map_err(Box::new) } +async fn copy_commitlog_range(src: CommitLogDir, dst: PathBuf, through: TxOffset) -> anyhow::Result<()> { + let src_repo = commitlog::repo::Fs::new(src, None)?; + let dst_repo = commitlog::repo::Fs::new(CommitLogDir::from_path_unchecked(dst), None)?; + let reader = tokio::io::BufReader::new(tokio_util::io::StreamReader::new(commitlog::stream::commits( + src_repo, + ..=through, + ))); + tokio::pin!(reader); + let writer = + commitlog::stream::StreamWriter::create(dst_repo, <_>::default(), commitlog::stream::OnTrailingData::Error)?; + let mut writer = writer.append_all(reader, |_| ()).await?; + writer.sync_all().await?; + Ok(()) +} + +async fn write_hot_backup_manifest( + runtime: &Handle, + output_dir: &Path, + manifest: &HotBackupManifest, +) -> anyhow::Result<()> { + let output_dir = output_dir.to_path_buf(); + let json = serde_json::to_vec_pretty(manifest)?; + asyncify(runtime, move || std::fs::write(output_dir.join("manifest.json"), json)).await?; + Ok(()) +} + +fn ensure_empty_dir(path: &Path) -> anyhow::Result<()> { + if path.exists() { + anyhow::ensure!( + path.is_dir(), + "backup output path is not a directory: {}", + path.display() + ); + anyhow::ensure!( + path.read_dir()?.next().is_none(), + "backup output directory must be empty: {}", + path.display() + ); + } else { + std::fs::create_dir_all(path)?; + } + Ok(()) +} + +fn ensure_backup_path( + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + output_dir: &Path, +) -> anyhow::Result<()> { + anyhow::ensure!( + output_dir.is_absolute(), + "backup output directory must be an absolute server path: {}", + output_dir.display() + ); + if let Some(server_data_dir) = server_data_dir { + anyhow::ensure!( + !output_dir.starts_with(&server_data_dir.0), + "backup output directory must not be inside the server data directory: {}", + output_dir.display() + ); + } + anyhow::ensure!( + !output_dir.starts_with(&replica_dir.0), + "backup output directory must not be inside the replica directory: {}", + replica_dir.display() + ); + Ok(()) +} + +async fn copy_server_state( + runtime: &Handle, + data_dir: &ServerDataDir, + output_dir: &Path, + copy_control_db: bool, +) -> anyhow::Result<()> { + let data_dir = data_dir.clone(); + let output_dir = output_dir.to_path_buf(); + asyncify(runtime, move || -> anyhow::Result<()> { + let server_dir = output_dir.join("server"); + std::fs::create_dir_all(&server_dir)?; + + copy_required_file(&data_dir.config_toml().0, &server_dir.join("config.toml"))?; + copy_required_file(&data_dir.metadata_toml().0, &server_dir.join("metadata.toml"))?; + if copy_control_db { + copy_required_dir(&data_dir.control_db().0, &server_dir.join("control-db"))?; + } + copy_required_dir(&data_dir.program_bytes().0, &server_dir.join("program-bytes"))?; + Ok(()) + }) + .await?; + Ok(()) +} + +fn copy_required_file(src: &Path, dst: &Path) -> anyhow::Result<()> { + anyhow::ensure!(src.is_file(), "server state file is missing: {}", src.display()); + if let Some(parent) = dst.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::copy(src, dst).with_context(|| format!("copying {} to {}", src.display(), dst.display()))?; + Ok(()) +} + +fn copy_required_dir(src: &Path, dst: &Path) -> anyhow::Result<()> { + anyhow::ensure!(src.is_dir(), "server state directory is missing: {}", src.display()); + copy_dir_all(src, dst).with_context(|| format!("copying {} to {}", src.display(), dst.display())) +} + +fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { + let src = src.as_ref(); + let dst = dst.as_ref(); + std::fs::create_dir_all(dst)?; + for entry in std::fs::read_dir(src)? { + let entry = entry?; + let ty = entry.file_type()?; + let dst = dst.join(entry.file_name()); + if ty.is_dir() { + copy_dir_all(entry.path(), dst)?; + } else { + std::fs::copy(entry.path(), dst)?; + } + } + Ok(()) +} + +fn copy_dir_all_retry(src: &Path, dst: &Path, required_file_name: &OsString) -> anyhow::Result<()> { + let start = Instant::now(); + loop { + match copy_snapshot_dir(src, dst, required_file_name) { + Ok(()) => return Ok(()), + Err(err) if start.elapsed() < Duration::from_secs(30) => { + std::thread::sleep(Duration::from_millis(10)); + drop(err); + } + Err(err) => { + return Err(err).with_context(|| format!("copying snapshot {} to {}", src.display(), dst.display())); + } + } + } +} + +fn copy_snapshot_dir(src: &Path, dst: &Path, required_file_name: &OsString) -> anyhow::Result<()> { + std::fs::create_dir_all(dst)?; + let src_required = src.join(required_file_name); + let dst_required = dst.join(required_file_name); + std::fs::copy(&src_required, &dst_required) + .with_context(|| format!("copying {} to {}", src_required.display(), dst_required.display()))?; + + for entry in std::fs::read_dir(src).with_context(|| format!("reading {}", src.display()))? { + let entry = entry?; + if entry.file_name() == *required_file_name { + continue; + } + let ty = entry.file_type()?; + let dst = dst.join(entry.file_name()); + if ty.is_dir() { + copy_dir_all(entry.path(), dst)?; + } else { + std::fs::copy(entry.path(), dst)?; + } + } + anyhow::ensure!( + dst_required.is_file(), + "copying snapshot {} to {} did not copy {}", + src.display(), + dst.display(), + required_file_name.to_string_lossy() + ); + Ok(()) +} + +fn dir_size(path: &Path) -> io::Result { + let mut bytes = 0; + for entry in std::fs::read_dir(path)? { + let entry = entry?; + let meta = entry.metadata()?; + if meta.is_dir() { + bytes += dir_size(&entry.path())?; + } else { + bytes += meta.len(); + } + } + Ok(bytes) +} + +fn elapsed_ms(duration: Duration) -> u64 { + duration.as_millis().try_into().unwrap_or(u64::MAX) +} + +async fn wait_for_dir(path: &Path) -> anyhow::Result<()> { + let start = Instant::now(); + while !path.is_dir() { + anyhow::ensure!( + start.elapsed() < Duration::from_secs(5), + "directory did not appear: {}", + path.display() + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + Ok(()) +} + +async fn wait_for_file(path: &Path) -> anyhow::Result<()> { + let start = Instant::now(); + while !path.is_file() { + anyhow::ensure!( + start.elapsed() < Duration::from_secs(5), + "file did not appear: {}", + path.display() + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + Ok(()) +} + fn default_row_count_fn(db: Identity) -> RowCountFn { Arc::new(move |table_id, table_name| { DB_METRICS @@ -2354,6 +2710,8 @@ mod tests { use std::rc::Rc; use std::time::{Duration, Instant}; + const HOT_BACKUP_TEST_TIMEOUT: Duration = Duration::from_secs(60); + use super::tests_utils::begin_mut_tx; use super::*; use crate::relational_db::tests_utils::{begin_tx, create_view_for_test, insert, make_snapshot, TestDB}; @@ -3782,6 +4140,133 @@ mod tests { Ok(()) } + #[test] + fn hot_backup_snapshot_waits_until_snapshot_is_durable() -> ResultTest<()> { + let stdb = TestDB::durable()?; + + let mut tx = begin_mut_tx(&stdb); + stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; + stdb.commit_tx(tx)?; + + let snapshot_offset = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout(HOT_BACKUP_TEST_TIMEOUT, stdb.request_hot_backup_snapshot()).await + })??; + + assert_eq!(Some(snapshot_offset), stdb.durable_tx_offset().unwrap().last_seen()); + let repo = open_snapshot_repo(stdb.path().unwrap().snapshots(), stdb.database_identity(), 0)?; + assert_eq!(repo.latest_snapshot()?, Some(snapshot_offset)); + + Ok(()) + } + + #[test] + fn hot_backup_create_writes_manifest_snapshot_and_commitlog() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + + let mut tx = begin_mut_tx(&stdb); + stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; + stdb.commit_tx(tx)?; + + let backup_dir = tempfile::tempdir()?; + let manifest = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + stdb.create_hot_backup(stdb.path().unwrap(), None, 0, backup_dir.path()), + ) + .await + })??; + + assert!(backup_dir.path().join("manifest.json").is_file()); + assert!(manifest.bytes > 0); + assert_eq!(manifest.durable_offset, manifest.snapshot_offset); + let manifest_json: serde_json::Value = + serde_json::from_slice(&std::fs::read(backup_dir.path().join("manifest.json"))?)?; + assert_eq!( + manifest_json["snapshot_offset"].as_u64(), + Some(manifest.snapshot_offset) + ); + assert_eq!(manifest_json["durable_offset"].as_u64(), Some(manifest.snapshot_offset)); + let repo = open_snapshot_repo( + SnapshotsPath::from_path_unchecked(backup_dir.path().join("snapshots")), + stdb.database_identity(), + 0, + )?; + let snapshot = repo.read_snapshot(manifest.snapshot_offset, &PagePool::new_for_test())?; + assert_eq!(snapshot.database_identity, manifest.database_identity); + assert_eq!(snapshot.replica_id, manifest.replica_id); + assert_eq!(snapshot.tx_offset, manifest.snapshot_offset); + let clog = Commitlog::::open( + CommitLogDir::from_path_unchecked(backup_dir.path().join("clog")), + Default::default(), + None, + )?; + assert_eq!(clog.max_committed_offset(), Some(manifest.snapshot_offset)); + + Ok(()) + } + + #[test] + fn hot_backup_create_copies_server_state_when_provided() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + + let mut tx = begin_mut_tx(&stdb); + stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; + stdb.commit_tx(tx)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + std::fs::write(data.path().join("config.toml"), b"config")?; + std::fs::write(data.path().join("metadata.toml"), b"metadata")?; + std::fs::create_dir_all(data.path().join("control-db"))?; + std::fs::write(data.path().join("control-db/control"), b"control")?; + std::fs::create_dir_all(data.path().join("program-bytes"))?; + std::fs::write(data.path().join("program-bytes/program"), b"program")?; + + let backup_dir = tempfile::tempdir()?; + stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + stdb.create_hot_backup(stdb.path().unwrap(), Some(&data_dir), 0, backup_dir.path()), + ) + .await + })??; + + assert_eq!( + std::fs::read_to_string(backup_dir.path().join("server/config.toml"))?, + "config" + ); + assert_eq!( + std::fs::read_to_string(backup_dir.path().join("server/metadata.toml"))?, + "metadata" + ); + assert!(backup_dir.path().join("server/control-db/control").is_file()); + assert!(backup_dir.path().join("server/program-bytes/program").is_file()); + + Ok(()) + } + + #[test] + fn hot_backup_rejects_unsafe_output_dirs() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + let replica_dir = stdb.path().unwrap(); + + let err = stdb + .runtime() + .unwrap() + .block_on(stdb.create_hot_backup(replica_dir, None, 0, PathBuf::from("relative"))) + .unwrap_err(); + assert!(err.to_string().contains("absolute server path")); + + let err = stdb + .runtime() + .unwrap() + .block_on(stdb.create_hot_backup(replica_dir, None, 0, replica_dir.0.join("backup"))) + .unwrap_err(); + assert!(err.to_string().contains("replica directory")); + + Ok(()) + } + // For test compression into an existing database. // Must supply the path to the database and the identity of the replica using the `ENV`: // - `SNAPSHOT` the path to the database, like `/tmp/db/replicas/.../8/database` diff --git a/crates/engine/src/snapshot.rs b/crates/engine/src/snapshot.rs index 11b08727b7d..102fbea071c 100644 --- a/crates/engine/src/snapshot.rs +++ b/crates/engine/src/snapshot.rs @@ -147,6 +147,28 @@ impl SnapshotWorker { pub fn subscribe(&self) -> watch::Receiver { self.snapshot_created.subscribe() } + + /// Request a snapshot if needed, and wait until one exists at or beyond `offset`. + pub async fn ensure_snapshot_at_least(&self, offset: TxOffset) -> anyhow::Result { + if let Some(latest) = self.snapshot_repository.latest_snapshot()? { + if latest >= offset { + return Ok(latest); + } + } + + let mut snapshot_created = self.subscribe(); + self.request_snapshot(); + loop { + snapshot_created + .changed() + .await + .context("snapshot worker closed before creating requested snapshot")?; + let latest = *snapshot_created.borrow_and_update(); + if latest >= offset { + return Ok(latest); + } + } + } } struct SnapshotMetrics { diff --git a/crates/standalone/Cargo.toml b/crates/standalone/Cargo.toml index 180b3a60b4c..f1d7305ed8b 100644 --- a/crates/standalone/Cargo.toml +++ b/crates/standalone/Cargo.toml @@ -43,6 +43,7 @@ dirs.workspace = true futures.workspace = true hostname.workspace = true http.workspace = true +humantime.workspace = true log.workspace = true netstat2.workspace = true openssl.workspace = true diff --git a/crates/standalone/config.toml b/crates/standalone/config.toml index 9eeef5d3535..6a5568da0b0 100644 --- a/crates/standalone/config.toml +++ b/crates/standalone/config.toml @@ -64,4 +64,10 @@ directives = [ # Size in bytes of the memory buffer holding commit data before flushing to storage. # write-buffer-size = 131072 +# [scheduled-backup] +# database = "mydb" +# output-dir = "/var/backups/stdb" +# interval = "1h" +# keep-last = 24 + # vim: set nowritebackup: << otherwise triggers cargo-watch diff --git a/crates/standalone/src/control_db.rs b/crates/standalone/src/control_db.rs index a637d38afcc..20b89a5c04b 100644 --- a/crates/standalone/src/control_db.rs +++ b/crates/standalone/src/control_db.rs @@ -75,6 +75,33 @@ impl ControlDb { Ok(Self { db }) } + pub fn export_to_path(&self, path: &ControlDbDir) -> Result<()> { + if path.is_dir() { + if path + .read_dir() + .with_context(|| format!("reading control db export dir {}", path.display()))? + .next() + .is_some() + { + return Err(anyhow::anyhow!("control db export directory must be empty: {}", path.display()).into()); + } + } else { + path.create() + .with_context(|| format!("creating control db export dir {}", path.display()))?; + } + + // ponytail: sled owns live files on Windows; export/import avoids copying locked files. + self.db.flush()?; + let dst = sled::Config::default() + .path(path) + .flush_every_ms(Some(50)) + .mode(sled::Mode::HighThroughput) + .open()?; + dst.import(self.db.export()); + dst.flush()?; + Ok(()) + } + #[cfg(test)] pub fn at(path: impl AsRef) -> Result { let config = sled::Config::default() diff --git a/crates/standalone/src/control_db/tests.rs b/crates/standalone/src/control_db/tests.rs index 171fcadffc7..9c539ae619e 100644 --- a/crates/standalone/src/control_db/tests.rs +++ b/crates/standalone/src/control_db/tests.rs @@ -152,3 +152,31 @@ fn test_decode() -> ResultTest<()> { Ok(()) } + +#[test] +fn test_export_to_path_can_be_opened() -> anyhow::Result<()> { + let src = TempDir::with_prefix("control-db-export-src")?; + let dst = TempDir::with_prefix("control-db-export-dst")?; + let cdb = ControlDb::at(src.path())?; + let database = Database { + id: 0, + database_identity: Identity::ZERO, + owner_identity: *ALICE, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + }; + let database_id = cdb.insert_database(database)?; + cdb.insert_replica(Replica { + id: 0, + database_id, + node_id: 0, + leader: true, + })?; + + cdb.export_to_path(&ControlDbDir(dst.path().to_path_buf()))?; + let exported = ControlDb::at(dst.path())?; + + assert!(exported.get_database_by_id(database_id)?.is_some()); + assert!(exported.get_leader_replica_by_database(database_id).is_some()); + Ok(()) +} diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index f806f18e995..7c3459abaa5 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -31,11 +31,14 @@ use spacetimedb_datastore::db_metrics::data_size::DATA_SIZE_METRICS; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::traits::Program; use spacetimedb_paths::server::{ModuleLogsDir, PidFile, ServerDataDir}; +use spacetimedb_paths::standalone::ControlDbDir; use spacetimedb_paths::standalone::StandaloneDataDirExt; +use spacetimedb_paths::FromPathUnchecked; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; use spacetimedb_table::page_pool::PagePool; +use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; pub use spacetimedb_client_api::routes::subscribe::{BIN_PROTOCOL, TEXT_PROTOCOL}; @@ -191,11 +194,54 @@ impl NodeDelegate for StandaloneEnv { Ok(Host::new(leader.id, self.host_controller.clone())) } + async fn create_hot_backup( + &self, + leader: Host, + output_dir: PathBuf, + ) -> anyhow::Result { + let total_start = Instant::now(); + let mut manifest = leader.create_hot_backup_without_control_db(&output_dir).await?; + + let export_start = Instant::now(); + self.control_db + .export_to_path(&ControlDbDir::from_path_unchecked( + output_dir.join("server").join("control-db"), + )) + .context("exporting control-db into backup")?; + manifest.copy_ms = manifest.copy_ms.saturating_add(elapsed_ms(export_start.elapsed())); + manifest.total_ms = elapsed_ms(total_start.elapsed()); + + let manifest_path = output_dir.join("manifest.json"); + let _ = std::fs::remove_file(&manifest_path); + manifest.bytes = dir_size(&output_dir).context("measuring backup size")?; + std::fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?) + .with_context(|| format!("writing {}", manifest_path.display()))?; + Ok(manifest) + } + fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir { self.data_dir().replica(replica_id).module_logs() } } +fn elapsed_ms(duration: Duration) -> u64 { + duration.as_millis().try_into().unwrap_or(u64::MAX) +} + +fn dir_size(path: &Path) -> std::io::Result { + let mut bytes = 0; + for entry in std::fs::read_dir(path)? { + let entry = entry?; + let meta = entry.metadata()?; + if meta.is_dir() { + bytes += dir_size(&entry.path())?; + } else { + bytes += meta.len(); + } + } + Ok(bytes) +} + #[async_trait] impl spacetimedb_client_api::ControlStateReadAccess for StandaloneEnv { // Nodes diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index 01078f7af8d..51f2d5cceee 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -1,9 +1,13 @@ use netstat2::{get_sockets_info, AddressFamilyFlags, ProtocolFlags, ProtocolSocketInfo, TcpState}; use spacetimedb_client_api::routes::identity::IdentityRoutes; +use spacetimedb_client_api::util::NameOrIdentity; +use spacetimedb_client_api::{ControlStateReadAccess, NodeDelegate}; use spacetimedb_pg::pg_server; use std::io::{self, Write}; use std::net::IpAddr; +use std::path::PathBuf; use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::{StandaloneEnv, StandaloneOptions}; use anyhow::Context; @@ -22,6 +26,7 @@ use spacetimedb_client_api::routes::subscribe::WebSocketOptions; use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; use spacetimedb_paths::server::{ConfigToml, ServerDataDir}; use tokio::net::TcpListener; +use tokio::task::JoinHandle; pub fn cli() -> clap::Command { clap::Command::new("start") @@ -103,6 +108,18 @@ struct ConfigFile { commitlog: CommitlogConfig, #[serde(default)] websocket: WebSocketOptions, + #[serde(default, alias = "scheduled-backup")] + scheduled_backup: Option, +} + +#[derive(Clone, Debug, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +struct ScheduledBackupConfig { + database: NameOrIdentity, + output_dir: PathBuf, + #[serde(deserialize_with = "deserialize_duration")] + interval: Duration, + keep_last: Option, } impl ConfigFile { @@ -111,6 +128,102 @@ impl ConfigFile { } } +fn deserialize_duration<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + #[derive(serde::Deserialize)] + #[serde(untagged)] + enum DurationValue { + String(String), + Seconds(u64), + } + + match ::deserialize(deserializer)? { + DurationValue::String(value) => humantime::parse_duration(&value).map_err(serde::de::Error::custom), + DurationValue::Seconds(value) => Ok(Duration::from_secs(value)), + } +} + +fn backup_dir_name() -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or_default(); + format!("stdb-{nanos}") +} + +fn validate_scheduled_backup_config(config: &ScheduledBackupConfig) -> anyhow::Result<()> { + anyhow::ensure!( + !config.interval.is_zero(), + "scheduled-backup interval must be greater than zero" + ); + anyhow::ensure!( + config.output_dir.is_absolute(), + "scheduled-backup output-dir must be an absolute server path: {}", + config.output_dir.display() + ); + if let Some(0) = config.keep_last { + anyhow::bail!("scheduled-backup keep-last must be greater than zero"); + } + Ok(()) +} + +fn prune_scheduled_backups(config: &ScheduledBackupConfig) -> anyhow::Result<()> { + let Some(keep_last) = config.keep_last else { + return Ok(()); + }; + let mut dirs: Vec<_> = std::fs::read_dir(&config.output_dir) + .with_context(|| format!("reading scheduled-backup output-dir {}", config.output_dir.display()))? + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_ok_and(|ty| ty.is_dir())) + .filter(|entry| entry.file_name().to_string_lossy().starts_with("stdb-")) + .collect(); + dirs.sort_by_key(|entry| entry.file_name()); + let remove_count = dirs.len().saturating_sub(keep_last); + for entry in dirs.into_iter().take(remove_count) { + std::fs::remove_dir_all(entry.path()) + .with_context(|| format!("removing old backup {}", entry.path().display()))?; + } + Ok(()) +} + +async fn run_scheduled_backup(ctx: &Arc, config: &ScheduledBackupConfig) -> anyhow::Result<()> { + validate_scheduled_backup_config(config)?; + let database_identity = config + .database + .try_resolve(ctx) + .await? + .map_err(|name| anyhow::anyhow!("scheduled-backup database `{name}` not found"))?; + let database = ctx + .get_database_by_identity(&database_identity) + .await? + .with_context(|| format!("scheduled-backup database `{}` not found", config.database))?; + let output_dir = config.output_dir.join(backup_dir_name()); + let leader = ctx.leader(database.id).await?; + ctx.create_hot_backup(leader, output_dir).await?; + prune_scheduled_backups(config)?; + Ok(()) +} + +fn spawn_scheduled_backup(ctx: Arc, config: ScheduledBackupConfig) -> JoinHandle<()> { + tokio::spawn(async move { + log::info!( + "scheduled backup enabled for {} every {:?} into {}", + config.database, + config.interval, + config.output_dir.display() + ); + + loop { + tokio::time::sleep(config.interval).await; + if let Err(err) = run_scheduled_backup(&ctx, &config).await { + log::error!("scheduled backup failed: {err:#}"); + } + } + }) +} + pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { let listen_addr = args.get_one::("listen_addr").unwrap(); let pg_port = args.get_one::("pg_port"); @@ -204,6 +317,19 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { ); worker_metrics::spawn_page_pool_stats(listen_addr.clone(), ctx.page_pool().clone()); worker_metrics::spawn_bsatn_rlb_pool_stats(listen_addr.clone(), ctx.bsatn_rlb_pool().clone()); + let backup_task = if matches!(storage, Storage::Disk) { + if let Some(scheduled_backup) = config.scheduled_backup.clone() { + validate_scheduled_backup_config(&scheduled_backup)?; + Some(spawn_scheduled_backup(ctx.clone(), scheduled_backup)) + } else { + None + } + } else { + None + }; + if config.scheduled_backup.is_some() && matches!(storage, Storage::Memory) { + log::warn!("scheduled backup disabled for --in-memory server"); + } let mut db_routes = DatabaseRoutes::default(); db_routes.root_post = db_routes.root_post.layer(DefaultBodyLimit::disable()); db_routes.db_put = db_routes.db_put.layer(DefaultBodyLimit::disable()); @@ -289,6 +415,9 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { }) .await?; } + if let Some(task) = backup_task { + task.abort(); + } Ok(()) } @@ -539,6 +668,12 @@ mod tests { offset-index-require-segment-fsync = false preallocate-segments = true write-buffer-size = 131072 + + [scheduled-backup] + database = "mydb" + output-dir = "/var/backups/stdb" + interval = "1h" + keep-last = 2 "#; let config: ConfigFile = toml::from_str(toml).unwrap(); @@ -581,6 +716,11 @@ mod tests { ..<_>::default() } ); + let scheduled_backup = config.scheduled_backup.unwrap(); + assert_eq!(scheduled_backup.database.to_string(), "mydb"); + assert_eq!(scheduled_backup.output_dir, PathBuf::from("/var/backups/stdb")); + assert_eq!(scheduled_backup.interval, Duration::from_secs(60 * 60)); + assert_eq!(scheduled_backup.keep_last, Some(2)); } #[test] From 88999fd2728cfab96570afc703e97d61acb9cec0 Mon Sep 17 00:00:00 2001 From: kdletters <61648117+kdletters@users.noreply.github.com> Date: Sat, 4 Jul 2026 12:18:11 +0800 Subject: [PATCH 2/8] Harden hot backup handling --- Cargo.lock | 2 + crates/cli/src/subcommands/backup.rs | 6 +- crates/client-api/Cargo.toml | 1 + crates/client-api/src/lib.rs | 24 +- crates/client-api/src/routes/database.rs | 95 +++++- crates/engine/src/relational_db.rs | 319 +++++++++++++++------ crates/engine/src/snapshot.rs | 27 +- crates/fs-utils/src/lib.rs | 111 ++++++- crates/standalone/Cargo.toml | 1 + crates/standalone/config.toml | 5 + crates/standalone/src/control_db/tests.rs | 3 +- crates/standalone/src/lib.rs | 73 ++--- crates/standalone/src/subcommands/start.rs | 114 +++++++- crates/testing/src/modules.rs | 1 + 14 files changed, 627 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5629dee8703..5a3c3ce43ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8141,6 +8141,7 @@ dependencies = [ "spacetimedb-core", "spacetimedb-data-structures", "spacetimedb-datastore", + "spacetimedb-fs-utils", "spacetimedb-lib", "spacetimedb-paths", "spacetimedb-schema", @@ -8936,6 +8937,7 @@ dependencies = [ "spacetimedb-client-api-messages", "spacetimedb-core", "spacetimedb-datastore", + "spacetimedb-fs-utils", "spacetimedb-lib", "spacetimedb-paths", "spacetimedb-pg", diff --git a/crates/cli/src/subcommands/backup.rs b/crates/cli/src/subcommands/backup.rs index 6dbbf0a8454..f1249ae03d0 100644 --- a/crates/cli/src/subcommands/backup.rs +++ b/crates/cli/src/subcommands/backup.rs @@ -25,10 +25,10 @@ pub fn cli() -> Command { .arg( Arg::new("output_dir") .long("output-dir") - .value_name("SERVER_OUTPUT_DIR") + .value_name("ROOT_RELATIVE_OUTPUT_DIR") .required(true) .value_parser(clap::value_parser!(PathBuf)) - .help("Directory on the server where the backup will be written; it must be empty"), + .help("Directory relative to the server's configured hot-backup root; it must be empty"), ) .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database")) .arg(common_args::anonymous()) @@ -103,7 +103,7 @@ async fn exec_create(mut config: Config, args: &ArgMatches) -> Result<(), anyhow let resolved = resolve_database_arg( database_arg, config_targets.as_deref(), - "spacetime backup create --database --output-dir [--no-config]", + "spacetime backup create --database --output-dir [--no-config]", )?; let server = server_from_cli.or(resolved.server.as_deref()); diff --git a/crates/client-api/Cargo.toml b/crates/client-api/Cargo.toml index afbb5a2a340..e34ffb0c0ed 100644 --- a/crates/client-api/Cargo.toml +++ b/crates/client-api/Cargo.toml @@ -11,6 +11,7 @@ spacetimedb-client-api-messages.workspace = true spacetimedb-core.workspace = true spacetimedb-data-structures.workspace = true spacetimedb-datastore.workspace = true +spacetimedb-fs-utils.workspace = true spacetimedb-lib = { workspace = true, features = ["serde"] } spacetimedb-paths.workspace = true spacetimedb-schema.workspace = true diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 93f0287dd9e..21d32ce8b24 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -67,6 +67,14 @@ pub trait NodeDelegate: Send + Sync { ) -> anyhow::Result { leader.create_hot_backup(output_dir).await } + /// Root directory under which HTTP-initiated hot backups may be written. + /// + /// Requested backup output directories are resolved against and confined + /// to this directory. Returning `None` (the default) disables the HTTP + /// backup endpoints entirely. + fn hot_backup_root(&self) -> Option { + None + } fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir; } @@ -119,16 +127,8 @@ impl Host { &self, output_dir: impl AsRef, ) -> anyhow::Result { - let module = self.module().await?; - module - .relational_db() - .create_hot_backup( - &self.host_controller.data_dir.replica(self.replica_id), - Some(&self.host_controller.data_dir), - self.replica_id, - output_dir, - ) - .await + let _ = output_dir; + anyhow::bail!("hot backup requires the node delegate to export control-db and finalize the backup manifest") } pub async fn create_hot_backup_without_control_db( @@ -544,6 +544,10 @@ impl NodeDelegate for Arc { (**self).create_hot_backup(leader, output_dir).await } + fn hot_backup_root(&self) -> Option { + (**self).hot_backup_root() + } + fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir { (**self).module_logs_dir(replica_id) } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 0e402c9de81..9c94ce5070f 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; use std::future::Future; use std::num::NonZeroU8; -use std::path::PathBuf; +use std::path::{Component, Path as StdPath, PathBuf}; use std::str::FromStr; use std::time::Duration; use std::{env, io}; @@ -43,6 +43,7 @@ use spacetimedb_client_api_messages::name::{ self, DatabaseName, DomainName, MigrationPolicy, PrePublishAutoMigrateResult, PrePublishManualMigrateResult, PrePublishResult, PrettyPrintStyle, PublishOp, PublishResult, }; +use spacetimedb_fs_utils::normalize_absolute_path; use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::{http as st_http, ConnectionId}; @@ -688,6 +689,45 @@ pub struct BackupRequest { server_output_dir: PathBuf, } +fn bad_backup_request(message: impl Into) -> ErrorResponse { + (StatusCode::BAD_REQUEST, message.into()).into() +} + +fn resolve_hot_backup_output_dir( + hot_backup_root: Option, + requested: &StdPath, +) -> axum::response::Result { + let root = hot_backup_root + .ok_or_else(|| ErrorResponse::from((StatusCode::NOT_FOUND, "hot backup endpoint is disabled")))?; + + if requested.as_os_str().is_empty() { + return Err(bad_backup_request("backup output path must not be empty")); + } + if requested.is_absolute() { + return Err(bad_backup_request( + "backup output path must be relative to the configured hot backup root", + )); + } + if requested + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(bad_backup_request( + "backup output path must contain only normal relative path components", + )); + } + + let root = normalize_absolute_path(&root).map_err(log_and_500)?; + let output_dir = normalize_absolute_path(&root.join(requested)).map_err(log_and_500)?; + if !output_dir.starts_with(&root) { + return Err(bad_backup_request( + "backup output path must remain within the configured hot backup root", + )); + } + + Ok(output_dir) +} + pub async fn backup( State(worker_ctx): State, Path(BackupParams { name_or_identity }): Path, @@ -702,6 +742,7 @@ where .authorize_action(auth.claims.identity, database.database_identity, Action::UpdateDatabase) .await?; + let server_output_dir = resolve_hot_backup_output_dir(worker_ctx.hot_backup_root(), &server_output_dir)?; let manifest = worker_ctx .create_hot_backup(leader, server_output_dir) .await @@ -1713,6 +1754,7 @@ mod tests { }; use async_trait::async_trait; use axum::body::Body; + use axum::response::IntoResponse; use http::Request; use spacetimedb::auth::identity::{JwtError, JwtErrorKind, SpacetimeIdentityClaims}; use spacetimedb::auth::token_validation::{TokenSigner, TokenValidationError, TokenValidator}; @@ -1726,6 +1768,7 @@ mod tests { use spacetimedb_paths::server::ModuleLogsDir; use spacetimedb_paths::FromPathUnchecked; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; + use std::path::Path as StdPath; use tower::util::ServiceExt; #[derive(Clone, Default)] struct DummyValidator; @@ -1969,6 +2012,56 @@ mod tests { } } + fn err_status(result: axum::response::Result) -> StatusCode { + axum::response::Result::<()>::Err(result.unwrap_err()) + .into_response() + .status() + } + + #[test] + fn backup_output_dir_requires_configured_root() { + assert_eq!( + err_status(resolve_hot_backup_output_dir(None, StdPath::new("backup-1"))), + StatusCode::NOT_FOUND + ); + } + + #[test] + fn backup_output_dir_resolves_relative_path_under_root() { + let root = tempfile::tempdir().unwrap(); + + let output = + resolve_hot_backup_output_dir(Some(root.path().to_path_buf()), StdPath::new("db/backup-1")).unwrap(); + + assert_eq!(output, root.path().canonicalize().unwrap().join("db/backup-1")); + } + + #[test] + fn backup_output_dir_rejects_absolute_path() { + let root = tempfile::tempdir().unwrap(); + + assert_eq!( + err_status(resolve_hot_backup_output_dir( + Some(root.path().to_path_buf()), + root.path() + )), + StatusCode::BAD_REQUEST + ); + } + + #[test] + fn backup_output_dir_rejects_parent_components() { + let root = tempfile::tempdir().unwrap(); + + assert_eq!( + err_status(resolve_hot_backup_output_dir( + Some(root.path().to_path_buf()), + StdPath::new("../escape") + )), + StatusCode::BAD_REQUEST + ); + } + /// Tests that requests to user-defined routes under `/database/:name-or-identity/routes` /// bypass the usual SpacetimeDB auth middleware, /// and accept requests with `Authorization` headers that SpacetimeDB would treat as malformed. diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index 36171f4b96c..46c1c959e05 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -7,6 +7,7 @@ use crate::util::asyncify; use crate::MetricsRecorderQueue; use anyhow::{anyhow, Context}; use enum_map::EnumMap; +use parking_lot::Mutex; use spacetimedb_commitlog::repo::OnNewSegmentFn; use spacetimedb_commitlog::{self as commitlog, Commitlog, SizeOnDisk}; use spacetimedb_data_structures::map::HashSet; @@ -36,6 +37,7 @@ use spacetimedb_datastore::{ traits::TxData, }; use spacetimedb_durability::{self as durability, History}; +use spacetimedb_fs_utils::{copy_dir_all, dir_size, normalize_absolute_path}; use spacetimedb_lib::bsatn::ToBsatn; use spacetimedb_lib::db::auth::StAccess; use spacetimedb_lib::db::raw_def::v9::{btree, RawModuleDefV9Builder, RawSql}; @@ -64,6 +66,7 @@ use spacetimedb_table::page_pool::PagePool; use spacetimedb_table::table::{RowRef, TableScanIter}; use spacetimedb_table::table_index::IndexKey; use std::borrow::Cow; +use std::collections::BTreeSet; use std::ffi::OsString; use std::io; use std::ops::RangeBounds; @@ -149,6 +152,15 @@ pub struct HotBackupManifest { pub bytes: u64, } +impl HotBackupManifest { + /// Convert a [`Duration`] to whole milliseconds, saturating at `u64::MAX`. + /// + /// Used to fill in the `*_ms` timing fields of the manifest. + pub fn elapsed_ms(duration: Duration) -> u64 { + duration.as_millis().try_into().unwrap_or(u64::MAX) + } +} + impl std::fmt::Debug for RelationalDB { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RelationalDB") @@ -989,6 +1001,13 @@ impl RelationalDB { /// Export a point-in-time hot backup into `output_dir`, leaving `server/control-db` /// for the caller to provide. + /// + /// Unlike [`Self::create_hot_backup`], this does **not** compute + /// [`HotBackupManifest::bytes`] (it is left at 0) and does **not** write + /// `manifest.json`: the caller is expected to add `server/control-db` and + /// then finalize the manifest itself. This both avoids traversing the + /// backup tree twice and makes `manifest.json` a reliable marker of a + /// fully-written backup. pub async fn create_hot_backup_without_control_db( &self, replica_dir: &ReplicaDir, @@ -1014,12 +1033,18 @@ impl RelationalDB { .as_ref() .context("durability runtime is not enabled for this database")? .clone(); - let output_dir = output_dir.as_ref().to_path_buf(); - ensure_backup_path(replica_dir, server_data_dir, &output_dir)?; + let output_dir = ensure_backup_path(replica_dir, server_data_dir, output_dir.as_ref())?; + // Guard against two concurrent backups interleaving writes into the same + // directory after both passed the `ensure_empty_dir` check. + let _dir_guard = BackupDirGuard::acquire(&output_dir)?; + anyhow::ensure!( + !(copy_control_db && server_data_dir.is_some()), + "copying live server/control-db is not supported by hot backup; export control-db separately and finalize the manifest after the export" + ); let snapshot_start = Instant::now(); let snapshot_offset = self.request_hot_backup_snapshot().await?; - let snapshot_ms = elapsed_ms(snapshot_start.elapsed()); + let snapshot_ms = HotBackupManifest::elapsed_ms(snapshot_start.elapsed()); let durable_offset = snapshot_offset; let copy_start = Instant::now(); @@ -1032,9 +1057,20 @@ impl RelationalDB { .file_name() .context("snapshot directory has no file name")?, ); - wait_for_dir(&src_snapshot_dir.0).await?; + // `request_hot_backup_snapshot` returns only after the snapshot worker has + // fsynced the snapshot to disk (see `SnapshotWorker::ensure_snapshot_at_least`), + // so both paths must already exist; a missing path is a bug, not a timing issue. + anyhow::ensure!( + src_snapshot_dir.0.is_dir(), + "snapshot directory does not exist: {}", + src_snapshot_dir.display() + ); let src_snapshot_file = src_snapshot_dir.snapshot_file(snapshot_offset); - wait_for_file(&src_snapshot_file.0).await?; + anyhow::ensure!( + src_snapshot_file.0.is_file(), + "snapshot file does not exist: {}", + src_snapshot_file.display() + ); let snapshot_file_name: OsString = src_snapshot_file .0 .file_name() @@ -1057,7 +1093,7 @@ impl RelationalDB { copy_server_state(&runtime, server_data_dir, &output_dir, copy_control_db).await?; } copy_commitlog_range(replica_dir.commit_log(), output_dir.join("clog"), snapshot_offset).await?; - let copy_ms = elapsed_ms(copy_start.elapsed()); + let copy_ms = HotBackupManifest::elapsed_ms(copy_start.elapsed()); let mut manifest = HotBackupManifest { version: 1, @@ -1068,15 +1104,20 @@ impl RelationalDB { output_dir: output_dir.clone(), snapshot_ms, copy_ms, - total_ms: elapsed_ms(total_start.elapsed()), + total_ms: HotBackupManifest::elapsed_ms(total_start.elapsed()), bytes: 0, }; - manifest.bytes = asyncify(&runtime, { - let output_dir = output_dir.clone(); - move || dir_size(&output_dir) - }) - .await?; - write_hot_backup_manifest(&runtime, &output_dir, &manifest).await?; + // When the caller provides `server/control-db` itself, leave `bytes` + // and `manifest.json` to it, so the backup tree is only traversed once + // and the manifest is written exactly once, after the backup is complete. + if copy_control_db { + manifest.bytes = asyncify(&runtime, { + let output_dir = output_dir.clone(); + move || dir_size(&output_dir) + }) + .await?; + write_hot_backup_manifest(&runtime, &output_dir, &manifest).await?; + } Ok(manifest) } @@ -2011,6 +2052,77 @@ async fn write_hot_backup_manifest( Ok(()) } +/// Write `manifest` as `manifest.json` into `output_dir` on a blocking thread. +/// +/// Exposed for callers of +/// [`RelationalDB::create_hot_backup_without_control_db`], which finalize the +/// manifest themselves after adding their own state to the backup. +pub async fn finalize_hot_backup_manifest(output_dir: &Path, manifest: &HotBackupManifest) -> anyhow::Result<()> { + write_hot_backup_manifest(&Handle::tokio_current(), output_dir, manifest).await +} + +/// Serializes concurrent hot backups into the same output directory within +/// this process. +/// +/// Two concurrent backups into the same (empty) directory would both pass the +/// `ensure_empty_dir` check and interleave their writes, producing a corrupt +/// backup. Registering the normalized output path in a process-wide set closes +/// that race; the path is released when the guard drops. +struct BackupDirGuard(PathBuf); + +static ACTIVE_BACKUP_DIRS: Mutex> = Mutex::new(BTreeSet::new()); + +impl BackupDirGuard { + fn acquire(output_dir: &Path) -> anyhow::Result { + let mut active = ACTIVE_BACKUP_DIRS.lock(); + if let Some(overlapping) = active + .iter() + .find(|active_dir| backup_paths_overlap(output_dir, active_dir)) + { + anyhow::bail!( + "backup output path {} overlaps with in-progress backup {}", + output_dir.display(), + overlapping.display() + ); + } + anyhow::ensure!( + active.insert(output_dir.to_path_buf()), + "a backup into {} is already in progress", + output_dir.display() + ); + Ok(Self(output_dir.to_path_buf())) + } +} + +#[cfg(windows)] +fn backup_path_components(path: &Path) -> Vec { + path.components() + .map(|component| component.as_os_str().to_string_lossy().to_ascii_lowercase()) + .collect() +} + +#[cfg(windows)] +fn backup_path_starts_with(path: &Path, base: &Path) -> bool { + let path = backup_path_components(path); + let base = backup_path_components(base); + base.len() <= path.len() && path.iter().zip(&base).all(|(path, base)| path == base) +} + +#[cfg(not(windows))] +fn backup_path_starts_with(path: &Path, base: &Path) -> bool { + path.starts_with(base) +} + +fn backup_paths_overlap(a: &Path, b: &Path) -> bool { + backup_path_starts_with(a, b) || backup_path_starts_with(b, a) +} + +impl Drop for BackupDirGuard { + fn drop(&mut self) { + ACTIVE_BACKUP_DIRS.lock().remove(&self.0); + } +} + fn ensure_empty_dir(path: &Path) -> anyhow::Result<()> { if path.exists() { anyhow::ensure!( @@ -2029,29 +2141,42 @@ fn ensure_empty_dir(path: &Path) -> anyhow::Result<()> { Ok(()) } +/// Validate `output_dir` as a hot backup destination and return its +/// normalized form. +/// +/// The path must be absolute and, after resolving symlinks in its existing +/// ancestors and rejecting `..` components (see +/// [`spacetimedb_fs_utils::normalize_absolute_path`]), must not point inside +/// the server data directory or the replica directory. All subsequent backup +/// I/O must use the returned path, so the checks cannot be bypassed via +/// symlinks or path traversal. fn ensure_backup_path( replica_dir: &ReplicaDir, server_data_dir: Option<&ServerDataDir>, output_dir: &Path, -) -> anyhow::Result<()> { +) -> anyhow::Result { anyhow::ensure!( output_dir.is_absolute(), "backup output directory must be an absolute server path: {}", output_dir.display() ); + let output_dir = normalize_absolute_path(output_dir).context("normalizing the backup output directory path")?; if let Some(server_data_dir) = server_data_dir { + let data_dir = normalize_absolute_path(&server_data_dir.0).unwrap_or_else(|_| server_data_dir.0.to_path_buf()); anyhow::ensure!( - !output_dir.starts_with(&server_data_dir.0), + !output_dir.starts_with(&data_dir), "backup output directory must not be inside the server data directory: {}", output_dir.display() ); } + let replica_dir_normalized = + normalize_absolute_path(&replica_dir.0).unwrap_or_else(|_| replica_dir.0.to_path_buf()); anyhow::ensure!( - !output_dir.starts_with(&replica_dir.0), + !output_dir.starts_with(&replica_dir_normalized), "backup output directory must not be inside the replica directory: {}", replica_dir.display() ); - Ok(()) + Ok(output_dir) } async fn copy_server_state( @@ -2063,14 +2188,12 @@ async fn copy_server_state( let data_dir = data_dir.clone(); let output_dir = output_dir.to_path_buf(); asyncify(runtime, move || -> anyhow::Result<()> { + debug_assert!(!copy_control_db); let server_dir = output_dir.join("server"); std::fs::create_dir_all(&server_dir)?; copy_required_file(&data_dir.config_toml().0, &server_dir.join("config.toml"))?; copy_required_file(&data_dir.metadata_toml().0, &server_dir.join("metadata.toml"))?; - if copy_control_db { - copy_required_dir(&data_dir.control_db().0, &server_dir.join("control-db"))?; - } copy_required_dir(&data_dir.program_bytes().0, &server_dir.join("program-bytes"))?; Ok(()) }) @@ -2092,23 +2215,6 @@ fn copy_required_dir(src: &Path, dst: &Path) -> anyhow::Result<()> { copy_dir_all(src, dst).with_context(|| format!("copying {} to {}", src.display(), dst.display())) } -fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { - let src = src.as_ref(); - let dst = dst.as_ref(); - std::fs::create_dir_all(dst)?; - for entry in std::fs::read_dir(src)? { - let entry = entry?; - let ty = entry.file_type()?; - let dst = dst.join(entry.file_name()); - if ty.is_dir() { - copy_dir_all(entry.path(), dst)?; - } else { - std::fs::copy(entry.path(), dst)?; - } - } - Ok(()) -} - fn copy_dir_all_retry(src: &Path, dst: &Path, required_file_name: &OsString) -> anyhow::Result<()> { let start = Instant::now(); loop { @@ -2155,50 +2261,6 @@ fn copy_snapshot_dir(src: &Path, dst: &Path, required_file_name: &OsString) -> a Ok(()) } -fn dir_size(path: &Path) -> io::Result { - let mut bytes = 0; - for entry in std::fs::read_dir(path)? { - let entry = entry?; - let meta = entry.metadata()?; - if meta.is_dir() { - bytes += dir_size(&entry.path())?; - } else { - bytes += meta.len(); - } - } - Ok(bytes) -} - -fn elapsed_ms(duration: Duration) -> u64 { - duration.as_millis().try_into().unwrap_or(u64::MAX) -} - -async fn wait_for_dir(path: &Path) -> anyhow::Result<()> { - let start = Instant::now(); - while !path.is_dir() { - anyhow::ensure!( - start.elapsed() < Duration::from_secs(5), - "directory did not appear: {}", - path.display() - ); - tokio::time::sleep(Duration::from_millis(10)).await; - } - Ok(()) -} - -async fn wait_for_file(path: &Path) -> anyhow::Result<()> { - let start = Instant::now(); - while !path.is_file() { - anyhow::ensure!( - start.elapsed() < Duration::from_secs(5), - "file did not appear: {}", - path.display() - ); - tokio::time::sleep(Duration::from_millis(10)).await; - } - Ok(()) -} - fn default_row_count_fn(db: Identity) -> RowCountFn { Arc::new(move |table_id, table_name| { DB_METRICS @@ -4159,6 +4221,36 @@ mod tests { Ok(()) } + #[test] + fn hot_backup_concurrent_snapshot_requests_do_not_rewrite_same_offset_snapshot() -> ResultTest<()> { + let stdb = TestDB::durable()?; + + let mut tx = begin_mut_tx(&stdb); + stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; + stdb.commit_tx(tx)?; + + let offsets = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout(HOT_BACKUP_TEST_TIMEOUT, async { + futures::future::join_all((0..8).map(|_| stdb.request_hot_backup_snapshot())) + .await + .into_iter() + .collect::, _>>() + }) + .await + })??; + let snapshot_offset = offsets[0]; + assert!(offsets.iter().all(|offset| *offset == snapshot_offset)); + let repo = open_snapshot_repo(stdb.path().unwrap().snapshots(), stdb.database_identity(), 0)?; + let snapshot_dir = repo.snapshot_dir_path(snapshot_offset); + let sentinel = snapshot_dir.0.join("hot-backup-sentinel"); + std::fs::write(&sentinel, b"keep")?; + + std::thread::sleep(Duration::from_millis(250)); + + assert!(sentinel.is_file(), "same-offset snapshot was rewritten"); + Ok(()) + } + #[test] fn hot_backup_create_writes_manifest_snapshot_and_commitlog() -> anyhow::Result<()> { let stdb = TestDB::durable()?; @@ -4206,7 +4298,7 @@ mod tests { } #[test] - fn hot_backup_create_copies_server_state_when_provided() -> anyhow::Result<()> { + fn hot_backup_create_without_control_db_copies_other_server_state() -> anyhow::Result<()> { let stdb = TestDB::durable()?; let mut tx = begin_mut_tx(&stdb); @@ -4223,14 +4315,16 @@ mod tests { std::fs::write(data.path().join("program-bytes/program"), b"program")?; let backup_dir = tempfile::tempdir()?; - stdb.runtime().unwrap().block_on(async { + let manifest = stdb.runtime().unwrap().block_on(async { tokio::time::timeout( HOT_BACKUP_TEST_TIMEOUT, - stdb.create_hot_backup(stdb.path().unwrap(), Some(&data_dir), 0, backup_dir.path()), + stdb.create_hot_backup_without_control_db(stdb.path().unwrap(), Some(&data_dir), 0, backup_dir.path()), ) .await })??; + assert_eq!(manifest.bytes, 0); + assert!(!backup_dir.path().join("manifest.json").exists()); assert_eq!( std::fs::read_to_string(backup_dir.path().join("server/config.toml"))?, "config" @@ -4239,12 +4333,44 @@ mod tests { std::fs::read_to_string(backup_dir.path().join("server/metadata.toml"))?, "metadata" ); - assert!(backup_dir.path().join("server/control-db/control").is_file()); + assert!(!backup_dir.path().join("server/control-db").exists()); assert!(backup_dir.path().join("server/program-bytes/program").is_file()); Ok(()) } + #[test] + fn hot_backup_rejects_raw_copy_of_live_control_db() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + + let mut tx = begin_mut_tx(&stdb); + stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; + stdb.commit_tx(tx)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + std::fs::write(data.path().join("config.toml"), b"config")?; + std::fs::write(data.path().join("metadata.toml"), b"metadata")?; + std::fs::create_dir_all(data.path().join("control-db"))?; + std::fs::write(data.path().join("control-db/control"), b"control")?; + std::fs::create_dir_all(data.path().join("program-bytes"))?; + std::fs::write(data.path().join("program-bytes/program"), b"program")?; + + let backup_dir = tempfile::tempdir()?; + let result = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + stdb.create_hot_backup(stdb.path().unwrap(), Some(&data_dir), 0, backup_dir.path()), + ) + .await + })?; + let err = result.unwrap_err(); + + assert!(err.to_string().contains("control-db")); + assert!(!backup_dir.path().join("manifest.json").exists()); + Ok(()) + } + #[test] fn hot_backup_rejects_unsafe_output_dirs() -> anyhow::Result<()> { let stdb = TestDB::durable()?; @@ -4267,6 +4393,29 @@ mod tests { Ok(()) } + #[test] + fn hot_backup_dir_guard_rejects_parent_child_overlap() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let parent = temp.path().join("backup"); + let child = parent.join("nested"); + + let parent_guard = BackupDirGuard::acquire(&parent)?; + let Err(err) = BackupDirGuard::acquire(&child) else { + panic!("child backup directory overlapped active parent"); + }; + assert!(err.to_string().contains("overlaps")); + drop(parent_guard); + + let child_guard = BackupDirGuard::acquire(&child)?; + let Err(err) = BackupDirGuard::acquire(&parent) else { + panic!("parent backup directory overlapped active child"); + }; + assert!(err.to_string().contains("overlaps")); + drop(child_guard); + + Ok(()) + } + // For test compression into an existing database. // Must supply the path to the database and the identity of the replica using the `ENV`: // - `SNAPSHOT` the path to the database, like `/tmp/db/replicas/.../8/database` diff --git a/crates/engine/src/snapshot.rs b/crates/engine/src/snapshot.rs index 102fbea071c..6432a6ccc27 100644 --- a/crates/engine/src/snapshot.rs +++ b/crates/engine/src/snapshot.rs @@ -127,7 +127,7 @@ impl SnapshotWorker { /// which is likely due to it having panicked. pub fn request_snapshot(&self) { self.request_snapshot - .unbounded_send(Request::TakeSnapshot) + .unbounded_send(Request::TakeSnapshot { min_offset: None }) .expect("snapshot worker panicked"); } @@ -136,7 +136,9 @@ impl SnapshotWorker { /// Used by the durability to request snapshots on commitlog segment rotation, /// since the durability should continue writing queued TXes even if the snapshot worker panics. pub fn request_snapshot_ignore_closed(&self) { - let _ = self.request_snapshot.unbounded_send(Request::TakeSnapshot); + let _ = self + .request_snapshot + .unbounded_send(Request::TakeSnapshot { min_offset: None }); } /// Subscribe to the [TxOffset]s of snapshots created by this worker. @@ -157,7 +159,11 @@ impl SnapshotWorker { } let mut snapshot_created = self.subscribe(); - self.request_snapshot(); + self.request_snapshot + .unbounded_send(Request::TakeSnapshot { + min_offset: Some(offset), + }) + .expect("snapshot worker panicked"); loop { snapshot_created .changed() @@ -190,7 +196,7 @@ impl SnapshotMetrics { type WeakDatabaseState = Weak>; enum Request { - TakeSnapshot, + TakeSnapshot { min_offset: Option }, ReplaceState(SnapshotDatabaseState), } @@ -223,13 +229,24 @@ impl SnapshotWorkerActor { let mut database_state: Option = None; while let Some(req) = self.snapshot_requests.next().await { match req { - Request::TakeSnapshot => { + Request::TakeSnapshot { min_offset } => { + if let Some(min_offset) = min_offset + && let Ok(Some(latest)) = self.snapshot_repo.latest_snapshot() + && latest >= min_offset + { + self.snapshot_created.send_replace(latest); + continue; + } let res = self .maybe_take_snapshot(database_state.as_ref()) .await .inspect_err(|e| warn!("database={database_identity} SnapshotWorker: {e:#}")); if let Ok(snapshot_offset) = res { self.maybe_compress_snapshots(snapshot_offset).await; + // INVARIANT: only signal `snapshot_created` after the snapshot + // is fully durable on disk (`take_snapshot` returns post-fsync). + // Hot backup relies on this to copy the snapshot directory as + // soon as it observes the offset. self.snapshot_created.send_replace(snapshot_offset); } } diff --git a/crates/fs-utils/src/lib.rs b/crates/fs-utils/src/lib.rs index c4d1a6ba0c1..e664bb27b23 100644 --- a/crates/fs-utils/src/lib.rs +++ b/crates/fs-utils/src/lib.rs @@ -1,5 +1,5 @@ use std::io::Write; -use std::path::Path; +use std::path::{Component, Path, PathBuf}; pub mod compression; pub mod dir_trie; @@ -44,3 +44,112 @@ pub fn atomic_write(file_path: &Path, data: String) -> anyhow::Result<()> { std::fs::rename(&temp_path, file_path)?; Ok(()) } + +/// Recursively compute the total size in bytes of all files under `path`. +pub fn dir_size(path: &Path) -> std::io::Result { + let mut bytes = 0; + for entry in std::fs::read_dir(path)? { + let entry = entry?; + let meta = entry.metadata()?; + if meta.is_dir() { + bytes += dir_size(&entry.path())?; + } else { + bytes += meta.len(); + } + } + Ok(bytes) +} + +/// Recursively copy the directory `src` into `dst`, creating `dst` if necessary. +pub fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + use anyhow::Context as _; + + let src = src.as_ref(); + let dst = dst.as_ref(); + std::fs::create_dir_all(dst).with_context(|| format!("creating {}", dst.display()))?; + for entry in std::fs::read_dir(src).with_context(|| format!("reading {}", src.display()))? { + let entry = entry?; + let ty = entry.file_type()?; + let dst = dst.join(entry.file_name()); + if ty.is_dir() { + copy_dir_all(entry.path(), &dst)?; + } else { + std::fs::copy(entry.path(), &dst) + .with_context(|| format!("copying {} to {}", entry.path().display(), dst.display()))?; + } + } + Ok(()) +} + +/// Normalize an absolute path that may not exist yet. +/// +/// Canonicalizes the deepest existing ancestor of `path` — resolving any +/// symlinks in it — and re-appends the remaining, not-yet-existing components. +/// +/// Errors if `path` is relative or contains `..` components, as those cannot +/// be resolved reliably for paths that do not exist yet. +pub fn normalize_absolute_path(path: &Path) -> anyhow::Result { + use anyhow::Context as _; + + anyhow::ensure!(path.is_absolute(), "path must be absolute: {}", path.display()); + anyhow::ensure!( + !path.components().any(|c| matches!(c, Component::ParentDir)), + "path must not contain `..` components: {}", + path.display() + ); + + // Split off trailing components until we find an existing ancestor. + // Use `symlink_metadata` so a dangling symlink counts as existing and + // fails the subsequent `canonicalize` instead of being skipped over. + let mut prefix = path.to_path_buf(); + let mut tail: Vec = Vec::new(); + while prefix.symlink_metadata().is_err() { + match (prefix.file_name().map(|n| n.to_owned()), prefix.parent()) { + (Some(name), Some(parent)) if parent != Path::new("") => { + tail.push(name); + prefix = parent.to_path_buf(); + } + _ => anyhow::bail!("path has no existing ancestor: {}", path.display()), + } + } + let mut normalized = prefix + .canonicalize() + .with_context(|| format!("canonicalizing {}", prefix.display()))?; + for name in tail.into_iter().rev() { + normalized.push(name); + } + Ok(normalized) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_absolute_path_rejects_relative_and_parent_components() { + assert!(normalize_absolute_path(Path::new("relative/path")).is_err()); + let dir = tempdir::TempDir::new("normalize").unwrap(); + assert!(normalize_absolute_path(&dir.path().join("a/../b")).is_err()); + } + + #[test] + fn normalize_absolute_path_resolves_nonexistent_suffix() { + let dir = tempdir::TempDir::new("normalize").unwrap(); + let root = dir.path().canonicalize().unwrap(); + let normalized = normalize_absolute_path(&dir.path().join("does/not/exist")).unwrap(); + assert_eq!(normalized, root.join("does/not/exist")); + } + + #[cfg(unix)] + #[test] + fn normalize_absolute_path_resolves_symlinks() { + let dir = tempdir::TempDir::new("normalize").unwrap(); + let target = dir.path().join("target"); + std::fs::create_dir(&target).unwrap(); + let link = dir.path().join("link"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + let normalized = normalize_absolute_path(&link.join("new")).unwrap(); + assert_eq!(normalized, target.canonicalize().unwrap().join("new")); + } +} diff --git a/crates/standalone/Cargo.toml b/crates/standalone/Cargo.toml index f1d7305ed8b..39e9ded8f87 100644 --- a/crates/standalone/Cargo.toml +++ b/crates/standalone/Cargo.toml @@ -29,6 +29,7 @@ spacetimedb-client-api-messages.workspace = true spacetimedb-client-api.workspace = true spacetimedb-core.workspace = true spacetimedb-datastore.workspace = true +spacetimedb-fs-utils.workspace = true spacetimedb-lib.workspace = true spacetimedb-paths.workspace = true spacetimedb-pg.workspace = true diff --git a/crates/standalone/config.toml b/crates/standalone/config.toml index 6a5568da0b0..da2b1404340 100644 --- a/crates/standalone/config.toml +++ b/crates/standalone/config.toml @@ -64,6 +64,11 @@ directives = [ # Size in bytes of the memory buffer holding commit data before flushing to storage. # write-buffer-size = 131072 +# [hot-backup] +# HTTP/CLI-triggered backup output paths are relative to this server-side root. +# Omit to disable POST /v1/database/:name/backup. +# root-dir = "/var/backups/stdb" + # [scheduled-backup] # database = "mydb" # output-dir = "/var/backups/stdb" diff --git a/crates/standalone/src/control_db/tests.rs b/crates/standalone/src/control_db/tests.rs index 9c539ae619e..58ebe04cfe4 100644 --- a/crates/standalone/src/control_db/tests.rs +++ b/crates/standalone/src/control_db/tests.rs @@ -5,6 +5,7 @@ use spacetimedb::messages::control_db::HostType; use spacetimedb_client_api::auth::LOCALHOST; use spacetimedb_lib::error::ResultTest; use spacetimedb_lib::Hash; +use spacetimedb_paths::FromPathUnchecked; use tempfile::TempDir; use super::*; @@ -173,7 +174,7 @@ fn test_export_to_path_can_be_opened() -> anyhow::Result<()> { leader: true, })?; - cdb.export_to_path(&ControlDbDir(dst.path().to_path_buf()))?; + cdb.export_to_path(&ControlDbDir::from_path_unchecked(dst.path()))?; let exported = ControlDb::at(dst.path())?; assert!(exported.get_database_by_id(database_id)?.is_some()); diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index 7c3459abaa5..8ffc5f578cb 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -23,6 +23,7 @@ use spacetimedb::util::jobs::JobCores; use spacetimedb::worker_metrics::WORKER_METRICS; use spacetimedb_client_api::auth::{self, LOCALHOST}; use spacetimedb_client_api::routes::subscribe::{HasWebSocketOptions, WebSocketOptions}; +pub use spacetimedb_client_api::routes::subscribe::{BIN_PROTOCOL, TEXT_PROTOCOL}; use spacetimedb_client_api::{ControlStateReadAccess, DatabaseResetDef, Host, NodeDelegate}; use spacetimedb_client_api_messages::name::{ DatabaseName, DomainName, InsertDomainResult, RegisterTldResult, SetDomainsResult, Tld, @@ -30,25 +31,25 @@ use spacetimedb_client_api_messages::name::{ use spacetimedb_datastore::db_metrics::data_size::DATA_SIZE_METRICS; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::traits::Program; +use spacetimedb_fs_utils::dir_size; use spacetimedb_paths::server::{ModuleLogsDir, PidFile, ServerDataDir}; use spacetimedb_paths::standalone::ControlDbDir; use spacetimedb_paths::standalone::StandaloneDataDirExt; use spacetimedb_paths::FromPathUnchecked; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; use spacetimedb_table::page_pool::PagePool; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::Arc; use std::time::{Duration, Instant}; -pub use spacetimedb_client_api::routes::subscribe::{BIN_PROTOCOL, TEXT_PROTOCOL}; - -#[derive(Clone, Copy)] +#[derive(Clone)] pub struct StandaloneOptions { pub db_config: db::Config, pub durability: DurabilityConfig, pub websocket: WebSocketOptions, pub wasm: WasmConfig, pub v8: V8Config, + pub hot_backup_root: Option, } pub struct StandaloneEnv { @@ -60,6 +61,7 @@ pub struct StandaloneEnv { _pid_file: PidFile, auth_provider: auth::DefaultJwtAuthProvider, websocket_options: WebSocketOptions, + hot_backup_root: Option, } impl StandaloneEnv { @@ -113,6 +115,7 @@ impl StandaloneEnv { _pid_file, auth_provider: auth_env, websocket_options: config.websocket, + hot_backup_root: config.hot_backup_root, })) } @@ -199,47 +202,44 @@ impl NodeDelegate for StandaloneEnv { leader: Host, output_dir: PathBuf, ) -> anyhow::Result { + use db::relational_db::HotBackupManifest; + let total_start = Instant::now(); + // Writes the snapshot, commitlog and server state, but leaves + // `server/control-db`, `bytes` and `manifest.json` to us. let mut manifest = leader.create_hot_backup_without_control_db(&output_dir).await?; + // Trust the normalized path used by the engine, not the raw request path. + let output_dir = manifest.output_dir.clone(); let export_start = Instant::now(); - self.control_db - .export_to_path(&ControlDbDir::from_path_unchecked( - output_dir.join("server").join("control-db"), - )) - .context("exporting control-db into backup")?; - manifest.copy_ms = manifest.copy_ms.saturating_add(elapsed_ms(export_start.elapsed())); - manifest.total_ms = elapsed_ms(total_start.elapsed()); - - let manifest_path = output_dir.join("manifest.json"); - let _ = std::fs::remove_file(&manifest_path); - manifest.bytes = dir_size(&output_dir).context("measuring backup size")?; - std::fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?) - .with_context(|| format!("writing {}", manifest_path.display()))?; + let control_db = self.control_db.clone(); + // sled export and the dir-size sweep are blocking; keep them off the async executor. + manifest.bytes = spacetimedb::util::asyncify(move || -> anyhow::Result { + control_db + .export_to_path(&ControlDbDir::from_path_unchecked( + output_dir.join("server").join("control-db"), + )) + .context("exporting control-db into backup")?; + dir_size(&output_dir).context("measuring backup size") + }) + .await?; + manifest.copy_ms = manifest + .copy_ms + .saturating_add(HotBackupManifest::elapsed_ms(export_start.elapsed())); + manifest.total_ms = HotBackupManifest::elapsed_ms(total_start.elapsed()); + + // Written last: a `manifest.json` marks a complete backup. + db::relational_db::finalize_hot_backup_manifest(&manifest.output_dir, &manifest).await?; Ok(manifest) } - fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir { - self.data_dir().replica(replica_id).module_logs() + fn hot_backup_root(&self) -> Option { + self.hot_backup_root.clone() } -} - -fn elapsed_ms(duration: Duration) -> u64 { - duration.as_millis().try_into().unwrap_or(u64::MAX) -} -fn dir_size(path: &Path) -> std::io::Result { - let mut bytes = 0; - for entry in std::fs::read_dir(path)? { - let entry = entry?; - let meta = entry.metadata()?; - if meta.is_dir() { - bytes += dir_size(&entry.path())?; - } else { - bytes += meta.len(); - } + fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir { + self.data_dir().replica(replica_id).module_logs() } - Ok(bytes) } #[async_trait] @@ -722,9 +722,10 @@ mod tests { websocket: WebSocketOptions::default(), wasm: WasmConfig::default(), v8: V8Config::default(), + hot_backup_root: None, }; - let _env = StandaloneEnv::init(config, &ca, data_dir.clone(), JobCores::without_pinned_cores()).await?; + let _env = StandaloneEnv::init(config.clone(), &ca, data_dir.clone(), JobCores::without_pinned_cores()).await?; // Ensure that we have a lock. assert!( StandaloneEnv::init(config, &ca, data_dir.clone(), JobCores::without_pinned_cores()) diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index 51f2d5cceee..2905ee26542 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -5,7 +5,7 @@ use spacetimedb_client_api::{ControlStateReadAccess, NodeDelegate}; use spacetimedb_pg::pg_server; use std::io::{self, Write}; use std::net::IpAddr; -use std::path::PathBuf; +use std::path::{Component, PathBuf}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -110,6 +110,14 @@ struct ConfigFile { websocket: WebSocketOptions, #[serde(default, alias = "scheduled-backup")] scheduled_backup: Option, + #[serde(default, alias = "hot-backup")] + hot_backup: HotBackupConfig, +} + +#[derive(Clone, Debug, Default, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +struct HotBackupConfig { + root_dir: Option, } #[derive(Clone, Debug, serde::Deserialize)] @@ -163,25 +171,52 @@ fn validate_scheduled_backup_config(config: &ScheduledBackupConfig) -> anyhow::R "scheduled-backup output-dir must be an absolute server path: {}", config.output_dir.display() ); + anyhow::ensure!( + !config + .output_dir + .components() + .any(|component| matches!(component, Component::ParentDir)), + "scheduled-backup output-dir must not contain `..` components: {}", + config.output_dir.display() + ); if let Some(0) = config.keep_last { anyhow::bail!("scheduled-backup keep-last must be greater than zero"); } Ok(()) } +/// Remove old scheduled backups, keeping the most recent `keep-last` complete ones. +/// +/// A backup is complete iff it contains `manifest.json`, which is written as +/// the final step of a backup. `stdb-*` directories without a manifest are +/// leftovers of failed or interrupted backups: they are deleted outright and +/// never occupy a `keep-last` slot, so they cannot crowd out good backups. fn prune_scheduled_backups(config: &ScheduledBackupConfig) -> anyhow::Result<()> { + let mut complete = Vec::new(); + for entry in std::fs::read_dir(&config.output_dir) + .with_context(|| format!("reading scheduled-backup output-dir {}", config.output_dir.display()))? + { + let entry = entry.with_context(|| format!("reading entry in {}", config.output_dir.display()))?; + let file_type = entry + .file_type() + .with_context(|| format!("reading file type for {}", entry.path().display()))?; + if !file_type.is_dir() || !entry.file_name().to_string_lossy().starts_with("stdb-") { + continue; + } + if entry.path().join("manifest.json").is_file() { + complete.push(entry); + } else { + log::warn!("removing incomplete scheduled backup {}", entry.path().display()); + std::fs::remove_dir_all(entry.path()) + .with_context(|| format!("removing incomplete backup {}", entry.path().display()))?; + } + } let Some(keep_last) = config.keep_last else { return Ok(()); }; - let mut dirs: Vec<_> = std::fs::read_dir(&config.output_dir) - .with_context(|| format!("reading scheduled-backup output-dir {}", config.output_dir.display()))? - .filter_map(Result::ok) - .filter(|entry| entry.file_type().is_ok_and(|ty| ty.is_dir())) - .filter(|entry| entry.file_name().to_string_lossy().starts_with("stdb-")) - .collect(); - dirs.sort_by_key(|entry| entry.file_name()); - let remove_count = dirs.len().saturating_sub(keep_last); - for entry in dirs.into_iter().take(remove_count) { + complete.sort_by_key(|entry| entry.file_name()); + let remove_count = complete.len().saturating_sub(keep_last); + for entry in complete.into_iter().take(remove_count) { std::fs::remove_dir_all(entry.path()) .with_context(|| format!("removing old backup {}", entry.path().display()))?; } @@ -201,9 +236,14 @@ async fn run_scheduled_backup(ctx: &Arc, config: &ScheduledBackup .with_context(|| format!("scheduled-backup database `{}` not found", config.database))?; let output_dir = config.output_dir.join(backup_dir_name()); let leader = ctx.leader(database.id).await?; - ctx.create_hot_backup(leader, output_dir).await?; - prune_scheduled_backups(config)?; - Ok(()) + let backup_result = ctx.create_hot_backup(leader, output_dir).await; + // Prune even if this run failed, so directories left behind by failed + // backups are cleaned up instead of accumulating; pruning is blocking + // filesystem work, so keep it off the async executor. + let prune_config = config.clone(); + let prune_result = spacetimedb::util::asyncify(move || prune_scheduled_backups(&prune_config)).await; + backup_result?; + prune_result } fn spawn_scheduled_backup(ctx: Arc, config: ScheduledBackupConfig) -> JoinHandle<()> { @@ -303,6 +343,7 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { websocket: config.websocket, wasm: config.common.wasm, v8: config.common.v8, + hot_backup_root: config.hot_backup.root_dir.clone(), }, &certs, data_dir, @@ -636,6 +677,46 @@ mod tests { use super::*; use std::time::Duration; + fn scheduled_backup_config(output_dir: PathBuf, keep_last: Option) -> ScheduledBackupConfig { + ScheduledBackupConfig { + database: NameOrIdentity::Name("mydb".parse().unwrap()), + output_dir, + interval: Duration::from_secs(60), + keep_last, + } + } + + #[test] + fn prune_scheduled_backups_deletes_incomplete_backups_without_keep_last() { + let temp = tempfile::tempdir().unwrap(); + let incomplete = temp.path().join("stdb-incomplete"); + let complete = temp.path().join("stdb-complete"); + let unrelated = temp.path().join("not-a-backup"); + + std::fs::create_dir(&incomplete).unwrap(); + std::fs::write(incomplete.join("partial"), b"not done").unwrap(); + std::fs::create_dir(&complete).unwrap(); + std::fs::write(complete.join("manifest.json"), b"{}").unwrap(); + std::fs::create_dir(&unrelated).unwrap(); + + prune_scheduled_backups(&scheduled_backup_config(temp.path().to_path_buf(), None)).unwrap(); + + assert!(!incomplete.exists()); + assert!(complete.exists()); + assert!(unrelated.exists()); + } + + #[test] + fn scheduled_backup_config_rejects_parent_components() { + let mut path = std::env::temp_dir(); + path.push(".."); + path.push("backups"); + + let err = validate_scheduled_backup_config(&scheduled_backup_config(path, Some(1))).unwrap_err(); + + assert!(err.to_string().contains("must not contain `..`")); + } + #[test] fn options_from_partial_toml() { let toml = r#" @@ -669,6 +750,9 @@ mod tests { preallocate-segments = true write-buffer-size = 131072 + [hot-backup] + root-dir = "/var/backups/stdb-http" + [scheduled-backup] database = "mydb" output-dir = "/var/backups/stdb" @@ -707,6 +791,10 @@ mod tests { config.commitlog.write_buffer_size.map(|val| val.get()), Some(128 * 1024) ); + assert_eq!( + config.hot_backup.root_dir, + Some(PathBuf::from("/var/backups/stdb-http")) + ); assert_eq!( config.websocket, diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index b03b87df9b5..c0d4c2ded43 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -277,6 +277,7 @@ impl CompiledModule { websocket: WebSocketOptions::default(), wasm: Default::default(), v8: Default::default(), + hot_backup_root: None, }, &certs, paths.data_dir.into(), From 4470c24783cb14f0fb1255da33bc9d55f2bf4291 Mon Sep 17 00:00:00 2001 From: kdletters <61648117+kdletters@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:20:58 +0800 Subject: [PATCH 3/8] Harden hot backup restore durability --- crates/cli/src/subcommands/backup.rs | 152 +++++++++++++++++++++++++-- crates/engine/src/relational_db.rs | 24 +++-- crates/engine/src/snapshot.rs | 8 +- crates/fs-utils/src/lib.rs | 135 ++++++++++++++++++++++-- crates/snapshot/src/lib.rs | 5 +- 5 files changed, 292 insertions(+), 32 deletions(-) diff --git a/crates/cli/src/subcommands/backup.rs b/crates/cli/src/subcommands/backup.rs index f1249ae03d0..423abafb9c3 100644 --- a/crates/cli/src/subcommands/backup.rs +++ b/crates/cli/src/subcommands/backup.rs @@ -1,4 +1,5 @@ use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use crate::common_args; use crate::config::Config; @@ -9,6 +10,8 @@ use clap::{Arg, ArgMatches, Command}; use serde::{Deserialize, Serialize}; use spacetimedb_paths::{server::ServerDataDir, SpacetimePaths}; +static RESTORE_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + pub fn cli() -> Command { Command::new("backup") .about("Create server-side database backups") @@ -155,12 +158,20 @@ fn restore_backup(input_dir: &Path, data_dir: &ServerDataDir, force: bool) -> an let _pid_file = data_dir .pid_file() .context("target data-dir must be offline before restore")?; - copy_missing_server_state(input_dir, data_dir)?; - let replica_dir = data_dir.replica(manifest.replica_id); - let tmp_dir = replica_dir - .0 - .with_file_name(format!("{}.restore_tmp_{}", manifest.replica_id, std::process::id())); + let restore_temp_id = RESTORE_TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); + let tmp_dir = replica_dir.0.with_file_name(format!( + "{}.restore_tmp_{}_{}", + manifest.replica_id, + std::process::id(), + restore_temp_id + )); + let old_tmp_dir = replica_dir.0.with_file_name(format!( + "{}.restore_old_{}_{}", + manifest.replica_id, + std::process::id(), + restore_temp_id + )); anyhow::ensure!( force || !replica_dir.0.exists(), @@ -172,28 +183,54 @@ fn restore_backup(input_dir: &Path, data_dir: &ServerDataDir, force: bool) -> an "temporary restore directory already exists: {}", tmp_dir.display() ); + if force { + anyhow::ensure!( + !old_tmp_dir.exists(), + "temporary old replica directory already exists: {}", + old_tmp_dir.display() + ); + } if let Some(parent) = tmp_dir.parent() { std::fs::create_dir_all(parent)?; } + copy_missing_server_state(input_dir, data_dir)?; + + let mut old_tmp_moved = false; + let mut restore_old_tmp_on_error = false; let res = (|| -> anyhow::Result<()> { std::fs::create_dir_all(&tmp_dir)?; copy_dir_all(input_dir.join("snapshots"), tmp_dir.join("snapshots"))?; copy_dir_all(input_dir.join("clog"), tmp_dir.join("clog"))?; std::fs::create_dir_all(tmp_dir.join("module_logs"))?; - if replica_dir.0.exists() { - std::fs::remove_dir_all(&replica_dir.0) - .with_context(|| format!("removing existing target replica directory {}", replica_dir.display()))?; + if force && replica_dir.0.exists() { + std::fs::rename(&replica_dir.0, &old_tmp_dir) + .with_context(|| format!("moving existing target replica directory {}", replica_dir.display()))?; + old_tmp_moved = true; + restore_old_tmp_on_error = true; } std::fs::rename(&tmp_dir, &replica_dir.0) .with_context(|| format!("moving restored replica into {}", replica_dir.display()))?; + restore_old_tmp_on_error = false; + if old_tmp_moved { + if let Err(err) = std::fs::remove_dir_all(&old_tmp_dir) { + tracing::warn!( + "restore completed, but failed to remove old replica directory {}: {err}", + old_tmp_dir.display() + ); + } + old_tmp_moved = false; + } Ok(()) })(); if res.is_err() { let _ = std::fs::remove_dir_all(&tmp_dir); + if restore_old_tmp_on_error && old_tmp_moved && old_tmp_dir.exists() && !replica_dir.0.exists() { + let _ = std::fs::rename(&old_tmp_dir, &replica_dir.0); + } } res?; @@ -350,6 +387,88 @@ mod tests { Ok(()) } + #[test] + fn backup_restore_requires_force_before_copying_missing_server_state() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + std::fs::create_dir_all(data.path().join("replicas/7"))?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + assert!(err.to_string().contains("--force")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + assert!(!data.path().join("config.toml").exists()); + assert!(!data.path().join("metadata.toml").exists()); + Ok(()) + } + + #[test] + fn backup_restore_force_replaces_existing_replica_without_leaving_old_tmp() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path())?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + restore_backup(backup.path(), &data_dir, true)?; + + assert!(!replica_dir.join("old-marker").exists()); + assert!(replica_dir + .join("snapshots/00000000000000000042.snapshot_dir/snapshot") + .is_file()); + assert!(replica_dir.join("clog/00000000000000000000.stdb.log").is_file()); + assert!(replica_dir.join("module_logs").is_dir()); + + for entry in std::fs::read_dir(data.path().join("replicas"))? { + let entry = entry?; + assert!(!entry.file_name().to_string_lossy().contains(".restore_old_")); + } + Ok(()) + } + + #[cfg(windows)] + #[test] + fn backup_restore_force_commits_when_old_replica_cleanup_fails() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path())?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + let old_marker = replica_dir.join("old-marker"); + std::fs::write(&old_marker, b"old")?; + let mut permissions = std::fs::metadata(&old_marker)?.permissions(); + permissions.set_readonly(true); + std::fs::set_permissions(&old_marker, permissions)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + restore_backup(backup.path(), &data_dir, true)?; + restore_backup(backup.path(), &data_dir, true)?; + + assert!(replica_dir + .join("snapshots/00000000000000000042.snapshot_dir/snapshot") + .is_file()); + + for entry in std::fs::read_dir(data.path().join("replicas"))? { + let entry = entry?; + if !entry.file_name().to_string_lossy().contains(".restore_old_") { + continue; + } + clear_readonly_recursively(&entry.path())?; + std::fs::remove_dir_all(entry.path())?; + } + Ok(()) + } + #[test] fn backup_restore_copies_server_state_into_empty_data_dir() -> anyhow::Result<()> { let backup = tempfile::tempdir()?; @@ -398,6 +517,23 @@ mod tests { Ok(()) } + #[cfg(windows)] + #[allow(clippy::permissions_set_readonly_false)] + fn clear_readonly_recursively(path: &Path) -> anyhow::Result<()> { + let metadata = std::fs::symlink_metadata(path)?; + let mut permissions = metadata.permissions(); + if permissions.readonly() { + permissions.set_readonly(false); + std::fs::set_permissions(path, permissions)?; + } + if metadata.is_dir() { + for entry in std::fs::read_dir(path)? { + clear_readonly_recursively(&entry?.path())?; + } + } + Ok(()) + } + fn make_backup_dir(path: &Path, replica_id: u64, offset: u64) -> anyhow::Result<()> { let snapshot_dir = path.join("snapshots").join(format!("{offset:020}.snapshot_dir")); std::fs::create_dir_all(&snapshot_dir)?; diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index 46c1c959e05..775bdd3e8ee 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -37,7 +37,9 @@ use spacetimedb_datastore::{ traits::TxData, }; use spacetimedb_durability::{self as durability, History}; -use spacetimedb_fs_utils::{copy_dir_all, dir_size, normalize_absolute_path}; +use spacetimedb_fs_utils::{ + atomic_write_bytes, copy_dir_all, copy_file_sync, create_dir_all_sync, dir_size, normalize_absolute_path, sync_dir, +}; use spacetimedb_lib::bsatn::ToBsatn; use spacetimedb_lib::db::auth::StAccess; use spacetimedb_lib::db::raw_def::v9::{btree, RawModuleDefV9Builder, RawSql}; @@ -2048,7 +2050,10 @@ async fn write_hot_backup_manifest( ) -> anyhow::Result<()> { let output_dir = output_dir.to_path_buf(); let json = serde_json::to_vec_pretty(manifest)?; - asyncify(runtime, move || std::fs::write(output_dir.join("manifest.json"), json)).await?; + asyncify(runtime, move || { + atomic_write_bytes(&output_dir.join("manifest.json"), &json) + }) + .await?; Ok(()) } @@ -2136,7 +2141,7 @@ fn ensure_empty_dir(path: &Path) -> anyhow::Result<()> { path.display() ); } else { - std::fs::create_dir_all(path)?; + create_dir_all_sync(path)?; } Ok(()) } @@ -2190,11 +2195,12 @@ async fn copy_server_state( asyncify(runtime, move || -> anyhow::Result<()> { debug_assert!(!copy_control_db); let server_dir = output_dir.join("server"); - std::fs::create_dir_all(&server_dir)?; + create_dir_all_sync(&server_dir)?; copy_required_file(&data_dir.config_toml().0, &server_dir.join("config.toml"))?; copy_required_file(&data_dir.metadata_toml().0, &server_dir.join("metadata.toml"))?; copy_required_dir(&data_dir.program_bytes().0, &server_dir.join("program-bytes"))?; + sync_dir(&server_dir)?; Ok(()) }) .await?; @@ -2204,9 +2210,9 @@ async fn copy_server_state( fn copy_required_file(src: &Path, dst: &Path) -> anyhow::Result<()> { anyhow::ensure!(src.is_file(), "server state file is missing: {}", src.display()); if let Some(parent) = dst.parent() { - std::fs::create_dir_all(parent)?; + create_dir_all_sync(parent)?; } - std::fs::copy(src, dst).with_context(|| format!("copying {} to {}", src.display(), dst.display()))?; + copy_file_sync(src, dst).with_context(|| format!("copying {} to {}", src.display(), dst.display()))?; Ok(()) } @@ -2232,10 +2238,10 @@ fn copy_dir_all_retry(src: &Path, dst: &Path, required_file_name: &OsString) -> } fn copy_snapshot_dir(src: &Path, dst: &Path, required_file_name: &OsString) -> anyhow::Result<()> { - std::fs::create_dir_all(dst)?; + create_dir_all_sync(dst)?; let src_required = src.join(required_file_name); let dst_required = dst.join(required_file_name); - std::fs::copy(&src_required, &dst_required) + copy_file_sync(&src_required, &dst_required) .with_context(|| format!("copying {} to {}", src_required.display(), dst_required.display()))?; for entry in std::fs::read_dir(src).with_context(|| format!("reading {}", src.display()))? { @@ -2248,7 +2254,7 @@ fn copy_snapshot_dir(src: &Path, dst: &Path, required_file_name: &OsString) -> a if ty.is_dir() { copy_dir_all(entry.path(), dst)?; } else { - std::fs::copy(entry.path(), dst)?; + copy_file_sync(&entry.path(), &dst)?; } } anyhow::ensure!( diff --git a/crates/engine/src/snapshot.rs b/crates/engine/src/snapshot.rs index 6432a6ccc27..281bcf3cc1f 100644 --- a/crates/engine/src/snapshot.rs +++ b/crates/engine/src/snapshot.rs @@ -152,10 +152,10 @@ impl SnapshotWorker { /// Request a snapshot if needed, and wait until one exists at or beyond `offset`. pub async fn ensure_snapshot_at_least(&self, offset: TxOffset) -> anyhow::Result { - if let Some(latest) = self.snapshot_repository.latest_snapshot()? { - if latest >= offset { - return Ok(latest); - } + if let Some(latest) = self.snapshot_repository.latest_snapshot()? + && latest >= offset + { + return Ok(latest); } let mut snapshot_created = self.subscribe(); diff --git a/crates/fs-utils/src/lib.rs b/crates/fs-utils/src/lib.rs index e664bb27b23..362ed41614f 100644 --- a/crates/fs-utils/src/lib.rs +++ b/crates/fs-utils/src/lib.rs @@ -1,4 +1,4 @@ -use std::io::Write; +use std::io::{ErrorKind, Write}; use std::path::{Component, Path, PathBuf}; pub mod compression; @@ -27,24 +27,96 @@ pub fn create_parent_dir(file: &Path) -> Result<(), std::io::Error> { } pub fn atomic_write(file_path: &Path, data: String) -> anyhow::Result<()> { + atomic_write_bytes(file_path, data.as_bytes()) +} + +/// Atomically write `data` to `file_path`. +/// +/// The bytes are written to a same-directory temporary file, fsynced, renamed +/// over `file_path`, and followed by a best-effort parent directory sync on +/// platforms that support opening directories as files. +pub fn atomic_write_bytes(file_path: &Path, data: &[u8]) -> anyhow::Result<()> { + const ATOMIC_WRITE_MAX_TEMP_ATTEMPTS: u32 = 1024; + let mut temp_path = file_path.to_path_buf(); - let mut temp_file: std::fs::File; - loop { + for _ in 0..ATOMIC_WRITE_MAX_TEMP_ATTEMPTS { temp_path.set_extension(format!(".tmp{}", rand::random::())); let opened = std::fs::OpenOptions::new() .write(true) .create_new(true) .open(&temp_path); - if let Ok(file) = opened { - temp_file = file; - break; + match opened { + Ok(file) => { + let mut temp_file = file; + let write_result = (|| -> anyhow::Result<()> { + temp_file.write_all(data)?; + temp_file.sync_all()?; + drop(temp_file); + std::fs::rename(&temp_path, file_path)?; + if let Some(parent) = non_empty_parent(file_path) { + sync_dir(parent)?; + } + Ok(()) + })(); + if write_result.is_err() { + let _ = std::fs::remove_file(&temp_path); + } + return write_result; + } + Err(err) if err.kind() == ErrorKind::AlreadyExists => {} + Err(err) => return Err(err.into()), } } - temp_file.write_all(data.as_bytes())?; - std::fs::rename(&temp_path, file_path)?; + anyhow::bail!( + "failed to create a temporary file for atomic write after repeated name collisions: {}", + file_path.display() + ) +} + +fn non_empty_parent(path: &Path) -> Option<&Path> { + path.parent().filter(|parent| !parent.as_os_str().is_empty()) +} + +/// Best-effort directory sync. +/// +/// On Unix, opening and syncing a directory persists directory-entry updates +/// such as file creation and rename. On platforms where directories cannot be +/// opened this is a no-op, so callers still get durable file contents without +/// losing Windows compatibility. +pub fn sync_dir(path: &Path) -> std::io::Result<()> { + #[cfg(unix)] + { + std::fs::File::open(path)?.sync_all() + } + #[cfg(not(unix))] + { + let _ = path; + Ok(()) + } +} + +/// Recursively create `path` and best-effort sync its parent directory. +pub fn create_dir_all_sync(path: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(path)?; + if let Some(parent) = non_empty_parent(path) { + sync_dir(parent)?; + } Ok(()) } +/// Copy a file and fsync the destination file before returning. +pub fn copy_file_sync(src: &Path, dst: &Path) -> std::io::Result { + if let Some(parent) = non_empty_parent(dst) { + std::fs::create_dir_all(parent)?; + } + let copied = std::fs::copy(src, dst)?; + std::fs::OpenOptions::new().write(true).open(dst)?.sync_all()?; + if let Some(parent) = non_empty_parent(dst) { + sync_dir(parent)?; + } + Ok(copied) +} + /// Recursively compute the total size in bytes of all files under `path`. pub fn dir_size(path: &Path) -> std::io::Result { let mut bytes = 0; @@ -66,7 +138,7 @@ pub fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> anyhow::Res let src = src.as_ref(); let dst = dst.as_ref(); - std::fs::create_dir_all(dst).with_context(|| format!("creating {}", dst.display()))?; + create_dir_all_sync(dst).with_context(|| format!("creating {}", dst.display()))?; for entry in std::fs::read_dir(src).with_context(|| format!("reading {}", src.display()))? { let entry = entry?; let ty = entry.file_type()?; @@ -74,10 +146,11 @@ pub fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> anyhow::Res if ty.is_dir() { copy_dir_all(entry.path(), &dst)?; } else { - std::fs::copy(entry.path(), &dst) + copy_file_sync(&entry.path(), &dst) .with_context(|| format!("copying {} to {}", entry.path().display(), dst.display()))?; } } + sync_dir(dst).with_context(|| format!("syncing {}", dst.display()))?; Ok(()) } @@ -152,4 +225,46 @@ mod tests { let normalized = normalize_absolute_path(&link.join("new")).unwrap(); assert_eq!(normalized, target.canonicalize().unwrap().join("new")); } + + #[test] + fn atomic_write_bytes_replaces_file_without_leaving_temps() { + let dir = tempdir::TempDir::new("atomic-write").unwrap(); + let path = dir.path().join("manifest.json"); + std::fs::write(&path, b"old").unwrap(); + + atomic_write_bytes(&path, b"new").unwrap(); + + assert_eq!(std::fs::read(&path).unwrap(), b"new"); + let leftover_temp = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(Result::ok) + .any(|entry| entry.file_name().to_string_lossy().contains(".tmp")); + assert!(!leftover_temp, "atomic write left a temp file behind"); + } + + #[test] + fn atomic_write_bytes_returns_error_for_missing_parent() { + let dir = tempdir::TempDir::new("atomic-write").unwrap(); + let path = dir.path().join("missing/manifest.json"); + + let err = atomic_write_bytes(&path, b"new").unwrap_err(); + + let io_err = err + .downcast_ref::() + .expect("expected missing parent to return the original io error"); + assert_eq!(io_err.kind(), std::io::ErrorKind::NotFound); + assert!(!path.exists()); + } + + #[test] + fn copy_dir_all_syncs_nested_file_contents() { + let src = tempdir::TempDir::new("copy-src").unwrap(); + let dst = tempdir::TempDir::new("copy-dst").unwrap(); + std::fs::create_dir_all(src.path().join("nested")).unwrap(); + std::fs::write(src.path().join("nested/file"), b"contents").unwrap(); + + copy_dir_all(src.path(), dst.path().join("out")).unwrap(); + + assert_eq!(std::fs::read(dst.path().join("out/nested/file")).unwrap(), b"contents"); + } } diff --git a/crates/snapshot/src/lib.rs b/crates/snapshot/src/lib.rs index b04ee20216f..4a84a3f6c7d 100644 --- a/crates/snapshot/src/lib.rs +++ b/crates/snapshot/src/lib.rs @@ -1574,7 +1574,10 @@ impl FileOrDirPath<'_> { fn sync_all(&self) -> io::Result<()> { match self { #[cfg(target_os = "windows")] - Self::Dir(path) => Ok(()), + Self::Dir(path) => { + let _ = path; + Ok(()) + } #[cfg(not(target_os = "windows"))] Self::Dir(path) => File::open(path) .map_err(|e| { From 2e3faf54611686e59b260a6287c1b4383e8df3bd Mon Sep 17 00:00:00 2001 From: kdletters <61648117+kdletters@users.noreply.github.com> Date: Mon, 6 Jul 2026 10:11:48 +0800 Subject: [PATCH 4/8] Harden backup restore validation and docs --- Cargo.lock | 3 + crates/cli/Cargo.toml | 3 + crates/cli/src/subcommands/backup.rs | 515 ++++++++++++++++-- crates/standalone/src/control_db.rs | 7 +- crates/standalone/src/subcommands/start.rs | 14 +- .../00100-deploy/00200-self-hosting.md | 61 ++- .../00100-cli-reference.md | 47 ++ .../00200-standalone-config.md | 45 ++ 8 files changed, 642 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5a3c3ce43ae..548af752948 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8072,12 +8072,15 @@ dependencies = [ "spacetimedb-auth", "spacetimedb-client-api-messages", "spacetimedb-codegen", + "spacetimedb-commitlog", "spacetimedb-data-structures", "spacetimedb-fs-utils", "spacetimedb-lib", "spacetimedb-paths", "spacetimedb-primitives", "spacetimedb-schema", + "spacetimedb-snapshot", + "spacetimedb-table", "syntect", "tabled", "tar", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index bb71fb8b6e5..bf951356a32 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -24,12 +24,15 @@ bench = false spacetimedb-auth.workspace = true spacetimedb-client-api-messages.workspace = true spacetimedb-codegen.workspace = true +spacetimedb-commitlog.workspace = true spacetimedb-data-structures.workspace = true spacetimedb-fs-utils.workspace = true spacetimedb-lib.workspace = true spacetimedb-paths.workspace = true spacetimedb-primitives.workspace = true spacetimedb-schema.workspace = true +spacetimedb-snapshot.workspace = true +spacetimedb-table.workspace = true anyhow.workspace = true base64.workspace = true diff --git a/crates/cli/src/subcommands/backup.rs b/crates/cli/src/subcommands/backup.rs index 423abafb9c3..51319a9e4f8 100644 --- a/crates/cli/src/subcommands/backup.rs +++ b/crates/cli/src/subcommands/backup.rs @@ -8,7 +8,15 @@ use crate::util::{add_auth_header_opt, database_identity, get_auth_header, Respo use anyhow::Context; use clap::{Arg, ArgMatches, Command}; use serde::{Deserialize, Serialize}; -use spacetimedb_paths::{server::ServerDataDir, SpacetimePaths}; +use spacetimedb_commitlog::{payload::Txdata, Commitlog}; +use spacetimedb_fs_utils::{copy_dir_all, copy_file_sync, create_dir_all_sync, sync_dir}; +use spacetimedb_lib::{Identity, ProductValue}; +use spacetimedb_paths::{ + server::{CommitLogDir, ServerDataDir, SnapshotsPath}, + FromPathUnchecked, SpacetimePaths, +}; +use spacetimedb_snapshot::SnapshotRepository; +use spacetimedb_table::page_pool::PagePool; static RESTORE_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); @@ -152,12 +160,21 @@ fn exec_restore(paths: &SpacetimePaths, args: &ArgMatches) -> Result<(), anyhow: fn restore_backup(input_dir: &Path, data_dir: &ServerDataDir, force: bool) -> anyhow::Result { let manifest = read_backup_manifest(input_dir)?; validate_backup(input_dir, &manifest)?; - std::fs::create_dir_all(data_dir)?; + create_dir_all_sync(&data_dir.0)?; - // ponytail: data-dir lock catches the common footgun; per-replica online restore needs a real restore service. + // The data-dir lock catches the common footgun; per-replica online restore needs a real restore service. let _pid_file = data_dir .pid_file() .context("target data-dir must be offline before restore")?; + restore_backup_inner(input_dir, data_dir, force, manifest) +} + +fn restore_backup_inner( + input_dir: &Path, + data_dir: &ServerDataDir, + force: bool, + manifest: BackupManifest, +) -> anyhow::Result { let replica_dir = data_dir.replica(manifest.replica_id); let restore_temp_id = RESTORE_TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); let tmp_dir = replica_dir.0.with_file_name(format!( @@ -192,34 +209,52 @@ fn restore_backup(input_dir: &Path, data_dir: &ServerDataDir, force: bool) -> an } if let Some(parent) = tmp_dir.parent() { - std::fs::create_dir_all(parent)?; + create_dir_all_sync(parent)?; } - copy_missing_server_state(input_dir, data_dir)?; + let staged_server_state = stage_missing_server_state(input_dir, data_dir, restore_temp_id)?; + anyhow::ensure!( + !(force && replica_dir.0.exists() && !staged_server_state.is_empty()), + "cannot restore over existing replica {} while target data-dir is missing server state; restore into an empty data-dir or complete the target server state first", + replica_dir.display() + ); let mut old_tmp_moved = false; let mut restore_old_tmp_on_error = false; + let mut committed_server_state = None; let res = (|| -> anyhow::Result<()> { - std::fs::create_dir_all(&tmp_dir)?; + create_dir_all_sync(&tmp_dir)?; copy_dir_all(input_dir.join("snapshots"), tmp_dir.join("snapshots"))?; copy_dir_all(input_dir.join("clog"), tmp_dir.join("clog"))?; - std::fs::create_dir_all(tmp_dir.join("module_logs"))?; + create_dir_all_sync(&tmp_dir.join("module_logs"))?; + sync_parent_dir(&tmp_dir)?; + + committed_server_state = Some(staged_server_state.commit()?); if force && replica_dir.0.exists() { std::fs::rename(&replica_dir.0, &old_tmp_dir) .with_context(|| format!("moving existing target replica directory {}", replica_dir.display()))?; + sync_parent_dir(&replica_dir.0)?; old_tmp_moved = true; restore_old_tmp_on_error = true; } std::fs::rename(&tmp_dir, &replica_dir.0) .with_context(|| format!("moving restored replica into {}", replica_dir.display()))?; + sync_parent_dir(&replica_dir.0)?; restore_old_tmp_on_error = false; + + if let Some(committed_server_state) = &mut committed_server_state { + committed_server_state.keep(); + } + if old_tmp_moved { if let Err(err) = std::fs::remove_dir_all(&old_tmp_dir) { tracing::warn!( "restore completed, but failed to remove old replica directory {}: {err}", old_tmp_dir.display() ); + } else { + let _ = sync_parent_dir(&old_tmp_dir); } old_tmp_moved = false; } @@ -228,8 +263,10 @@ fn restore_backup(input_dir: &Path, data_dir: &ServerDataDir, force: bool) -> an if res.is_err() { let _ = std::fs::remove_dir_all(&tmp_dir); + let _ = sync_parent_dir(&tmp_dir); if restore_old_tmp_on_error && old_tmp_moved && old_tmp_dir.exists() && !replica_dir.0.exists() { let _ = std::fs::rename(&old_tmp_dir, &replica_dir.0); + let _ = sync_parent_dir(&replica_dir.0); } } res?; @@ -237,6 +274,13 @@ fn restore_backup(input_dir: &Path, data_dir: &ServerDataDir, force: bool) -> an Ok(manifest) } +fn sync_parent_dir(path: &Path) -> anyhow::Result<()> { + if let Some(parent) = path.parent().filter(|parent| !parent.as_os_str().is_empty()) { + sync_dir(parent).with_context(|| format!("syncing parent directory {}", parent.display()))?; + } + Ok(()) +} + fn read_backup_manifest(input_dir: &Path) -> anyhow::Result { let manifest_path = input_dir.join("manifest.json"); let bytes = std::fs::read(&manifest_path) @@ -256,39 +300,213 @@ fn validate_backup(input_dir: &Path, manifest: &BackupManifest) -> anyhow::Resul manifest.durable_offset, manifest.snapshot_offset ); + let database_identity: Identity = manifest + .database_identity + .parse() + .with_context(|| format!("parsing backup database identity {}", manifest.database_identity))?; - let snapshot_dir = input_dir - .join("snapshots") - .join(format!("{:020}.snapshot_dir", manifest.snapshot_offset)); + let snapshots = SnapshotsPath::from_path_unchecked(input_dir.join("snapshots")); + let snapshot_dir = snapshots.snapshot_dir(manifest.snapshot_offset); anyhow::ensure!( - snapshot_dir.is_dir(), + snapshot_dir.0.is_dir(), "backup snapshot directory is missing: {}", snapshot_dir.display() ); + let snapshot_file = snapshot_dir.snapshot_file(manifest.snapshot_offset); + anyhow::ensure!( + snapshot_file.0.is_file(), + "backup snapshot file is missing: {}", + snapshot_file.display() + ); + let snapshot_repo = + SnapshotRepository::open(snapshots, database_identity, manifest.replica_id).with_context(|| { + format!( + "opening backup snapshots directory {}", + input_dir.join("snapshots").display() + ) + })?; + let snapshot = snapshot_repo + .read_snapshot(manifest.snapshot_offset, &PagePool::new(None)) + .with_context(|| format!("reading backup snapshot {}", snapshot_file.display()))?; + anyhow::ensure!( + snapshot.database_identity == database_identity, + "backup snapshot database identity {} does not match manifest {}", + snapshot.database_identity, + database_identity + ); + anyhow::ensure!( + snapshot.replica_id == manifest.replica_id, + "backup snapshot replica_id {} does not match manifest {}", + snapshot.replica_id, + manifest.replica_id + ); + anyhow::ensure!( + snapshot.tx_offset == manifest.snapshot_offset, + "backup snapshot tx_offset {} does not match manifest {}", + snapshot.tx_offset, + manifest.snapshot_offset + ); - let clog_dir = input_dir.join("clog"); + let clog_dir = CommitLogDir::from_path_unchecked(input_dir.join("clog")); anyhow::ensure!( - clog_dir.is_dir(), + clog_dir.0.is_dir(), "backup clog directory is missing: {}", clog_dir.display() ); + let initial_segment = clog_dir.segment(0); + anyhow::ensure!( + initial_segment.0.is_file(), + "backup commitlog segment is missing: {}", + initial_segment.display() + ); + let mut segment_count = 0u64; + for entry in std::fs::read_dir(&clog_dir.0).with_context(|| format!("reading {}", clog_dir.display()))? { + let entry = entry?; + let file_name = entry.file_name(); + let file_name = file_name.to_string_lossy(); + if file_name.ends_with(".stdb.log") { + anyhow::ensure!( + is_commitlog_segment_name(&file_name) && entry.file_type()?.is_file(), + "invalid backup commitlog segment: {}", + entry.path().display() + ); + segment_count += 1; + } + } anyhow::ensure!( - clog_dir.read_dir()?.next().is_some(), - "backup clog directory is empty: {}", + segment_count > 0, + "backup clog directory contains no segment files: {}", clog_dir.display() ); + let clog = Commitlog::>::open(clog_dir, Default::default(), None) + .with_context(|| format!("opening backup commitlog {}", input_dir.join("clog").display()))?; + anyhow::ensure!( + clog.max_committed_offset() == Some(manifest.snapshot_offset), + "backup commitlog max committed offset {:?} does not match manifest snapshot_offset {}", + clog.max_committed_offset(), + manifest.snapshot_offset + ); + for commit in clog.commits() { + commit.with_context(|| format!("reading backup commitlog {}", input_dir.join("clog").display()))?; + } Ok(()) } -fn copy_missing_server_state(input_dir: &Path, data_dir: &ServerDataDir) -> anyhow::Result<()> { +fn is_commitlog_segment_name(file_name: &str) -> bool { + let Some(offset) = file_name.strip_suffix(".stdb.log") else { + return false; + }; + offset.len() == 20 && offset.bytes().all(|byte| byte.is_ascii_digit()) +} + +#[derive(Debug)] +struct StagedServerStateEntry { + final_path: PathBuf, + staged_path: PathBuf, +} + +#[derive(Debug)] +struct StagedServerState { + entries: Vec, +} + +impl StagedServerState { + fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn cleanup(&self) { + for entry in &self.entries { + if entry.staged_path.is_dir() { + let _ = std::fs::remove_dir_all(&entry.staged_path); + } else { + let _ = std::fs::remove_file(&entry.staged_path); + } + let _ = sync_parent_dir(&entry.staged_path); + } + } + + fn commit(self) -> anyhow::Result { + let mut committed = CommittedServerState::default(); + let res = (|| -> anyhow::Result<()> { + for entry in &self.entries { + anyhow::ensure!( + !entry.final_path.exists(), + "target data-dir server state appeared during restore: {}", + entry.final_path.display() + ); + std::fs::rename(&entry.staged_path, &entry.final_path).with_context(|| { + format!( + "moving staged server state {} to {}", + entry.staged_path.display(), + entry.final_path.display() + ) + })?; + committed.paths.push(entry.final_path.clone()); + sync_parent_dir(&entry.final_path)?; + } + Ok(()) + })(); + if res.is_err() { + self.cleanup(); + committed.rollback(); + } + res?; + Ok(committed) + } +} + +impl Drop for StagedServerState { + fn drop(&mut self) { + self.cleanup(); + } +} + +#[derive(Debug, Default)] +struct CommittedServerState { + paths: Vec, + keep: bool, +} + +impl CommittedServerState { + fn keep(&mut self) { + self.keep = true; + } + + fn rollback(&self) { + for path in self.paths.iter().rev() { + if path.is_dir() { + let _ = std::fs::remove_dir_all(path); + } else { + let _ = std::fs::remove_file(path); + } + let _ = sync_parent_dir(path); + } + } +} + +impl Drop for CommittedServerState { + fn drop(&mut self) { + if !self.keep { + self.rollback(); + } + } +} + +fn stage_missing_server_state( + input_dir: &Path, + data_dir: &ServerDataDir, + restore_temp_id: u64, +) -> anyhow::Result { let server_dir = input_dir.join("server"); let needs_required_dirs = ["control-db", "program-bytes"] .into_iter() .any(|required_dir| !data_dir.0.join(required_dir).exists()); if !needs_required_dirs && !server_dir.exists() { - return Ok(()); + return Ok(StagedServerState { entries: Vec::new() }); } + let mut entries = Vec::new(); for required_dir in ["control-db", "program-bytes"] { let dst = data_dir.0.join(required_dir); if dst.exists() { @@ -301,7 +519,13 @@ fn copy_missing_server_state(input_dir: &Path, data_dir: &ServerDataDir) -> anyh dst.display(), src.display() ); - copy_dir_all(src, dst)?; + let staged = staged_server_state_path(&dst, restore_temp_id); + anyhow::ensure!( + !staged.exists(), + "temporary server state path already exists: {}", + staged.display() + ); + entries.push((src, dst, staged, true)); } for file in ["config.toml", "metadata.toml"] { let dst = data_dir.0.join(file); @@ -318,33 +542,69 @@ fn copy_missing_server_state(input_dir: &Path, data_dir: &ServerDataDir) -> anyh dst.display(), src.display() ); - std::fs::copy(&src, &dst).with_context(|| format!("copying {} to {}", src.display(), dst.display()))?; + let staged = staged_server_state_path(&dst, restore_temp_id); + anyhow::ensure!( + !staged.exists(), + "temporary server state path already exists: {}", + staged.display() + ); + entries.push((src, dst, staged, false)); } - Ok(()) -} -fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { - let src = src.as_ref(); - let dst = dst.as_ref(); - std::fs::create_dir_all(dst)?; - for entry in std::fs::read_dir(src).with_context(|| format!("reading {}", src.display()))? { - let entry = entry?; - let ty = entry.file_type()?; - let dst = dst.join(entry.file_name()); - if ty.is_dir() { - copy_dir_all(entry.path(), dst)?; - } else { - std::fs::copy(entry.path(), &dst) - .with_context(|| format!("copying {} to {}", entry.path().display(), dst.display()))?; + let mut staged = StagedServerState { entries: Vec::new() }; + let res = (|| -> anyhow::Result<()> { + for (src, final_path, staged_path, is_dir) in entries { + copy_staged_server_state_entry(&mut staged, &src, final_path, staged_path, is_dir)?; } + Ok(()) + })(); + if res.is_err() { + staged.cleanup(); + } + res?; + Ok(staged) +} + +fn copy_staged_server_state_entry( + staged: &mut StagedServerState, + src: &Path, + final_path: PathBuf, + staged_path: PathBuf, + is_dir: bool, +) -> anyhow::Result<()> { + staged.entries.push(StagedServerStateEntry { + final_path, + staged_path, + }); + let entry = staged.entries.last().expect("staged entry was just pushed"); + let res = if is_dir { + copy_dir_all(src, &entry.staged_path) + .with_context(|| format!("copying {} to {}", src.display(), entry.staged_path.display())) + } else { + copy_file_sync(src, &entry.staged_path) + .with_context(|| format!("copying {} to {}", src.display(), entry.staged_path.display())) + .map(|_| ()) + }; + if res.is_err() { + staged.cleanup(); + } + res +} + +fn staged_server_state_path(path: &Path, restore_temp_id: u64) -> PathBuf { + let suffix = format!("restore_tmp_{}_{}", std::process::id(), restore_temp_id); + if let Some(file_name) = path.file_name() { + path.with_file_name(format!("{}.{}", file_name.to_string_lossy(), suffix)) + } else { + path.with_extension(suffix) } - Ok(()) } #[cfg(test)] mod tests { use super::*; use spacetimedb_paths::FromPathUnchecked; + use spacetimedb_table::{blob_store::HashMapBlobStore, table::Table}; #[test] fn backup_restore_copies_replica_into_existing_data_dir() -> anyhow::Result<()> { @@ -360,7 +620,7 @@ mod tests { assert_eq!(manifest.replica_id, 7); assert!(data .path() - .join("replicas/7/snapshots/00000000000000000042.snapshot_dir/snapshot") + .join("replicas/7/snapshots/00000000000000000042.snapshot_dir/00000000000000000042.snapshot_bsatn") .is_file()); assert!(data .path() @@ -422,7 +682,7 @@ mod tests { assert!(!replica_dir.join("old-marker").exists()); assert!(replica_dir - .join("snapshots/00000000000000000042.snapshot_dir/snapshot") + .join("snapshots/00000000000000000042.snapshot_dir/00000000000000000042.snapshot_bsatn") .is_file()); assert!(replica_dir.join("clog/00000000000000000000.stdb.log").is_file()); assert!(replica_dir.join("module_logs").is_dir()); @@ -455,7 +715,7 @@ mod tests { restore_backup(backup.path(), &data_dir, true)?; assert!(replica_dir - .join("snapshots/00000000000000000042.snapshot_dir/snapshot") + .join("snapshots/00000000000000000042.snapshot_dir/00000000000000000042.snapshot_bsatn") .is_file()); for entry in std::fs::read_dir(data.path().join("replicas"))? { @@ -501,6 +761,154 @@ mod tests { Ok(()) } + #[test] + fn backup_restore_rejects_missing_snapshot_file_before_copying_server_state() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + std::fs::remove_file( + backup + .path() + .join("snapshots/00000000000000000042.snapshot_dir/00000000000000000042.snapshot_bsatn"), + )?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("backup snapshot file is missing")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_corrupt_snapshot_before_copying_server_state() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + std::fs::write( + backup + .path() + .join("snapshots/00000000000000000042.snapshot_dir/00000000000000000042.snapshot_bsatn"), + b"not a snapshot", + )?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("reading backup snapshot")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_clog_without_segment_file_before_copying_server_state() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + std::fs::remove_file(backup.path().join("clog/00000000000000000000.stdb.log"))?; + std::fs::write(backup.path().join("clog/not-a-segment"), b"log")?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("backup commitlog segment is missing")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_invalid_commitlog_segment_name() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + std::fs::write(backup.path().join("clog/bad.stdb.log"), b"log")?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path())?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("invalid backup commitlog segment")); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_does_not_leave_partial_server_state_when_server_state_is_incomplete() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + std::fs::create_dir_all(backup.path().join("server/control-db"))?; + std::fs::write(backup.path().join("server/control-db/control"), b"control")?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("backup is missing server state")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + assert!(!data.path().join("config.toml").exists()); + assert!(!data.path().join("metadata.toml").exists()); + assert!(!data.path().join("replicas/7").exists()); + for entry in std::fs::read_dir(data.path())? { + let entry = entry?; + assert!(!entry.file_name().to_string_lossy().contains(".restore_tmp_")); + } + Ok(()) + } + + #[test] + fn backup_restore_staged_server_state_cleanup_removes_current_entry_after_copy_error() -> anyhow::Result<()> { + let data = tempfile::tempdir()?; + let final_path = data.path().join("control-db"); + let staged_path = staged_server_state_path(&final_path, 99); + let mut staged = StagedServerState { entries: Vec::new() }; + + let err = copy_staged_server_state_entry( + &mut staged, + &data.path().join("missing-control-db"), + final_path, + staged_path.clone(), + true, + ) + .unwrap_err(); + + assert!(err.to_string().contains("copying")); + assert_eq!(staged.entries.len(), 1); + assert!(!staged_path.exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_force_existing_replica_when_server_state_is_missing() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + + assert!(err.to_string().contains("cannot restore over existing replica")); + assert!(replica_dir.is_dir()); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + Ok(()) + } + fn make_target_data_dir(path: &Path) -> anyhow::Result<()> { std::fs::create_dir_all(path.join("control-db"))?; std::fs::create_dir_all(path.join("program-bytes"))?; @@ -535,17 +943,38 @@ mod tests { } fn make_backup_dir(path: &Path, replica_id: u64, offset: u64) -> anyhow::Result<()> { - let snapshot_dir = path.join("snapshots").join(format!("{offset:020}.snapshot_dir")); - std::fs::create_dir_all(&snapshot_dir)?; - std::fs::write(snapshot_dir.join("snapshot"), b"snapshot")?; + let database_identity: Identity = "c200000000000000000000000000000000000000000000000000000000000000".parse()?; + + let snapshots_path = SnapshotsPath::from_path_unchecked(path.join("snapshots")); + std::fs::create_dir_all(&snapshots_path.0)?; + let snapshot_repo = SnapshotRepository::open(snapshots_path, database_identity, replica_id)?; + let blobs = HashMapBlobStore::default(); + snapshot_repo + .create_snapshot(std::iter::empty::<&mut Table>(), &blobs, offset)? + .sync_all()?; let clog_dir = path.join("clog"); std::fs::create_dir_all(&clog_dir)?; - std::fs::write(clog_dir.join("00000000000000000000.stdb.log"), b"log")?; + let clog = Commitlog::>::open( + CommitLogDir::from_path_unchecked(&clog_dir), + Default::default(), + None, + )?; + for tx_offset in 0..=offset { + clog.commit([( + tx_offset, + Txdata { + inputs: None, + outputs: None, + mutations: None, + }, + )])?; + } + clog.flush_and_sync()?; let manifest = serde_json::json!({ "version": 1, - "database_identity": "c200000000000000000000000000000000000000000000000000000000000000", + "database_identity": database_identity, "replica_id": replica_id, "snapshot_offset": offset, "durable_offset": offset, diff --git a/crates/standalone/src/control_db.rs b/crates/standalone/src/control_db.rs index 20b89a5c04b..615df8d8737 100644 --- a/crates/standalone/src/control_db.rs +++ b/crates/standalone/src/control_db.rs @@ -10,6 +10,7 @@ use spacetimedb::messages::control_db::{Database, EnergyBalance, Node, Replica}; use spacetimedb_client_api_messages::name::{ DomainName, DomainParsingError, InsertDomainResult, RegisterTldResult, SetDomainsResult, Tld, TldRef, }; +use spacetimedb_fs_utils::sync_dir; use spacetimedb_lib::bsatn; use spacetimedb_paths::standalone::ControlDbDir; @@ -90,7 +91,7 @@ impl ControlDb { .with_context(|| format!("creating control db export dir {}", path.display()))?; } - // ponytail: sled owns live files on Windows; export/import avoids copying locked files. + // Sled owns live files on Windows; export/import avoids copying locked files. self.db.flush()?; let dst = sled::Config::default() .path(path) @@ -99,6 +100,10 @@ impl ControlDb { .open()?; dst.import(self.db.export()); dst.flush()?; + sync_dir(path).with_context(|| format!("syncing control db export dir {}", path.display()))?; + if let Some(parent) = path.parent() { + sync_dir(parent).with_context(|| format!("syncing control db export parent {}", parent.display()))?; + } Ok(()) } diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index 2905ee26542..85a4df599bc 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -189,8 +189,10 @@ fn validate_scheduled_backup_config(config: &ScheduledBackupConfig) -> anyhow::R /// /// A backup is complete iff it contains `manifest.json`, which is written as /// the final step of a backup. `stdb-*` directories without a manifest are -/// leftovers of failed or interrupted backups: they are deleted outright and -/// never occupy a `keep-last` slot, so they cannot crowd out good backups. +/// ignored by pruning and never occupy a `keep-last` slot, so they cannot +/// crowd out good backups. They are not deleted here because a concurrent +/// manual backup may still be writing into a `stdb-*` directory in the same +/// output root. fn prune_scheduled_backups(config: &ScheduledBackupConfig) -> anyhow::Result<()> { let mut complete = Vec::new(); for entry in std::fs::read_dir(&config.output_dir) @@ -206,9 +208,7 @@ fn prune_scheduled_backups(config: &ScheduledBackupConfig) -> anyhow::Result<()> if entry.path().join("manifest.json").is_file() { complete.push(entry); } else { - log::warn!("removing incomplete scheduled backup {}", entry.path().display()); - std::fs::remove_dir_all(entry.path()) - .with_context(|| format!("removing incomplete backup {}", entry.path().display()))?; + log::warn!("ignoring incomplete scheduled backup {}", entry.path().display()); } } let Some(keep_last) = config.keep_last else { @@ -687,7 +687,7 @@ mod tests { } #[test] - fn prune_scheduled_backups_deletes_incomplete_backups_without_keep_last() { + fn prune_scheduled_backups_ignores_incomplete_backups_without_keep_last() { let temp = tempfile::tempdir().unwrap(); let incomplete = temp.path().join("stdb-incomplete"); let complete = temp.path().join("stdb-complete"); @@ -701,7 +701,7 @@ mod tests { prune_scheduled_backups(&scheduled_backup_config(temp.path().to_path_buf(), None)).unwrap(); - assert!(!incomplete.exists()); + assert!(incomplete.exists()); assert!(complete.exists()); assert!(unrelated.exists()); } diff --git a/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md b/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md index 62d40384ece..9aed48c350e 100644 --- a/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md +++ b/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md @@ -223,7 +223,64 @@ ssh ubuntu@ spacetime publish -s local --bin-path spacetime_module.wasm \ + --output-dir manual/ +``` + +Restore is an offline operation. Stop the service, restore into the server data directory, then start the service again: + +```sh +sudo systemctl stop spacetimedb +sudo -u spacetimedb /stdb/spacetime --root-dir=/stdb backup restore \ + --input-dir /var/backups/stdb/manual/ \ + --data-dir /stdb/data \ + --force +sudo systemctl start spacetimedb +``` + +Keep the backup directory private. Backups contain the database snapshot, commitlog, and any server state needed to reopen the restored database. + +## Step 7: Updating SpacetimeDB Version To update SpacetimeDB to the latest version, first stop the service: @@ -249,7 +306,7 @@ Finally, restart the service: sudo systemctl start spacetimedb ``` -## Step 7: Troubleshooting +## Step 8: Troubleshooting ### SpacetimeDB Service Fails to Start diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index 319a8288108..c1c452d3ed0 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -10,6 +10,9 @@ This document contains the help content for the `spacetime` command-line program **Command Overview:** * [`spacetime`↴](#spacetime) +* [`spacetime backup`↴](#spacetime-backup) +* [`spacetime backup create`↴](#spacetime-backup-create) +* [`spacetime backup restore`↴](#spacetime-backup-restore) * [`spacetime publish`↴](#spacetime-publish) * [`spacetime delete`↴](#spacetime-delete) * [`spacetime logs`↴](#spacetime-logs) @@ -46,6 +49,7 @@ This document contains the help content for the `spacetime` command-line program ###### **Subcommands:** +* `backup` — Create server-side database backups * `publish` — Create and update a SpacetimeDB database * `delete` — Deletes a SpacetimeDB database * `logs` — Prints logs from a SpacetimeDB database @@ -74,6 +78,49 @@ This document contains the help content for the `spacetime` command-line program +## `spacetime backup` + +Create server-side database backups + +**Usage:** `spacetime backup ` + +###### **Subcommands:** + +* `create` — Create a hot backup of a running database +* `restore` — Restore a hot backup into an offline server data directory + + + +## `spacetime backup create` + +Create a hot backup of a running database + +**Usage:** `spacetime backup create [OPTIONS] --output-dir ` + +###### **Options:** + +* `--database ` — The name or identity of the database to back up +* `--output-dir ` — Directory relative to the server's configured hot-backup root; it must be empty +* `-s`, `--server ` — The nickname, host name or URL of the server hosting the database +* `--anonymous` — Perform this action with an anonymous identity +* `--no-config` — Ignore spacetime.json configuration + + + +## `spacetime backup restore` + +Restore a hot backup into an offline server data directory + +**Usage:** `spacetime backup restore [OPTIONS] --input-dir ` + +###### **Options:** + +* `--input-dir ` — Directory containing manifest.json, snapshots/, and clog/ +* `--data-dir ` — Offline server data directory whose matching replica will be restored; defaults to the CLI root data-dir +* `--force` — Overwrite the existing target replica directory + + + ## `spacetime publish` Create and update a SpacetimeDB database diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md index 6863d642a98..cdd820d0234 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md @@ -20,6 +20,10 @@ On Linux and macOS, this directory is by default `~/.local/share/spacetime/data` - [`commitlog`](#commitlog) +- [`hot-backup`](#hot-backup) + +- [`scheduled-backup`](#scheduled-backup) + - [`websocket`](#websocket) ### `certificate-authority` @@ -101,6 +105,47 @@ If `true`, preallocate disk space for commitlog segments up to `commitlog.max-se Size in bytes of the memory buffer holding commit data before flushing to storage. +### `hot-backup` + +```toml +[hot-backup] +root-dir = "/var/backups/stdb" +``` + +The `hot-backup` section configures server-side backups created through `spacetime backup create` or the HTTP backup endpoint. + +#### `hot-backup.root-dir` + +An absolute server path that acts as the root directory for CLI-triggered hot backups. Backup creation requests choose an output directory relative to this root. Omit `root-dir` to disable CLI-triggered hot backups. + +### `scheduled-backup` + +```toml +[scheduled-backup] +database = "mydb" +output-dir = "/var/backups/stdb" +interval = "1h" +keep-last = 24 +``` + +The `scheduled-backup` section configures a background task that periodically backs up one local database. + +#### `scheduled-backup.database` + +The database name or identity to back up. + +#### `scheduled-backup.output-dir` + +An absolute server path where scheduled backups are created. Each backup is written into a timestamped `stdb-*` subdirectory. + +#### `scheduled-backup.interval` + +How often to create a backup. Values are strings of any format the [`humantime`] crate can parse, such as `"15m"`, `"1h"`, or `"1day"`. + +#### `scheduled-backup.keep-last` + +The number of complete scheduled backups to retain. Incomplete `stdb-*` directories left by failed, interrupted, or concurrent backups are ignored during pruning and do not count toward this limit. Omit `keep-last` to keep all complete scheduled backups. + ### `websocket` ```toml From a45ffeb0a1505388a9e0c4dc5ff6418294b073d5 Mon Sep 17 00:00:00 2001 From: kdletters <61648117+kdletters@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:27:23 +0800 Subject: [PATCH 5/8] Address final hot backup review feedback --- Cargo.lock | 1 + crates/cli/Cargo.toml | 1 + crates/cli/src/subcommands/backup.rs | 327 ++++++++++++++++-- crates/engine/src/relational_db.rs | 25 +- crates/engine/src/snapshot.rs | 95 ++++- crates/standalone/src/control_db.rs | 122 ++++++- crates/standalone/src/control_db/tests.rs | 53 +++ crates/standalone/src/lib.rs | 10 +- crates/standalone/src/subcommands/start.rs | 62 +++- .../00100-deploy/00200-self-hosting.md | 2 + .../00200-standalone-config.md | 4 +- 11 files changed, 630 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 548af752948..f2282e297c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8069,6 +8069,7 @@ dependencies = [ "serde_json", "serde_with", "slab", + "sled", "spacetimedb-auth", "spacetimedb-client-api-messages", "spacetimedb-codegen", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index bf951356a32..bf83b5d0fa8 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -64,6 +64,7 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value", "preserve_order", "arbitrary_precision"] } serde_with = { workspace = true, features = ["chrono_0_4"] } slab.workspace = true +sled.workspace = true syntect.workspace = true tabled.workspace = true tar.workspace = true diff --git a/crates/cli/src/subcommands/backup.rs b/crates/cli/src/subcommands/backup.rs index 51319a9e4f8..c368b4b0e88 100644 --- a/crates/cli/src/subcommands/backup.rs +++ b/crates/cli/src/subcommands/backup.rs @@ -7,10 +7,10 @@ use crate::subcommands::db_arg_resolution::{load_config_db_targets, resolve_data use crate::util::{add_auth_header_opt, database_identity, get_auth_header, ResponseExt}; use anyhow::Context; use clap::{Arg, ArgMatches, Command}; -use serde::{Deserialize, Serialize}; -use spacetimedb_commitlog::{payload::Txdata, Commitlog}; +use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize}; +use spacetimedb_commitlog::{commits, committed_meta}; use spacetimedb_fs_utils::{copy_dir_all, copy_file_sync, create_dir_all_sync, sync_dir}; -use spacetimedb_lib::{Identity, ProductValue}; +use spacetimedb_lib::{bsatn, de::Deserialize as BsatnDeserialize, ser::Serialize as BsatnSerialize, Identity}; use spacetimedb_paths::{ server::{CommitLogDir, ServerDataDir, SnapshotsPath}, FromPathUnchecked, SpacetimePaths, @@ -86,12 +86,12 @@ pub async fn exec(config: Config, paths: &SpacetimePaths, args: &ArgMatches) -> } } -#[derive(Serialize)] +#[derive(SerdeSerialize)] struct BackupRequest { server_output_dir: PathBuf, } -#[derive(Debug, Deserialize)] +#[derive(Debug, SerdeDeserialize)] struct BackupManifest { version: u32, database_identity: String, @@ -212,7 +212,8 @@ fn restore_backup_inner( create_dir_all_sync(parent)?; } - let staged_server_state = stage_missing_server_state(input_dir, data_dir, restore_temp_id)?; + let staged_server_state = stage_missing_server_state(input_dir, data_dir, restore_temp_id, &manifest)?; + validate_target_control_db(input_dir, data_dir, &manifest)?; anyhow::ensure!( !(force && replica_dir.0.exists() && !staged_server_state.is_empty()), "cannot restore over existing replica {} while target data-dir is missing server state; restore into an empty data-dir or complete the target server state first", @@ -378,15 +379,21 @@ fn validate_backup(input_dir: &Path, manifest: &BackupManifest) -> anyhow::Resul "backup clog directory contains no segment files: {}", clog_dir.display() ); - let clog = Commitlog::>::open(clog_dir, Default::default(), None) - .with_context(|| format!("opening backup commitlog {}", input_dir.join("clog").display()))?; + let commitlog_meta = committed_meta(clog_dir.clone()) + .with_context(|| format!("reading backup commitlog metadata {}", clog_dir.display()))?; + let max_committed_offset = commitlog_meta.and_then(|meta| meta.metadata().tx_range.end.checked_sub(1)); anyhow::ensure!( - clog.max_committed_offset() == Some(manifest.snapshot_offset), + max_committed_offset == Some(manifest.snapshot_offset), "backup commitlog max committed offset {:?} does not match manifest snapshot_offset {}", - clog.max_committed_offset(), + max_committed_offset, manifest.snapshot_offset ); - for commit in clog.commits() { + for commit in commits(clog_dir).with_context(|| { + format!( + "opening backup commitlog for traversal {}", + input_dir.join("clog").display() + ) + })? { commit.with_context(|| format!("reading backup commitlog {}", input_dir.join("clog").display()))?; } Ok(()) @@ -399,6 +406,166 @@ fn is_commitlog_segment_name(file_name: &str) -> bool { offset.len() == 20 && offset.bytes().all(|byte| byte.is_ascii_digit()) } +fn validate_scoped_backup_control_db( + control_db_dir: &Path, + manifest: &BackupManifest, + database_identity: Identity, +) -> anyhow::Result<()> { + if !control_db_dir.exists() { + return Ok(()); + } + validate_control_db_records( + control_db_dir, + database_identity, + manifest.replica_id, + ControlDbValidationScope::ScopedBackup, + ) + .with_context(|| format!("validating backup control-db {}", control_db_dir.display())) +} + +fn validate_target_control_db( + input_dir: &Path, + data_dir: &ServerDataDir, + manifest: &BackupManifest, +) -> anyhow::Result<()> { + let control_db_dir = data_dir.0.join("control-db"); + if !control_db_dir.exists() { + return Ok(()); + } + let database_identity: Identity = manifest + .database_identity + .parse() + .with_context(|| format!("parsing backup database identity {}", manifest.database_identity))?; + validate_control_db_records( + &control_db_dir, + database_identity, + manifest.replica_id, + ControlDbValidationScope::ExistingTarget, + ) + .with_context(|| { + format!( + "target data-dir already has {}; restore into an empty data-dir or a data-dir whose control-db contains the backed up database from {}", + control_db_dir.display(), + input_dir.display() + ) + }) +} + +#[derive(Debug, Clone, Copy)] +enum ControlDbValidationScope { + ScopedBackup, + ExistingTarget, +} + +#[allow(dead_code)] +#[derive(BsatnDeserialize, BsatnSerialize)] +struct ControlDbReplica { + id: u64, + database_id: u64, + node_id: u64, + leader: bool, +} + +fn validate_control_db_records( + control_db_dir: &Path, + database_identity: Identity, + replica_id: u64, + scope: ControlDbValidationScope, +) -> anyhow::Result<()> { + anyhow::ensure!( + control_db_dir.is_dir(), + "control-db path is not a directory: {}", + control_db_dir.display() + ); + let db = sled::Config::default() + .path(control_db_dir) + .flush_every_ms(Some(50)) + .mode(sled::Mode::HighThroughput) + .open() + .with_context(|| format!("opening control-db {}", control_db_dir.display()))?; + + let database_by_identity = open_existing_control_db_tree(&db, "database_by_identity")?; + let database_key = database_identity.to_be_byte_array(); + let Some(database_record) = database_by_identity + .get(database_key) + .with_context(|| format!("reading database identity {database_identity} from control-db"))? + else { + anyhow::bail!("control-db is missing database identity {database_identity}"); + }; + + let databases = open_existing_control_db_tree(&db, "database")?; + let mut database_count = 0usize; + let mut database_id = None; + for item in databases.iter() { + let (key, value) = item.with_context(|| format!("reading database records in {}", control_db_dir.display()))?; + database_count += 1; + if value == database_record { + database_id = Some(control_db_u64_key(&key, "database id")?); + } + } + let Some(database_id) = database_id else { + anyhow::bail!("control-db database table is missing database identity {database_identity}"); + }; + + let replicas = open_existing_control_db_tree(&db, "replica")?; + let replica_key = replica_id.to_be_bytes(); + let mut replica_count = 0usize; + let mut matching_replica = None; + for item in replicas.iter() { + let (key, value) = item.with_context(|| format!("reading replica records in {}", control_db_dir.display()))?; + replica_count += 1; + if key.as_ref() == replica_key.as_slice() { + let replica: ControlDbReplica = + bsatn::from_slice(&value).with_context(|| format!("decoding replica {replica_id} in control-db"))?; + matching_replica = Some(replica); + } + } + let Some(replica) = matching_replica else { + anyhow::bail!("control-db is missing replica {replica_id}"); + }; + anyhow::ensure!( + replica.id == replica_id, + "control-db replica record id {} does not match key {}", + replica.id, + replica_id + ); + anyhow::ensure!( + replica.database_id == database_id, + "control-db replica {} belongs to database {}, not {}", + replica_id, + replica.database_id, + database_id + ); + + if matches!(scope, ControlDbValidationScope::ScopedBackup) { + anyhow::ensure!( + database_count == 1, + "backup control-db must contain exactly one database record, found {database_count}" + ); + anyhow::ensure!( + replica_count == 1, + "backup control-db must contain exactly one replica record, found {replica_count}" + ); + } + Ok(()) +} + +fn control_db_u64_key(key: &[u8], label: &str) -> anyhow::Result { + let bytes: [u8; 8] = key + .try_into() + .with_context(|| format!("invalid control-db {label} key length {}", key.len()))?; + Ok(u64::from_be_bytes(bytes)) +} + +fn open_existing_control_db_tree(db: &sled::Db, tree_name: &str) -> anyhow::Result { + anyhow::ensure!( + db.tree_names().iter().any(|name| name.as_ref() == tree_name.as_bytes()), + "control-db is missing `{tree_name}` tree" + ); + db.open_tree(tree_name) + .with_context(|| format!("opening control-db `{tree_name}` tree")) +} + #[derive(Debug)] struct StagedServerStateEntry { final_path: PathBuf, @@ -497,6 +664,7 @@ fn stage_missing_server_state( input_dir: &Path, data_dir: &ServerDataDir, restore_temp_id: u64, + manifest: &BackupManifest, ) -> anyhow::Result { let server_dir = input_dir.join("server"); let needs_required_dirs = ["control-db", "program-bytes"] @@ -556,6 +724,17 @@ fn stage_missing_server_state( for (src, final_path, staged_path, is_dir) in entries { copy_staged_server_state_entry(&mut staged, &src, final_path, staged_path, is_dir)?; } + if let Some(entry) = staged + .entries + .iter() + .find(|entry| entry.final_path.ends_with("control-db")) + { + let database_identity: Identity = manifest + .database_identity + .parse() + .with_context(|| format!("parsing backup database identity {}", manifest.database_identity))?; + validate_scoped_backup_control_db(&entry.staged_path, manifest, database_identity)?; + } Ok(()) })(); if res.is_err() { @@ -603,6 +782,8 @@ fn staged_server_state_path(path: &Path, restore_temp_id: u64) -> PathBuf { #[cfg(test)] mod tests { use super::*; + use spacetimedb_commitlog::{payload::Txdata, Commitlog}; + use spacetimedb_lib::ProductValue; use spacetimedb_paths::FromPathUnchecked; use spacetimedb_table::{blob_store::HashMapBlobStore, table::Table}; @@ -612,7 +793,7 @@ mod tests { make_backup_dir(backup.path(), 7, 42)?; let data = tempfile::tempdir()?; - make_target_data_dir(data.path())?; + make_target_data_dir(data.path(), 7)?; let data_dir = ServerDataDir::from_path_unchecked(data.path()); let manifest = restore_backup(backup.path(), &data_dir, false)?; @@ -637,7 +818,7 @@ mod tests { make_backup_dir(backup.path(), 7, 42)?; let data = tempfile::tempdir()?; - make_target_data_dir(data.path())?; + make_target_data_dir(data.path(), 7)?; std::fs::create_dir_all(data.path().join("replicas/7"))?; let data_dir = ServerDataDir::from_path_unchecked(data.path()); @@ -672,7 +853,7 @@ mod tests { make_backup_dir(backup.path(), 7, 42)?; let data = tempfile::tempdir()?; - make_target_data_dir(data.path())?; + make_target_data_dir(data.path(), 7)?; let replica_dir = data.path().join("replicas/7"); std::fs::create_dir_all(&replica_dir)?; std::fs::write(replica_dir.join("old-marker"), b"old")?; @@ -701,7 +882,7 @@ mod tests { make_backup_dir(backup.path(), 7, 42)?; let data = tempfile::tempdir()?; - make_target_data_dir(data.path())?; + make_target_data_dir(data.path(), 7)?; let replica_dir = data.path().join("replicas/7"); std::fs::create_dir_all(&replica_dir)?; let old_marker = replica_dir.join("old-marker"); @@ -740,7 +921,7 @@ mod tests { restore_backup(backup.path(), &data_dir, false)?; - assert!(data.path().join("control-db/control").is_file()); + assert!(data.path().join("control-db").is_dir()); assert!(data.path().join("program-bytes/program").is_file()); assert_eq!(std::fs::read_to_string(data.path().join("config.toml"))?, "config"); assert_eq!(std::fs::read_to_string(data.path().join("metadata.toml"))?, "metadata"); @@ -832,7 +1013,7 @@ mod tests { std::fs::write(backup.path().join("clog/bad.stdb.log"), b"log")?; let data = tempfile::tempdir()?; - make_target_data_dir(data.path())?; + make_target_data_dir(data.path(), 7)?; let data_dir = ServerDataDir::from_path_unchecked(data.path()); let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); @@ -846,8 +1027,12 @@ mod tests { fn backup_restore_does_not_leave_partial_server_state_when_server_state_is_incomplete() -> anyhow::Result<()> { let backup = tempfile::tempdir()?; make_backup_dir(backup.path(), 7, 42)?; - std::fs::create_dir_all(backup.path().join("server/control-db"))?; - std::fs::write(backup.path().join("server/control-db/control"), b"control")?; + make_test_control_db( + &backup.path().join("server/control-db"), + test_database_identity()?, + 7, + false, + )?; let data = tempfile::tempdir()?; let data_dir = ServerDataDir::from_path_unchecked(data.path()); @@ -867,6 +1052,48 @@ mod tests { Ok(()) } + #[test] + fn backup_restore_rejects_unscoped_backup_control_db() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_test_control_db( + &backup.path().join("server/control-db"), + test_database_identity()?, + 7, + true, + )?; + std::fs::create_dir_all(backup.path().join("server/program-bytes"))?; + std::fs::write(backup.path().join("server/program-bytes/program"), b"program")?; + std::fs::write(backup.path().join("server/config.toml"), b"config")?; + std::fs::write(backup.path().join("server/metadata.toml"), b"metadata")?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("backup control-db must contain exactly one database record")); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_existing_control_db_without_database_metadata() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + std::fs::create_dir_all(data.path().join("control-db"))?; + std::fs::create_dir_all(data.path().join("program-bytes"))?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("target data-dir already has")); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + #[test] fn backup_restore_staged_server_state_cleanup_removes_current_entry_after_copy_error() -> anyhow::Result<()> { let data = tempfile::tempdir()?; @@ -909,15 +1136,14 @@ mod tests { Ok(()) } - fn make_target_data_dir(path: &Path) -> anyhow::Result<()> { - std::fs::create_dir_all(path.join("control-db"))?; + fn make_target_data_dir(path: &Path, replica_id: u64) -> anyhow::Result<()> { + make_test_control_db(&path.join("control-db"), test_database_identity()?, replica_id, false)?; std::fs::create_dir_all(path.join("program-bytes"))?; Ok(()) } fn make_backup_server_state(path: &Path) -> anyhow::Result<()> { - std::fs::create_dir_all(path.join("server/control-db"))?; - std::fs::write(path.join("server/control-db/control"), b"control")?; + make_test_control_db(&path.join("server/control-db"), test_database_identity()?, 7, false)?; std::fs::create_dir_all(path.join("server/program-bytes"))?; std::fs::write(path.join("server/program-bytes/program"), b"program")?; std::fs::write(path.join("server/config.toml"), b"config")?; @@ -925,6 +1151,59 @@ mod tests { Ok(()) } + fn make_test_control_db( + path: &Path, + database_identity: Identity, + replica_id: u64, + include_extra_database: bool, + ) -> anyhow::Result<()> { + std::fs::create_dir_all(path)?; + let db = sled::Config::default() + .path(path) + .flush_every_ms(Some(50)) + .mode(sled::Mode::HighThroughput) + .open()?; + let database_record = sled::IVec::from(vec![1, 2, 3]); + db.open_tree("database_by_identity")? + .insert(database_identity.to_be_byte_array(), database_record.clone())?; + db.open_tree("database")? + .insert(1u64.to_be_bytes(), database_record.clone())?; + db.open_tree("replica")?.insert( + replica_id.to_be_bytes(), + bsatn::to_vec(&ControlDbReplica { + id: replica_id, + database_id: 1, + node_id: 0, + leader: true, + })?, + )?; + if include_extra_database { + let extra_identity: Identity = + "c300000000000000000000000000000000000000000000000000000000000000".parse()?; + let extra_database_record = sled::IVec::from(vec![7, 8, 9]); + db.open_tree("database_by_identity")? + .insert(extra_identity.to_be_byte_array(), extra_database_record.clone())?; + db.open_tree("database")? + .insert(2u64.to_be_bytes(), extra_database_record)?; + db.open_tree("replica")?.insert( + (replica_id + 1).to_be_bytes(), + bsatn::to_vec(&ControlDbReplica { + id: replica_id + 1, + database_id: 2, + node_id: 0, + leader: true, + })?, + )?; + } + db.flush()?; + drop(db); + Ok(()) + } + + fn test_database_identity() -> anyhow::Result { + Ok("c200000000000000000000000000000000000000000000000000000000000000".parse()?) + } + #[cfg(windows)] #[allow(clippy::permissions_set_readonly_false)] fn clear_readonly_recursively(path: &Path) -> anyhow::Result<()> { @@ -943,7 +1222,7 @@ mod tests { } fn make_backup_dir(path: &Path, replica_id: u64, offset: u64) -> anyhow::Result<()> { - let database_identity: Identity = "c200000000000000000000000000000000000000000000000000000000000000".parse()?; + let database_identity: Identity = test_database_identity()?; let snapshots_path = SnapshotsPath::from_path_unchecked(path.join("snapshots")); std::fs::create_dir_all(&snapshots_path.0)?; diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index 775bdd3e8ee..5f5a88f814e 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -1094,7 +1094,13 @@ impl RelationalDB { if let Some(server_data_dir) = server_data_dir { copy_server_state(&runtime, server_data_dir, &output_dir, copy_control_db).await?; } - copy_commitlog_range(replica_dir.commit_log(), output_dir.join("clog"), snapshot_offset).await?; + copy_commitlog_range( + &runtime, + replica_dir.commit_log(), + output_dir.join("clog"), + snapshot_offset, + ) + .await?; let copy_ms = HotBackupManifest::elapsed_ms(copy_start.elapsed()); let mut manifest = HotBackupManifest { @@ -2028,9 +2034,14 @@ pub fn open_snapshot_repo( .map_err(Box::new) } -async fn copy_commitlog_range(src: CommitLogDir, dst: PathBuf, through: TxOffset) -> anyhow::Result<()> { +async fn copy_commitlog_range( + runtime: &Handle, + src: CommitLogDir, + dst: PathBuf, + through: TxOffset, +) -> anyhow::Result<()> { let src_repo = commitlog::repo::Fs::new(src, None)?; - let dst_repo = commitlog::repo::Fs::new(CommitLogDir::from_path_unchecked(dst), None)?; + let dst_repo = commitlog::repo::Fs::new(CommitLogDir::from_path_unchecked(&dst), None)?; let reader = tokio::io::BufReader::new(tokio_util::io::StreamReader::new(commitlog::stream::commits( src_repo, ..=through, @@ -2040,6 +2051,14 @@ async fn copy_commitlog_range(src: CommitLogDir, dst: PathBuf, through: TxOffset commitlog::stream::StreamWriter::create(dst_repo, <_>::default(), commitlog::stream::OnTrailingData::Error)?; let mut writer = writer.append_all(reader, |_| ()).await?; writer.sync_all().await?; + asyncify(runtime, move || -> anyhow::Result<()> { + sync_dir(&dst).with_context(|| format!("syncing backup commitlog dir {}", dst.display()))?; + if let Some(parent) = dst.parent() { + sync_dir(parent).with_context(|| format!("syncing backup commitlog parent {}", parent.display()))?; + } + Ok(()) + }) + .await?; Ok(()) } diff --git a/crates/engine/src/snapshot.rs b/crates/engine/src/snapshot.rs index 281bcf3cc1f..9943917f5c0 100644 --- a/crates/engine/src/snapshot.rs +++ b/crates/engine/src/snapshot.rs @@ -7,7 +7,10 @@ use std::{ }; use anyhow::Context as _; -use futures::{channel::mpsc, StreamExt as _}; +use futures::{ + channel::{mpsc, oneshot}, + StreamExt as _, +}; use log::{info, warn}; use parking_lot::RwLock; use prometheus::{Histogram, IntGauge}; @@ -127,7 +130,10 @@ impl SnapshotWorker { /// which is likely due to it having panicked. pub fn request_snapshot(&self) { self.request_snapshot - .unbounded_send(Request::TakeSnapshot { min_offset: None }) + .unbounded_send(Request::TakeSnapshot { + min_offset: None, + result: None, + }) .expect("snapshot worker panicked"); } @@ -136,9 +142,10 @@ impl SnapshotWorker { /// Used by the durability to request snapshots on commitlog segment rotation, /// since the durability should continue writing queued TXes even if the snapshot worker panics. pub fn request_snapshot_ignore_closed(&self) { - let _ = self - .request_snapshot - .unbounded_send(Request::TakeSnapshot { min_offset: None }); + let _ = self.request_snapshot.unbounded_send(Request::TakeSnapshot { + min_offset: None, + result: None, + }); } /// Subscribe to the [TxOffset]s of snapshots created by this worker. @@ -159,11 +166,19 @@ impl SnapshotWorker { } let mut snapshot_created = self.subscribe(); + let (result_tx, result_rx) = oneshot::channel(); self.request_snapshot .unbounded_send(Request::TakeSnapshot { min_offset: Some(offset), + result: Some(result_tx), }) .expect("snapshot worker panicked"); + let snapshot_offset = result_rx + .await + .context("snapshot worker closed before creating requested snapshot")??; + if snapshot_offset >= offset { + return Ok(snapshot_offset); + } loop { snapshot_created .changed() @@ -196,7 +211,10 @@ impl SnapshotMetrics { type WeakDatabaseState = Weak>; enum Request { - TakeSnapshot { min_offset: Option }, + TakeSnapshot { + min_offset: Option, + result: Option>>, + }, ReplaceState(SnapshotDatabaseState), } @@ -209,6 +227,15 @@ struct SnapshotWorkerActor { compression: Option, } +fn send_snapshot_request_result( + result: Option>>, + value: anyhow::Result, +) { + if let Some(result) = result { + let _ = result.send(value); + } +} + impl SnapshotWorkerActor { /// Read messages from `snapshot_requests` indefinitely. /// @@ -229,25 +256,30 @@ impl SnapshotWorkerActor { let mut database_state: Option = None; while let Some(req) = self.snapshot_requests.next().await { match req { - Request::TakeSnapshot { min_offset } => { + Request::TakeSnapshot { min_offset, result } => { if let Some(min_offset) = min_offset && let Ok(Some(latest)) = self.snapshot_repo.latest_snapshot() && latest >= min_offset { self.snapshot_created.send_replace(latest); + send_snapshot_request_result(result, Ok(latest)); continue; } - let res = self - .maybe_take_snapshot(database_state.as_ref()) - .await - .inspect_err(|e| warn!("database={database_identity} SnapshotWorker: {e:#}")); - if let Ok(snapshot_offset) = res { - self.maybe_compress_snapshots(snapshot_offset).await; - // INVARIANT: only signal `snapshot_created` after the snapshot - // is fully durable on disk (`take_snapshot` returns post-fsync). - // Hot backup relies on this to copy the snapshot directory as - // soon as it observes the offset. - self.snapshot_created.send_replace(snapshot_offset); + let res = self.maybe_take_snapshot(database_state.as_ref()).await; + match res { + Ok(snapshot_offset) => { + self.maybe_compress_snapshots(snapshot_offset).await; + // INVARIANT: only signal `snapshot_created` after the snapshot + // is fully durable on disk (`take_snapshot` returns post-fsync). + // Hot backup relies on this to copy the snapshot directory as + // soon as it observes the offset. + self.snapshot_created.send_replace(snapshot_offset); + send_snapshot_request_result(result, Ok(snapshot_offset)); + } + Err(err) => { + warn!("database={database_identity} SnapshotWorker: {err:#}"); + send_snapshot_request_result(result, Err(err)); + } } } Request::ReplaceState(new_state) => { @@ -424,3 +456,30 @@ impl Compressor { } } } + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb_paths::{server::SnapshotsPath, FromPathUnchecked}; + use spacetimedb_snapshot::SnapshotRepository; + + #[test] + fn ensure_snapshot_at_least_returns_snapshot_errors() -> anyhow::Result<()> { + let rt = tokio::runtime::Runtime::new()?; + let temp = tempfile::tempdir()?; + std::fs::create_dir_all(temp.path().join("snapshots"))?; + let repo: Arc = Arc::new(SnapshotRepository::open( + SnapshotsPath::from_path_unchecked(temp.path().join("snapshots")), + Identity::ZERO, + 0, + )?); + let worker = SnapshotWorker::new(repo, Compression::Disabled, Handle::tokio(rt.handle().clone())); + + let err = rt + .block_on(async { tokio::time::timeout(Duration::from_secs(5), worker.ensure_snapshot_at_least(1)).await })? + .unwrap_err(); + + assert!(format!("{err:#}").contains("database state not set")); + Ok(()) + } +} diff --git a/crates/standalone/src/control_db.rs b/crates/standalone/src/control_db.rs index 615df8d8737..2205e5a9445 100644 --- a/crates/standalone/src/control_db.rs +++ b/crates/standalone/src/control_db.rs @@ -77,19 +77,7 @@ impl ControlDb { } pub fn export_to_path(&self, path: &ControlDbDir) -> Result<()> { - if path.is_dir() { - if path - .read_dir() - .with_context(|| format!("reading control db export dir {}", path.display()))? - .next() - .is_some() - { - return Err(anyhow::anyhow!("control db export directory must be empty: {}", path.display()).into()); - } - } else { - path.create() - .with_context(|| format!("creating control db export dir {}", path.display()))?; - } + ensure_empty_control_db_export_dir(path)?; // Sled owns live files on Windows; export/import avoids copying locked files. self.db.flush()?; @@ -100,8 +88,80 @@ impl ControlDb { .open()?; dst.import(self.db.export()); dst.flush()?; - sync_dir(path).with_context(|| format!("syncing control db export dir {}", path.display()))?; - if let Some(parent) = path.parent() { + sync_dir(path.as_ref()).with_context(|| format!("syncing control db export dir {}", path.display()))?; + if let Some(parent) = path.as_ref().parent() { + sync_dir(parent).with_context(|| format!("syncing control db export parent {}", parent.display()))?; + } + Ok(()) + } + + pub fn export_database_to_path( + &self, + path: &ControlDbDir, + database_identity: &Identity, + replica_id: u64, + ) -> Result<()> { + ensure_empty_control_db_export_dir(path)?; + let database = self + .get_database_by_identity(database_identity)? + .with_context(|| format!("database {database_identity} not found in control-db"))?; + let replica = self + .get_replica_by_id(replica_id)? + .with_context(|| format!("replica {replica_id} not found in control-db"))?; + anyhow::ensure!( + replica.database_id == database.id, + "replica {} belongs to database {}, not {}", + replica.id, + replica.database_id, + database.id + ); + + self.db.flush()?; + let dst = sled::Config::default() + .path(path) + .flush_every_ms(Some(50)) + .mode(sled::Mode::HighThroughput) + .open()?; + + let database_buf = sled::IVec::from(compat::Database::from(database.clone()).to_vec()?); + let database_by_identity = dst.open_tree("database_by_identity")?; + database_by_identity.insert(database_identity.to_be_byte_array(), database_buf.clone())?; + database_by_identity.flush()?; + + let databases = dst.open_tree("database")?; + databases.insert(database.id.to_be_bytes(), database_buf)?; + databases.flush()?; + + let replicas = dst.open_tree("replica")?; + replicas.insert(replica.id.to_be_bytes(), bsatn::to_vec(&replica)?)?; + replicas.flush()?; + + copy_optional_control_db_tree_entry(&self.db, &dst, "database_locks", &database_identity.to_be_byte_array())?; + copy_optional_control_db_tree_entry( + &self.db, + &dst, + "energy_budget", + &database.owner_identity.to_byte_array(), + )?; + + let domains = self.spacetime_reverse_dns(database_identity)?; + if !domains.is_empty() { + let identity_bytes = database_identity.to_byte_array(); + copy_optional_control_db_tree_entry(&self.db, &dst, "reverse_dns", &identity_bytes)?; + for domain in domains { + copy_optional_control_db_tree_entry(&self.db, &dst, "dns", domain.to_lowercase().as_bytes())?; + copy_optional_control_db_tree_entry( + &self.db, + &dst, + "top_level_domains", + domain.tld().to_string().to_lowercase().as_bytes(), + )?; + } + } + + dst.flush()?; + sync_dir(path.as_ref()).with_context(|| format!("syncing control db export dir {}", path.display()))?; + if let Some(parent) = path.as_ref().parent() { sync_dir(parent).with_context(|| format!("syncing control db export parent {}", parent.display()))?; } Ok(()) @@ -118,6 +178,38 @@ impl ControlDb { } } +fn ensure_empty_control_db_export_dir(path: &ControlDbDir) -> Result<()> { + if path.is_dir() { + if path + .read_dir() + .with_context(|| format!("reading control db export dir {}", path.display()))? + .next() + .is_some() + { + return Err(anyhow::anyhow!("control db export directory must be empty: {}", path.display()).into()); + } + } else { + path.create() + .with_context(|| format!("creating control db export dir {}", path.display()))?; + } + Ok(()) +} + +fn copy_optional_control_db_tree_entry( + src: &sled::Db, + dst: &sled::Db, + tree_name: &str, + key: impl AsRef<[u8]>, +) -> Result<()> { + let src_tree = src.open_tree(tree_name)?; + if let Some(value) = src_tree.get(key.as_ref())? { + let dst_tree = dst.open_tree(tree_name)?; + dst_tree.insert(key.as_ref(), value)?; + dst_tree.flush()?; + } + Ok(()) +} + /// A helper to convert a `sled::IVec` into an `Identity`. /// This expects the identity to be in LITTLE_ENDIAN format. /// This fails if the `sled::IVec` is not 32 bytes long. diff --git a/crates/standalone/src/control_db/tests.rs b/crates/standalone/src/control_db/tests.rs index 58ebe04cfe4..f3ddbfeea9e 100644 --- a/crates/standalone/src/control_db/tests.rs +++ b/crates/standalone/src/control_db/tests.rs @@ -181,3 +181,56 @@ fn test_export_to_path_can_be_opened() -> anyhow::Result<()> { assert!(exported.get_leader_replica_by_database(database_id).is_some()); Ok(()) } + +#[test] +fn test_export_database_to_path_is_scoped_to_one_database() -> anyhow::Result<()> { + let src = TempDir::with_prefix("control-db-export-scoped-src")?; + let dst = TempDir::with_prefix("control-db-export-scoped-dst")?; + let cdb = ControlDb::at(src.path())?; + let first = Database { + id: 0, + database_identity: Identity::ZERO, + owner_identity: *ALICE, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + }; + let first_database_id = cdb.insert_database(first)?; + let first_replica_id = cdb.insert_replica(Replica { + id: 0, + database_id: first_database_id, + node_id: 0, + leader: true, + })?; + let second_identity = Identity::from_claims(LOCALHOST, "second"); + let second_database_id = cdb.insert_database(Database { + id: 0, + database_identity: second_identity, + owner_identity: *BOB, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + })?; + cdb.insert_replica(Replica { + id: 0, + database_id: second_database_id, + node_id: 0, + leader: true, + })?; + cdb.set_database_lock(&Identity::ZERO, true)?; + cdb.set_database_lock(&second_identity, true)?; + + cdb.export_database_to_path( + &ControlDbDir::from_path_unchecked(dst.path()), + &Identity::ZERO, + first_replica_id, + )?; + let exported = ControlDb::at(dst.path())?; + + assert!(exported.get_database_by_identity(&Identity::ZERO)?.is_some()); + assert!(exported.get_database_by_identity(&second_identity)?.is_none()); + assert_eq!(exported.get_databases()?.len(), 1); + assert!(exported.get_replica_by_id(first_replica_id)?.is_some()); + assert_eq!(exported.get_replicas()?.len(), 1); + assert!(exported.is_database_locked(&Identity::ZERO)?); + assert!(!exported.is_database_locked(&second_identity)?); + Ok(()) +} diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index 8ffc5f578cb..6eedb518bba 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -213,12 +213,16 @@ impl NodeDelegate for StandaloneEnv { let export_start = Instant::now(); let control_db = self.control_db.clone(); + let database_identity = manifest.database_identity; + let replica_id = manifest.replica_id; // sled export and the dir-size sweep are blocking; keep them off the async executor. manifest.bytes = spacetimedb::util::asyncify(move || -> anyhow::Result { control_db - .export_to_path(&ControlDbDir::from_path_unchecked( - output_dir.join("server").join("control-db"), - )) + .export_database_to_path( + &ControlDbDir::from_path_unchecked(output_dir.join("server").join("control-db")), + &database_identity, + replica_id, + ) .context("exporting control-db into backup")?; dir_size(&output_dir).context("measuring backup size") }) diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index 85a4df599bc..c6f42d8a0a0 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -5,7 +5,7 @@ use spacetimedb_client_api::{ControlStateReadAccess, NodeDelegate}; use spacetimedb_pg::pg_server; use std::io::{self, Write}; use std::net::IpAddr; -use std::path::{Component, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -23,6 +23,7 @@ use spacetimedb::worker_metrics; use spacetimedb_client_api::routes::database::DatabaseRoutes; use spacetimedb_client_api::routes::router; use spacetimedb_client_api::routes::subscribe::WebSocketOptions; +use spacetimedb_fs_utils::sync_dir; use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; use spacetimedb_paths::server::{ConfigToml, ServerDataDir}; use tokio::net::TcpListener; @@ -190,9 +191,8 @@ fn validate_scheduled_backup_config(config: &ScheduledBackupConfig) -> anyhow::R /// A backup is complete iff it contains `manifest.json`, which is written as /// the final step of a backup. `stdb-*` directories without a manifest are /// ignored by pruning and never occupy a `keep-last` slot, so they cannot -/// crowd out good backups. They are not deleted here because a concurrent -/// manual backup may still be writing into a `stdb-*` directory in the same -/// output root. +/// crowd out good backups. They are not deleted here because another backup may +/// still be writing into a `stdb-*` directory in the same output root. fn prune_scheduled_backups(config: &ScheduledBackupConfig) -> anyhow::Result<()> { let mut complete = Vec::new(); for entry in std::fs::read_dir(&config.output_dir) @@ -223,6 +223,17 @@ fn prune_scheduled_backups(config: &ScheduledBackupConfig) -> anyhow::Result<()> Ok(()) } +fn cleanup_failed_scheduled_backup_output(output_dir: &Path) -> anyhow::Result<()> { + if output_dir.exists() && !output_dir.join("manifest.json").is_file() { + std::fs::remove_dir_all(output_dir) + .with_context(|| format!("removing incomplete scheduled backup {}", output_dir.display()))?; + if let Some(parent) = output_dir.parent() { + sync_dir(parent).with_context(|| format!("syncing scheduled-backup output-dir {}", parent.display()))?; + } + } + Ok(()) +} + async fn run_scheduled_backup(ctx: &Arc, config: &ScheduledBackupConfig) -> anyhow::Result<()> { validate_scheduled_backup_config(config)?; let database_identity = config @@ -235,14 +246,27 @@ async fn run_scheduled_backup(ctx: &Arc, config: &ScheduledBackup .await? .with_context(|| format!("scheduled-backup database `{}` not found", config.database))?; let output_dir = config.output_dir.join(backup_dir_name()); + let cleanup_output_dir = output_dir.clone(); let leader = ctx.leader(database.id).await?; let backup_result = ctx.create_hot_backup(leader, output_dir).await; - // Prune even if this run failed, so directories left behind by failed - // backups are cleaned up instead of accumulating; pruning is blocking - // filesystem work, so keep it off the async executor. + // Remove only this scheduled run's incomplete output on failure. Pruning + // still ignores other manifest-less `stdb-*` dirs because they may belong + // to a concurrent manual or HTTP-triggered backup. + let cleanup_result = if backup_result.is_err() { + Some(spacetimedb::util::asyncify(move || cleanup_failed_scheduled_backup_output(&cleanup_output_dir)).await) + } else { + None + }; + if let Some(Err(err)) = &cleanup_result { + log::warn!("failed to remove incomplete scheduled backup: {err:#}"); + } + // Pruning is blocking filesystem work, so keep it off the async executor. let prune_config = config.clone(); let prune_result = spacetimedb::util::asyncify(move || prune_scheduled_backups(&prune_config)).await; backup_result?; + if let Some(cleanup_result) = cleanup_result { + cleanup_result?; + } prune_result } @@ -706,6 +730,30 @@ mod tests { assert!(unrelated.exists()); } + #[test] + fn cleanup_failed_scheduled_backup_output_removes_incomplete_dir() { + let temp = tempfile::tempdir().unwrap(); + let incomplete = temp.path().join("stdb-incomplete"); + std::fs::create_dir(&incomplete).unwrap(); + std::fs::write(incomplete.join("partial"), b"not done").unwrap(); + + cleanup_failed_scheduled_backup_output(&incomplete).unwrap(); + + assert!(!incomplete.exists()); + } + + #[test] + fn cleanup_failed_scheduled_backup_output_preserves_complete_dir() { + let temp = tempfile::tempdir().unwrap(); + let complete = temp.path().join("stdb-complete"); + std::fs::create_dir(&complete).unwrap(); + std::fs::write(complete.join("manifest.json"), b"{}").unwrap(); + + cleanup_failed_scheduled_backup_output(&complete).unwrap(); + + assert!(complete.exists()); + } + #[test] fn scheduled_backup_config_rejects_parent_components() { let mut path = std::env::temp_dir(); diff --git a/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md b/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md index 9aed48c350e..0454ebb7681 100644 --- a/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md +++ b/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md @@ -278,6 +278,8 @@ sudo -u spacetimedb /stdb/spacetime --root-dir=/stdb backup restore \ sudo systemctl start spacetimedb ``` +Restore into an empty data directory when you want the backup's saved server state to be used. Restoring into an existing data directory requires that its `control-db` already contains the backed up database identity and replica id; otherwise the CLI rejects the restore instead of creating an orphaned replica. + Keep the backup directory private. Backups contain the database snapshot, commitlog, and any server state needed to reopen the restored database. ## Step 7: Updating SpacetimeDB Version diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md index cdd820d0234..b71f2f5b5e2 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md @@ -116,7 +116,7 @@ The `hot-backup` section configures server-side backups created through `spaceti #### `hot-backup.root-dir` -An absolute server path that acts as the root directory for CLI-triggered hot backups. Backup creation requests choose an output directory relative to this root. Omit `root-dir` to disable CLI-triggered hot backups. +An absolute server path that acts as the root directory for CLI/HTTP-triggered hot backups. Backup creation requests choose an output directory relative to this root. Omit `root-dir` to disable CLI/HTTP-triggered hot backups. ### `scheduled-backup` @@ -144,7 +144,7 @@ How often to create a backup. Values are strings of any format the [`humantime`] #### `scheduled-backup.keep-last` -The number of complete scheduled backups to retain. Incomplete `stdb-*` directories left by failed, interrupted, or concurrent backups are ignored during pruning and do not count toward this limit. Omit `keep-last` to keep all complete scheduled backups. +The number of complete scheduled backups to retain. Incomplete `stdb-*` directories are ignored during pruning and do not count toward this limit. A failed scheduled run removes its own incomplete output directory; other manifest-less directories are preserved because they may belong to a concurrent manual or HTTP-triggered backup. Omit `keep-last` to keep all complete scheduled backups. ### `websocket` From 89376e74bfd93cd97cb98a3331c627c43f426228 Mon Sep 17 00:00:00 2001 From: kdletters <61648117+kdletters@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:39:23 +0800 Subject: [PATCH 6/8] Address hot backup restore review blockers --- crates/cli/src/subcommands/backup.rs | 240 +++++++++++++++++++-- crates/standalone/src/control_db.rs | 108 +++++++++- crates/standalone/src/control_db/tests.rs | 49 +++++ crates/standalone/src/subcommands/start.rs | 21 ++ 4 files changed, 402 insertions(+), 16 deletions(-) diff --git a/crates/cli/src/subcommands/backup.rs b/crates/cli/src/subcommands/backup.rs index c368b4b0e88..8cb66f36e5f 100644 --- a/crates/cli/src/subcommands/backup.rs +++ b/crates/cli/src/subcommands/backup.rs @@ -10,7 +10,7 @@ use clap::{Arg, ArgMatches, Command}; use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize}; use spacetimedb_commitlog::{commits, committed_meta}; use spacetimedb_fs_utils::{copy_dir_all, copy_file_sync, create_dir_all_sync, sync_dir}; -use spacetimedb_lib::{bsatn, de::Deserialize as BsatnDeserialize, ser::Serialize as BsatnSerialize, Identity}; +use spacetimedb_lib::{bsatn, de::Deserialize as BsatnDeserialize, ser::Serialize as BsatnSerialize, Hash, Identity}; use spacetimedb_paths::{ server::{CommitLogDir, ServerDataDir, SnapshotsPath}, FromPathUnchecked, SpacetimePaths, @@ -222,6 +222,7 @@ fn restore_backup_inner( let mut old_tmp_moved = false; let mut restore_old_tmp_on_error = false; + let mut restored_replica_moved = false; let mut committed_server_state = None; let res = (|| -> anyhow::Result<()> { create_dir_all_sync(&tmp_dir)?; @@ -235,12 +236,13 @@ fn restore_backup_inner( if force && replica_dir.0.exists() { std::fs::rename(&replica_dir.0, &old_tmp_dir) .with_context(|| format!("moving existing target replica directory {}", replica_dir.display()))?; - sync_parent_dir(&replica_dir.0)?; old_tmp_moved = true; restore_old_tmp_on_error = true; + sync_parent_dir(&replica_dir.0)?; } std::fs::rename(&tmp_dir, &replica_dir.0) .with_context(|| format!("moving restored replica into {}", replica_dir.display()))?; + restored_replica_moved = true; sync_parent_dir(&replica_dir.0)?; restore_old_tmp_on_error = false; @@ -265,6 +267,10 @@ fn restore_backup_inner( if res.is_err() { let _ = std::fs::remove_dir_all(&tmp_dir); let _ = sync_parent_dir(&tmp_dir); + if restored_replica_moved && replica_dir.0.exists() { + let _ = std::fs::remove_dir_all(&replica_dir.0); + let _ = sync_parent_dir(&replica_dir.0); + } if restore_old_tmp_on_error && old_tmp_moved && old_tmp_dir.exists() && !replica_dir.0.exists() { let _ = std::fs::rename(&old_tmp_dir, &replica_dir.0); let _ = sync_parent_dir(&replica_dir.0); @@ -277,11 +283,30 @@ fn restore_backup_inner( fn sync_parent_dir(path: &Path) -> anyhow::Result<()> { if let Some(parent) = path.parent().filter(|parent| !parent.as_os_str().is_empty()) { + #[cfg(test)] + maybe_fail_sync_parent_dir(parent)?; sync_dir(parent).with_context(|| format!("syncing parent directory {}", parent.display()))?; } Ok(()) } +#[cfg(test)] +fn maybe_fail_sync_parent_dir(parent: &Path) -> anyhow::Result<()> { + let marker = parent.join(".fail-next-restore-sync-parent"); + if !marker.exists() { + return Ok(()); + } + let remaining = std::fs::read_to_string(&marker) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if remaining == 0 { + anyhow::bail!("injected sync_parent_dir failure for {}", parent.display()); + } + std::fs::write(marker, (remaining - 1).to_string())?; + Ok(()) +} + fn read_backup_manifest(input_dir: &Path) -> anyhow::Result { let manifest_path = input_dir.join("manifest.json"); let bytes = std::fs::read(&manifest_path) @@ -457,6 +482,24 @@ enum ControlDbValidationScope { ExistingTarget, } +#[allow(dead_code)] +#[derive(BsatnDeserialize, BsatnSerialize)] +struct ControlDbDatabase { + id: u64, + database_identity: Identity, + owner_identity: Identity, + host_type: ControlDbHostType, + initial_program: Hash, +} + +#[allow(dead_code)] +#[derive(BsatnDeserialize, BsatnSerialize)] +#[repr(i32)] +enum ControlDbHostType { + Wasm = 0, + Js = 1, +} + #[allow(dead_code)] #[derive(BsatnDeserialize, BsatnSerialize)] struct ControlDbReplica { @@ -492,20 +535,61 @@ fn validate_control_db_records( else { anyhow::bail!("control-db is missing database identity {database_identity}"); }; + let database_from_identity: ControlDbDatabase = bsatn::from_slice(&database_record) + .with_context(|| format!("decoding database identity {database_identity} in control-db"))?; + anyhow::ensure!( + database_from_identity.database_identity == database_identity, + "control-db database_by_identity record identity {} does not match key {}", + database_from_identity.database_identity, + database_identity + ); let databases = open_existing_control_db_tree(&db, "database")?; let mut database_count = 0usize; - let mut database_id = None; for item in databases.iter() { let (key, value) = item.with_context(|| format!("reading database records in {}", control_db_dir.display()))?; database_count += 1; - if value == database_record { - database_id = Some(control_db_u64_key(&key, "database id")?); - } + let key_id = control_db_u64_key(&key, "database id")?; + let database: ControlDbDatabase = + bsatn::from_slice(&value).with_context(|| format!("decoding database {key_id} in control-db"))?; + anyhow::ensure!( + database.id == key_id, + "control-db database record id {} does not match key {}", + database.id, + key_id + ); } - let Some(database_id) = database_id else { + let database_id = database_from_identity.id; + let Some(database_record_by_id) = databases + .get(database_id.to_be_bytes()) + .with_context(|| format!("reading database {database_id} from control-db"))? + else { anyhow::bail!("control-db database table is missing database identity {database_identity}"); }; + anyhow::ensure!( + database_record_by_id == database_record, + "control-db database_by_identity record for {database_identity} does not match database table record {database_id}" + ); + + let mut database_by_identity_count = 0usize; + for item in database_by_identity.iter() { + let (key, value) = + item.with_context(|| format!("reading database_by_identity records in {}", control_db_dir.display()))?; + database_by_identity_count += 1; + let key_bytes: [u8; 32] = key + .as_ref() + .try_into() + .with_context(|| format!("invalid control-db database identity key length {}", key.len()))?; + let key_identity = Identity::from_be_byte_array(key_bytes); + let database: ControlDbDatabase = bsatn::from_slice(&value) + .with_context(|| format!("decoding database identity {key_identity} in control-db"))?; + anyhow::ensure!( + database.database_identity == key_identity, + "control-db database_by_identity record identity {} does not match key {}", + database.database_identity, + key_identity + ); + } let replicas = open_existing_control_db_tree(&db, "replica")?; let replica_key = replica_id.to_be_bytes(); @@ -514,9 +598,16 @@ fn validate_control_db_records( for item in replicas.iter() { let (key, value) = item.with_context(|| format!("reading replica records in {}", control_db_dir.display()))?; replica_count += 1; + let key_id = control_db_u64_key(&key, "replica id")?; + let replica: ControlDbReplica = + bsatn::from_slice(&value).with_context(|| format!("decoding replica {key_id} in control-db"))?; + anyhow::ensure!( + replica.id == key_id, + "control-db replica record id {} does not match key {}", + replica.id, + key_id + ); if key.as_ref() == replica_key.as_slice() { - let replica: ControlDbReplica = - bsatn::from_slice(&value).with_context(|| format!("decoding replica {replica_id} in control-db"))?; matching_replica = Some(replica); } } @@ -538,6 +629,10 @@ fn validate_control_db_records( ); if matches!(scope, ControlDbValidationScope::ScopedBackup) { + anyhow::ensure!( + database_by_identity_count == 1, + "backup control-db must contain exactly one database_by_identity record, found {database_by_identity_count}" + ); anyhow::ensure!( database_count == 1, "backup control-db must contain exactly one database record, found {database_count}" @@ -875,6 +970,50 @@ mod tests { Ok(()) } + #[test] + fn backup_restore_force_restores_old_replica_when_sync_after_old_rename_fails() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + std::fs::write(data.path().join("replicas/.fail-next-restore-sync-parent"), b"1")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + + assert!(err.to_string().contains("injected sync_parent_dir failure")); + assert_eq!(std::fs::read(replica_dir.join("old-marker"))?, b"old"); + assert!(!replica_dir.join("snapshots").exists()); + assert_no_restore_temps(data.path())?; + Ok(()) + } + + #[test] + fn backup_restore_force_restores_old_replica_when_sync_after_new_rename_fails() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + std::fs::write(data.path().join("replicas/.fail-next-restore-sync-parent"), b"2")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + + assert!(err.to_string().contains("injected sync_parent_dir failure")); + assert_eq!(std::fs::read(replica_dir.join("old-marker"))?, b"old"); + assert!(!replica_dir.join("snapshots").exists()); + assert_no_restore_temps(data.path())?; + Ok(()) + } + #[cfg(windows)] #[test] fn backup_restore_force_commits_when_old_replica_cleanup_fails() -> anyhow::Result<()> { @@ -1072,7 +1211,55 @@ mod tests { let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); - assert!(format!("{err:#}").contains("backup control-db must contain exactly one database record")); + assert!(format!("{err:#}").contains("backup control-db must contain exactly one database")); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_control_db_database_record_identity_mismatch() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + let manifest_identity = test_database_identity()?; + let other_identity: Identity = "c300000000000000000000000000000000000000000000000000000000000000".parse()?; + std::fs::create_dir_all(backup.path().join("server/control-db"))?; + let db = sled::Config::default() + .path(backup.path().join("server/control-db")) + .flush_every_ms(Some(50)) + .mode(sled::Mode::HighThroughput) + .open()?; + let database_record = sled::IVec::from(bsatn::to_vec(&ControlDbDatabase { + id: 1, + database_identity: other_identity, + owner_identity: Identity::ZERO, + host_type: ControlDbHostType::Wasm, + initial_program: Hash::ZERO, + })?); + db.open_tree("database_by_identity")? + .insert(manifest_identity.to_be_byte_array(), database_record.clone())?; + db.open_tree("database")?.insert(1u64.to_be_bytes(), database_record)?; + db.open_tree("replica")?.insert( + 7u64.to_be_bytes(), + bsatn::to_vec(&ControlDbReplica { + id: 7, + database_id: 1, + node_id: 0, + leader: true, + })?, + )?; + db.flush()?; + drop(db); + std::fs::create_dir_all(backup.path().join("server/program-bytes"))?; + std::fs::write(backup.path().join("server/program-bytes/program"), b"program")?; + std::fs::write(backup.path().join("server/config.toml"), b"config")?; + std::fs::write(backup.path().join("server/metadata.toml"), b"metadata")?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("database_by_identity record identity")); assert!(!data.path().join("replicas/7").exists()); Ok(()) } @@ -1151,6 +1338,23 @@ mod tests { Ok(()) } + fn assert_no_restore_temps(data_dir: &Path) -> anyhow::Result<()> { + for entry in std::fs::read_dir(data_dir.join("replicas"))? { + let entry = entry?; + let file_name = entry.file_name(); + let file_name = file_name.to_string_lossy(); + assert!( + !file_name.contains(".restore_tmp_"), + "unexpected restore tmp {file_name}" + ); + assert!( + !file_name.contains(".restore_old_"), + "unexpected restore old tmp {file_name}" + ); + } + Ok(()) + } + fn make_test_control_db( path: &Path, database_identity: Identity, @@ -1163,7 +1367,13 @@ mod tests { .flush_every_ms(Some(50)) .mode(sled::Mode::HighThroughput) .open()?; - let database_record = sled::IVec::from(vec![1, 2, 3]); + let database_record = sled::IVec::from(bsatn::to_vec(&ControlDbDatabase { + id: 1, + database_identity, + owner_identity: Identity::ZERO, + host_type: ControlDbHostType::Wasm, + initial_program: Hash::ZERO, + })?); db.open_tree("database_by_identity")? .insert(database_identity.to_be_byte_array(), database_record.clone())?; db.open_tree("database")? @@ -1180,7 +1390,13 @@ mod tests { if include_extra_database { let extra_identity: Identity = "c300000000000000000000000000000000000000000000000000000000000000".parse()?; - let extra_database_record = sled::IVec::from(vec![7, 8, 9]); + let extra_database_record = sled::IVec::from(bsatn::to_vec(&ControlDbDatabase { + id: 2, + database_identity: extra_identity, + owner_identity: Identity::ZERO, + host_type: ControlDbHostType::Wasm, + initial_program: Hash::ZERO, + })?); db.open_tree("database_by_identity")? .insert(extra_identity.to_be_byte_array(), extra_database_record.clone())?; db.open_tree("database")? diff --git a/crates/standalone/src/control_db.rs b/crates/standalone/src/control_db.rs index 2205e5a9445..969b96b251a 100644 --- a/crates/standalone/src/control_db.rs +++ b/crates/standalone/src/control_db.rs @@ -29,6 +29,9 @@ pub struct ControlDb { pub type Result = core::result::Result; +const CONTROL_ID_COUNTER_TREE: &str = "control_id_counter"; +const CONTROL_ID_COUNTER_KEY: &[u8] = b"next"; + #[derive(thiserror::Error, Debug)] pub enum Error { #[error("collection not found: {0}")] @@ -136,7 +139,15 @@ impl ControlDb { replicas.insert(replica.id.to_be_bytes(), bsatn::to_vec(&replica)?)?; replicas.flush()?; - copy_optional_control_db_tree_entry(&self.db, &dst, "database_locks", &database_identity.to_be_byte_array())?; + let next_control_id = database + .id + .max(replica.id) + .max(replica.node_id) + .checked_add(1) + .context("control db id counter overflow")?; + set_control_id_counter_at_least(&dst, next_control_id)?; + + copy_optional_control_db_tree_entry(&self.db, &dst, "database_locks", database_identity.to_byte_array())?; copy_optional_control_db_tree_entry( &self.db, &dst, @@ -201,6 +212,9 @@ fn copy_optional_control_db_tree_entry( tree_name: &str, key: impl AsRef<[u8]>, ) -> Result<()> { + if !control_db_has_tree(src, tree_name) { + return Ok(()); + } let src_tree = src.open_tree(tree_name)?; if let Some(value) = src_tree.get(key.as_ref())? { let dst_tree = dst.open_tree(tree_name)?; @@ -210,6 +224,92 @@ fn copy_optional_control_db_tree_entry( Ok(()) } +fn control_db_has_tree(db: &sled::Db, tree_name: &str) -> bool { + db.tree_names().iter().any(|name| name.as_ref() == tree_name.as_bytes()) +} + +fn decode_control_id(value: &[u8]) -> anyhow::Result { + let bytes: [u8; 8] = value + .try_into() + .with_context(|| format!("invalid control db id counter length: {}", value.len()))?; + Ok(u64::from_be_bytes(bytes)) +} + +fn max_id_key_in_tree(db: &sled::Db, tree_name: &str) -> Result> { + if !control_db_has_tree(db, tree_name) { + return Ok(None); + } + let tree = db.open_tree(tree_name)?; + let mut max_id = None; + for entry in tree.iter() { + let (key, _) = entry?; + let key = key.as_ref(); + let id = decode_control_id(key).with_context(|| format!("invalid id key in control-db `{tree_name}` tree"))?; + max_id = Some(max_id.map_or(id, |max_id: u64| max_id.max(id))); + } + Ok(max_id) +} + +fn next_control_id_seed(db: &sled::Db) -> Result { + let generated_id = db.generate_id()?; + let mut next_id = generated_id; + for tree_name in ["database", "replica", "node"] { + if let Some(id) = max_id_key_in_tree(db, tree_name)? { + next_id = next_id.max(id.checked_add(1).context("control db id counter overflow")?); + } + } + Ok(next_id) +} + +fn set_control_id_counter_at_least(db: &sled::Db, next_id: u64) -> Result<()> { + let counter = db.open_tree(CONTROL_ID_COUNTER_TREE)?; + let result: TransactionResult<(), anyhow::Error> = (&counter).transaction(|counter_tx| { + let stored_next_id = match counter_tx.get(CONTROL_ID_COUNTER_KEY)? { + Some(value) => decode_control_id(&value).map_err(ConflictableTransactionError::Abort)?, + None => 0, + }; + if stored_next_id < next_id { + counter_tx.insert(CONTROL_ID_COUNTER_KEY, next_id.to_be_bytes().to_vec())?; + counter_tx.flush(); + } + Ok::<_, ConflictableTransactionError>(()) + }); + match result { + Ok(()) => { + counter.flush()?; + Ok(()) + } + Err(TransactionError::Storage(err)) => Err(err.into()), + Err(TransactionError::Abort(err)) => Err(err.into()), + } +} + +fn reserve_control_id(db: &sled::Db) -> Result { + let seed = next_control_id_seed(db)?; + let counter = db.open_tree(CONTROL_ID_COUNTER_TREE)?; + let result: TransactionResult = (&counter).transaction(|counter_tx| { + let stored_next_id = match counter_tx.get(CONTROL_ID_COUNTER_KEY)? { + Some(value) => decode_control_id(&value).map_err(ConflictableTransactionError::Abort)?, + None => seed, + }; + let id = stored_next_id.max(seed); + let next_id = id + .checked_add(1) + .ok_or_else(|| ConflictableTransactionError::Abort(anyhow::anyhow!("control db id counter overflow")))?; + counter_tx.insert(CONTROL_ID_COUNTER_KEY, next_id.to_be_bytes().to_vec())?; + counter_tx.flush(); + Ok(id) + }); + match result { + Ok(id) => { + counter.flush()?; + Ok(id) + } + Err(TransactionError::Storage(err)) => Err(err.into()), + Err(TransactionError::Abort(err)) => Err(err.into()), + } +} + /// A helper to convert a `sled::IVec` into an `Identity`. /// This expects the identity to be in LITTLE_ENDIAN format. /// This fails if the `sled::IVec` is not 32 bytes long. @@ -490,7 +590,7 @@ impl ControlDb { } pub fn insert_database(&self, mut database: Database) -> Result { - let id = self.db.generate_id()?; + let id = reserve_control_id(&self.db)?; let tree = self.db.open_tree("database_by_identity")?; let key = database.database_identity.to_be_byte_array(); @@ -609,7 +709,7 @@ impl ControlDb { pub fn insert_replica(&self, mut replica: Replica) -> Result { let tree = self.db.open_tree("replica")?; - let id = self.db.generate_id()?; + let id = reserve_control_id(&self.db)?; replica.id = id; let buf = bsatn::to_vec(&replica).unwrap(); @@ -654,7 +754,7 @@ impl ControlDb { pub fn _insert_node(&self, mut node: Node) -> Result { let tree = self.db.open_tree("node")?; - let id = self.db.generate_id()?; + let id = reserve_control_id(&self.db)?; node.id = id; let buf = bsatn::to_vec(&node).unwrap(); diff --git a/crates/standalone/src/control_db/tests.rs b/crates/standalone/src/control_db/tests.rs index f3ddbfeea9e..cd77ee69499 100644 --- a/crates/standalone/src/control_db/tests.rs +++ b/crates/standalone/src/control_db/tests.rs @@ -234,3 +234,52 @@ fn test_export_database_to_path_is_scoped_to_one_database() -> anyhow::Result<() assert!(!exported.is_database_locked(&second_identity)?); Ok(()) } + +#[test] +fn test_export_database_to_path_advances_control_id_counter() -> anyhow::Result<()> { + let src = TempDir::with_prefix("control-db-export-id-src")?; + let dst = TempDir::with_prefix("control-db-export-id-dst")?; + let cdb = ControlDb::at(src.path())?; + let first_database_id = cdb.insert_database(Database { + id: 0, + database_identity: Identity::ZERO, + owner_identity: *ALICE, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + })?; + let first_replica_id = cdb.insert_replica(Replica { + id: 0, + database_id: first_database_id, + node_id: 0, + leader: true, + })?; + + cdb.export_database_to_path( + &ControlDbDir::from_path_unchecked(dst.path()), + &Identity::ZERO, + first_replica_id, + )?; + let exported = ControlDb::at(dst.path())?; + let second_identity = Identity::from_claims(LOCALHOST, "after-restore"); + let second_database_id = exported.insert_database(Database { + id: 0, + database_identity: second_identity, + owner_identity: *BOB, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + })?; + let second_replica_id = exported.insert_replica(Replica { + id: 0, + database_id: second_database_id, + node_id: 0, + leader: true, + })?; + + assert_ne!(second_database_id, first_database_id); + assert_ne!(second_replica_id, first_replica_id); + assert!(exported.get_database_by_identity(&Identity::ZERO)?.is_some()); + assert!(exported.get_replica_by_id(first_replica_id)?.is_some()); + assert_eq!(exported.get_databases()?.len(), 2); + assert_eq!(exported.get_replicas()?.len(), 2); + Ok(()) +} diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index c6f42d8a0a0..f52c667ea85 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -730,6 +730,27 @@ mod tests { assert!(unrelated.exists()); } + #[test] + fn prune_scheduled_backups_keeps_latest_complete_and_ignores_incomplete_with_keep_last() { + let temp = tempfile::tempdir().unwrap(); + let old_complete = temp.path().join("stdb-0001"); + let new_complete = temp.path().join("stdb-0002"); + let incomplete = temp.path().join("stdb-0003"); + + std::fs::create_dir(&old_complete).unwrap(); + std::fs::write(old_complete.join("manifest.json"), b"{}").unwrap(); + std::fs::create_dir(&new_complete).unwrap(); + std::fs::write(new_complete.join("manifest.json"), b"{}").unwrap(); + std::fs::create_dir(&incomplete).unwrap(); + std::fs::write(incomplete.join("partial"), b"not done").unwrap(); + + prune_scheduled_backups(&scheduled_backup_config(temp.path().to_path_buf(), Some(1))).unwrap(); + + assert!(!old_complete.exists()); + assert!(new_complete.exists()); + assert!(incomplete.exists()); + } + #[test] fn cleanup_failed_scheduled_backup_output_removes_incomplete_dir() { let temp = tempfile::tempdir().unwrap(); From cb2a69f868bbc923b5eeedd7b32a67c002cc3a9a Mon Sep 17 00:00:00 2001 From: kdletters <61648117+kdletters@users.noreply.github.com> Date: Mon, 6 Jul 2026 15:10:33 +0800 Subject: [PATCH 7/8] Fix scoped backup lock export --- crates/cli/src/subcommands/backup.rs | 23 +++++++++++++++++++++++ crates/standalone/src/control_db.rs | 2 +- crates/standalone/src/control_db/tests.rs | 11 ++++++----- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/crates/cli/src/subcommands/backup.rs b/crates/cli/src/subcommands/backup.rs index 8cb66f36e5f..25bd3f2adcb 100644 --- a/crates/cli/src/subcommands/backup.rs +++ b/crates/cli/src/subcommands/backup.rs @@ -1014,6 +1014,29 @@ mod tests { Ok(()) } + #[test] + fn backup_restore_rolls_back_staged_server_state_when_replica_commit_sync_fails() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + std::fs::create_dir_all(data.path().join("replicas"))?; + std::fs::write(data.path().join("replicas/.fail-next-restore-sync-parent"), b"1")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(err.to_string().contains("injected sync_parent_dir failure")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + assert!(!data.path().join("config.toml").exists()); + assert!(!data.path().join("metadata.toml").exists()); + assert!(!data.path().join("replicas/7").exists()); + assert_no_restore_temps(data.path())?; + Ok(()) + } + #[cfg(windows)] #[test] fn backup_restore_force_commits_when_old_replica_cleanup_fails() -> anyhow::Result<()> { diff --git a/crates/standalone/src/control_db.rs b/crates/standalone/src/control_db.rs index 969b96b251a..585768a5240 100644 --- a/crates/standalone/src/control_db.rs +++ b/crates/standalone/src/control_db.rs @@ -147,7 +147,7 @@ impl ControlDb { .context("control db id counter overflow")?; set_control_id_counter_at_least(&dst, next_control_id)?; - copy_optional_control_db_tree_entry(&self.db, &dst, "database_locks", database_identity.to_byte_array())?; + copy_optional_control_db_tree_entry(&self.db, &dst, "database_locks", database_identity.to_be_byte_array())?; copy_optional_control_db_tree_entry( &self.db, &dst, diff --git a/crates/standalone/src/control_db/tests.rs b/crates/standalone/src/control_db/tests.rs index cd77ee69499..930234a99cb 100644 --- a/crates/standalone/src/control_db/tests.rs +++ b/crates/standalone/src/control_db/tests.rs @@ -187,9 +187,10 @@ fn test_export_database_to_path_is_scoped_to_one_database() -> anyhow::Result<() let src = TempDir::with_prefix("control-db-export-scoped-src")?; let dst = TempDir::with_prefix("control-db-export-scoped-dst")?; let cdb = ControlDb::at(src.path())?; + let first_identity = Identity::from_claims(LOCALHOST, "first"); let first = Database { id: 0, - database_identity: Identity::ZERO, + database_identity: first_identity, owner_identity: *ALICE, host_type: HostType::Wasm, initial_program: Hash::ZERO, @@ -215,22 +216,22 @@ fn test_export_database_to_path_is_scoped_to_one_database() -> anyhow::Result<() node_id: 0, leader: true, })?; - cdb.set_database_lock(&Identity::ZERO, true)?; + cdb.set_database_lock(&first_identity, true)?; cdb.set_database_lock(&second_identity, true)?; cdb.export_database_to_path( &ControlDbDir::from_path_unchecked(dst.path()), - &Identity::ZERO, + &first_identity, first_replica_id, )?; let exported = ControlDb::at(dst.path())?; - assert!(exported.get_database_by_identity(&Identity::ZERO)?.is_some()); + assert!(exported.get_database_by_identity(&first_identity)?.is_some()); assert!(exported.get_database_by_identity(&second_identity)?.is_none()); assert_eq!(exported.get_databases()?.len(), 1); assert!(exported.get_replica_by_id(first_replica_id)?.is_some()); assert_eq!(exported.get_replicas()?.len(), 1); - assert!(exported.is_database_locked(&Identity::ZERO)?); + assert!(exported.is_database_locked(&first_identity)?); assert!(!exported.is_database_locked(&second_identity)?); Ok(()) } From c78fb951871f4819b21dc452ad6ef75be939343f Mon Sep 17 00:00:00 2001 From: kdletters <61648117+kdletters@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:38:07 +0800 Subject: [PATCH 8/8] Address hot backup PR review findings Move backup orchestration out of the engine deep core. Harden snapshot durability, cancellation, and output path claims. Validate restore metadata and aggregate rollback failures. Scope scheduled retention by database and preserve ownership markers. Keep control-db storage compatibility and invalidate snapshots without durable history. Update Windows atomic replacement and operator documentation. --- Cargo.lock | 3 +- crates/cli/src/subcommands/backup.rs | 910 ++++++++++++-- crates/client-api/src/lib.rs | 37 +- crates/client-api/src/routes/database.rs | 130 +- crates/core/src/host/host_controller.rs | 20 + crates/core/src/host/hot_backup.rs | 1090 +++++++++++++++++ crates/core/src/host/mod.rs | 2 + crates/engine/Cargo.toml | 5 +- crates/engine/src/relational_db.rs | 742 ++--------- crates/engine/src/snapshot.rs | 102 +- crates/fs-utils/Cargo.toml | 3 + crates/fs-utils/src/lib.rs | 37 +- crates/snapshot/src/lib.rs | 35 +- crates/standalone/config.toml | 1 + crates/standalone/src/control_db.rs | 38 +- crates/standalone/src/control_db/tests.rs | 5 + crates/standalone/src/lib.rs | 245 +++- crates/standalone/src/subcommands/start.rs | 322 +++-- .../00100-deploy/00200-self-hosting.md | 44 +- .../00200-standalone-config.md | 14 +- 20 files changed, 2838 insertions(+), 947 deletions(-) create mode 100644 crates/core/src/host/hot_backup.rs diff --git a/Cargo.lock b/Cargo.lock index f2282e297c4..ca14d4cf68d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8477,7 +8477,6 @@ dependencies = [ "pretty_assertions", "prometheus", "serde", - "serde_json", "sled", "spacetimedb-commitlog", "spacetimedb-data-structures", @@ -8498,7 +8497,6 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tokio", - "tokio-util", "tracing", ] @@ -8550,6 +8548,7 @@ dependencies = [ "tempdir", "thiserror 1.0.69", "tokio", + "windows-sys 0.59.0", "zstd-framed", ] diff --git a/crates/cli/src/subcommands/backup.rs b/crates/cli/src/subcommands/backup.rs index 25bd3f2adcb..0b2bb985c8b 100644 --- a/crates/cli/src/subcommands/backup.rs +++ b/crates/cli/src/subcommands/backup.rs @@ -8,9 +8,11 @@ use crate::util::{add_auth_header_opt, database_identity, get_auth_header, Respo use anyhow::Context; use clap::{Arg, ArgMatches, Command}; use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize}; -use spacetimedb_commitlog::{commits, committed_meta}; +use spacetimedb_commitlog::{commits, committed_meta, segment, DEFAULT_LOG_FORMAT_VERSION}; use spacetimedb_fs_utils::{copy_dir_all, copy_file_sync, create_dir_all_sync, sync_dir}; -use spacetimedb_lib::{bsatn, de::Deserialize as BsatnDeserialize, ser::Serialize as BsatnSerialize, Hash, Identity}; +use spacetimedb_lib::{ + bsatn, de::Deserialize as BsatnDeserialize, hash_bytes, ser::Serialize as BsatnSerialize, Hash, Identity, +}; use spacetimedb_paths::{ server::{CommitLogDir, ServerDataDir, SnapshotsPath}, FromPathUnchecked, SpacetimePaths, @@ -160,6 +162,8 @@ fn exec_restore(paths: &SpacetimePaths, args: &ArgMatches) -> Result<(), anyhow: fn restore_backup(input_dir: &Path, data_dir: &ServerDataDir, force: bool) -> anyhow::Result { let manifest = read_backup_manifest(input_dir)?; validate_backup(input_dir, &manifest)?; + let backup_log_format_version = read_backup_max_log_format_version(input_dir)?; + validate_target_log_format_version(data_dir, backup_log_format_version)?; create_dir_all_sync(&data_dir.0)?; // The data-dir lock catches the common footgun; per-replica online restore needs a real restore service. @@ -169,6 +173,96 @@ fn restore_backup(input_dir: &Path, data_dir: &ServerDataDir, force: bool) -> an restore_backup_inner(input_dir, data_dir, force, manifest) } +#[derive(SerdeDeserialize)] +struct RestoreTargetConfig { + #[serde(default)] + commitlog: RestoreTargetCommitlogConfig, +} + +#[derive(SerdeDeserialize)] +struct RestoreTargetCommitlogConfig { + #[serde(default = "default_log_format_version", rename = "log-format-version")] + log_format_version: u8, +} + +impl Default for RestoreTargetCommitlogConfig { + fn default() -> Self { + Self { + log_format_version: DEFAULT_LOG_FORMAT_VERSION, + } + } +} + +const fn default_log_format_version() -> u8 { + DEFAULT_LOG_FORMAT_VERSION +} + +fn read_backup_max_log_format_version(input_dir: &Path) -> anyhow::Result { + let clog_dir = CommitLogDir::from_path_unchecked(input_dir.join("clog")); + anyhow::ensure!( + clog_dir.0.is_dir(), + "backup clog directory is missing: {}", + clog_dir.display() + ); + let initial_segment = clog_dir.segment(0); + anyhow::ensure!( + initial_segment.0.is_file(), + "backup commitlog segment is missing: {}", + initial_segment.display() + ); + + let mut max_log_format_version: Option = None; + for entry in std::fs::read_dir(&clog_dir.0).with_context(|| format!("reading {}", clog_dir.display()))? { + let entry = entry?; + let file_name = entry.file_name(); + let file_name = file_name.to_string_lossy(); + if !file_name.ends_with(".stdb.log") { + continue; + } + anyhow::ensure!( + is_commitlog_segment_name(&file_name) && entry.file_type()?.is_file(), + "invalid backup commitlog segment: {}", + entry.path().display() + ); + let path = entry.path(); + let file = std::fs::File::open(&path) + .with_context(|| format!("opening backup commitlog segment header {}", path.display()))?; + let header = segment::Header::decode(file) + .with_context(|| format!("reading backup commitlog segment header {}", path.display()))?; + max_log_format_version = Some( + max_log_format_version + .unwrap_or_default() + .max(header.log_format_version), + ); + } + max_log_format_version.with_context(|| { + format!( + "backup clog directory contains no segment files: {}", + clog_dir.display() + ) + }) +} + +fn validate_target_log_format_version(data_dir: &ServerDataDir, backup_log_format_version: u8) -> anyhow::Result<()> { + let config_path = data_dir.0.join("config.toml"); + let target_log_format_version = if config_path.exists() { + let config = std::fs::read_to_string(&config_path) + .with_context(|| format!("reading target server config {}", config_path.display()))?; + toml::from_str::(&config) + .with_context(|| format!("parsing target server config {}", config_path.display()))? + .commitlog + .log_format_version + } else { + DEFAULT_LOG_FORMAT_VERSION + }; + anyhow::ensure!( + target_log_format_version >= backup_log_format_version, + "target commitlog log format version {target_log_format_version} is lower than backup commitlog requires version {backup_log_format_version}; update [commitlog].log-format-version in {} before restoring", + config_path.display() + ); + Ok(()) +} + fn restore_backup_inner( input_dir: &Path, data_dir: &ServerDataDir, @@ -212,13 +306,19 @@ fn restore_backup_inner( create_dir_all_sync(parent)?; } - let staged_server_state = stage_missing_server_state(input_dir, data_dir, restore_temp_id, &manifest)?; - validate_target_control_db(input_dir, data_dir, &manifest)?; - anyhow::ensure!( - !(force && replica_dir.0.exists() && !staged_server_state.is_empty()), - "cannot restore over existing replica {} while target data-dir is missing server state; restore into an empty data-dir or complete the target server state first", - replica_dir.display() - ); + let mut staged_server_state = stage_missing_server_state(input_dir, data_dir, restore_temp_id, &manifest)?; + let validation = (|| -> anyhow::Result<()> { + validate_target_control_db(input_dir, data_dir, &manifest)?; + anyhow::ensure!( + !(force && replica_dir.0.exists() && !staged_server_state.is_empty()), + "cannot restore over existing replica {} while target data-dir is missing server state; restore into an empty data-dir or complete the target server state first", + replica_dir.display() + ); + Ok(()) + })(); + if let Err(error) = validation { + return Err(staged_server_state.cleanup().attach(error)); + } let mut old_tmp_moved = false; let mut restore_old_tmp_on_error = false; @@ -264,23 +364,181 @@ fn restore_backup_inner( Ok(()) })(); - if res.is_err() { - let _ = std::fs::remove_dir_all(&tmp_dir); - let _ = sync_parent_dir(&tmp_dir); - if restored_replica_moved && replica_dir.0.exists() { - let _ = std::fs::remove_dir_all(&replica_dir.0); - let _ = sync_parent_dir(&replica_dir.0); - } - if restore_old_tmp_on_error && old_tmp_moved && old_tmp_dir.exists() && !replica_dir.0.exists() { - let _ = std::fs::rename(&old_tmp_dir, &replica_dir.0); - let _ = sync_parent_dir(&replica_dir.0); + if let Err(error) = res { + let mut rollback = rollback_failed_replica_restore( + &tmp_dir, + &replica_dir.0, + &old_tmp_dir, + restored_replica_moved, + restore_old_tmp_on_error && old_tmp_moved, + ); + if let Some(committed_server_state) = &mut committed_server_state { + rollback.extend(committed_server_state.rollback()); } + return Err(rollback.attach(error)); } - res?; Ok(manifest) } +#[derive(Debug, Default)] +struct RollbackFailures { + errors: Vec, + recovery_paths: Vec, +} + +impl RollbackFailures { + fn capture(&mut self, result: anyhow::Result<()>, recovery_paths: &[&Path]) -> bool { + match result { + Ok(()) => true, + Err(error) => { + self.errors.push(format!("{error:#}")); + for path in recovery_paths { + let path = (*path).to_path_buf(); + if !self.recovery_paths.contains(&path) { + self.recovery_paths.push(path); + } + } + false + } + } + } + + fn extend(&mut self, other: Self) { + self.errors.extend(other.errors); + for path in other.recovery_paths { + if !self.recovery_paths.contains(&path) { + self.recovery_paths.push(path); + } + } + } + + fn is_empty(&self) -> bool { + self.errors.is_empty() + } + + fn summary(&self) -> String { + let paths = self + .recovery_paths + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(", "); + format!( + "restore rollback was incomplete; possible leftover/recoverable paths: {paths}; rollback errors: {}", + self.errors.join("; ") + ) + } + + fn attach(self, error: anyhow::Error) -> anyhow::Error { + if self.is_empty() { + error + } else { + error.context(self.summary()) + } + } + + fn report_unattached(&self) { + if !self.is_empty() { + tracing::error!("{}", self.summary()); + } + } +} + +fn rollback_failed_replica_restore( + tmp_dir: &Path, + replica_dir: &Path, + old_tmp_dir: &Path, + restored_replica_moved: bool, + restore_old_tmp: bool, +) -> RollbackFailures { + let mut failures = RollbackFailures::default(); + + if tmp_dir.exists() { + failures.capture(rollback_remove_dir_all(tmp_dir), &[tmp_dir]); + failures.capture(rollback_sync_parent_dir(tmp_dir), &[tmp_dir]); + } + if restored_replica_moved && replica_dir.exists() { + failures.capture(rollback_remove_dir_all(replica_dir), &[replica_dir, old_tmp_dir]); + failures.capture(rollback_sync_parent_dir(replica_dir), &[replica_dir, old_tmp_dir]); + } + if restore_old_tmp { + if !old_tmp_dir.exists() { + if !replica_dir.exists() { + failures.capture( + Err(anyhow::anyhow!( + "recoverable old replica directory disappeared during rollback: {}", + old_tmp_dir.display() + )), + &[replica_dir, old_tmp_dir], + ); + } + } else if replica_dir.exists() { + failures.capture( + Err(anyhow::anyhow!( + "could not restore old replica because rollback target still exists: {}", + replica_dir.display() + )), + &[replica_dir, old_tmp_dir], + ); + } else { + let renamed = failures.capture(rollback_rename(old_tmp_dir, replica_dir), &[replica_dir, old_tmp_dir]); + if renamed { + failures.capture(rollback_sync_parent_dir(replica_dir), &[replica_dir, old_tmp_dir]); + } + } + } + + failures +} + +fn rollback_remove_dir_all(path: &Path) -> anyhow::Result<()> { + #[cfg(test)] + maybe_fail_restore_rollback_operation(path, "remove")?; + std::fs::remove_dir_all(path).with_context(|| format!("removing directory {} during rollback", path.display())) +} + +fn rollback_remove_file(path: &Path) -> anyhow::Result<()> { + #[cfg(test)] + maybe_fail_restore_rollback_operation(path, "remove")?; + std::fs::remove_file(path).with_context(|| format!("removing file {} during rollback", path.display())) +} + +fn rollback_rename(from: &Path, to: &Path) -> anyhow::Result<()> { + #[cfg(test)] + maybe_fail_restore_rollback_operation(from, "rename")?; + std::fs::rename(from, to).with_context(|| { + format!( + "restoring recoverable path {} to {} during rollback", + from.display(), + to.display() + ) + }) +} + +fn rollback_sync_parent_dir(path: &Path) -> anyhow::Result<()> { + if let Some(parent) = path.parent().filter(|parent| !parent.as_os_str().is_empty()) { + #[cfg(test)] + maybe_fail_restore_rollback_operation(path, "sync-parent")?; + sync_dir(parent).with_context(|| format!("syncing parent directory {} during rollback", parent.display()))?; + } + Ok(()) +} + +#[cfg(test)] +fn maybe_fail_restore_rollback_operation(path: &Path, operation: &str) -> anyhow::Result<()> { + let Some(parent) = path.parent().filter(|parent| !parent.as_os_str().is_empty()) else { + return Ok(()); + }; + let marker = parent.join(format!(".fail-next-restore-rollback-{operation}")); + anyhow::ensure!( + !marker.exists(), + "injected rollback {operation} failure for {}", + path.display() + ); + Ok(()) +} + fn sync_parent_dir(path: &Path) -> anyhow::Result<()> { if let Some(parent) = path.parent().filter(|parent| !parent.as_os_str().is_empty()) { #[cfg(test)] @@ -431,23 +689,6 @@ fn is_commitlog_segment_name(file_name: &str) -> bool { offset.len() == 20 && offset.bytes().all(|byte| byte.is_ascii_digit()) } -fn validate_scoped_backup_control_db( - control_db_dir: &Path, - manifest: &BackupManifest, - database_identity: Identity, -) -> anyhow::Result<()> { - if !control_db_dir.exists() { - return Ok(()); - } - validate_control_db_records( - control_db_dir, - database_identity, - manifest.replica_id, - ControlDbValidationScope::ScopedBackup, - ) - .with_context(|| format!("validating backup control-db {}", control_db_dir.display())) -} - fn validate_target_control_db( input_dir: &Path, data_dir: &ServerDataDir, @@ -461,7 +702,7 @@ fn validate_target_control_db( .database_identity .parse() .with_context(|| format!("parsing backup database identity {}", manifest.database_identity))?; - validate_control_db_records( + let database = validate_control_db_records( &control_db_dir, database_identity, manifest.replica_id, @@ -473,7 +714,53 @@ fn validate_target_control_db( control_db_dir.display(), input_dir.display() ) - }) + })?; + let backup_control_db_dir = input_dir.join("server").join("control-db"); + let backup_database = validate_control_db_records( + &backup_control_db_dir, + database_identity, + manifest.replica_id, + ControlDbValidationScope::ScopedBackup, + ) + .with_context(|| format!("validating backup control-db {}", backup_control_db_dir.display()))?; + validate_control_db_metadata_match(&backup_database, &database)?; + let backup_program_bytes_dir = input_dir.join("server").join("program-bytes"); + validate_program_bytes(&backup_program_bytes_dir, backup_database.initial_program) + .with_context(|| format!("validating backup program-bytes {}", backup_program_bytes_dir.display()))?; + let program_bytes_dir = data_dir.0.join("program-bytes"); + if program_bytes_dir.exists() { + validate_program_bytes(&program_bytes_dir, database.initial_program) + .with_context(|| format!("validating target program-bytes {}", program_bytes_dir.display()))?; + } + Ok(()) +} + +fn validate_control_db_metadata_match(backup: &ControlDbDatabase, target: &ControlDbDatabase) -> anyhow::Result<()> { + anyhow::ensure!( + backup.database_identity == target.database_identity, + "backup control-db database identity {} does not match target {}", + backup.database_identity, + target.database_identity + ); + anyhow::ensure!( + backup.owner_identity == target.owner_identity, + "backup control-db owner identity {} does not match target {}", + backup.owner_identity, + target.owner_identity + ); + anyhow::ensure!( + backup.host_type == target.host_type, + "backup control-db host_type {:?} does not match target {:?}", + backup.host_type, + target.host_type + ); + anyhow::ensure!( + backup.initial_program == target.initial_program, + "backup control-db initial_program {} does not match target {}", + backup.initial_program, + target.initial_program + ); + Ok(()) } #[derive(Debug, Clone, Copy)] @@ -483,7 +770,7 @@ enum ControlDbValidationScope { } #[allow(dead_code)] -#[derive(BsatnDeserialize, BsatnSerialize)] +#[derive(Debug, Clone, BsatnDeserialize, BsatnSerialize)] struct ControlDbDatabase { id: u64, database_identity: Identity, @@ -493,7 +780,7 @@ struct ControlDbDatabase { } #[allow(dead_code)] -#[derive(BsatnDeserialize, BsatnSerialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, BsatnDeserialize, BsatnSerialize)] #[repr(i32)] enum ControlDbHostType { Wasm = 0, @@ -514,7 +801,7 @@ fn validate_control_db_records( database_identity: Identity, replica_id: u64, scope: ControlDbValidationScope, -) -> anyhow::Result<()> { +) -> anyhow::Result { anyhow::ensure!( control_db_dir.is_dir(), "control-db path is not a directory: {}", @@ -642,9 +929,36 @@ fn validate_control_db_records( "backup control-db must contain exactly one replica record, found {replica_count}" ); } + Ok(database_from_identity) +} + +fn validate_program_bytes(program_bytes_dir: &Path, program_hash: Hash) -> anyhow::Result<()> { + anyhow::ensure!( + program_bytes_dir.is_dir(), + "program-bytes path is not a directory: {}", + program_bytes_dir.display() + ); + let relative_path = program_bytes_relative_path(&program_hash); + let path = program_bytes_dir.join(relative_path); + let program_bytes = std::fs::read(&path) + .with_context(|| format!("reading program bytes {} referenced by control-db", path.display()))?; + let actual_hash = hash_bytes(&program_bytes); + anyhow::ensure!( + actual_hash == program_hash, + "program bytes hash mismatch at {}: expected {}, got {}", + path.display(), + program_hash, + actual_hash + ); Ok(()) } +fn program_bytes_relative_path(program_hash: &Hash) -> PathBuf { + let hex = program_hash.to_hex(); + let (prefix, suffix) = hex.split_at(2); + PathBuf::from(prefix).join(suffix) +} + fn control_db_u64_key(key: &[u8], label: &str) -> anyhow::Result { let bytes: [u8; 8] = key .try_into() @@ -670,25 +984,42 @@ struct StagedServerStateEntry { #[derive(Debug)] struct StagedServerState { entries: Vec, + cleanup_on_drop: bool, } impl StagedServerState { + fn new() -> Self { + Self { + entries: Vec::new(), + cleanup_on_drop: true, + } + } + fn is_empty(&self) -> bool { self.entries.is_empty() } - fn cleanup(&self) { + fn cleanup(&mut self) -> RollbackFailures { + if !self.cleanup_on_drop { + return RollbackFailures::default(); + } + self.cleanup_on_drop = false; + let mut failures = RollbackFailures::default(); for entry in &self.entries { + if !entry.staged_path.exists() { + continue; + } if entry.staged_path.is_dir() { - let _ = std::fs::remove_dir_all(&entry.staged_path); + failures.capture(rollback_remove_dir_all(&entry.staged_path), &[&entry.staged_path]) } else { - let _ = std::fs::remove_file(&entry.staged_path); - } - let _ = sync_parent_dir(&entry.staged_path); + failures.capture(rollback_remove_file(&entry.staged_path), &[&entry.staged_path]) + }; + failures.capture(rollback_sync_parent_dir(&entry.staged_path), &[&entry.staged_path]); } + failures } - fn commit(self) -> anyhow::Result { + fn commit(mut self) -> anyhow::Result { let mut committed = CommittedServerState::default(); let res = (|| -> anyhow::Result<()> { for entry in &self.entries { @@ -709,18 +1040,19 @@ impl StagedServerState { } Ok(()) })(); - if res.is_err() { - self.cleanup(); - committed.rollback(); + if let Err(error) = res { + let mut rollback = self.cleanup(); + rollback.extend(committed.rollback()); + return Err(rollback.attach(error)); } - res?; + self.cleanup_on_drop = false; Ok(committed) } } impl Drop for StagedServerState { fn drop(&mut self) { - self.cleanup(); + self.cleanup().report_unattached(); } } @@ -735,22 +1067,31 @@ impl CommittedServerState { self.keep = true; } - fn rollback(&self) { + fn rollback(&mut self) -> RollbackFailures { + if self.keep { + return RollbackFailures::default(); + } + self.keep = true; + let mut failures = RollbackFailures::default(); for path in self.paths.iter().rev() { + if !path.exists() { + continue; + } if path.is_dir() { - let _ = std::fs::remove_dir_all(path); + failures.capture(rollback_remove_dir_all(path), &[path]) } else { - let _ = std::fs::remove_file(path); - } - let _ = sync_parent_dir(path); + failures.capture(rollback_remove_file(path), &[path]) + }; + failures.capture(rollback_sync_parent_dir(path), &[path]); } + failures } } impl Drop for CommittedServerState { fn drop(&mut self) { if !self.keep { - self.rollback(); + self.rollback().report_unattached(); } } } @@ -766,7 +1107,7 @@ fn stage_missing_server_state( .into_iter() .any(|required_dir| !data_dir.0.join(required_dir).exists()); if !needs_required_dirs && !server_dir.exists() { - return Ok(StagedServerState { entries: Vec::new() }); + return Ok(StagedServerState::new()); } let mut entries = Vec::new(); @@ -814,28 +1155,45 @@ fn stage_missing_server_state( entries.push((src, dst, staged, false)); } - let mut staged = StagedServerState { entries: Vec::new() }; + let mut staged = StagedServerState::new(); let res = (|| -> anyhow::Result<()> { for (src, final_path, staged_path, is_dir) in entries { copy_staged_server_state_entry(&mut staged, &src, final_path, staged_path, is_dir)?; } - if let Some(entry) = staged + let control_db_entry = staged .entries .iter() - .find(|entry| entry.final_path.ends_with("control-db")) - { + .find(|entry| entry.final_path.ends_with("control-db")); + let program_bytes_entry = staged + .entries + .iter() + .find(|entry| entry.final_path.ends_with("program-bytes")); + if control_db_entry.is_some() || program_bytes_entry.is_some() { let database_identity: Identity = manifest .database_identity .parse() .with_context(|| format!("parsing backup database identity {}", manifest.database_identity))?; - validate_scoped_backup_control_db(&entry.staged_path, manifest, database_identity)?; + let control_db_path = control_db_entry + .map(|entry| entry.staged_path.clone()) + .unwrap_or_else(|| data_dir.0.join("control-db")); + let scope = if control_db_entry.is_some() { + ControlDbValidationScope::ScopedBackup + } else { + ControlDbValidationScope::ExistingTarget + }; + let database = validate_control_db_records(&control_db_path, database_identity, manifest.replica_id, scope) + .with_context(|| format!("validating control-db {}", control_db_path.display()))?; + let program_bytes_dir = program_bytes_entry + .map(|entry| entry.staged_path.clone()) + .unwrap_or_else(|| data_dir.0.join("program-bytes")); + validate_program_bytes(&program_bytes_dir, database.initial_program) + .with_context(|| format!("validating program-bytes {}", program_bytes_dir.display()))?; } Ok(()) })(); - if res.is_err() { - staged.cleanup(); + if let Err(error) = res { + return Err(staged.cleanup().attach(error)); } - res?; Ok(staged) } @@ -859,10 +1217,10 @@ fn copy_staged_server_state_entry( .with_context(|| format!("copying {} to {}", src.display(), entry.staged_path.display())) .map(|_| ()) }; - if res.is_err() { - staged.cleanup(); + if let Err(error) = res { + return Err(staged.cleanup().attach(error)); } - res + Ok(()) } fn staged_server_state_path(path: &Path, restore_temp_id: u64) -> PathBuf { @@ -886,6 +1244,7 @@ mod tests { fn backup_restore_copies_replica_into_existing_data_dir() -> anyhow::Result<()> { let backup = tempfile::tempdir()?; make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; let data = tempfile::tempdir()?; make_target_data_dir(data.path(), 7)?; @@ -907,6 +1266,28 @@ mod tests { Ok(()) } + #[test] + fn backup_restore_rejects_v1_backup_for_v0_target_before_mutation() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let config = "[commitlog]\nlog-format-version = 0\n"; + std::fs::write(data.path().join("config.toml"), config)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + let rendered = format!("{err:#}"); + + assert!(rendered.contains("target commitlog log format version 0")); + assert!(rendered.contains("backup commitlog requires version 1")); + assert_eq!(std::fs::read_to_string(data.path().join("config.toml"))?, config); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + #[test] fn backup_restore_requires_force_for_existing_replica() -> anyhow::Result<()> { let backup = tempfile::tempdir()?; @@ -946,6 +1327,7 @@ mod tests { fn backup_restore_force_replaces_existing_replica_without_leaving_old_tmp() -> anyhow::Result<()> { let backup = tempfile::tempdir()?; make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; let data = tempfile::tempdir()?; make_target_data_dir(data.path(), 7)?; @@ -974,6 +1356,7 @@ mod tests { fn backup_restore_force_restores_old_replica_when_sync_after_old_rename_fails() -> anyhow::Result<()> { let backup = tempfile::tempdir()?; make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; let data = tempfile::tempdir()?; make_target_data_dir(data.path(), 7)?; @@ -996,6 +1379,7 @@ mod tests { fn backup_restore_force_restores_old_replica_when_sync_after_new_rename_fails() -> anyhow::Result<()> { let backup = tempfile::tempdir()?; make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; let data = tempfile::tempdir()?; make_target_data_dir(data.path(), 7)?; @@ -1014,6 +1398,94 @@ mod tests { Ok(()) } + #[test] + fn backup_restore_reports_rollback_remove_failure_with_recovery_paths() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + let replicas_dir = data.path().join("replicas"); + std::fs::write(replicas_dir.join(".fail-next-restore-sync-parent"), b"2")?; + std::fs::write(replicas_dir.join(".fail-next-restore-rollback-remove"), b"fail")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + let rendered = format!("{err:#}"); + let recoverable_old = find_restore_old_dir(&replicas_dir)?; + + assert!(rendered.contains("restore rollback was incomplete")); + assert!(rendered.contains("injected rollback remove failure")); + assert!(rendered.contains("injected sync_parent_dir failure")); + assert!(rendered.contains("possible leftover/recoverable paths")); + assert!(rendered.contains(&replica_dir.display().to_string())); + assert!(rendered.contains(&recoverable_old.display().to_string())); + assert!(replica_dir.join("snapshots").is_dir()); + assert_eq!(std::fs::read(recoverable_old.join("old-marker"))?, b"old"); + Ok(()) + } + + #[test] + fn backup_restore_reports_rollback_rename_failure_with_recoverable_old_replica() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + let replicas_dir = data.path().join("replicas"); + std::fs::write(replicas_dir.join(".fail-next-restore-sync-parent"), b"2")?; + std::fs::write(replicas_dir.join(".fail-next-restore-rollback-rename"), b"fail")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + let rendered = format!("{err:#}"); + let recoverable_old = find_restore_old_dir(&replicas_dir)?; + + assert!(rendered.contains("restore rollback was incomplete")); + assert!(rendered.contains("injected rollback rename failure")); + assert!(rendered.contains("injected sync_parent_dir failure")); + assert!(rendered.contains(&recoverable_old.display().to_string())); + assert!(!replica_dir.exists()); + assert_eq!(std::fs::read(recoverable_old.join("old-marker"))?, b"old"); + Ok(()) + } + + #[test] + fn backup_restore_reports_rollback_fsync_failure_and_preserves_original_error_chain() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_target_data_dir(data.path(), 7)?; + let replica_dir = data.path().join("replicas/7"); + std::fs::create_dir_all(&replica_dir)?; + std::fs::write(replica_dir.join("old-marker"), b"old")?; + let replicas_dir = data.path().join("replicas"); + std::fs::write(replicas_dir.join(".fail-next-restore-sync-parent"), b"2")?; + std::fs::write(replicas_dir.join(".fail-next-restore-rollback-sync-parent"), b"fail")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, true).unwrap_err(); + let rendered = format!("{err:#}"); + + assert!(rendered.contains("restore rollback was incomplete")); + assert!(rendered.contains("injected rollback sync-parent failure")); + assert!(rendered.contains("injected sync_parent_dir failure")); + assert!(rendered.contains(&replica_dir.display().to_string())); + assert_eq!(std::fs::read(replica_dir.join("old-marker"))?, b"old"); + assert_no_restore_temps(data.path())?; + Ok(()) + } + #[test] fn backup_restore_rolls_back_staged_server_state_when_replica_commit_sync_fails() -> anyhow::Result<()> { let backup = tempfile::tempdir()?; @@ -1037,6 +1509,31 @@ mod tests { Ok(()) } + #[test] + fn backup_restore_reports_committed_server_state_rollback_failure() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + std::fs::create_dir_all(data.path().join("replicas"))?; + std::fs::write(data.path().join("replicas/.fail-next-restore-sync-parent"), b"1")?; + std::fs::write(data.path().join(".fail-next-restore-rollback-remove"), b"fail")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + let rendered = format!("{err:#}"); + let metadata_path = data.path().join("metadata.toml"); + + assert!(rendered.contains("restore rollback was incomplete")); + assert!(rendered.contains("injected rollback remove failure")); + assert!(rendered.contains("injected sync_parent_dir failure")); + assert!(rendered.contains(&metadata_path.display().to_string())); + assert!(metadata_path.is_file()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + #[cfg(windows)] #[test] fn backup_restore_force_commits_when_old_replica_cleanup_fails() -> anyhow::Result<()> { @@ -1084,12 +1581,174 @@ mod tests { restore_backup(backup.path(), &data_dir, false)?; assert!(data.path().join("control-db").is_dir()); - assert!(data.path().join("program-bytes/program").is_file()); + assert!(data + .path() + .join("program-bytes") + .join(program_bytes_relative_path(&test_program_hash())) + .is_file()); assert_eq!(std::fs::read_to_string(data.path().join("config.toml"))?, "config"); assert_eq!(std::fs::read_to_string(data.path().join("metadata.toml"))?, "metadata"); Ok(()) } + #[test] + fn backup_restore_copies_program_bytes_when_target_has_control_db() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_test_control_db(&data.path().join("control-db"), test_database_identity()?, 7, false)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + restore_backup(backup.path(), &data_dir, false)?; + + assert!(data.path().join("control-db").is_dir()); + assert!(data + .path() + .join("program-bytes") + .join(program_bytes_relative_path(&test_program_hash())) + .is_file()); + assert!(data.path().join("replicas/7").is_dir()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_missing_program_bytes_referenced_by_control_db() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + std::fs::remove_file( + backup + .path() + .join("server/program-bytes") + .join(program_bytes_relative_path(&test_program_hash())), + )?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("referenced by control-db")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_program_bytes_hash_mismatch() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + std::fs::write( + backup + .path() + .join("server/program-bytes") + .join(program_bytes_relative_path(&test_program_hash())), + b"not the program bytes", + )?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("program bytes hash mismatch")); + assert!(!data.path().join("control-db").exists()); + assert!(!data.path().join("program-bytes").exists()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_existing_target_missing_program_bytes_referenced_by_control_db() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_test_control_db(&data.path().join("control-db"), test_database_identity()?, 7, false)?; + std::fs::create_dir_all(data.path().join("program-bytes"))?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("referenced by control-db")); + assert!(data.path().join("control-db").is_dir()); + assert!(data.path().join("program-bytes").is_dir()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_existing_target_program_bytes_hash_mismatch() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + make_test_control_db(&data.path().join("control-db"), test_database_identity()?, 7, false)?; + let program_path = data + .path() + .join("program-bytes") + .join(program_bytes_relative_path(&test_program_hash())); + std::fs::create_dir_all(program_path.parent().context("program bytes path has no parent")?)?; + std::fs::write(&program_path, b"not the program bytes")?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("program bytes hash mismatch")); + assert!(data.path().join("control-db").is_dir()); + assert!(program_path.is_file()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + + #[test] + fn backup_restore_rejects_existing_target_control_db_metadata_mismatch() -> anyhow::Result<()> { + let backup = tempfile::tempdir()?; + make_backup_dir(backup.path(), 7, 42)?; + make_backup_server_state(backup.path())?; + + let data = tempfile::tempdir()?; + let target_program_bytes = b"different program"; + let target_program_hash = hash_bytes(target_program_bytes); + make_test_control_db_with_database( + &data.path().join("control-db"), + ControlDbDatabase { + id: 1, + database_identity: test_database_identity()?, + owner_identity: Identity::ZERO, + host_type: ControlDbHostType::Wasm, + initial_program: target_program_hash, + }, + 7, + false, + )?; + let target_program_path = data + .path() + .join("program-bytes") + .join(program_bytes_relative_path(&target_program_hash)); + std::fs::create_dir_all( + target_program_path + .parent() + .context("program bytes path has no parent")?, + )?; + std::fs::write(&target_program_path, target_program_bytes)?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + + let err = restore_backup(backup.path(), &data_dir, false).unwrap_err(); + + assert!(format!("{err:#}").contains("initial_program")); + assert!(data.path().join("control-db").is_dir()); + assert!(target_program_path.is_file()); + assert!(!data.path().join("replicas/7").exists()); + Ok(()) + } + #[test] fn backup_restore_errors_when_empty_data_dir_lacks_server_state() -> anyhow::Result<()> { let backup = tempfile::tempdir()?; @@ -1224,8 +1883,7 @@ mod tests { 7, true, )?; - std::fs::create_dir_all(backup.path().join("server/program-bytes"))?; - std::fs::write(backup.path().join("server/program-bytes/program"), b"program")?; + write_test_program_bytes(&backup.path().join("server/program-bytes"))?; std::fs::write(backup.path().join("server/config.toml"), b"config")?; std::fs::write(backup.path().join("server/metadata.toml"), b"metadata")?; @@ -1251,13 +1909,14 @@ mod tests { .flush_every_ms(Some(50)) .mode(sled::Mode::HighThroughput) .open()?; - let database_record = sled::IVec::from(bsatn::to_vec(&ControlDbDatabase { + let database = ControlDbDatabase { id: 1, database_identity: other_identity, owner_identity: Identity::ZERO, host_type: ControlDbHostType::Wasm, initial_program: Hash::ZERO, - })?); + }; + let database_record = sled::IVec::from(encode_stored_control_db_database(&database)?); db.open_tree("database_by_identity")? .insert(manifest_identity.to_be_byte_array(), database_record.clone())?; db.open_tree("database")?.insert(1u64.to_be_bytes(), database_record)?; @@ -1272,8 +1931,7 @@ mod tests { )?; db.flush()?; drop(db); - std::fs::create_dir_all(backup.path().join("server/program-bytes"))?; - std::fs::write(backup.path().join("server/program-bytes/program"), b"program")?; + write_test_program_bytes(&backup.path().join("server/program-bytes"))?; std::fs::write(backup.path().join("server/config.toml"), b"config")?; std::fs::write(backup.path().join("server/metadata.toml"), b"metadata")?; @@ -1309,7 +1967,7 @@ mod tests { let data = tempfile::tempdir()?; let final_path = data.path().join("control-db"); let staged_path = staged_server_state_path(&final_path, 99); - let mut staged = StagedServerState { entries: Vec::new() }; + let mut staged = StagedServerState::new(); let err = copy_staged_server_state_entry( &mut staged, @@ -1348,19 +2006,38 @@ mod tests { fn make_target_data_dir(path: &Path, replica_id: u64) -> anyhow::Result<()> { make_test_control_db(&path.join("control-db"), test_database_identity()?, replica_id, false)?; - std::fs::create_dir_all(path.join("program-bytes"))?; + write_test_program_bytes(&path.join("program-bytes"))?; + std::fs::write(path.join("config.toml"), b"[commitlog]\nlog-format-version = 1\n")?; + std::fs::write(path.join("metadata.toml"), b"metadata")?; Ok(()) } fn make_backup_server_state(path: &Path) -> anyhow::Result<()> { make_test_control_db(&path.join("server/control-db"), test_database_identity()?, 7, false)?; - std::fs::create_dir_all(path.join("server/program-bytes"))?; - std::fs::write(path.join("server/program-bytes/program"), b"program")?; + write_test_program_bytes(&path.join("server/program-bytes"))?; std::fs::write(path.join("server/config.toml"), b"config")?; std::fs::write(path.join("server/metadata.toml"), b"metadata")?; Ok(()) } + fn test_program_bytes() -> &'static [u8] { + b"program" + } + + fn test_program_hash() -> Hash { + hash_bytes(test_program_bytes()) + } + + fn write_test_program_bytes(program_bytes_dir: &Path) -> anyhow::Result<()> { + let program_hash = test_program_hash(); + let path = program_bytes_dir.join(program_bytes_relative_path(&program_hash)); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, test_program_bytes())?; + Ok(()) + } + fn assert_no_restore_temps(data_dir: &Path) -> anyhow::Result<()> { for entry in std::fs::read_dir(data_dir.join("replicas"))? { let entry = entry?; @@ -1378,11 +2055,39 @@ mod tests { Ok(()) } + fn find_restore_old_dir(replicas_dir: &Path) -> anyhow::Result { + std::fs::read_dir(replicas_dir)? + .filter_map(Result::ok) + .find(|entry| entry.file_name().to_string_lossy().contains(".restore_old_")) + .map(|entry| entry.path()) + .context("expected recoverable restore-old directory") + } + fn make_test_control_db( path: &Path, database_identity: Identity, replica_id: u64, include_extra_database: bool, + ) -> anyhow::Result<()> { + make_test_control_db_with_database( + path, + ControlDbDatabase { + id: 1, + database_identity, + owner_identity: Identity::ZERO, + host_type: ControlDbHostType::Wasm, + initial_program: test_program_hash(), + }, + replica_id, + include_extra_database, + ) + } + + fn make_test_control_db_with_database( + path: &Path, + database: ControlDbDatabase, + replica_id: u64, + include_extra_database: bool, ) -> anyhow::Result<()> { std::fs::create_dir_all(path)?; let db = sled::Config::default() @@ -1390,22 +2095,18 @@ mod tests { .flush_every_ms(Some(50)) .mode(sled::Mode::HighThroughput) .open()?; - let database_record = sled::IVec::from(bsatn::to_vec(&ControlDbDatabase { - id: 1, - database_identity, - owner_identity: Identity::ZERO, - host_type: ControlDbHostType::Wasm, - initial_program: Hash::ZERO, - })?); + let database_identity = database.database_identity; + let database_id = database.id; + let database_record = sled::IVec::from(encode_stored_control_db_database(&database)?); db.open_tree("database_by_identity")? .insert(database_identity.to_be_byte_array(), database_record.clone())?; db.open_tree("database")? - .insert(1u64.to_be_bytes(), database_record.clone())?; + .insert(database_id.to_be_bytes(), database_record.clone())?; db.open_tree("replica")?.insert( replica_id.to_be_bytes(), bsatn::to_vec(&ControlDbReplica { id: replica_id, - database_id: 1, + database_id, node_id: 0, leader: true, })?, @@ -1413,13 +2114,14 @@ mod tests { if include_extra_database { let extra_identity: Identity = "c300000000000000000000000000000000000000000000000000000000000000".parse()?; - let extra_database_record = sled::IVec::from(bsatn::to_vec(&ControlDbDatabase { + let extra_database = ControlDbDatabase { id: 2, database_identity: extra_identity, owner_identity: Identity::ZERO, host_type: ControlDbHostType::Wasm, - initial_program: Hash::ZERO, - })?); + initial_program: test_program_hash(), + }; + let extra_database_record = sled::IVec::from(encode_stored_control_db_database(&extra_database)?); db.open_tree("database_by_identity")? .insert(extra_identity.to_be_byte_array(), extra_database_record.clone())?; db.open_tree("database")? @@ -1439,6 +2141,26 @@ mod tests { Ok(()) } + #[derive(BsatnSerialize)] + struct StoredControlDbDatabaseFixture { + id: u64, + database_identity: Identity, + owner_identity: Identity, + host_type: ControlDbHostType, + initial_program: Hash, + } + + fn encode_stored_control_db_database(database: &ControlDbDatabase) -> anyhow::Result> { + bsatn::to_vec(&StoredControlDbDatabaseFixture { + id: database.id, + database_identity: database.database_identity, + owner_identity: database.owner_identity, + host_type: database.host_type, + initial_program: database.initial_program, + }) + .map_err(Into::into) + } + fn test_database_identity() -> anyhow::Result { Ok("c200000000000000000000000000000000000000000000000000000000000000".parse()?) } diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 21d32ce8b24..cc9012efd39 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -12,7 +12,10 @@ use http::StatusCode; use spacetimedb::client::ClientActorIndex; use spacetimedb::energy::{EnergyBalance, EnergyQuanta}; -use spacetimedb::host::{HostController, MigratePlanResult, ModuleHost, NoSuchModule, UpdateDatabaseResult}; +use spacetimedb::host::{ + HostController, HotBackupInProgress, HotBackupManifest, MigratePlanResult, ModuleHost, NoSuchModule, + UpdateDatabaseResult, +}; use spacetimedb::identity::{AuthCtx, Identity}; use spacetimedb::messages::control_db::{Database, HostType, Node, Replica}; use spacetimedb::sql; @@ -60,11 +63,7 @@ pub trait NodeDelegate: Send + Sync { /// /// The [`Host`] is spawned implicitly if not already running. async fn leader(&self, database_id: u64) -> Result; - async fn create_hot_backup( - &self, - leader: Host, - output_dir: PathBuf, - ) -> anyhow::Result { + async fn create_hot_backup(&self, leader: Host, output_dir: PathBuf) -> anyhow::Result { leader.create_hot_backup(output_dir).await } /// Root directory under which HTTP-initiated hot backups may be written. @@ -123,10 +122,7 @@ impl Host { self.host_controller.get_module_host(self.replica_id).await } - pub async fn create_hot_backup( - &self, - output_dir: impl AsRef, - ) -> anyhow::Result { + pub async fn create_hot_backup(&self, output_dir: impl AsRef) -> anyhow::Result { let _ = output_dir; anyhow::bail!("hot backup requires the node delegate to export control-db and finalize the backup manifest") } @@ -134,16 +130,9 @@ impl Host { pub async fn create_hot_backup_without_control_db( &self, output_dir: impl AsRef, - ) -> anyhow::Result { - let module = self.module().await?; - module - .relational_db() - .create_hot_backup_without_control_db( - &self.host_controller.data_dir.replica(self.replica_id), - Some(&self.host_controller.data_dir), - self.replica_id, - output_dir, - ) + ) -> anyhow::Result { + self.host_controller + .create_hot_backup_without_control_db(self.replica_id, output_dir) .await } @@ -536,11 +525,7 @@ impl NodeDelegate for Arc { (**self).leader(database_id).await } - async fn create_hot_backup( - &self, - leader: Host, - output_dir: PathBuf, - ) -> anyhow::Result { + async fn create_hot_backup(&self, leader: Host, output_dir: PathBuf) -> anyhow::Result { (**self).create_hot_backup(leader, output_dir).await } @@ -621,6 +606,7 @@ pub enum Action { DeleteDatabase, RenameDatabase, ViewModuleLogs, + CreateHotBackup, } impl fmt::Display for Action { @@ -639,6 +625,7 @@ impl fmt::Display for Action { Self::DeleteDatabase => f.write_str("delete database"), Self::RenameDatabase => f.write_str("rename database"), Self::ViewModuleLogs => f.write_str("view module logs"), + Self::CreateHotBackup => f.write_str("create hot backup"), } } } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 9c94ce5070f..2992ec70128 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -737,12 +737,20 @@ pub async fn backup( where S: ControlStateDelegate + NodeDelegate + Authorization, { - let (leader, database) = find_leader_and_database(&worker_ctx, name_or_identity).await?; + let server_output_dir = resolve_hot_backup_output_dir(worker_ctx.hot_backup_root(), &server_output_dir)?; + let database_identity = name_or_identity.resolve(&worker_ctx).await?; + let database = worker_ctx_find_database(&worker_ctx, &database_identity) + .await? + .ok_or(NO_SUCH_DATABASE)?; worker_ctx - .authorize_action(auth.claims.identity, database.database_identity, Action::UpdateDatabase) + .authorize_action( + auth.claims.identity, + database.database_identity, + Action::CreateHotBackup, + ) .await?; - let server_output_dir = resolve_hot_backup_output_dir(worker_ctx.hot_backup_root(), &server_output_dir)?; + let leader = worker_ctx.leader(database.id).await.map_err(Into::into)?; let manifest = worker_ctx .create_hot_backup(leader, server_output_dir) .await @@ -1747,7 +1755,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::auth::JwtAuthProvider; + use crate::auth::{JwtAuthProvider, SpacetimeCreds}; use crate::routes::subscribe::{HasWebSocketOptions, WebSocketOptions}; use crate::{ Action, Authorization, ControlStateReadAccess, ControlStateWriteAccess, MaybeMisdirected, Unauthorized, @@ -1812,6 +1820,16 @@ mod tests { jwt: DummyJwtProvider, client_actor_index: std::sync::Arc, module_logs_dir: ModuleLogsDir, + database: Option, + hot_backup_root: Option, + leader_calls: std::sync::Arc, + authorize_action_result: DummyAuthorizeActionResult, + } + + #[derive(Clone, Copy)] + enum DummyAuthorizeActionResult { + Deny, + InternalError, } impl DummyState { @@ -1822,8 +1840,31 @@ mod tests { }, client_actor_index: std::sync::Arc::new(ClientActorIndex::new()), module_logs_dir: ModuleLogsDir::from_path_unchecked(std::env::temp_dir()), + database: None, + hot_backup_root: None, + leader_calls: std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)), + authorize_action_result: DummyAuthorizeActionResult::InternalError, } } + + fn with_database(mut self, database: Database) -> Self { + self.database = Some(database); + self + } + + fn with_hot_backup_root(mut self, root: PathBuf) -> Self { + self.hot_backup_root = Some(root); + self + } + + fn with_authorize_action_result(mut self, result: DummyAuthorizeActionResult) -> Self { + self.authorize_action_result = result; + self + } + + fn leader_calls(&self) -> usize { + self.leader_calls.load(std::sync::atomic::Ordering::SeqCst) + } } impl HasWebSocketOptions for DummyState { @@ -1850,9 +1891,14 @@ mod tests { } async fn leader(&self, _database_id: u64) -> Result { + self.leader_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); Err(DummyLeaderError) } + fn hot_backup_root(&self) -> Option { + self.hot_backup_root.clone() + } + fn module_logs_dir(&self, _replica_id: u64) -> ModuleLogsDir { self.module_logs_dir.clone() } @@ -1894,7 +1940,10 @@ mod tests { Ok(None) } async fn get_database_by_identity(&self, _database_identity: &Identity) -> anyhow::Result> { - Ok(None) + Ok(self + .database + .clone() + .filter(|database| database.database_identity == *_database_identity)) } async fn get_databases(&self) -> anyhow::Result> { Ok(Vec::new()) @@ -2000,11 +2049,21 @@ mod tests { impl Authorization for DummyState { async fn authorize_action( &self, - _subject: Identity, - _database: Identity, - _action: Action, + subject: Identity, + database: Identity, + action: Action, ) -> Result<(), Unauthorized> { - Err(Unauthorized::InternalError(anyhow::anyhow!("unused"))) + match self.authorize_action_result { + DummyAuthorizeActionResult::Deny => Err(Unauthorized::Unauthorized { + subject, + action, + database: Some(database), + source: None, + }), + DummyAuthorizeActionResult::InternalError => { + Err(Unauthorized::InternalError(anyhow::anyhow!("unused"))) + } + } } async fn authorize_sql(&self, _subject: Identity, _database: Identity) -> Result { @@ -2018,6 +2077,22 @@ mod tests { .status() } + fn test_auth(identity: Identity) -> SpacetimeAuth { + SpacetimeAuth { + creds: SpacetimeCreds::from_signed_token("unused.header.signature".to_owned()), + claims: spacetimedb::auth::identity::SpacetimeIdentityClaims { + identity, + subject: "subject".into(), + issuer: "issuer".into(), + audience: [].into(), + extra: None, + iat: std::time::SystemTime::now(), + exp: None, + }, + jwt_payload: "{}".into(), + } + } + #[test] fn backup_output_dir_requires_configured_root() { assert_eq!( @@ -2062,6 +2137,43 @@ mod tests { ); } + #[tokio::test] + async fn backup_authorizes_before_starting_leader() { + let root = tempfile::tempdir().unwrap(); + let subject = Identity::from_claims("issuer", "owner"); + let database_identity = Identity::from_claims("issuer", "database"); + let state = DummyState::new() + .with_hot_backup_root(root.path().to_path_buf()) + .with_database(Database { + id: 1, + database_identity, + owner_identity: subject, + host_type: HostType::Wasm, + initial_program: spacetimedb_lib::Hash::ZERO, + bootstrap_generation: 0, + }) + .with_authorize_action_result(DummyAuthorizeActionResult::Deny); + + let result = backup( + State(state.clone()), + Path(BackupParams { + name_or_identity: NameOrIdentity::Identity(database_identity.into()), + }), + Extension(test_auth(subject)), + axum::Json(BackupRequest { + server_output_dir: PathBuf::from("manual/backup-1"), + }), + ) + .await; + + let response = match result { + Ok(response) => response.into_response(), + Err(err) => axum::response::Result::<()>::Err(err).into_response(), + }; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(state.leader_calls(), 0); + } + /// Tests that requests to user-defined routes under `/database/:name-or-identity/routes` /// bypass the usual SpacetimeDB auth middleware, /// and accept requests with `Authorization` headers that SpacetimeDB would treat as malformed. diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 2f9c232bb0b..029988b47f7 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -1,3 +1,4 @@ +use super::hot_backup::{self, HotBackupInProgress}; use super::module_host::{DurableOffset, EventStatus, InitDatabaseResult, ModuleHost, ModuleInfo, NoSuchModule}; use super::scheduler::SchedulerStarter; use super::v8::V8HeapMetrics; @@ -46,6 +47,7 @@ use spacetimedb_schema::def::{ModuleDef, RawModuleDefVersion}; use spacetimedb_table::page_pool::PagePool; use std::future::Future; use std::ops::Deref; +use std::path::Path; use std::sync::Arc; use std::time::Duration; use tokio::sync::{watch, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock as AsyncRwLock}; @@ -690,6 +692,24 @@ impl HostController { .ok_or(NoSuchModule) } + /// Create a hot backup for `replica_id`, leaving scoped server state for + /// the caller to export before finalizing the manifest. + pub async fn create_hot_backup_without_control_db( + &self, + replica_id: u64, + output_dir: impl AsRef, + ) -> anyhow::Result { + let module = self.get_module_host(replica_id).await?; + hot_backup::create_hot_backup_without_control_db( + module.relational_db(), + &self.data_dir.replica(replica_id), + Some(&self.data_dir), + replica_id, + output_dir, + ) + .await + } + /// Subscribe to updates of the [`ModuleHost`] identified by `replica_id`, /// or return an error if it is not registered with the controller. /// diff --git a/crates/core/src/host/hot_backup.rs b/crates/core/src/host/hot_backup.rs new file mode 100644 index 00000000000..8c679d53ce9 --- /dev/null +++ b/crates/core/src/host/hot_backup.rs @@ -0,0 +1,1090 @@ +use crate::db::relational_db::{open_snapshot_repo, RelationalDB, TxOffset}; +use crate::util::asyncify; +use anyhow::{anyhow, Context}; +use parking_lot::Mutex; +use spacetimedb_commitlog::repo::Repo; +use spacetimedb_commitlog::{self as commitlog}; +use spacetimedb_fs_utils::{ + atomic_write_bytes, copy_dir_all, copy_file_sync, create_dir_all_sync, dir_size, normalize_absolute_path, sync_dir, +}; +use spacetimedb_lib::Identity; +use spacetimedb_paths::server::{CommitLogDir, ReplicaDir, ServerDataDir}; +use spacetimedb_paths::FromPathUnchecked; +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::fs::OpenOptions; +use std::io::{ErrorKind, Write as _}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +#[derive(Debug, serde::Serialize)] +pub struct HotBackupManifest { + pub version: u32, + pub database_identity: Identity, + pub replica_id: u64, + pub snapshot_offset: TxOffset, + pub durable_offset: TxOffset, + pub output_dir: PathBuf, + pub snapshot_ms: u64, + pub copy_ms: u64, + pub total_ms: u64, + pub bytes: u64, +} + +impl HotBackupManifest { + /// Convert a [`Duration`] to whole milliseconds, saturating at `u64::MAX`. + /// + /// Used to fill in the `*_ms` timing fields of the manifest. + pub fn elapsed_ms(duration: Duration) -> u64 { + duration.as_millis().try_into().unwrap_or(u64::MAX) + } +} + +/// A hot backup whose output directory is still being finalized. +/// +/// Dropping this value releases the in-process guard for its output directory. +/// Call [`Self::finalize_with_blocking`] to add caller-owned state and write the +/// final `manifest.json` while the guard is still held. +pub struct HotBackupInProgress { + manifest: HotBackupManifest, + _dir_guard: BackupDirGuard, +} + +impl HotBackupInProgress { + /// The manifest that will be written when this backup is finalized. + pub fn manifest(&self) -> &HotBackupManifest { + &self.manifest + } + + fn into_manifest(self) -> HotBackupManifest { + self.manifest + } + + /// Run caller-provided blocking finalization work, then write + /// `manifest.json` while still holding the backup output directory guard. + /// + /// The guard is moved into the blocking task with the manifest. If the + /// awaiting future is cancelled, the blocking task continues to own the + /// guard until the caller's work and final manifest write have returned. + pub async fn finalize_with_blocking(self, f: F) -> anyhow::Result + where + F: FnOnce(&mut HotBackupManifest) -> anyhow::Result<()> + Send + 'static, + { + asyncify(move || { + let mut backup = self; + f(&mut backup.manifest)?; + write_hot_backup_manifest_sync(&backup.manifest.output_dir, &backup.manifest)?; + Ok(backup.into_manifest()) + }) + .await + } +} + +/// Export a point-in-time hot backup into `output_dir`. +#[cfg(test)] +async fn create_hot_backup( + db: &RelationalDB, + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + replica_id: u64, + output_dir: impl AsRef, +) -> anyhow::Result { + let backup = create_hot_backup_inner(db, replica_dir, server_data_dir, replica_id, output_dir, true).await?; + Ok(backup.into_manifest()) +} + +/// Export a point-in-time hot backup into `output_dir`, leaving scoped server +/// state for the caller to provide. +/// +/// Unlike a finalized hot backup, this does **not** compute +/// [`HotBackupManifest::bytes`] (it is left at 0) and does **not** write +/// `manifest.json`: the caller is expected to add server state that must be +/// exported with broader context, such as `server/control-db`, and then finalize +/// the manifest itself. This both avoids traversing the backup tree twice and +/// makes `manifest.json` a reliable marker of a fully-written backup. +pub(super) async fn create_hot_backup_without_control_db( + db: &RelationalDB, + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + replica_id: u64, + output_dir: impl AsRef, +) -> anyhow::Result { + create_hot_backup_inner(db, replica_dir, server_data_dir, replica_id, output_dir, false).await +} + +async fn create_hot_backup_inner( + db: &RelationalDB, + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + replica_id: u64, + output_dir: impl AsRef, + copy_control_db: bool, +) -> anyhow::Result { + let total_start = Instant::now(); + let output_dir = ensure_backup_path(replica_dir, server_data_dir, output_dir.as_ref())?; + // Guard against two concurrent backups interleaving writes into the same + // directory after both passed the `ensure_empty_dir` check. + let dir_guard = BackupDirGuard::acquire(&output_dir)?; + anyhow::ensure!( + !(copy_control_db && server_data_dir.is_some()), + "copying live server/control-db is not supported by hot backup; export control-db separately and finalize the manifest after the export" + ); + + let snapshot_start = Instant::now(); + let snapshot_offset = db.request_hot_backup_snapshot().await?; + let snapshot_ms = HotBackupManifest::elapsed_ms(snapshot_start.elapsed()); + let durable_offset = snapshot_offset; + + let copy_start = Instant::now(); + let snapshot_repo = open_snapshot_repo(replica_dir.snapshots(), db.database_identity(), replica_id)?; + let src_snapshot_dir = snapshot_repo.snapshot_dir_path(snapshot_offset); + let dst_snapshots = output_dir.join("snapshots"); + let dst_snapshot_dir = dst_snapshots.join( + src_snapshot_dir + .0 + .file_name() + .context("snapshot directory has no file name")?, + ); + // `request_hot_backup_snapshot` returns only after the snapshot worker has + // fsynced the snapshot to disk (see `SnapshotWorker::ensure_snapshot_at_least`), + // so both paths must already exist; a missing path is a bug, not a timing issue. + anyhow::ensure!( + src_snapshot_dir.0.is_dir(), + "snapshot directory does not exist: {}", + src_snapshot_dir.display() + ); + let src_snapshot_file = src_snapshot_dir.snapshot_file(snapshot_offset); + anyhow::ensure!( + src_snapshot_file.0.is_file(), + "snapshot file does not exist: {}", + src_snapshot_file.display() + ); + let snapshot_file_name: OsString = src_snapshot_file + .0 + .file_name() + .context("snapshot file has no file name")? + .to_owned(); + + asyncify({ + let output_dir = output_dir.clone(); + let src_snapshot_dir = src_snapshot_dir.clone(); + let snapshot_file_name = snapshot_file_name.clone(); + let dir_guard = dir_guard.clone(); + move || -> anyhow::Result<()> { + ensure_empty_dir(&output_dir, &dir_guard)?; + copy_dir_all_retry(&src_snapshot_dir.0, &dst_snapshot_dir, &snapshot_file_name)?; + Ok(()) + } + }) + .await?; + + if let Some(server_data_dir) = server_data_dir { + copy_server_state( + server_data_dir, + &output_dir, + copy_control_db, + commitlog::DEFAULT_LOG_FORMAT_VERSION, + dir_guard.clone(), + ) + .await?; + } + copy_commitlog_range( + replica_dir.commit_log(), + output_dir.join("clog"), + snapshot_offset, + dir_guard.clone(), + ) + .await?; + let copy_ms = HotBackupManifest::elapsed_ms(copy_start.elapsed()); + + let mut manifest = HotBackupManifest { + version: 1, + database_identity: db.database_identity(), + replica_id, + snapshot_offset, + durable_offset, + output_dir: output_dir.clone(), + snapshot_ms, + copy_ms, + total_ms: HotBackupManifest::elapsed_ms(total_start.elapsed()), + bytes: 0, + }; + // When the caller provides `server/control-db` itself, leave `bytes` + // and `manifest.json` to it, so the backup tree is only traversed once + // and the manifest is written exactly once, after the backup is complete. + if copy_control_db { + manifest.bytes = asyncify({ + let output_dir = output_dir.clone(); + let dir_guard = dir_guard.clone(); + move || { + let _dir_guard = dir_guard; + dir_size(&output_dir) + } + }) + .await?; + write_hot_backup_manifest_guarded(&output_dir, &manifest, dir_guard.clone()).await?; + } + Ok(HotBackupInProgress { + manifest, + _dir_guard: dir_guard, + }) +} + +async fn copy_commitlog_range( + src: CommitLogDir, + dst: PathBuf, + through: TxOffset, + dir_guard: BackupDirGuard, +) -> anyhow::Result<()> { + asyncify(move || -> anyhow::Result<()> { + let _dir_guard = dir_guard; + copy_commitlog_range_sync(src, &dst, through)?; + sync_dir(&dst).with_context(|| format!("syncing backup commitlog dir {}", dst.display()))?; + if let Some(parent) = dst.parent() { + sync_dir(parent).with_context(|| format!("syncing backup commitlog parent {}", parent.display()))?; + } + Ok(()) + }) + .await?; + Ok(()) +} + +fn copy_commitlog_range_sync(src: CommitLogDir, dst: &Path, through: TxOffset) -> anyhow::Result<()> { + let src_repo = commitlog::repo::Fs::new(src, None)?; + let dst_repo = commitlog::repo::Fs::new(CommitLogDir::from_path_unchecked(dst), None)?; + let mut copied_any_commit = false; + let mut next_tx_offset = None; + for segment_offset in src_repo.existing_offsets()? { + let reader = + commitlog::repo::open_segment_reader(&src_repo, commitlog::DEFAULT_LOG_FORMAT_VERSION, segment_offset) + .with_context(|| format!("opening source commitlog segment {segment_offset}"))?; + let mut writer = None; + for commit in reader.commits() { + let commit = commit.with_context(|| format!("reading source commitlog segment {segment_offset}"))?; + if commit.min_tx_offset > through { + break; + } + let commit_range = commit.tx_range(); + if let Some(expected_tx_offset) = next_tx_offset { + anyhow::ensure!( + commit.min_tx_offset == expected_tx_offset, + "source commitlog is not contiguous at segment {segment_offset}: expected tx offset {expected_tx_offset}, got {}", + commit.min_tx_offset + ); + } + next_tx_offset = Some(commit_range.end); + let writer = writer.get_or_insert_with(|| { + dst_repo.create_segment( + segment_offset, + commitlog::segment::Header { + log_format_version: commitlog::DEFAULT_LOG_FORMAT_VERSION, + checksum_algorithm: commitlog::Commit::CHECKSUM_ALGORITHM, + }, + ) + }); + let writer = writer + .as_mut() + .map_err(|err| anyhow!("creating target commitlog segment {segment_offset}: {err}"))?; + commitlog::Commit::from(commit) + .write(&mut *writer) + .with_context(|| format!("writing target commitlog segment {segment_offset}"))?; + copied_any_commit = true; + } + if let Some(writer) = writer { + let mut writer = writer.with_context(|| format!("creating target commitlog segment {segment_offset}"))?; + writer + .flush() + .with_context(|| format!("flushing target commitlog segment {segment_offset}"))?; + writer + .sync_data() + .with_context(|| format!("syncing target commitlog segment {segment_offset}"))?; + } + } + anyhow::ensure!( + copied_any_commit, + "source commitlog does not contain commits through snapshot offset {through}" + ); + Ok(()) +} + +async fn write_hot_backup_manifest_guarded( + output_dir: &Path, + manifest: &HotBackupManifest, + dir_guard: BackupDirGuard, +) -> anyhow::Result<()> { + let output_dir = output_dir.to_path_buf(); + let json = serde_json::to_vec_pretty(manifest)?; + asyncify(move || { + let _dir_guard = dir_guard; + write_hot_backup_manifest_bytes(&output_dir, &json) + }) + .await?; + Ok(()) +} + +fn write_hot_backup_manifest_sync(output_dir: &Path, manifest: &HotBackupManifest) -> anyhow::Result<()> { + let json = serde_json::to_vec_pretty(manifest)?; + write_hot_backup_manifest_bytes(output_dir, &json) +} + +fn write_hot_backup_manifest_bytes(output_dir: &Path, json: &[u8]) -> anyhow::Result<()> { + atomic_write_bytes(&output_dir.join("manifest.json"), json) + .with_context(|| format!("writing hot backup manifest in {}", output_dir.display())) +} + +const BACKUP_DIR_CLAIM_MARKER: &str = ".spacetimedb-hot-backup-in-progress"; + +/// Serializes concurrent hot backups into overlapping output directories in +/// this process and claims the exact output directory across processes. +/// +/// Two concurrent backups into the same (empty) directory would both pass the +/// `ensure_empty_dir` check and interleave their writes, producing a corrupt +/// backup. The process-wide map rejects overlapping paths locally, while a +/// hidden `create_new` marker atomically claims the exact path on the +/// filesystem. Both claims are released when the last guard clone drops. +struct BackupDirGuard(PathBuf); + +static ACTIVE_BACKUP_DIRS: Mutex> = Mutex::new(BTreeMap::new()); + +impl BackupDirGuard { + fn acquire(output_dir: &Path) -> anyhow::Result { + let mut active = ACTIVE_BACKUP_DIRS.lock(); + if let Some(overlapping) = active + .keys() + .find(|active_dir| backup_paths_overlap(output_dir, active_dir)) + { + anyhow::bail!( + "backup output path {} overlaps with in-progress backup {}", + output_dir.display(), + overlapping.display() + ); + } + + ensure_no_claimed_ancestor(output_dir)?; + create_dir_all_sync(output_dir) + .with_context(|| format!("creating backup output directory {}", output_dir.display()))?; + let marker_path = Self::marker_path_for(output_dir); + let marker = match OpenOptions::new().write(true).create_new(true).open(&marker_path) { + Ok(marker) => marker, + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + anyhow::bail!( + "backup output directory {} is already claimed by marker {}", + output_dir.display(), + marker_path.display() + ) + } + Err(err) => { + return Err(err).with_context(|| format!("claiming backup output directory {}", output_dir.display())) + } + }; + let claim_result = marker + .sync_all() + .map_err(anyhow::Error::from) + .and_then(|()| ensure_no_claimed_ancestor(output_dir)) + .and_then(|()| sync_dir(output_dir).map_err(anyhow::Error::from)); + drop(marker); + if let Err(err) = claim_result { + let _ = std::fs::remove_file(&marker_path); + let _ = sync_dir(output_dir); + return Err(err).with_context(|| format!("persisting backup output claim {}", marker_path.display())); + } + + active.insert(output_dir.to_path_buf(), 1); + Ok(Self(output_dir.to_path_buf())) + } + + fn marker_path_for(output_dir: &Path) -> PathBuf { + output_dir.join(BACKUP_DIR_CLAIM_MARKER) + } + + fn marker_path(&self) -> PathBuf { + Self::marker_path_for(&self.0) + } +} + +fn ensure_no_claimed_ancestor(output_dir: &Path) -> anyhow::Result<()> { + for ancestor in output_dir.ancestors().skip(1) { + let marker_path = BackupDirGuard::marker_path_for(ancestor); + anyhow::ensure!( + !marker_path + .try_exists() + .with_context(|| format!("checking backup output claim {}", marker_path.display()))?, + "backup output directory {} is inside claimed backup directory {}", + output_dir.display(), + ancestor.display() + ); + } + Ok(()) +} + +impl Clone for BackupDirGuard { + fn clone(&self) -> Self { + let mut active = ACTIVE_BACKUP_DIRS.lock(); + let count = active + .get_mut(&self.0) + .expect("cloning active backup directory guard after it was released"); + *count = count + .checked_add(1) + .expect("active backup directory guard refcount overflow"); + Self(self.0.clone()) + } +} + +#[cfg(windows)] +fn backup_path_components(path: &Path) -> Vec { + path.components() + .map(|component| component.as_os_str().to_string_lossy().to_ascii_lowercase()) + .collect() +} + +#[cfg(windows)] +fn backup_path_starts_with(path: &Path, base: &Path) -> bool { + let path = backup_path_components(path); + let base = backup_path_components(base); + base.len() <= path.len() && path.iter().zip(&base).all(|(path, base)| path == base) +} + +#[cfg(not(windows))] +fn backup_path_starts_with(path: &Path, base: &Path) -> bool { + path.starts_with(base) +} + +fn backup_paths_overlap(a: &Path, b: &Path) -> bool { + backup_path_starts_with(a, b) || backup_path_starts_with(b, a) +} + +impl Drop for BackupDirGuard { + fn drop(&mut self) { + let mut active = ACTIVE_BACKUP_DIRS.lock(); + let Some(count) = active.get_mut(&self.0) else { + panic!("dropping backup directory guard after it was released"); + }; + if *count == 1 { + let _ = std::fs::remove_file(self.marker_path()); + let _ = sync_dir(&self.0); + active.remove(&self.0); + } else { + *count -= 1; + } + } +} + +fn ensure_empty_dir(path: &Path, dir_guard: &BackupDirGuard) -> anyhow::Result<()> { + anyhow::ensure!( + path == dir_guard.0.as_path(), + "backup output guard does not own directory: {}", + path.display() + ); + anyhow::ensure!( + path.is_dir(), + "backup output path is not a directory: {}", + path.display() + ); + + let marker_path = dir_guard.marker_path(); + let mut found_marker = false; + for entry in path.read_dir()? { + let entry = entry?; + anyhow::ensure!( + entry.path() == marker_path, + "backup output directory must be empty: {}", + path.display() + ); + let file_type = entry.file_type()?; + let metadata = entry.metadata()?; + anyhow::ensure!( + file_type.is_file() && metadata.len() == 0, + "backup output claim marker is invalid: {}", + marker_path.display() + ); + found_marker = true; + } + anyhow::ensure!( + found_marker, + "backup output claim marker is missing: {}", + marker_path.display() + ); + Ok(()) +} + +/// Validate `output_dir` as a hot backup destination and return its normalized +/// form. +/// +/// The path must be absolute and, after resolving symlinks in its existing +/// ancestors and rejecting `..` components (see +/// [`spacetimedb_fs_utils::normalize_absolute_path`]), must not point inside +/// the server data directory or the replica directory. All subsequent backup +/// I/O must use the returned path, so the checks cannot be bypassed via +/// symlinks or path traversal. +fn ensure_backup_path( + replica_dir: &ReplicaDir, + server_data_dir: Option<&ServerDataDir>, + output_dir: &Path, +) -> anyhow::Result { + anyhow::ensure!( + output_dir.is_absolute(), + "backup output directory must be an absolute server path: {}", + output_dir.display() + ); + let output_dir = normalize_absolute_path(output_dir).context("normalizing the backup output directory path")?; + if let Some(server_data_dir) = server_data_dir { + let data_dir = normalize_absolute_path(&server_data_dir.0).unwrap_or_else(|_| server_data_dir.0.to_path_buf()); + anyhow::ensure!( + !output_dir.starts_with(&data_dir), + "backup output directory must not be inside the server data directory: {}", + output_dir.display() + ); + } + let replica_dir_normalized = + normalize_absolute_path(&replica_dir.0).unwrap_or_else(|_| replica_dir.0.to_path_buf()); + anyhow::ensure!( + !output_dir.starts_with(&replica_dir_normalized), + "backup output directory must not be inside the replica directory: {}", + output_dir.display() + ); + Ok(output_dir) +} + +async fn copy_server_state( + data_dir: &ServerDataDir, + output_dir: &Path, + copy_control_db: bool, + output_log_format_version: u8, + dir_guard: BackupDirGuard, +) -> anyhow::Result<()> { + let data_dir = data_dir.clone(); + let output_dir = output_dir.to_path_buf(); + asyncify(move || -> anyhow::Result<()> { + let _dir_guard = dir_guard; + assert!(!copy_control_db); + let server_dir = output_dir.join("server"); + create_dir_all_sync(&server_dir)?; + + copy_server_config( + &data_dir.config_toml().0, + &server_dir.join("config.toml"), + output_log_format_version, + )?; + copy_required_file(&data_dir.metadata_toml().0, &server_dir.join("metadata.toml"))?; + sync_dir(&server_dir)?; + Ok(()) + }) + .await?; + Ok(()) +} + +fn copy_server_config(src: &Path, dst: &Path, output_log_format_version: u8) -> anyhow::Result<()> { + anyhow::ensure!(src.is_file(), "server state file is missing: {}", src.display()); + let source = std::fs::read_to_string(src).with_context(|| format!("reading {}", src.display()))?; + let mut config: toml::Table = + toml::from_str(&source).with_context(|| format!("parsing server config {}", src.display()))?; + let commitlog = config + .entry("commitlog".to_owned()) + .or_insert_with(|| toml::Value::Table(Default::default())) + .as_table_mut() + .with_context(|| format!("server config [commitlog] is not a table in {}", src.display()))?; + commitlog.insert( + "log-format-version".to_owned(), + toml::Value::Integer(i64::from(output_log_format_version)), + ); + + let serialized = toml::to_string_pretty(&config).context("serializing backup server config")?; + atomic_write_bytes(dst, serialized.as_bytes()) + .with_context(|| format!("writing backup server config {}", dst.display()))?; + Ok(()) +} + +fn copy_required_file(src: &Path, dst: &Path) -> anyhow::Result<()> { + anyhow::ensure!(src.is_file(), "server state file is missing: {}", src.display()); + if let Some(parent) = dst.parent() { + create_dir_all_sync(parent)?; + } + copy_file_sync(src, dst).with_context(|| format!("copying {} to {}", src.display(), dst.display()))?; + Ok(()) +} + +fn copy_dir_all_retry(src: &Path, dst: &Path, required_file_name: &OsString) -> anyhow::Result<()> { + const RETRY_LIMIT: u32 = 3_000; + const RETRY_SLEEP: Duration = Duration::from_millis(10); + + let mut last_err = None; + for _ in 0..RETRY_LIMIT { + match copy_snapshot_dir(src, dst, required_file_name) { + Ok(()) => return Ok(()), + Err(err) => { + last_err = Some(err); + std::thread::sleep(RETRY_SLEEP); + } + } + } + let err = last_err.expect("copy_dir_all_retry loop must run at least once"); + Err(err).with_context(|| format!("copying snapshot {} to {}", src.display(), dst.display())) +} + +fn copy_snapshot_dir(src: &Path, dst: &Path, required_file_name: &OsString) -> anyhow::Result<()> { + create_dir_all_sync(dst)?; + let src_required = src.join(required_file_name); + let dst_required = dst.join(required_file_name); + copy_file_sync(&src_required, &dst_required) + .with_context(|| format!("copying {} to {}", src_required.display(), dst_required.display()))?; + + for entry in std::fs::read_dir(src).with_context(|| format!("reading {}", src.display()))? { + let entry = entry?; + if entry.file_name() == *required_file_name { + continue; + } + let ty = entry.file_type()?; + let dst = dst.join(entry.file_name()); + if ty.is_dir() { + copy_dir_all(entry.path(), dst)?; + } else { + copy_file_sync(&entry.path(), &dst)?; + } + } + anyhow::ensure!( + dst_required.is_file(), + "copying snapshot {} to {} did not copy {}", + src.display(), + dst.display(), + required_file_name.to_string_lossy() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::disallowed_macros)] + + use super::*; + use crate::db::relational_db::tests_utils::{begin_mut_tx, TestDB}; + use crate::db::relational_db::Txdata; + use spacetimedb_commitlog::Commitlog; + use spacetimedb_lib::db::raw_def::v9::{RawModuleDefV9Builder, RawTableDefBuilder}; + use spacetimedb_paths::server::{CommitLogDir, SnapshotsPath}; + use spacetimedb_paths::FromPathUnchecked; + use spacetimedb_primitives::TableId; + use spacetimedb_sats::raw_identifier::RawIdentifier; + use spacetimedb_sats::{AlgebraicType, ProductType}; + use spacetimedb_schema::def::ModuleDef; + use spacetimedb_schema::schema::{Schema as _, TableSchema}; + use spacetimedb_table::page_pool::PagePool; + + const HOT_BACKUP_TEST_TIMEOUT: Duration = Duration::from_secs(60); + + fn my_table(col_type: AlgebraicType) -> TableSchema { + table("MyTable", ProductType::from([("my_col", col_type)]), |builder| builder) + } + + fn table( + name: &str, + columns: ProductType, + f: impl FnOnce(RawTableDefBuilder<'_>) -> RawTableDefBuilder, + ) -> TableSchema { + let mut builder = RawModuleDefV9Builder::new(); + f(builder.build_table_with_new_type(RawIdentifier::new(name), columns, true)); + let raw = builder.finish(); + let def: ModuleDef = raw.try_into().expect("table validation failed"); + let table = def.table(name).expect("table not found"); + TableSchema::from_module_def(&def, table, (), TableId::SENTINEL) + } + + fn add_table(stdb: &RelationalDB) -> anyhow::Result<()> { + let mut tx = begin_mut_tx(stdb); + stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; + stdb.commit_tx(tx)?; + Ok(()) + } + + fn write_test_server_state(data_dir: &Path, config: &str) -> std::io::Result<()> { + std::fs::write(data_dir.join("config.toml"), config)?; + std::fs::write(data_dir.join("metadata.toml"), b"metadata")?; + Ok(()) + } + + #[test] + fn hot_backup_create_writes_manifest_snapshot_and_commitlog() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + add_table(&stdb)?; + + let backup_dir = tempfile::tempdir()?; + let manifest = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + create_hot_backup(&stdb, stdb.path().unwrap(), None, 0, backup_dir.path()), + ) + .await + })??; + + assert!(backup_dir.path().join("manifest.json").is_file()); + assert!(manifest.bytes > 0); + assert_eq!(manifest.durable_offset, manifest.snapshot_offset); + let manifest_json: serde_json::Value = + serde_json::from_slice(&std::fs::read(backup_dir.path().join("manifest.json"))?)?; + assert_eq!( + manifest_json["snapshot_offset"].as_u64(), + Some(manifest.snapshot_offset) + ); + assert_eq!(manifest_json["durable_offset"].as_u64(), Some(manifest.snapshot_offset)); + let repo = open_snapshot_repo( + SnapshotsPath::from_path_unchecked(backup_dir.path().join("snapshots")), + stdb.database_identity(), + 0, + )?; + let snapshot = repo.read_snapshot(manifest.snapshot_offset, &PagePool::new_for_test())?; + assert_eq!(snapshot.database_identity, manifest.database_identity); + assert_eq!(snapshot.replica_id, manifest.replica_id); + assert_eq!(snapshot.tx_offset, manifest.snapshot_offset); + let clog = Commitlog::::open( + CommitLogDir::from_path_unchecked(backup_dir.path().join("clog")), + Default::default(), + None, + )?; + assert_eq!(clog.max_committed_offset(), Some(manifest.snapshot_offset)); + + Ok(()) + } + + #[test] + fn hot_backup_create_without_control_db_rewrites_v0_server_config() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + add_table(&stdb)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + let source_config = r#"[commitlog] +log-format-version = 0 +max-segment-size = 4096 + +[logs] +directives = ["info"] +"#; + write_test_server_state(data.path(), source_config)?; + std::fs::create_dir_all(data.path().join("control-db"))?; + std::fs::write(data.path().join("control-db/control"), b"control")?; + + let backup_dir = tempfile::tempdir()?; + let backup = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + create_hot_backup_without_control_db( + &stdb, + stdb.path().unwrap(), + Some(&data_dir), + 0, + backup_dir.path(), + ), + ) + .await + })??; + + assert_eq!(backup.manifest().bytes, 0); + assert!(!backup_dir.path().join("manifest.json").exists()); + assert_eq!(std::fs::read_to_string(data.path().join("config.toml"))?, source_config); + let backup_config: toml::Value = + toml::from_str(&std::fs::read_to_string(backup_dir.path().join("server/config.toml"))?)?; + assert_eq!( + backup_config["commitlog"]["log-format-version"].as_integer(), + Some(i64::from(commitlog::DEFAULT_LOG_FORMAT_VERSION)) + ); + assert_eq!(backup_config["commitlog"]["max-segment-size"].as_integer(), Some(4096)); + assert_eq!(backup_config["logs"]["directives"][0].as_str(), Some("info")); + assert_eq!( + std::fs::read_to_string(backup_dir.path().join("server/metadata.toml"))?, + "metadata" + ); + assert!(!backup_dir.path().join("server/control-db").exists()); + assert!(!backup_dir.path().join("server/program-bytes").exists()); + + let output_commitlog = + commitlog::repo::Fs::new(CommitLogDir::from_path_unchecked(backup_dir.path().join("clog")), None)?; + let segment_offset = output_commitlog + .existing_offsets()? + .into_iter() + .next() + .context("backup commitlog has no segments")?; + let segment = output_commitlog.open_segment_reader(segment_offset)?; + let header = commitlog::segment::Header::decode(segment)?; + assert_eq!(header.log_format_version, commitlog::DEFAULT_LOG_FORMAT_VERSION); + + Ok(()) + } + + #[test] + fn hot_backup_without_control_db_keeps_output_guard_until_manifest_finalization() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + add_table(&stdb)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + write_test_server_state(data.path(), "[commitlog]\nlog-format-version = 1\n")?; + + let backup_dir = tempfile::tempdir()?; + let backup = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + create_hot_backup_without_control_db( + &stdb, + stdb.path().unwrap(), + Some(&data_dir), + 0, + backup_dir.path(), + ), + ) + .await + })??; + let output_dir = backup.manifest().output_dir.clone(); + + let err = match BackupDirGuard::acquire(&output_dir.join("nested")) { + Ok(_) => panic!("overlapping backup directory guard was accepted"), + Err(err) => err, + }; + assert!(err.to_string().contains("overlaps")); + + drop(backup); + let guard = BackupDirGuard::acquire(&output_dir.join("nested"))?; + drop(guard); + + Ok(()) + } + + #[test] + fn hot_backup_finalize_with_blocking_keeps_output_guard_after_cancellation() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + add_table(&stdb)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + write_test_server_state(data.path(), "[commitlog]\nlog-format-version = 1\n")?; + + let backup_dir = tempfile::tempdir()?; + let backup = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + create_hot_backup_without_control_db( + &stdb, + stdb.path().unwrap(), + Some(&data_dir), + 0, + backup_dir.path(), + ), + ) + .await + })??; + let output_dir = backup.manifest().output_dir.clone(); + + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let (finish_tx, finish_rx) = std::sync::mpsc::channel(); + let handle = stdb.runtime().unwrap().spawn(async move { + backup + .finalize_with_blocking(move |_| { + started_tx + .send(()) + .map_err(|_| anyhow!("finalize start receiver dropped"))?; + finish_rx + .recv() + .context("waiting for test to release hot backup finalize")?; + Ok(()) + }) + .await + }); + + started_rx + .recv_timeout(HOT_BACKUP_TEST_TIMEOUT) + .context("hot backup finalize did not start")?; + handle.abort(); + + let err = match BackupDirGuard::acquire(&output_dir.join("nested")) { + Ok(_) => panic!("overlapping backup directory guard was accepted after cancellation"), + Err(err) => err, + }; + assert!(err.to_string().contains("overlaps")); + + finish_tx + .send(()) + .map_err(|_| anyhow!("finalize blocking task finished before release signal"))?; + let deadline = Instant::now() + HOT_BACKUP_TEST_TIMEOUT; + loop { + match BackupDirGuard::acquire(&output_dir.join("nested")) { + Ok(guard) => { + drop(guard); + break; + } + Err(err) if err.to_string().contains("overlaps") && Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(err) => return Err(err), + } + } + assert!(output_dir.join("manifest.json").is_file()); + + Ok(()) + } + + #[test] + fn hot_backup_rejects_raw_copy_of_live_control_db() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + add_table(&stdb)?; + + let data = tempfile::tempdir()?; + let data_dir = ServerDataDir::from_path_unchecked(data.path()); + write_test_server_state(data.path(), "[commitlog]\nlog-format-version = 1\n")?; + std::fs::create_dir_all(data.path().join("control-db"))?; + std::fs::write(data.path().join("control-db/control"), b"control")?; + + let backup_dir = tempfile::tempdir()?; + let result = stdb.runtime().unwrap().block_on(async { + tokio::time::timeout( + HOT_BACKUP_TEST_TIMEOUT, + create_hot_backup(&stdb, stdb.path().unwrap(), Some(&data_dir), 0, backup_dir.path()), + ) + .await + })?; + let err = result.unwrap_err(); + + assert!(err.to_string().contains("control-db")); + assert!(!backup_dir.path().join("manifest.json").exists()); + Ok(()) + } + + #[test] + fn hot_backup_rejects_unsafe_output_dirs() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + let replica_dir = stdb.path().unwrap(); + + let err = stdb + .runtime() + .unwrap() + .block_on(create_hot_backup( + &stdb, + replica_dir, + None, + 0, + PathBuf::from("relative"), + )) + .unwrap_err(); + assert!(err.to_string().contains("absolute server path")); + + let err = stdb + .runtime() + .unwrap() + .block_on(create_hot_backup( + &stdb, + replica_dir, + None, + 0, + replica_dir.0.join("backup"), + )) + .unwrap_err(); + assert!(err.to_string().contains("replica directory")); + + Ok(()) + } + + #[test] + fn hot_backup_dir_guard_rejects_parent_child_overlap() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let parent = temp.path().join("backup"); + let child = parent.join("nested"); + + let parent_guard = BackupDirGuard::acquire(&parent)?; + let Err(err) = BackupDirGuard::acquire(&child) else { + panic!("child backup directory overlapped active parent"); + }; + assert!(err.to_string().contains("overlaps")); + drop(parent_guard); + + let child_guard = BackupDirGuard::acquire(&child)?; + let Err(err) = BackupDirGuard::acquire(&parent) else { + panic!("parent backup directory overlapped active child"); + }; + assert!(err.to_string().contains("overlaps")); + drop(child_guard); + + Ok(()) + } + + #[test] + fn hot_backup_dir_guard_clone_keeps_path_active() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let parent = temp.path().join("backup"); + let child = parent.join("nested"); + + let parent_guard = BackupDirGuard::acquire(&parent)?; + let blocking_guard = parent_guard.clone(); + drop(parent_guard); + + let Err(err) = BackupDirGuard::acquire(&child) else { + panic!("child backup directory overlapped cloned active parent"); + }; + assert!(err.to_string().contains("overlaps")); + + drop(blocking_guard); + let child_guard = BackupDirGuard::acquire(&child)?; + drop(child_guard); + + Ok(()) + } + + #[test] + fn hot_backup_dir_guard_rejects_existing_filesystem_claim() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let output_dir = temp.path().join("backup"); + std::fs::create_dir(&output_dir)?; + let marker_path = BackupDirGuard::marker_path_for(&output_dir); + std::fs::write(&marker_path, [])?; + + let Err(err) = BackupDirGuard::acquire(&output_dir) else { + panic!("existing filesystem claim marker was accepted"); + }; + assert!(err.to_string().contains("already claimed")); + assert!(marker_path.is_file()); + + Ok(()) + } + + #[test] + fn hot_backup_dir_guard_rejects_claimed_ancestor() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let parent = temp.path().join("parent"); + std::fs::create_dir(&parent)?; + let marker_path = BackupDirGuard::marker_path_for(&parent); + std::fs::write(&marker_path, [])?; + + let child = parent.join("child"); + let err = match BackupDirGuard::acquire(&child) { + Ok(_) => panic!("backup directory inside a claimed ancestor was accepted"), + Err(err) => err, + }; + + assert!(err.to_string().contains("inside claimed backup directory")); + assert!(!child.exists()); + Ok(()) + } + + #[test] + fn hot_backup_dir_guard_marker_is_only_allowed_entry_and_cleans_up() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let output_dir = temp.path().join("backup"); + let guard = BackupDirGuard::acquire(&output_dir)?; + let marker_path = guard.marker_path(); + + assert!(marker_path.is_file()); + assert_eq!(std::fs::metadata(&marker_path)?.len(), 0); + ensure_empty_dir(&output_dir, &guard)?; + + let other_path = output_dir.join("other"); + std::fs::write(&other_path, b"not empty")?; + let err = ensure_empty_dir(&output_dir, &guard).unwrap_err(); + assert!(err.to_string().contains("must be empty")); + std::fs::remove_file(other_path)?; + + let cloned_guard = guard.clone(); + drop(guard); + assert!(marker_path.is_file()); + drop(cloned_guard); + assert!(!marker_path.exists()); + assert!(output_dir.read_dir()?.next().is_none()); + + Ok(()) + } +} diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 68f6035df80..4ddd2ea0996 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -12,6 +12,7 @@ use spacetimedb_schema::def::ModuleDef; mod disk_storage; mod host_controller; +mod hot_backup; mod module_common; #[allow(clippy::too_many_arguments)] pub mod module_host; @@ -29,6 +30,7 @@ pub use host_controller::{ HostController, HostRuntimeConfig, MigratePlanResult, ModuleHostWithBootstrap, ProcedureCallResult, ProgramStorage, ReducerCallResult, ReducerCallResultWithTxOffset, ReducerOutcome, }; +pub use hot_backup::{HotBackupInProgress, HotBackupManifest}; pub use module_host::{ InitDatabaseResult, ModuleHost, NoSuchModule, ProcedureCallError, ReducerCallError, UpdateDatabaseResult, }; diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index dee353c444b..740e719b388 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -24,9 +24,8 @@ once_cell.workspace = true parking_lot.workspace = true prometheus.workspace = true serde.workspace = true -serde_json.workspace = true sled.workspace = true -spacetimedb-commitlog = { workspace = true, features = ["streaming"] } +spacetimedb-commitlog.workspace = true spacetimedb-data-structures.workspace = true spacetimedb-datastore.workspace = true spacetimedb-durability.workspace = true @@ -45,8 +44,6 @@ sqlparser.workspace = true tempfile.workspace = true thiserror.workspace = true tracing.workspace = true -tokio.workspace = true -tokio-util = { workspace = true, features = ["io-util"] } [dev-dependencies] bytes.workspace = true diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index 5f5a88f814e..7efbc4b168f 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -7,9 +7,8 @@ use crate::util::asyncify; use crate::MetricsRecorderQueue; use anyhow::{anyhow, Context}; use enum_map::EnumMap; -use parking_lot::Mutex; use spacetimedb_commitlog::repo::OnNewSegmentFn; -use spacetimedb_commitlog::{self as commitlog, Commitlog, SizeOnDisk}; +use spacetimedb_commitlog::{self as commitlog, Commitlog, CompressionStats, SizeOnDisk}; use spacetimedb_data_structures::map::HashSet; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::error::{DatastoreError, TableError, ViewError}; @@ -37,18 +36,13 @@ use spacetimedb_datastore::{ traits::TxData, }; use spacetimedb_durability::{self as durability, History}; -use spacetimedb_fs_utils::{ - atomic_write_bytes, copy_dir_all, copy_file_sync, create_dir_all_sync, dir_size, normalize_absolute_path, sync_dir, -}; use spacetimedb_lib::bsatn::ToBsatn; use spacetimedb_lib::db::auth::StAccess; use spacetimedb_lib::db::raw_def::v9::{btree, RawModuleDefV9Builder, RawSql}; use spacetimedb_lib::st_var::StVarValue; use spacetimedb_lib::ConnectionId; use spacetimedb_lib::Identity; -use spacetimedb_paths::server::{CommitLogDir, ReplicaDir, ServerDataDir, SnapshotsPath}; -use spacetimedb_paths::standalone::StandaloneDataDirExt; -use spacetimedb_paths::FromPathUnchecked; +use spacetimedb_paths::server::{ReplicaDir, SnapshotsPath}; use spacetimedb_primitives::*; use spacetimedb_runtime::sync::watch; use spacetimedb_runtime::Handle; @@ -68,13 +62,9 @@ use spacetimedb_table::page_pool::PagePool; use spacetimedb_table::table::{RowRef, TableScanIter}; use spacetimedb_table::table_index::IndexKey; use std::borrow::Cow; -use std::collections::BTreeSet; -use std::ffi::OsString; use std::io; use std::ops::RangeBounds; -use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::{Duration, Instant}; pub use super::persistence::{DiskSizeFn, Durability, Persistence}; pub use super::snapshot::SnapshotWorker; @@ -140,29 +130,6 @@ pub struct RelationalDB { // compiler to find external dependencies. pub const SNAPSHOT_FREQUENCY: u64 = 1_000_000; -#[derive(Debug, serde::Serialize)] -pub struct HotBackupManifest { - pub version: u32, - pub database_identity: Identity, - pub replica_id: u64, - pub snapshot_offset: TxOffset, - pub durable_offset: TxOffset, - pub output_dir: PathBuf, - pub snapshot_ms: u64, - pub copy_ms: u64, - pub total_ms: u64, - pub bytes: u64, -} - -impl HotBackupManifest { - /// Convert a [`Duration`] to whole milliseconds, saturating at `u64::MAX`. - /// - /// Used to fill in the `*_ms` timing fields of the manifest. - pub fn elapsed_ms(duration: Duration) -> u64 { - duration.as_millis().try_into().unwrap_or(u64::MAX) - } -} - impl std::fmt::Debug for RelationalDB { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RelationalDB") @@ -354,6 +321,11 @@ impl RelationalDB { } apply_history(&inner, database_identity, history)?; + if durable_tx_offset.is_some() + && let Some(snapshot_worker) = persistence.as_ref().and_then(|p| p.snapshots.as_ref()) + { + snapshot_worker.request_snapshot(); + } let elapsed_time = start_time.elapsed(); ENGINE_METRICS @@ -626,17 +598,24 @@ impl RelationalDB { } } - if let Some((snapshot_repo, durable_tx_offset)) = snapshot_repo.zip(durable_tx_offset) { - // Mark any newer snapshots as invalid, as the history past - // `durable_tx_offset` may have been reset and thus diverge from - // any snapshots taken earlier. - snapshot_repo - .invalidate_newer_snapshots(durable_tx_offset) - .map_err(|e| RestoreSnapshotError::Invalidate { - offset: durable_tx_offset, - source: Box::new(e), + if let Some(snapshot_repo) = snapshot_repo { + // Mark every snapshot without durable history as invalid. Otherwise, + // invalidate only snapshots after the last durable transaction. + let first_invalid_offset = match durable_tx_offset { + None => Some(0), + Some(offset) => offset.checked_add(1), + }; + if let Some(offset) = first_invalid_offset { + snapshot_repo.invalidate_snapshots_at_or_after(offset).map_err(|e| { + RestoreSnapshotError::Invalidate { + offset, + source: Box::new(e), + } })?; + } + } + if let Some((snapshot_repo, durable_tx_offset)) = snapshot_repo.zip(durable_tx_offset) { // Try to restore from any snapshot that was taken within the // range `(min_commitlog_offset + 1)..=durable_tx_offset`. let mut upper_bound = durable_tx_offset; @@ -989,146 +968,6 @@ impl RelationalDB { Ok(snapshot_offset) } - /// Export a point-in-time hot backup into `output_dir`. - pub async fn create_hot_backup( - &self, - replica_dir: &ReplicaDir, - server_data_dir: Option<&ServerDataDir>, - replica_id: u64, - output_dir: impl AsRef, - ) -> anyhow::Result { - self.create_hot_backup_inner(replica_dir, server_data_dir, replica_id, output_dir, true) - .await - } - - /// Export a point-in-time hot backup into `output_dir`, leaving `server/control-db` - /// for the caller to provide. - /// - /// Unlike [`Self::create_hot_backup`], this does **not** compute - /// [`HotBackupManifest::bytes`] (it is left at 0) and does **not** write - /// `manifest.json`: the caller is expected to add `server/control-db` and - /// then finalize the manifest itself. This both avoids traversing the - /// backup tree twice and makes `manifest.json` a reliable marker of a - /// fully-written backup. - pub async fn create_hot_backup_without_control_db( - &self, - replica_dir: &ReplicaDir, - server_data_dir: Option<&ServerDataDir>, - replica_id: u64, - output_dir: impl AsRef, - ) -> anyhow::Result { - self.create_hot_backup_inner(replica_dir, server_data_dir, replica_id, output_dir, false) - .await - } - - async fn create_hot_backup_inner( - &self, - replica_dir: &ReplicaDir, - server_data_dir: Option<&ServerDataDir>, - replica_id: u64, - output_dir: impl AsRef, - copy_control_db: bool, - ) -> anyhow::Result { - let total_start = Instant::now(); - let runtime = self - .durability_runtime - .as_ref() - .context("durability runtime is not enabled for this database")? - .clone(); - let output_dir = ensure_backup_path(replica_dir, server_data_dir, output_dir.as_ref())?; - // Guard against two concurrent backups interleaving writes into the same - // directory after both passed the `ensure_empty_dir` check. - let _dir_guard = BackupDirGuard::acquire(&output_dir)?; - anyhow::ensure!( - !(copy_control_db && server_data_dir.is_some()), - "copying live server/control-db is not supported by hot backup; export control-db separately and finalize the manifest after the export" - ); - - let snapshot_start = Instant::now(); - let snapshot_offset = self.request_hot_backup_snapshot().await?; - let snapshot_ms = HotBackupManifest::elapsed_ms(snapshot_start.elapsed()); - let durable_offset = snapshot_offset; - - let copy_start = Instant::now(); - let snapshot_repo = open_snapshot_repo(replica_dir.snapshots(), self.database_identity(), replica_id)?; - let src_snapshot_dir = snapshot_repo.snapshot_dir_path(snapshot_offset); - let dst_snapshots = output_dir.join("snapshots"); - let dst_snapshot_dir = dst_snapshots.join( - src_snapshot_dir - .0 - .file_name() - .context("snapshot directory has no file name")?, - ); - // `request_hot_backup_snapshot` returns only after the snapshot worker has - // fsynced the snapshot to disk (see `SnapshotWorker::ensure_snapshot_at_least`), - // so both paths must already exist; a missing path is a bug, not a timing issue. - anyhow::ensure!( - src_snapshot_dir.0.is_dir(), - "snapshot directory does not exist: {}", - src_snapshot_dir.display() - ); - let src_snapshot_file = src_snapshot_dir.snapshot_file(snapshot_offset); - anyhow::ensure!( - src_snapshot_file.0.is_file(), - "snapshot file does not exist: {}", - src_snapshot_file.display() - ); - let snapshot_file_name: OsString = src_snapshot_file - .0 - .file_name() - .context("snapshot file has no file name")? - .to_owned(); - - asyncify(&runtime, { - let output_dir = output_dir.clone(); - let src_snapshot_dir = src_snapshot_dir.clone(); - let snapshot_file_name = snapshot_file_name.clone(); - move || -> anyhow::Result<()> { - ensure_empty_dir(&output_dir)?; - copy_dir_all_retry(&src_snapshot_dir.0, &dst_snapshot_dir, &snapshot_file_name)?; - Ok(()) - } - }) - .await?; - - if let Some(server_data_dir) = server_data_dir { - copy_server_state(&runtime, server_data_dir, &output_dir, copy_control_db).await?; - } - copy_commitlog_range( - &runtime, - replica_dir.commit_log(), - output_dir.join("clog"), - snapshot_offset, - ) - .await?; - let copy_ms = HotBackupManifest::elapsed_ms(copy_start.elapsed()); - - let mut manifest = HotBackupManifest { - version: 1, - database_identity: self.database_identity(), - replica_id, - snapshot_offset, - durable_offset, - output_dir: output_dir.clone(), - snapshot_ms, - copy_ms, - total_ms: HotBackupManifest::elapsed_ms(total_start.elapsed()), - bytes: 0, - }; - // When the caller provides `server/control-db` itself, leave `bytes` - // and `manifest.json` to it, so the backup tree is only traversed once - // and the manifest is written exactly once, after the backup is complete. - if copy_control_db { - manifest.bytes = asyncify(&runtime, { - let output_dir = output_dir.clone(); - move || dir_size(&output_dir) - }) - .await?; - write_hot_backup_manifest(&runtime, &output_dir, &manifest).await?; - } - Ok(manifest) - } - /// Run a fallible function in a transaction. /// /// If the supplied function returns `Ok`, the transaction is automatically @@ -1972,10 +1811,15 @@ pub async fn snapshot_watching_commitlog_compressor( durability: LocalDurability, runtime: Handle, ) { - let mut prev_snapshot_offset = *snapshot_rx.borrow_and_update(); + let snapshot_offset = *snapshot_rx.borrow_and_update(); + let res = + compress_commitlog_segments_older_than_snapshot(snapshot_offset, durability.clone(), runtime.clone()).await; + if let Err(err) = res { + tracing::warn!("failed to compress commitlog segments on startup for snapshot offset {snapshot_offset}: {err}"); + } + while snapshot_rx.changed().await.is_ok() { let snapshot_offset = *snapshot_rx.borrow_and_update(); - let durability = durability.clone(); if let Some(snap_tx) = &mut snap_tx && let Err(err) = snap_tx.try_send(snapshot_offset) @@ -1983,35 +1827,17 @@ pub async fn snapshot_watching_commitlog_compressor( tracing::warn!("failed to send offset {snapshot_offset} after snapshot creation: {err}"); } - let res: io::Result<_> = asyncify(&runtime, move || { - let segment_offsets = durability.existing_segment_offsets()?; - let start_idx = segment_offsets - .binary_search(&prev_snapshot_offset) - // if the snapshot is in the middle of a segment, we want to round down. - // [0, 2].binary_search(1) will return Err(1), so we subtract 1. - .unwrap_or_else(|i| i.saturating_sub(1)); - let segment_offsets = &segment_offsets[start_idx..]; - let end_idx = segment_offsets - .binary_search(&snapshot_offset) - .unwrap_or_else(|i| i.saturating_sub(1)); - // in this case, segment_offsets[end_idx] is the segment that contains the snapshot, - // which we don't want to compress, so an exclusive range is correct. - let segment_offsets = &segment_offsets[..end_idx]; - durability.compress_segments(segment_offsets)?; - let n = segment_offsets.len(); - let last_compressed_segment = if n > 0 { Some(segment_offsets[n - 1]) } else { None }; - Ok(last_compressed_segment) - }) - .await; + let res = + compress_commitlog_segments_older_than_snapshot(snapshot_offset, durability.clone(), runtime.clone()).await; - let last_compressed_segment = match res { - Ok(opt_offset) => opt_offset, + let (last_compressed_segment, stats) = match res { + Ok(res) => res, Err(err) => { tracing::warn!("failed to compress segments: {err}"); continue; } }; - prev_snapshot_offset = snapshot_offset; + log_commitlog_compression_result(snapshot_offset, last_compressed_segment, stats); if let Some((clog_tx, last_compressed_segment)) = clog_tx.as_mut().zip(last_compressed_segment) && let Err(err) = clog_tx.try_send(last_compressed_segment) @@ -2021,6 +1847,46 @@ pub async fn snapshot_watching_commitlog_compressor( } } +async fn compress_commitlog_segments_older_than_snapshot( + snapshot_offset: TxOffset, + durability: LocalDurability, + runtime: Handle, +) -> io::Result<(Option, CompressionStats)> { + asyncify(&runtime, move || { + let segment_offsets = durability.existing_segment_offsets()?; + let segment_offsets = commitlog_segments_older_than_snapshot(&segment_offsets, snapshot_offset); + let last_compressed_segment = segment_offsets.last().copied(); + let stats = durability.compress_segments(segment_offsets)?; + Ok((last_compressed_segment, stats)) + }) + .await +} + +fn commitlog_segments_older_than_snapshot(segment_offsets: &[TxOffset], snapshot_offset: TxOffset) -> &[TxOffset] { + let end_idx = segment_offsets + .binary_search(&snapshot_offset) + // If the snapshot is in the middle of a segment, round down to the segment + // containing it. That segment is still needed for replay from the snapshot. + .unwrap_or_else(|i| i.saturating_sub(1)); + &segment_offsets[..end_idx] +} + +fn log_commitlog_compression_result( + snapshot_offset: TxOffset, + last_compressed_segment: Option, + stats: CompressionStats, +) { + if let Some(last_compressed_segment) = last_compressed_segment { + tracing::info!( + snapshot_offset, + last_compressed_segment, + bytes_read = stats.bytes_read, + bytes_written = stats.bytes_written, + "compressed commitlog segments older than snapshot" + ); + } +} + /// Open a [`SnapshotRepository`] at `db_path/snapshots`, /// configured to store snapshots of the database `database_identity`/`replica_id`. pub fn open_snapshot_repo( @@ -2034,258 +1900,6 @@ pub fn open_snapshot_repo( .map_err(Box::new) } -async fn copy_commitlog_range( - runtime: &Handle, - src: CommitLogDir, - dst: PathBuf, - through: TxOffset, -) -> anyhow::Result<()> { - let src_repo = commitlog::repo::Fs::new(src, None)?; - let dst_repo = commitlog::repo::Fs::new(CommitLogDir::from_path_unchecked(&dst), None)?; - let reader = tokio::io::BufReader::new(tokio_util::io::StreamReader::new(commitlog::stream::commits( - src_repo, - ..=through, - ))); - tokio::pin!(reader); - let writer = - commitlog::stream::StreamWriter::create(dst_repo, <_>::default(), commitlog::stream::OnTrailingData::Error)?; - let mut writer = writer.append_all(reader, |_| ()).await?; - writer.sync_all().await?; - asyncify(runtime, move || -> anyhow::Result<()> { - sync_dir(&dst).with_context(|| format!("syncing backup commitlog dir {}", dst.display()))?; - if let Some(parent) = dst.parent() { - sync_dir(parent).with_context(|| format!("syncing backup commitlog parent {}", parent.display()))?; - } - Ok(()) - }) - .await?; - Ok(()) -} - -async fn write_hot_backup_manifest( - runtime: &Handle, - output_dir: &Path, - manifest: &HotBackupManifest, -) -> anyhow::Result<()> { - let output_dir = output_dir.to_path_buf(); - let json = serde_json::to_vec_pretty(manifest)?; - asyncify(runtime, move || { - atomic_write_bytes(&output_dir.join("manifest.json"), &json) - }) - .await?; - Ok(()) -} - -/// Write `manifest` as `manifest.json` into `output_dir` on a blocking thread. -/// -/// Exposed for callers of -/// [`RelationalDB::create_hot_backup_without_control_db`], which finalize the -/// manifest themselves after adding their own state to the backup. -pub async fn finalize_hot_backup_manifest(output_dir: &Path, manifest: &HotBackupManifest) -> anyhow::Result<()> { - write_hot_backup_manifest(&Handle::tokio_current(), output_dir, manifest).await -} - -/// Serializes concurrent hot backups into the same output directory within -/// this process. -/// -/// Two concurrent backups into the same (empty) directory would both pass the -/// `ensure_empty_dir` check and interleave their writes, producing a corrupt -/// backup. Registering the normalized output path in a process-wide set closes -/// that race; the path is released when the guard drops. -struct BackupDirGuard(PathBuf); - -static ACTIVE_BACKUP_DIRS: Mutex> = Mutex::new(BTreeSet::new()); - -impl BackupDirGuard { - fn acquire(output_dir: &Path) -> anyhow::Result { - let mut active = ACTIVE_BACKUP_DIRS.lock(); - if let Some(overlapping) = active - .iter() - .find(|active_dir| backup_paths_overlap(output_dir, active_dir)) - { - anyhow::bail!( - "backup output path {} overlaps with in-progress backup {}", - output_dir.display(), - overlapping.display() - ); - } - anyhow::ensure!( - active.insert(output_dir.to_path_buf()), - "a backup into {} is already in progress", - output_dir.display() - ); - Ok(Self(output_dir.to_path_buf())) - } -} - -#[cfg(windows)] -fn backup_path_components(path: &Path) -> Vec { - path.components() - .map(|component| component.as_os_str().to_string_lossy().to_ascii_lowercase()) - .collect() -} - -#[cfg(windows)] -fn backup_path_starts_with(path: &Path, base: &Path) -> bool { - let path = backup_path_components(path); - let base = backup_path_components(base); - base.len() <= path.len() && path.iter().zip(&base).all(|(path, base)| path == base) -} - -#[cfg(not(windows))] -fn backup_path_starts_with(path: &Path, base: &Path) -> bool { - path.starts_with(base) -} - -fn backup_paths_overlap(a: &Path, b: &Path) -> bool { - backup_path_starts_with(a, b) || backup_path_starts_with(b, a) -} - -impl Drop for BackupDirGuard { - fn drop(&mut self) { - ACTIVE_BACKUP_DIRS.lock().remove(&self.0); - } -} - -fn ensure_empty_dir(path: &Path) -> anyhow::Result<()> { - if path.exists() { - anyhow::ensure!( - path.is_dir(), - "backup output path is not a directory: {}", - path.display() - ); - anyhow::ensure!( - path.read_dir()?.next().is_none(), - "backup output directory must be empty: {}", - path.display() - ); - } else { - create_dir_all_sync(path)?; - } - Ok(()) -} - -/// Validate `output_dir` as a hot backup destination and return its -/// normalized form. -/// -/// The path must be absolute and, after resolving symlinks in its existing -/// ancestors and rejecting `..` components (see -/// [`spacetimedb_fs_utils::normalize_absolute_path`]), must not point inside -/// the server data directory or the replica directory. All subsequent backup -/// I/O must use the returned path, so the checks cannot be bypassed via -/// symlinks or path traversal. -fn ensure_backup_path( - replica_dir: &ReplicaDir, - server_data_dir: Option<&ServerDataDir>, - output_dir: &Path, -) -> anyhow::Result { - anyhow::ensure!( - output_dir.is_absolute(), - "backup output directory must be an absolute server path: {}", - output_dir.display() - ); - let output_dir = normalize_absolute_path(output_dir).context("normalizing the backup output directory path")?; - if let Some(server_data_dir) = server_data_dir { - let data_dir = normalize_absolute_path(&server_data_dir.0).unwrap_or_else(|_| server_data_dir.0.to_path_buf()); - anyhow::ensure!( - !output_dir.starts_with(&data_dir), - "backup output directory must not be inside the server data directory: {}", - output_dir.display() - ); - } - let replica_dir_normalized = - normalize_absolute_path(&replica_dir.0).unwrap_or_else(|_| replica_dir.0.to_path_buf()); - anyhow::ensure!( - !output_dir.starts_with(&replica_dir_normalized), - "backup output directory must not be inside the replica directory: {}", - replica_dir.display() - ); - Ok(output_dir) -} - -async fn copy_server_state( - runtime: &Handle, - data_dir: &ServerDataDir, - output_dir: &Path, - copy_control_db: bool, -) -> anyhow::Result<()> { - let data_dir = data_dir.clone(); - let output_dir = output_dir.to_path_buf(); - asyncify(runtime, move || -> anyhow::Result<()> { - debug_assert!(!copy_control_db); - let server_dir = output_dir.join("server"); - create_dir_all_sync(&server_dir)?; - - copy_required_file(&data_dir.config_toml().0, &server_dir.join("config.toml"))?; - copy_required_file(&data_dir.metadata_toml().0, &server_dir.join("metadata.toml"))?; - copy_required_dir(&data_dir.program_bytes().0, &server_dir.join("program-bytes"))?; - sync_dir(&server_dir)?; - Ok(()) - }) - .await?; - Ok(()) -} - -fn copy_required_file(src: &Path, dst: &Path) -> anyhow::Result<()> { - anyhow::ensure!(src.is_file(), "server state file is missing: {}", src.display()); - if let Some(parent) = dst.parent() { - create_dir_all_sync(parent)?; - } - copy_file_sync(src, dst).with_context(|| format!("copying {} to {}", src.display(), dst.display()))?; - Ok(()) -} - -fn copy_required_dir(src: &Path, dst: &Path) -> anyhow::Result<()> { - anyhow::ensure!(src.is_dir(), "server state directory is missing: {}", src.display()); - copy_dir_all(src, dst).with_context(|| format!("copying {} to {}", src.display(), dst.display())) -} - -fn copy_dir_all_retry(src: &Path, dst: &Path, required_file_name: &OsString) -> anyhow::Result<()> { - let start = Instant::now(); - loop { - match copy_snapshot_dir(src, dst, required_file_name) { - Ok(()) => return Ok(()), - Err(err) if start.elapsed() < Duration::from_secs(30) => { - std::thread::sleep(Duration::from_millis(10)); - drop(err); - } - Err(err) => { - return Err(err).with_context(|| format!("copying snapshot {} to {}", src.display(), dst.display())); - } - } - } -} - -fn copy_snapshot_dir(src: &Path, dst: &Path, required_file_name: &OsString) -> anyhow::Result<()> { - create_dir_all_sync(dst)?; - let src_required = src.join(required_file_name); - let dst_required = dst.join(required_file_name); - copy_file_sync(&src_required, &dst_required) - .with_context(|| format!("copying {} to {}", src_required.display(), dst_required.display()))?; - - for entry in std::fs::read_dir(src).with_context(|| format!("reading {}", src.display()))? { - let entry = entry?; - if entry.file_name() == *required_file_name { - continue; - } - let ty = entry.file_type()?; - let dst = dst.join(entry.file_name()); - if ty.is_dir() { - copy_dir_all(entry.path(), dst)?; - } else { - copy_file_sync(&entry.path(), &dst)?; - } - } - anyhow::ensure!( - dst_required.is_file(), - "copying snapshot {} to {} did not copy {}", - src.display(), - dst.display(), - required_file_name.to_string_lossy() - ); - Ok(()) -} - fn default_row_count_fn(db: Identity) -> RowCountFn { Arc::new(move |table_id, table_name| { DB_METRICS @@ -2829,7 +2443,7 @@ mod tests { #[cfg(unix)] use spacetimedb_snapshot::Snapshot; use spacetimedb_table::read_column::ReadColumn; - use spacetimedb_table::table::RowRef; + use spacetimedb_table::table::{RowRef, Table}; use tempfile::TempDir; use tests::tests_utils::TestHistory; @@ -2912,6 +2526,21 @@ mod tests { Ok(()) } + #[test] + fn commitlog_segments_older_than_snapshot_excludes_segment_containing_snapshot() { + let segment_offsets = [0, 100, 200, 300]; + + assert_eq!(commitlog_segments_older_than_snapshot(&segment_offsets, 0), &[]); + assert_eq!(commitlog_segments_older_than_snapshot(&segment_offsets, 99), &[]); + assert_eq!(commitlog_segments_older_than_snapshot(&segment_offsets, 100), &[0]); + assert_eq!(commitlog_segments_older_than_snapshot(&segment_offsets, 199), &[0]); + assert_eq!(commitlog_segments_older_than_snapshot(&segment_offsets, 200), &[0, 100]); + assert_eq!( + commitlog_segments_older_than_snapshot(&segment_offsets, 350), + &[0, 100, 200] + ); + } + #[test] fn test_open_twice() -> ResultTest<()> { let stdb = TestDB::durable()?; @@ -4276,171 +3905,6 @@ mod tests { Ok(()) } - #[test] - fn hot_backup_create_writes_manifest_snapshot_and_commitlog() -> anyhow::Result<()> { - let stdb = TestDB::durable()?; - - let mut tx = begin_mut_tx(&stdb); - stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; - stdb.commit_tx(tx)?; - - let backup_dir = tempfile::tempdir()?; - let manifest = stdb.runtime().unwrap().block_on(async { - tokio::time::timeout( - HOT_BACKUP_TEST_TIMEOUT, - stdb.create_hot_backup(stdb.path().unwrap(), None, 0, backup_dir.path()), - ) - .await - })??; - - assert!(backup_dir.path().join("manifest.json").is_file()); - assert!(manifest.bytes > 0); - assert_eq!(manifest.durable_offset, manifest.snapshot_offset); - let manifest_json: serde_json::Value = - serde_json::from_slice(&std::fs::read(backup_dir.path().join("manifest.json"))?)?; - assert_eq!( - manifest_json["snapshot_offset"].as_u64(), - Some(manifest.snapshot_offset) - ); - assert_eq!(manifest_json["durable_offset"].as_u64(), Some(manifest.snapshot_offset)); - let repo = open_snapshot_repo( - SnapshotsPath::from_path_unchecked(backup_dir.path().join("snapshots")), - stdb.database_identity(), - 0, - )?; - let snapshot = repo.read_snapshot(manifest.snapshot_offset, &PagePool::new_for_test())?; - assert_eq!(snapshot.database_identity, manifest.database_identity); - assert_eq!(snapshot.replica_id, manifest.replica_id); - assert_eq!(snapshot.tx_offset, manifest.snapshot_offset); - let clog = Commitlog::::open( - CommitLogDir::from_path_unchecked(backup_dir.path().join("clog")), - Default::default(), - None, - )?; - assert_eq!(clog.max_committed_offset(), Some(manifest.snapshot_offset)); - - Ok(()) - } - - #[test] - fn hot_backup_create_without_control_db_copies_other_server_state() -> anyhow::Result<()> { - let stdb = TestDB::durable()?; - - let mut tx = begin_mut_tx(&stdb); - stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; - stdb.commit_tx(tx)?; - - let data = tempfile::tempdir()?; - let data_dir = ServerDataDir::from_path_unchecked(data.path()); - std::fs::write(data.path().join("config.toml"), b"config")?; - std::fs::write(data.path().join("metadata.toml"), b"metadata")?; - std::fs::create_dir_all(data.path().join("control-db"))?; - std::fs::write(data.path().join("control-db/control"), b"control")?; - std::fs::create_dir_all(data.path().join("program-bytes"))?; - std::fs::write(data.path().join("program-bytes/program"), b"program")?; - - let backup_dir = tempfile::tempdir()?; - let manifest = stdb.runtime().unwrap().block_on(async { - tokio::time::timeout( - HOT_BACKUP_TEST_TIMEOUT, - stdb.create_hot_backup_without_control_db(stdb.path().unwrap(), Some(&data_dir), 0, backup_dir.path()), - ) - .await - })??; - - assert_eq!(manifest.bytes, 0); - assert!(!backup_dir.path().join("manifest.json").exists()); - assert_eq!( - std::fs::read_to_string(backup_dir.path().join("server/config.toml"))?, - "config" - ); - assert_eq!( - std::fs::read_to_string(backup_dir.path().join("server/metadata.toml"))?, - "metadata" - ); - assert!(!backup_dir.path().join("server/control-db").exists()); - assert!(backup_dir.path().join("server/program-bytes/program").is_file()); - - Ok(()) - } - - #[test] - fn hot_backup_rejects_raw_copy_of_live_control_db() -> anyhow::Result<()> { - let stdb = TestDB::durable()?; - - let mut tx = begin_mut_tx(&stdb); - stdb.create_table(&mut tx, my_table(AlgebraicType::I32))?; - stdb.commit_tx(tx)?; - - let data = tempfile::tempdir()?; - let data_dir = ServerDataDir::from_path_unchecked(data.path()); - std::fs::write(data.path().join("config.toml"), b"config")?; - std::fs::write(data.path().join("metadata.toml"), b"metadata")?; - std::fs::create_dir_all(data.path().join("control-db"))?; - std::fs::write(data.path().join("control-db/control"), b"control")?; - std::fs::create_dir_all(data.path().join("program-bytes"))?; - std::fs::write(data.path().join("program-bytes/program"), b"program")?; - - let backup_dir = tempfile::tempdir()?; - let result = stdb.runtime().unwrap().block_on(async { - tokio::time::timeout( - HOT_BACKUP_TEST_TIMEOUT, - stdb.create_hot_backup(stdb.path().unwrap(), Some(&data_dir), 0, backup_dir.path()), - ) - .await - })?; - let err = result.unwrap_err(); - - assert!(err.to_string().contains("control-db")); - assert!(!backup_dir.path().join("manifest.json").exists()); - Ok(()) - } - - #[test] - fn hot_backup_rejects_unsafe_output_dirs() -> anyhow::Result<()> { - let stdb = TestDB::durable()?; - let replica_dir = stdb.path().unwrap(); - - let err = stdb - .runtime() - .unwrap() - .block_on(stdb.create_hot_backup(replica_dir, None, 0, PathBuf::from("relative"))) - .unwrap_err(); - assert!(err.to_string().contains("absolute server path")); - - let err = stdb - .runtime() - .unwrap() - .block_on(stdb.create_hot_backup(replica_dir, None, 0, replica_dir.0.join("backup"))) - .unwrap_err(); - assert!(err.to_string().contains("replica directory")); - - Ok(()) - } - - #[test] - fn hot_backup_dir_guard_rejects_parent_child_overlap() -> anyhow::Result<()> { - let temp = tempfile::tempdir()?; - let parent = temp.path().join("backup"); - let child = parent.join("nested"); - - let parent_guard = BackupDirGuard::acquire(&parent)?; - let Err(err) = BackupDirGuard::acquire(&child) else { - panic!("child backup directory overlapped active parent"); - }; - assert!(err.to_string().contains("overlaps")); - drop(parent_guard); - - let child_guard = BackupDirGuard::acquire(&child)?; - let Err(err) = BackupDirGuard::acquire(&parent) else { - panic!("parent backup directory overlapped active child"); - }; - assert!(err.to_string().contains("overlaps")); - drop(child_guard); - - Ok(()) - } - // For test compression into an existing database. // Must supply the path to the database and the identity of the replica using the `ENV`: // - `SNAPSHOT` the path to the database, like `/tmp/db/replicas/.../8/database` @@ -4539,6 +4003,30 @@ mod tests { Ok(()) } + #[test] + fn restore_without_durable_history_invalidates_all_snapshots() -> ResultTest<()> { + let temp = TempDir::with_prefix("snapshot-without-durable-history")?; + let snapshots_path = SnapshotsPath::from_path_unchecked(temp.path()); + snapshots_path.create()?; + let repo = SnapshotRepository::open(snapshots_path, Identity::ZERO, 0)?; + let blobs = spacetimedb_table::blob_store::HashMapBlobStore::default(); + + repo.create_snapshot(std::iter::empty::<&mut Table>(), &blobs, 0)? + .sync_all()?; + assert_eq!(repo.latest_snapshot()?, Some(0)); + + RelationalDB::restore_from_snapshot_or_bootstrap( + Identity::ZERO, + Some(&repo), + None, + 0, + PagePool::new_for_test(), + )?; + + assert_eq!(repo.latest_snapshot()?, None); + Ok(()) + } + #[test] /// Test that we can create a table after replaying a durable database /// without a snapshot. diff --git a/crates/engine/src/snapshot.rs b/crates/engine/src/snapshot.rs index 9943917f5c0..efb7b3713c2 100644 --- a/crates/engine/src/snapshot.rs +++ b/crates/engine/src/snapshot.rs @@ -131,8 +131,8 @@ impl SnapshotWorker { pub fn request_snapshot(&self) { self.request_snapshot .unbounded_send(Request::TakeSnapshot { - min_offset: None, - result: None, + minimum_offset: None, + result_sender: None, }) .expect("snapshot worker panicked"); } @@ -143,8 +143,8 @@ impl SnapshotWorker { /// since the durability should continue writing queued TXes even if the snapshot worker panics. pub fn request_snapshot_ignore_closed(&self) { let _ = self.request_snapshot.unbounded_send(Request::TakeSnapshot { - min_offset: None, - result: None, + minimum_offset: None, + result_sender: None, }); } @@ -165,30 +165,21 @@ impl SnapshotWorker { return Ok(latest); } - let mut snapshot_created = self.subscribe(); - let (result_tx, result_rx) = oneshot::channel(); + let (result_sender, result_receiver) = oneshot::channel(); self.request_snapshot .unbounded_send(Request::TakeSnapshot { - min_offset: Some(offset), - result: Some(result_tx), + minimum_offset: Some(offset), + result_sender: Some(result_sender), }) - .expect("snapshot worker panicked"); - let snapshot_offset = result_rx + .context("snapshot worker closed before accepting requested snapshot")?; + let snapshot_offset = result_receiver .await .context("snapshot worker closed before creating requested snapshot")??; - if snapshot_offset >= offset { - return Ok(snapshot_offset); - } - loop { - snapshot_created - .changed() - .await - .context("snapshot worker closed before creating requested snapshot")?; - let latest = *snapshot_created.borrow_and_update(); - if latest >= offset { - return Ok(latest); - } - } + assert!( + snapshot_offset >= offset, + "snapshot worker returned offset {snapshot_offset}, below requested offset {offset}" + ); + Ok(snapshot_offset) } } @@ -212,8 +203,8 @@ type WeakDatabaseState = Weak>; enum Request { TakeSnapshot { - min_offset: Option, - result: Option>>, + minimum_offset: Option, + result_sender: Option>>, }, ReplaceState(SnapshotDatabaseState), } @@ -228,11 +219,11 @@ struct SnapshotWorkerActor { } fn send_snapshot_request_result( - result: Option>>, + result_sender: Option>>, value: anyhow::Result, ) { - if let Some(result) = result { - let _ = result.send(value); + if let Some(result_sender) = result_sender { + let _ = result_sender.send(value); } } @@ -256,29 +247,45 @@ impl SnapshotWorkerActor { let mut database_state: Option = None; while let Some(req) = self.snapshot_requests.next().await { match req { - Request::TakeSnapshot { min_offset, result } => { - if let Some(min_offset) = min_offset - && let Ok(Some(latest)) = self.snapshot_repo.latest_snapshot() - && latest >= min_offset - { - self.snapshot_created.send_replace(latest); - send_snapshot_request_result(result, Ok(latest)); - continue; + Request::TakeSnapshot { + minimum_offset, + result_sender, + } => { + if let Some(minimum_offset) = minimum_offset { + match self.snapshot_repo.latest_snapshot() { + Ok(Some(latest)) if latest >= minimum_offset => { + self.snapshot_created.send_replace(latest); + send_snapshot_request_result(result_sender, Ok(latest)); + continue; + } + Ok(_) => {} + Err(err) => { + warn!("database={database_identity} SnapshotWorker: {err:#}"); + send_snapshot_request_result(result_sender, Err(err.into())); + continue; + } + } } let res = self.maybe_take_snapshot(database_state.as_ref()).await; match res { Ok(snapshot_offset) => { + if let Some(minimum_offset) = minimum_offset { + assert!( + snapshot_offset >= minimum_offset, + "captured snapshot offset {snapshot_offset} below requested offset {minimum_offset}" + ); + } self.maybe_compress_snapshots(snapshot_offset).await; // INVARIANT: only signal `snapshot_created` after the snapshot // is fully durable on disk (`take_snapshot` returns post-fsync). // Hot backup relies on this to copy the snapshot directory as // soon as it observes the offset. self.snapshot_created.send_replace(snapshot_offset); - send_snapshot_request_result(result, Ok(snapshot_offset)); + send_snapshot_request_result(result_sender, Ok(snapshot_offset)); } Err(err) => { warn!("database={database_identity} SnapshotWorker: {err:#}"); - send_snapshot_request_result(result, Err(err)); + send_snapshot_request_result(result_sender, Err(err)); } } } @@ -482,4 +489,23 @@ mod tests { assert!(format!("{err:#}").contains("database state not set")); Ok(()) } + + #[test] + fn ensure_snapshot_at_least_returns_error_when_worker_is_closed() -> anyhow::Result<()> { + let rt = tokio::runtime::Runtime::new()?; + let temp = tempfile::tempdir()?; + std::fs::create_dir_all(temp.path().join("snapshots"))?; + let repo: Arc = Arc::new(SnapshotRepository::open( + SnapshotsPath::from_path_unchecked(temp.path().join("snapshots")), + Identity::ZERO, + 0, + )?); + let worker = SnapshotWorker::new(repo, Compression::Disabled, Handle::tokio(rt.handle().clone())); + drop(rt); + + let err = futures::executor::block_on(worker.ensure_snapshot_at_least(1)).unwrap_err(); + + assert!(format!("{err:#}").contains("closed before accepting requested snapshot")); + Ok(()) + } } diff --git a/crates/fs-utils/Cargo.toml b/crates/fs-utils/Cargo.toml index 2446d6c888b..9827225b670 100644 --- a/crates/fs-utils/Cargo.toml +++ b/crates/fs-utils/Cargo.toml @@ -17,6 +17,9 @@ thiserror.workspace = true tokio.workspace = true zstd-framed.workspace = true +[target.'cfg(windows)'.dependencies] +windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem"] } + [dev-dependencies] tempdir.workspace = true diff --git a/crates/fs-utils/src/lib.rs b/crates/fs-utils/src/lib.rs index 362ed41614f..261b8ed6bf1 100644 --- a/crates/fs-utils/src/lib.rs +++ b/crates/fs-utils/src/lib.rs @@ -52,7 +52,7 @@ pub fn atomic_write_bytes(file_path: &Path, data: &[u8]) -> anyhow::Result<()> { temp_file.write_all(data)?; temp_file.sync_all()?; drop(temp_file); - std::fs::rename(&temp_path, file_path)?; + replace_file(&temp_path, file_path)?; if let Some(parent) = non_empty_parent(file_path) { sync_dir(parent)?; } @@ -73,6 +73,41 @@ pub fn atomic_write_bytes(file_path: &Path, data: &[u8]) -> anyhow::Result<()> { ) } +#[cfg(windows)] +fn replace_file(source: &Path, destination: &Path) -> std::io::Result<()> { + use std::os::windows::ffi::OsStrExt as _; + use windows_sys::Win32::Storage::FileSystem::{MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH}; + + fn encode_path(path: &Path) -> std::io::Result> { + let mut path: Vec<_> = path.as_os_str().encode_wide().collect(); + if path.contains(&0) { + return Err(std::io::Error::new( + ErrorKind::InvalidInput, + "path contains a null character", + )); + } + path.push(0); + Ok(path) + } + + let source = encode_path(source)?; + let destination = encode_path(destination)?; + let flags = MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH; + + // SAFETY: Both paths are valid, null-terminated UTF-16 strings that live + // for the duration of the call. + if unsafe { MoveFileExW(source.as_ptr(), destination.as_ptr(), flags) } == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(not(windows))] +fn replace_file(source: &Path, destination: &Path) -> std::io::Result<()> { + std::fs::rename(source, destination) +} + fn non_empty_parent(path: &Path) -> Option<&Path> { path.parent().filter(|parent| !parent.as_os_str().is_empty()) } diff --git a/crates/snapshot/src/lib.rs b/crates/snapshot/src/lib.rs index 4a84a3f6c7d..485644a1a69 100644 --- a/crates/snapshot/src/lib.rs +++ b/crates/snapshot/src/lib.rs @@ -860,7 +860,7 @@ impl SnapshotRepository { // failover causes the commitlog to be reset to offset 9. The next // transaction (also offset 10) will trigger snapshot creation, but we'd // mistake the existing snapshot (now invalid) as the previous snapshot. - self.invalidate_newer_snapshots(tx_offset.saturating_sub(1))?; + self.invalidate_snapshots_at_or_after(tx_offset)?; // If a previous snapshot exists in this snapshot repo, // get a handle on its object repo in order to hardlink shared objects into the new snapshot. @@ -1242,9 +1242,23 @@ impl SnapshotRepository { /// /// If this method returns an error, some snapshots may have been invalidated, but not all will have been. pub fn invalidate_newer_snapshots(&self, upper_bound: TxOffset) -> Result<(), SnapshotError> { + let Some(lower_bound) = upper_bound.checked_add(1) else { + return Ok(()); + }; + self.invalidate_snapshots_at_or_after(lower_bound) + } + + /// Mark every snapshot at or after `lower_bound` invalid. + /// + /// Does not invalidate snapshots which are locked. + /// + /// This may overwrite previously-invalidated snapshots. + /// + /// If this method returns an error, some snapshots may have been invalidated, but not all will have been. + pub fn invalidate_snapshots_at_or_after(&self, lower_bound: TxOffset) -> Result<(), SnapshotError> { let newer_snapshots = self .all_snapshots()? - .filter(|tx_offset| *tx_offset > upper_bound) + .filter(|tx_offset| *tx_offset >= lower_bound) // Collect to a vec to avoid iterator invalidation, // as the subsequent `for` loop will mutate the directory. .collect::>(); @@ -1485,6 +1499,19 @@ pub trait SnapshotRepo: Send + Sync { /// Invalidate every snapshot newer than `upper_bound`. fn invalidate_newer_snapshots(&self, upper_bound: TxOffset) -> Result<(), SnapshotError>; + /// Invalidate every snapshot at or after `lower_bound`. + fn invalidate_snapshots_at_or_after(&self, lower_bound: TxOffset) -> Result<(), SnapshotError> { + if let Some(upper_bound) = lower_bound.checked_sub(1) { + return self.invalidate_newer_snapshots(upper_bound); + } + + self.invalidate_newer_snapshots(0)?; + if self.latest_snapshot_older_than(0)?.is_some() { + self.invalidate_snapshot(0)?; + } + Ok(()) + } + /// Invalidate the snapshot at `tx_offset`. fn invalidate_snapshot(&self, tx_offset: TxOffset) -> Result<(), SnapshotError>; } @@ -1527,6 +1554,10 @@ impl SnapshotRepo for SnapshotRepository { SnapshotRepository::invalidate_newer_snapshots(self, upper_bound) } + fn invalidate_snapshots_at_or_after(&self, lower_bound: TxOffset) -> Result<(), SnapshotError> { + SnapshotRepository::invalidate_snapshots_at_or_after(self, lower_bound) + } + fn invalidate_snapshot(&self, tx_offset: TxOffset) -> Result<(), SnapshotError> { SnapshotRepository::invalidate_snapshot(self, tx_offset) } diff --git a/crates/standalone/config.toml b/crates/standalone/config.toml index da2b1404340..679591b24ff 100644 --- a/crates/standalone/config.toml +++ b/crates/standalone/config.toml @@ -66,6 +66,7 @@ directives = [ # [hot-backup] # HTTP/CLI-triggered backup output paths are relative to this server-side root. +# Standalone database-owner tokens are not authorized to use this endpoint. # Omit to disable POST /v1/database/:name/backup. # root-dir = "/var/backups/stdb" diff --git a/crates/standalone/src/control_db.rs b/crates/standalone/src/control_db.rs index 585768a5240..0b7131d53da 100644 --- a/crates/standalone/src/control_db.rs +++ b/crates/standalone/src/control_db.rs @@ -79,6 +79,7 @@ impl ControlDb { Ok(Self { db }) } + #[cfg(test)] pub fn export_to_path(&self, path: &ControlDbDir) -> Result<()> { ensure_empty_control_db_export_dir(path)?; @@ -92,7 +93,7 @@ impl ControlDb { dst.import(self.db.export()); dst.flush()?; sync_dir(path.as_ref()).with_context(|| format!("syncing control db export dir {}", path.display()))?; - if let Some(parent) = path.as_ref().parent() { + if let Some(parent) = path.0.parent() { sync_dir(parent).with_context(|| format!("syncing control db export parent {}", parent.display()))?; } Ok(()) @@ -103,7 +104,7 @@ impl ControlDb { path: &ControlDbDir, database_identity: &Identity, replica_id: u64, - ) -> Result<()> { + ) -> Result { ensure_empty_control_db_export_dir(path)?; let database = self .get_database_by_identity(database_identity)? @@ -111,13 +112,15 @@ impl ControlDb { let replica = self .get_replica_by_id(replica_id)? .with_context(|| format!("replica {replica_id} not found in control-db"))?; - anyhow::ensure!( - replica.database_id == database.id, - "replica {} belongs to database {}, not {}", - replica.id, - replica.database_id, - database.id - ); + if replica.database_id != database.id { + return Err(anyhow::anyhow!( + "replica {} belongs to database {}, not {}", + replica.id, + replica.database_id, + database.id + ) + .into()); + } self.db.flush()?; let dst = sled::Config::default() @@ -148,17 +151,12 @@ impl ControlDb { set_control_id_counter_at_least(&dst, next_control_id)?; copy_optional_control_db_tree_entry(&self.db, &dst, "database_locks", database_identity.to_be_byte_array())?; - copy_optional_control_db_tree_entry( - &self.db, - &dst, - "energy_budget", - &database.owner_identity.to_byte_array(), - )?; + copy_optional_control_db_tree_entry(&self.db, &dst, "energy_budget", database.owner_identity.to_byte_array())?; let domains = self.spacetime_reverse_dns(database_identity)?; if !domains.is_empty() { let identity_bytes = database_identity.to_byte_array(); - copy_optional_control_db_tree_entry(&self.db, &dst, "reverse_dns", &identity_bytes)?; + copy_optional_control_db_tree_entry(&self.db, &dst, "reverse_dns", identity_bytes)?; for domain in domains { copy_optional_control_db_tree_entry(&self.db, &dst, "dns", domain.to_lowercase().as_bytes())?; copy_optional_control_db_tree_entry( @@ -172,10 +170,10 @@ impl ControlDb { dst.flush()?; sync_dir(path.as_ref()).with_context(|| format!("syncing control db export dir {}", path.display()))?; - if let Some(parent) = path.as_ref().parent() { + if let Some(parent) = path.0.parent() { sync_dir(parent).with_context(|| format!("syncing control db export parent {}", parent.display()))?; } - Ok(()) + Ok(database) } #[cfg(test)] @@ -263,7 +261,7 @@ fn next_control_id_seed(db: &sled::Db) -> Result { fn set_control_id_counter_at_least(db: &sled::Db, next_id: u64) -> Result<()> { let counter = db.open_tree(CONTROL_ID_COUNTER_TREE)?; - let result: TransactionResult<(), anyhow::Error> = (&counter).transaction(|counter_tx| { + let result: TransactionResult<(), anyhow::Error> = counter.transaction(|counter_tx| { let stored_next_id = match counter_tx.get(CONTROL_ID_COUNTER_KEY)? { Some(value) => decode_control_id(&value).map_err(ConflictableTransactionError::Abort)?, None => 0, @@ -287,7 +285,7 @@ fn set_control_id_counter_at_least(db: &sled::Db, next_id: u64) -> Result<()> { fn reserve_control_id(db: &sled::Db) -> Result { let seed = next_control_id_seed(db)?; let counter = db.open_tree(CONTROL_ID_COUNTER_TREE)?; - let result: TransactionResult = (&counter).transaction(|counter_tx| { + let result: TransactionResult = counter.transaction(|counter_tx| { let stored_next_id = match counter_tx.get(CONTROL_ID_COUNTER_KEY)? { Some(value) => decode_control_id(&value).map_err(ConflictableTransactionError::Abort)?, None => seed, diff --git a/crates/standalone/src/control_db/tests.rs b/crates/standalone/src/control_db/tests.rs index 930234a99cb..ff3d9a4b8d4 100644 --- a/crates/standalone/src/control_db/tests.rs +++ b/crates/standalone/src/control_db/tests.rs @@ -165,6 +165,7 @@ fn test_export_to_path_can_be_opened() -> anyhow::Result<()> { owner_identity: *ALICE, host_type: HostType::Wasm, initial_program: Hash::ZERO, + bootstrap_generation: 0, }; let database_id = cdb.insert_database(database)?; cdb.insert_replica(Replica { @@ -194,6 +195,7 @@ fn test_export_database_to_path_is_scoped_to_one_database() -> anyhow::Result<() owner_identity: *ALICE, host_type: HostType::Wasm, initial_program: Hash::ZERO, + bootstrap_generation: 0, }; let first_database_id = cdb.insert_database(first)?; let first_replica_id = cdb.insert_replica(Replica { @@ -209,6 +211,7 @@ fn test_export_database_to_path_is_scoped_to_one_database() -> anyhow::Result<() owner_identity: *BOB, host_type: HostType::Wasm, initial_program: Hash::ZERO, + bootstrap_generation: 0, })?; cdb.insert_replica(Replica { id: 0, @@ -247,6 +250,7 @@ fn test_export_database_to_path_advances_control_id_counter() -> anyhow::Result< owner_identity: *ALICE, host_type: HostType::Wasm, initial_program: Hash::ZERO, + bootstrap_generation: 0, })?; let first_replica_id = cdb.insert_replica(Replica { id: 0, @@ -268,6 +272,7 @@ fn test_export_database_to_path_advances_control_id_counter() -> anyhow::Result< owner_identity: *BOB, host_type: HostType::Wasm, initial_program: Hash::ZERO, + bootstrap_generation: 0, })?; let second_replica_id = exported.insert_replica(Replica { id: 0, diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index 6eedb518bba..47b249dac4d 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -31,16 +31,18 @@ use spacetimedb_client_api_messages::name::{ use spacetimedb_datastore::db_metrics::data_size::DATA_SIZE_METRICS; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::traits::Program; -use spacetimedb_fs_utils::dir_size; +use spacetimedb_fs_utils::{atomic_write_bytes, create_dir_all_sync, dir_size, sync_dir}; +use spacetimedb_lib::{hash_bytes, Hash}; use spacetimedb_paths::server::{ModuleLogsDir, PidFile, ServerDataDir}; use spacetimedb_paths::standalone::ControlDbDir; use spacetimedb_paths::standalone::StandaloneDataDirExt; use spacetimedb_paths::FromPathUnchecked; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; use spacetimedb_table::page_pool::PagePool; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::{Duration, Instant}; +use tokio::sync::Mutex as AsyncMutex; #[derive(Clone)] pub struct StandaloneOptions { @@ -62,6 +64,7 @@ pub struct StandaloneEnv { auth_provider: auth::DefaultJwtAuthProvider, websocket_options: WebSocketOptions, hot_backup_root: Option, + database_lifecycle_lock: Arc>, } impl StandaloneEnv { @@ -116,6 +119,7 @@ impl StandaloneEnv { auth_provider: auth_env, websocket_options: config.websocket, hot_backup_root: config.hot_backup_root, + database_lifecycle_lock: Arc::new(AsyncMutex::new(())), })) } @@ -201,40 +205,41 @@ impl NodeDelegate for StandaloneEnv { &self, leader: Host, output_dir: PathBuf, - ) -> anyhow::Result { - use db::relational_db::HotBackupManifest; + ) -> anyhow::Result { + use spacetimedb::host::HotBackupManifest; + let database_lifecycle_guard = self.database_lifecycle_lock.clone().lock_owned().await; let total_start = Instant::now(); - // Writes the snapshot, commitlog and server state, but leaves - // `server/control-db`, `bytes` and `manifest.json` to us. - let mut manifest = leader.create_hot_backup_without_control_db(&output_dir).await?; + // Writes the snapshot, commitlog and generic server state, but leaves + // scoped program bytes, `server/control-db`, `bytes` and `manifest.json` to us. + let backup = leader.create_hot_backup_without_control_db(&output_dir).await?; // Trust the normalized path used by the engine, not the raw request path. - let output_dir = manifest.output_dir.clone(); + let output_dir = backup.manifest().output_dir.clone(); let export_start = Instant::now(); let control_db = self.control_db.clone(); - let database_identity = manifest.database_identity; - let replica_id = manifest.replica_id; - // sled export and the dir-size sweep are blocking; keep them off the async executor. - manifest.bytes = spacetimedb::util::asyncify(move || -> anyhow::Result { - control_db - .export_database_to_path( - &ControlDbDir::from_path_unchecked(output_dir.join("server").join("control-db")), - &database_identity, - replica_id, - ) - .context("exporting control-db into backup")?; - dir_size(&output_dir).context("measuring backup size") - }) - .await?; - manifest.copy_ms = manifest - .copy_ms - .saturating_add(HotBackupManifest::elapsed_ms(export_start.elapsed())); - manifest.total_ms = HotBackupManifest::elapsed_ms(total_start.elapsed()); - - // Written last: a `manifest.json` marks a complete backup. - db::relational_db::finalize_hot_backup_manifest(&manifest.output_dir, &manifest).await?; - Ok(manifest) + let database_identity = backup.manifest().database_identity; + let replica_id = backup.manifest().replica_id; + let program_bytes_dir = self.data_dir().program_bytes().0; + backup + .finalize_with_blocking(move |manifest| { + let _database_lifecycle_guard = database_lifecycle_guard; + let database = control_db + .export_database_to_path( + &ControlDbDir::from_path_unchecked(output_dir.join("server").join("control-db")), + &database_identity, + replica_id, + ) + .context("exporting scoped server/control-db")?; + copy_program_bytes_to_backup(&program_bytes_dir, &output_dir, database.initial_program)?; + manifest.bytes = dir_size(&output_dir).context("measuring backup size")?; + manifest.copy_ms = manifest + .copy_ms + .saturating_add(HotBackupManifest::elapsed_ms(export_start.elapsed())); + manifest.total_ms = HotBackupManifest::elapsed_ms(total_start.elapsed()); + Ok(()) + }) + .await } fn hot_backup_root(&self) -> Option { @@ -246,6 +251,38 @@ impl NodeDelegate for StandaloneEnv { } } +fn copy_program_bytes_to_backup(program_bytes_dir: &Path, output_dir: &Path, program_hash: Hash) -> anyhow::Result<()> { + let relative_path = program_bytes_relative_path(&program_hash); + let src = program_bytes_dir.join(&relative_path); + let program_bytes = std::fs::read(&src).with_context(|| format!("reading program bytes {}", src.display()))?; + let actual_hash = hash_bytes(&program_bytes); + anyhow::ensure!( + actual_hash == program_hash, + "program bytes hash mismatch at {}: expected {}, got {}", + src.display(), + program_hash, + actual_hash + ); + + let dst = output_dir.join("server").join("program-bytes").join(relative_path); + let program_bytes_root = output_dir.join("server").join("program-bytes"); + if let Some(parent) = dst.parent() { + create_dir_all_sync(parent)?; + } + atomic_write_bytes(&dst, &program_bytes)?; + sync_dir(&program_bytes_root) + .with_context(|| format!("syncing backup program-bytes dir {}", program_bytes_root.display()))?; + let server_dir = output_dir.join("server"); + sync_dir(&server_dir).with_context(|| format!("syncing backup server dir {}", server_dir.display()))?; + Ok(()) +} + +fn program_bytes_relative_path(program_hash: &Hash) -> PathBuf { + let hex = program_hash.to_hex(); + let (prefix, suffix) = hex.split_at(2); + PathBuf::from(prefix).join(suffix) +} + #[async_trait] impl spacetimedb_client_api::ControlStateReadAccess for StandaloneEnv { // Nodes @@ -326,6 +363,7 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { spec: spacetimedb_client_api::DatabaseDef, policy: MigrationPolicy, ) -> anyhow::Result> { + let _database_lifecycle_guard = self.database_lifecycle_lock.lock().await; let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; // standalone does not support replication. @@ -449,6 +487,7 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { } async fn delete_database(&self, _caller_identity: &Identity, database_identity: &Identity) -> anyhow::Result<()> { + let _database_lifecycle_guard = self.database_lifecycle_lock.lock().await; let Some(database) = self.control_db.get_database_by_identity(database_identity)? else { return Ok(()); }; @@ -462,6 +501,7 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { } async fn reset_database(&self, _caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { + let _database_lifecycle_guard = self.database_lifecycle_lock.lock().await; let mut database = self .control_db .get_database_by_identity(&spec.database_identity)? @@ -571,6 +611,14 @@ impl spacetimedb_client_api::Authorization for StandaloneEnv { .await? .with_context(|| format!("database {database} not found")) .with_context(|| format!("Unable to authorize {subject} to perform {action:?})"))?; + if matches!(action, spacetimedb_client_api::Action::CreateHotBackup) { + return Err(spacetimedb_client_api::Unauthorized::Unauthorized { + subject, + action, + database: Some(database.database_identity), + source: Some(anyhow::anyhow!("hot backup is a server-operator action in standalone")), + }); + } if subject == database.owner_identity { return Ok(()); } @@ -693,10 +741,147 @@ mod tests { use super::*; use anyhow::Result; use spacetimedb::db::Storage; + use spacetimedb::messages::control_db::HostType; + use spacetimedb_client_api::Authorization; use spacetimedb_paths::{cli::*, FromPathUnchecked}; use std::fs; use tempfile::TempDir; + #[test] + fn program_bytes_relative_path_matches_disk_storage_layout() { + assert_eq!( + program_bytes_relative_path(&Hash::ZERO), + PathBuf::from("00").join("0".repeat(62)) + ); + } + + #[test] + fn copy_program_bytes_to_backup_uses_disk_storage_layout() -> Result<()> { + let program_bytes_dir = TempDir::new()?; + let output_dir = TempDir::new()?; + let program_bytes = b"test program bytes"; + let program_hash = hash_bytes(program_bytes); + let relative_path = program_bytes_relative_path(&program_hash); + let src = program_bytes_dir.path().join(&relative_path); + fs::create_dir_all(src.parent().unwrap())?; + fs::write(&src, program_bytes)?; + + copy_program_bytes_to_backup(program_bytes_dir.path(), output_dir.path(), program_hash)?; + + assert_eq!( + fs::read( + output_dir + .path() + .join("server") + .join("program-bytes") + .join(relative_path) + )?, + program_bytes + ); + Ok(()) + } + + #[tokio::test] + async fn database_lifecycle_lock_survives_cancelled_blocking_finalizer() -> Result<()> { + let lock = Arc::new(AsyncMutex::new(())); + let guard = lock.clone().lock_owned().await; + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (finish_tx, finish_rx) = std::sync::mpsc::channel(); + + let handle = tokio::spawn(spacetimedb::util::asyncify(move || -> Result<()> { + let _guard = guard; + started_tx + .send(()) + .map_err(|_| anyhow::anyhow!("finalizer start receiver dropped"))?; + finish_rx.recv().context("waiting for test to release finalizer")?; + Ok(()) + })); + + started_rx.await.context("finalizer did not start")?; + handle.abort(); + + assert!( + lock.try_lock().is_err(), + "database lifecycle lock was released after awaiter cancellation" + ); + + finish_tx + .send(()) + .map_err(|_| anyhow::anyhow!("finalizer finished before release signal"))?; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Ok(guard) = lock.try_lock() { + drop(guard); + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .context("database lifecycle lock was not released after finalizer completed")?; + + Ok(()) + } + + #[tokio::test] + async fn standalone_owner_cannot_authorize_hot_backup() -> Result<()> { + let tempdir = TempDir::new()?; + let keys = tempdir.path().join("keys"); + let root = tempdir.path().join("data"); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(root)); + + fs::create_dir(&keys)?; + data_dir.create()?; + + let ca = CertificateAuthority { + jwt_pub_key_path: PubKeyPath(keys.join("public")), + jwt_priv_key_path: PrivKeyPath(keys.join("private")), + }; + ca.get_or_create_keys()?; + let env = StandaloneEnv::init( + StandaloneOptions { + db_config: db::Config { + storage: Storage::Memory, + page_pool_max_size: None, + }, + durability: DurabilityConfig::default(), + websocket: WebSocketOptions::default(), + wasm: WasmConfig::default(), + v8: V8Config::default(), + hot_backup_root: None, + }, + &ca, + data_dir, + JobCores::without_pinned_cores(), + ) + .await?; + + let owner = Identity::from_claims(LOCALHOST, "owner"); + let database_identity = Identity::from_claims(LOCALHOST, "database"); + env.control_db.insert_database(Database { + id: 0, + database_identity, + owner_identity: owner, + host_type: HostType::Wasm, + initial_program: Hash::ZERO, + bootstrap_generation: 0, + })?; + + env.authorize_action(owner, database_identity, spacetimedb_client_api::Action::UpdateDatabase) + .await?; + let err = env + .authorize_action( + owner, + database_identity, + spacetimedb_client_api::Action::CreateHotBackup, + ) + .await + .unwrap_err(); + assert!(matches!(err, spacetimedb_client_api::Unauthorized::Unauthorized { .. })); + + Ok(()) + } + #[tokio::test] async fn ensure_init_grabs_lock() -> Result<()> { let tempdir = TempDir::new()?; diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index f52c667ea85..5be11ded608 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -6,6 +6,7 @@ use spacetimedb_pg::pg_server; use std::io::{self, Write}; use std::net::IpAddr; use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -23,10 +24,11 @@ use spacetimedb::worker_metrics; use spacetimedb_client_api::routes::database::DatabaseRoutes; use spacetimedb_client_api::routes::router; use spacetimedb_client_api::routes::subscribe::WebSocketOptions; -use spacetimedb_fs_utils::sync_dir; +use spacetimedb_fs_utils::{create_dir_all_sync, sync_dir}; use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; use spacetimedb_paths::server::{ConfigToml, ServerDataDir}; use tokio::net::TcpListener; +use tokio::sync::watch; use tokio::task::JoinHandle; pub fn cli() -> clap::Command { @@ -154,12 +156,47 @@ where } } +const SCHEDULED_BACKUP_DIR: &str = "scheduled"; +const SCHEDULED_BACKUP_MARKER: &str = ".scheduled-backup"; +const SCHEDULED_BACKUP_CREATE_ATTEMPTS: u32 = 1024; +static SCHEDULED_BACKUP_SEQUENCE: AtomicU64 = AtomicU64::new(0); + fn backup_dir_name() -> String { let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|duration| duration.as_nanos()) .unwrap_or_default(); - format!("stdb-{nanos}") + let sequence = SCHEDULED_BACKUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + format!("stdb-{nanos:039}-{:010}-{sequence:020}", std::process::id()) +} + +fn scheduled_backup_database_dir(output_dir: &Path, database_identity: &spacetimedb::Identity) -> PathBuf { + output_dir + .join(SCHEDULED_BACKUP_DIR) + .join(database_identity.to_hex().as_str()) +} + +fn create_scheduled_backup_output(database_dir: &Path) -> anyhow::Result { + create_dir_all_sync(database_dir) + .with_context(|| format!("creating scheduled-backup database dir {}", database_dir.display()))?; + for _ in 0..SCHEDULED_BACKUP_CREATE_ATTEMPTS { + let output_dir = database_dir.join(backup_dir_name()); + match std::fs::create_dir(&output_dir) { + Ok(()) => { + sync_dir(database_dir) + .with_context(|| format!("syncing scheduled-backup database dir {}", database_dir.display()))?; + return Ok(output_dir); + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).with_context(|| format!("creating scheduled-backup output {}", output_dir.display())) + } + } + } + anyhow::bail!( + "failed to reserve a unique scheduled-backup output under {} after {SCHEDULED_BACKUP_CREATE_ATTEMPTS} attempts", + database_dir.display() + ) } fn validate_scheduled_backup_config(config: &ScheduledBackupConfig) -> anyhow::Result<()> { @@ -186,47 +223,55 @@ fn validate_scheduled_backup_config(config: &ScheduledBackupConfig) -> anyhow::R Ok(()) } -/// Remove old scheduled backups, keeping the most recent `keep-last` complete ones. +/// Remove old scheduler-owned backups for one database. /// -/// A backup is complete iff it contains `manifest.json`, which is written as -/// the final step of a backup. `stdb-*` directories without a manifest are -/// ignored by pruning and never occupy a `keep-last` slot, so they cannot -/// crowd out good backups. They are not deleted here because another backup may -/// still be writing into a `stdb-*` directory in the same output root. -fn prune_scheduled_backups(config: &ScheduledBackupConfig) -> anyhow::Result<()> { - let mut complete = Vec::new(); - for entry in std::fs::read_dir(&config.output_dir) - .with_context(|| format!("reading scheduled-backup output-dir {}", config.output_dir.display()))? +/// Both `manifest.json` and the scheduler ownership marker must be present. +/// Manual, API-created, and incomplete `stdb-*` directories are preserved and +/// do not occupy a `keep-last` slot. +fn prune_scheduled_backups(database_dir: &Path, keep_last: Option) -> anyhow::Result<()> { + let Some(keep_last) = keep_last else { + return Ok(()); + }; + + let mut complete_owned = Vec::new(); + for entry in std::fs::read_dir(database_dir) + .with_context(|| format!("reading scheduled-backup database dir {}", database_dir.display()))? { - let entry = entry.with_context(|| format!("reading entry in {}", config.output_dir.display()))?; + let entry = entry.with_context(|| format!("reading entry in {}", database_dir.display()))?; let file_type = entry .file_type() .with_context(|| format!("reading file type for {}", entry.path().display()))?; if !file_type.is_dir() || !entry.file_name().to_string_lossy().starts_with("stdb-") { continue; } - if entry.path().join("manifest.json").is_file() { - complete.push(entry); - } else { - log::warn!("ignoring incomplete scheduled backup {}", entry.path().display()); + + let marker = entry.path().join(SCHEDULED_BACKUP_MARKER); + let has_ownership_marker = marker + .metadata() + .map(|metadata| metadata.is_file() && metadata.len() == 0) + .unwrap_or(false); + if entry.path().join("manifest.json").is_file() && has_ownership_marker { + complete_owned.push(entry); } } - let Some(keep_last) = config.keep_last else { - return Ok(()); - }; - complete.sort_by_key(|entry| entry.file_name()); - let remove_count = complete.len().saturating_sub(keep_last); - for entry in complete.into_iter().take(remove_count) { + + complete_owned.sort_by_key(|entry| entry.file_name()); + let remove_count = complete_owned.len().saturating_sub(keep_last); + for entry in complete_owned.into_iter().take(remove_count) { std::fs::remove_dir_all(entry.path()) .with_context(|| format!("removing old backup {}", entry.path().display()))?; } + if remove_count > 0 { + sync_dir(database_dir) + .with_context(|| format!("syncing scheduled-backup database dir {}", database_dir.display()))?; + } Ok(()) } fn cleanup_failed_scheduled_backup_output(output_dir: &Path) -> anyhow::Result<()> { - if output_dir.exists() && !output_dir.join("manifest.json").is_file() { + if output_dir.exists() { std::fs::remove_dir_all(output_dir) - .with_context(|| format!("removing incomplete scheduled backup {}", output_dir.display()))?; + .with_context(|| format!("removing failed scheduled backup {}", output_dir.display()))?; if let Some(parent) = output_dir.parent() { sync_dir(parent).with_context(|| format!("syncing scheduled-backup output-dir {}", parent.display()))?; } @@ -234,6 +279,20 @@ fn cleanup_failed_scheduled_backup_output(output_dir: &Path) -> anyhow::Result<( Ok(()) } +fn write_scheduled_backup_marker(output_dir: &Path) -> anyhow::Result<()> { + let marker_path = output_dir.join(SCHEDULED_BACKUP_MARKER); + let marker = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&marker_path) + .with_context(|| format!("creating scheduled-backup marker {}", marker_path.display()))?; + marker + .sync_all() + .with_context(|| format!("syncing scheduled-backup marker {}", marker_path.display()))?; + sync_dir(output_dir).with_context(|| format!("syncing scheduled-backup output dir {}", output_dir.display()))?; + Ok(()) +} + async fn run_scheduled_backup(ctx: &Arc, config: &ScheduledBackupConfig) -> anyhow::Result<()> { validate_scheduled_backup_config(config)?; let database_identity = config @@ -245,24 +304,30 @@ async fn run_scheduled_backup(ctx: &Arc, config: &ScheduledBackup .get_database_by_identity(&database_identity) .await? .with_context(|| format!("scheduled-backup database `{}` not found", config.database))?; - let output_dir = config.output_dir.join(backup_dir_name()); - let cleanup_output_dir = output_dir.clone(); + let database_dir = scheduled_backup_database_dir(&config.output_dir, &database_identity); let leader = ctx.leader(database.id).await?; - let backup_result = ctx.create_hot_backup(leader, output_dir).await; - // Remove only this scheduled run's incomplete output on failure. Pruning - // still ignores other manifest-less `stdb-*` dirs because they may belong - // to a concurrent manual or HTTP-triggered backup. + let create_in = database_dir.clone(); + let output_dir = spacetimedb::util::asyncify(move || create_scheduled_backup_output(&create_in)).await?; + let cleanup_output_dir = output_dir.clone(); + let marker_output_dir = output_dir.clone(); + let backup_result = async { + ctx.create_hot_backup(leader, output_dir).await?; + spacetimedb::util::asyncify(move || write_scheduled_backup_marker(&marker_output_dir)).await + } + .await; + // This path is unique to the current run. Other scheduler, manual, API, + // and incomplete outputs are never cleaned from the failure path. let cleanup_result = if backup_result.is_err() { Some(spacetimedb::util::asyncify(move || cleanup_failed_scheduled_backup_output(&cleanup_output_dir)).await) } else { None }; if let Some(Err(err)) = &cleanup_result { - log::warn!("failed to remove incomplete scheduled backup: {err:#}"); + log::warn!("failed to remove failed scheduled backup output: {err:#}"); } // Pruning is blocking filesystem work, so keep it off the async executor. - let prune_config = config.clone(); - let prune_result = spacetimedb::util::asyncify(move || prune_scheduled_backups(&prune_config)).await; + let keep_last = config.keep_last; + let prune_result = spacetimedb::util::asyncify(move || prune_scheduled_backups(&database_dir, keep_last)).await; backup_result?; if let Some(cleanup_result) = cleanup_result { cleanup_result?; @@ -270,8 +335,22 @@ async fn run_scheduled_backup(ctx: &Arc, config: &ScheduledBackup prune_result } -fn spawn_scheduled_backup(ctx: Arc, config: ScheduledBackupConfig) -> JoinHandle<()> { - tokio::spawn(async move { +struct ScheduledBackupTask { + shutdown: watch::Sender, + handle: JoinHandle<()>, +} + +impl ScheduledBackupTask { + async fn shutdown(self) -> anyhow::Result<()> { + let _ = self.shutdown.send(true); + log::info!("waiting for scheduled backup task to stop"); + self.handle.await.context("waiting for scheduled backup task to stop") + } +} + +fn spawn_scheduled_backup(ctx: Arc, config: ScheduledBackupConfig) -> ScheduledBackupTask { + let (shutdown, mut shutdown_rx) = watch::channel(false); + let handle = tokio::spawn(async move { log::info!( "scheduled backup enabled for {} every {:?} into {}", config.database, @@ -280,12 +359,22 @@ fn spawn_scheduled_backup(ctx: Arc, config: ScheduledBackupConfig ); loop { - tokio::time::sleep(config.interval).await; + tokio::select! { + _ = shutdown_rx.changed() => break, + _ = tokio::time::sleep(config.interval) => {} + } + if *shutdown_rx.borrow() { + break; + } if let Err(err) = run_scheduled_backup(&ctx, &config).await { log::error!("scheduled backup failed: {err:#}"); } + if *shutdown_rx.borrow() { + break; + } } - }) + }); + ScheduledBackupTask { shutdown, handle } } pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { @@ -358,6 +447,15 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { .context("cannot omit --jwt-{pub,priv}-key-path when those options are not specified in config.toml")?; let data_dir = Arc::new(data_dir.clone()); + let hot_backup_root = if matches!(storage, Storage::Disk) { + config.hot_backup.root_dir.clone() + } else { + None + }; + if config.hot_backup.root_dir.is_some() && matches!(storage, Storage::Memory) { + log::warn!("hot backup disabled for --in-memory server"); + } + let ctx = StandaloneEnv::init( StandaloneOptions { db_config, @@ -367,7 +465,7 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { websocket: config.websocket, wasm: config.common.wasm, v8: config.common.v8, - hot_backup_root: config.hot_backup.root_dir.clone(), + hot_backup_root, }, &certs, data_dir, @@ -481,7 +579,7 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { .await?; } if let Some(task) = backup_task { - task.abort(); + task.shutdown().await?; } Ok(()) @@ -710,69 +808,139 @@ mod tests { } } + fn create_owned_scheduled_backup(path: &Path) { + std::fs::create_dir(path).unwrap(); + std::fs::write(path.join("manifest.json"), b"{}").unwrap(); + write_scheduled_backup_marker(path).unwrap(); + } + + #[test] + fn scheduled_backup_database_dir_is_scoped_by_identity() { + let identity = spacetimedb::Identity::ONE; + let output_dir = Path::new("/var/backups/stdb"); + + assert_eq!( + scheduled_backup_database_dir(output_dir, &identity), + output_dir.join("scheduled").join(identity.to_hex().as_str()) + ); + } + #[test] - fn prune_scheduled_backups_ignores_incomplete_backups_without_keep_last() { + fn write_scheduled_backup_marker_creates_zero_byte_file() { let temp = tempfile::tempdir().unwrap(); - let incomplete = temp.path().join("stdb-incomplete"); - let complete = temp.path().join("stdb-complete"); - let unrelated = temp.path().join("not-a-backup"); + let backup = temp.path().join("stdb-0001"); + std::fs::create_dir(&backup).unwrap(); - std::fs::create_dir(&incomplete).unwrap(); - std::fs::write(incomplete.join("partial"), b"not done").unwrap(); - std::fs::create_dir(&complete).unwrap(); - std::fs::write(complete.join("manifest.json"), b"{}").unwrap(); - std::fs::create_dir(&unrelated).unwrap(); + write_scheduled_backup_marker(&backup).unwrap(); - prune_scheduled_backups(&scheduled_backup_config(temp.path().to_path_buf(), None)).unwrap(); + let marker = std::fs::metadata(backup.join(SCHEDULED_BACKUP_MARKER)).unwrap(); + assert!(marker.is_file()); + assert_eq!(marker.len(), 0); + } - assert!(incomplete.exists()); - assert!(complete.exists()); - assert!(unrelated.exists()); + #[test] + fn create_scheduled_backup_output_reserves_an_empty_unique_directory() { + let temp = tempfile::tempdir().unwrap(); + let database_dir = temp.path().join("database"); + + let first = create_scheduled_backup_output(&database_dir).unwrap(); + let second = create_scheduled_backup_output(&database_dir).unwrap(); + + assert_ne!(first, second); + assert!(first.is_dir()); + assert!(second.is_dir()); + assert_eq!(first.read_dir().unwrap().count(), 0); + assert_eq!(second.read_dir().unwrap().count(), 0); } #[test] - fn prune_scheduled_backups_keeps_latest_complete_and_ignores_incomplete_with_keep_last() { + fn prune_scheduled_backups_only_removes_complete_owned_backups() { let temp = tempfile::tempdir().unwrap(); - let old_complete = temp.path().join("stdb-0001"); - let new_complete = temp.path().join("stdb-0002"); + let old_owned = temp.path().join("stdb-0001"); + let manual = temp.path().join("stdb-0002"); let incomplete = temp.path().join("stdb-0003"); + let marker_only = temp.path().join("stdb-0004"); + let new_owned = temp.path().join("stdb-0005"); - std::fs::create_dir(&old_complete).unwrap(); - std::fs::write(old_complete.join("manifest.json"), b"{}").unwrap(); - std::fs::create_dir(&new_complete).unwrap(); - std::fs::write(new_complete.join("manifest.json"), b"{}").unwrap(); + create_owned_scheduled_backup(&old_owned); + std::fs::create_dir(&manual).unwrap(); + std::fs::write(manual.join("manifest.json"), b"{}").unwrap(); std::fs::create_dir(&incomplete).unwrap(); std::fs::write(incomplete.join("partial"), b"not done").unwrap(); + std::fs::create_dir(&marker_only).unwrap(); + write_scheduled_backup_marker(&marker_only).unwrap(); + create_owned_scheduled_backup(&new_owned); - prune_scheduled_backups(&scheduled_backup_config(temp.path().to_path_buf(), Some(1))).unwrap(); + prune_scheduled_backups(temp.path(), Some(1)).unwrap(); - assert!(!old_complete.exists()); - assert!(new_complete.exists()); + assert!(!old_owned.exists()); + assert!(manual.exists(), "manual stdb-* backup must be preserved"); assert!(incomplete.exists()); + assert!(marker_only.exists()); + assert!(new_owned.exists()); } #[test] - fn cleanup_failed_scheduled_backup_output_removes_incomplete_dir() { + fn prune_scheduled_backups_is_scoped_to_one_database_dir() { let temp = tempfile::tempdir().unwrap(); - let incomplete = temp.path().join("stdb-incomplete"); - std::fs::create_dir(&incomplete).unwrap(); - std::fs::write(incomplete.join("partial"), b"not done").unwrap(); - - cleanup_failed_scheduled_backup_output(&incomplete).unwrap(); - - assert!(!incomplete.exists()); + let database_a = temp.path().join("database-a"); + let database_b = temp.path().join("database-b"); + std::fs::create_dir(&database_a).unwrap(); + std::fs::create_dir(&database_b).unwrap(); + let a_old = database_a.join("stdb-0001"); + let a_new = database_a.join("stdb-0002"); + let b_old = database_b.join("stdb-0001"); + let b_new = database_b.join("stdb-0002"); + create_owned_scheduled_backup(&a_old); + create_owned_scheduled_backup(&a_new); + create_owned_scheduled_backup(&b_old); + create_owned_scheduled_backup(&b_new); + + prune_scheduled_backups(&database_a, Some(1)).unwrap(); + + assert!(!a_old.exists()); + assert!(a_new.exists()); + assert!(b_old.exists()); + assert!(b_new.exists()); } #[test] - fn cleanup_failed_scheduled_backup_output_preserves_complete_dir() { + fn cleanup_failed_scheduled_backup_output_only_removes_current_output() { let temp = tempfile::tempdir().unwrap(); - let complete = temp.path().join("stdb-complete"); - std::fs::create_dir(&complete).unwrap(); - std::fs::write(complete.join("manifest.json"), b"{}").unwrap(); + let current = temp.path().join("stdb-current"); + let manual = temp.path().join("stdb-manual"); + std::fs::create_dir(¤t).unwrap(); + std::fs::write(current.join("manifest.json"), b"{}").unwrap(); + std::fs::create_dir(&manual).unwrap(); + std::fs::write(manual.join("manifest.json"), b"{}").unwrap(); + + cleanup_failed_scheduled_backup_output(¤t).unwrap(); - cleanup_failed_scheduled_backup_output(&complete).unwrap(); + assert!(!current.exists()); + assert!(manual.exists()); + } + + #[tokio::test] + async fn scheduled_backup_shutdown_waits_for_in_flight_task() { + let (shutdown, _shutdown_rx) = watch::channel(false); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (finish_tx, finish_rx) = tokio::sync::oneshot::channel(); + let handle = tokio::spawn(async move { + started_tx.send(()).unwrap(); + finish_rx.await.unwrap(); + }); + let task = ScheduledBackupTask { shutdown, handle }; + + started_rx.await.unwrap(); + let shutdown = task.shutdown(); + tokio::pin!(shutdown); + tokio::select! { + result = &mut shutdown => panic!("scheduled backup shutdown returned early: {result:?}"), + _ = tokio::time::sleep(Duration::from_millis(10)) => {} + } + finish_tx.send(()).unwrap(); - assert!(complete.exists()); + shutdown.await.unwrap(); } #[test] diff --git a/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md b/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md index 0454ebb7681..8b38f462975 100644 --- a/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md +++ b/docs/docs/00300-resources/00100-how-to/00100-deploy/00200-self-hosting.md @@ -239,14 +239,23 @@ Edit the standalone config: sudo -u spacetimedb nano /stdb/data/config.toml ``` -To enable CLI-triggered hot backups and a scheduled backup for one database, add: +To enable a scheduled backup for one database, add: ```toml -[hot-backup] -root-dir = "/var/backups/stdb" +[scheduled-backup] +database = "{database-name}" +output-dir = "/var/backups/stdb" +interval = "1h" +keep-last = 24 +``` +where _database-name_ is the database name or identity to back up. + +For example: + +```toml [scheduled-backup] -database = "" +database = "module-chat" output-dir = "/var/backups/stdb" interval = "1h" keep-last = 24 @@ -258,21 +267,34 @@ Restart the service after changing the config: sudo systemctl restart spacetimedb ``` -Create a backup from the server host. The `--output-dir` value is relative to `[hot-backup].root-dir` and must name an empty or non-existent directory: +Standalone scheduled backups are the supported self-hosted backup path. The HTTP/CLI hot-backup endpoint requires operator authorization, and standalone's ordinary database-owner tokens cannot create hot backups through that endpoint. + +Each run resolves the configured database to its identity and creates the backup at: + +```text +/var/backups/stdb/scheduled/{database-identity}/{backup-name} +``` + +Here, _backup-name_ is the generated `stdb-*` directory name. Keeping the identity in the path gives each database its own retention history, even when `database` is configured with a name. + +Restore is an offline operation. Stop the service, restore into the server data directory, then start the service again. If the target data directory already has `control-db` and `program-bytes`, they must match the database metadata stored in the backup. ```sh -sudo -u spacetimedb /stdb/spacetime --root-dir=/stdb backup create \ - --server http://127.0.0.1:3000 \ - --database \ - --output-dir manual/ +sudo systemctl stop spacetimedb +sudo -u spacetimedb /stdb/spacetime --root-dir=/stdb backup restore \ + --input-dir /var/backups/stdb/scheduled/{database-identity}/{backup-name} \ + --data-dir /stdb/data \ + --force +sudo systemctl start spacetimedb ``` -Restore is an offline operation. Stop the service, restore into the server data directory, then start the service again: +For example, to restore a selected scheduled backup: ```sh +BACKUP_DIR=/var/backups/stdb/scheduled/c200000000000000000000000000000000000000000000000000000000000000/stdb-000000000000000000000001788000000000000-0000004242-00000000000000000000 sudo systemctl stop spacetimedb sudo -u spacetimedb /stdb/spacetime --root-dir=/stdb backup restore \ - --input-dir /var/backups/stdb/manual/ \ + --input-dir "$BACKUP_DIR" \ --data-dir /stdb/data \ --force sudo systemctl start spacetimedb diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md index b71f2f5b5e2..0ca9db1e450 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md @@ -4,7 +4,7 @@ slug: /cli-reference/standalone-config --- -A local database instance (as started by `spacetime start`) can be configured in `{data-dir}/config.toml`, where `{data-dir}` is the database's data directory. This directory is printed when you run `spacetime start`: +A local database instance (as started by `spacetime start`) can be configured in the `config.toml` file inside _data-dir_, where _data-dir_ is the database's data directory. This directory is printed when you run `spacetime start`: 
spacetimedb-standalone version: 1.0.0
 spacetimedb-standalone path: /home/user/.local/share/spacetime/bin/1.0.0/spacetimedb-standalone
@@ -112,11 +112,11 @@ Size in bytes of the memory buffer holding commit data before flushing to storag
 root-dir = "/var/backups/stdb"
 ```
 
-The `hot-backup` section configures server-side backups created through `spacetime backup create` or the HTTP backup endpoint.
+The `hot-backup` section configures the server-side root for backups created through `spacetime backup create` or the HTTP backup endpoint. These entry points require the server to provide operator authorization; standalone's ordinary database-owner tokens cannot create hot backups through this endpoint. Use scheduled backups for standalone self-hosted servers.
 
 #### `hot-backup.root-dir`
 
-An absolute server path that acts as the root directory for CLI/HTTP-triggered hot backups. Backup creation requests choose an output directory relative to this root. Omit `root-dir` to disable CLI/HTTP-triggered hot backups.
+Sets the absolute server path that acts as the root directory for CLI/HTTP-triggered hot backups. Backup creation requests choose an output directory relative to this root. Omit `root-dir` to disable CLI/HTTP-triggered hot backups.
 
 ### `scheduled-backup`
 
@@ -132,19 +132,19 @@ The `scheduled-backup` section configures a background task that periodically ba
 
 #### `scheduled-backup.database`
 
-The database name or identity to back up.
+Selects the database name or identity to back up.
 
 #### `scheduled-backup.output-dir`
 
-An absolute server path where scheduled backups are created. Each backup is written into a timestamped `stdb-*` subdirectory.
+Sets the absolute server path used as the scheduled-backup root. Each backup is written to `output-dir/scheduled/{database-identity}/stdb-*`, where _database-identity_ is the resolved identity even when `scheduled-backup.database` is configured with a name.
 
 #### `scheduled-backup.interval`
 
-How often to create a backup. Values are strings of any format the [`humantime`] crate can parse, such as `"15m"`, `"1h"`, or `"1day"`.
+Sets how often to create a backup. Values are strings of any format the [`humantime`] crate can parse, such as `"15m"`, `"1h"`, or `"1day"`.
 
 #### `scheduled-backup.keep-last`
 
-The number of complete scheduled backups to retain. Incomplete `stdb-*` directories are ignored during pruning and do not count toward this limit. A failed scheduled run removes its own incomplete output directory; other manifest-less directories are preserved because they may belong to a concurrent manual or HTTP-triggered backup. Omit `keep-last` to keep all complete scheduled backups.
+Sets the number of complete scheduler-owned backups to retain for this database identity. A backup is eligible for pruning only when its `stdb-*` directory contains both `manifest.json` and the scheduler's zero-byte `.scheduled-backup` ownership marker. Manual, API-created, and incomplete directories are preserved and do not count toward this limit. A failed scheduled run removes only its own output directory. Omit `keep-last` to keep all complete scheduled backups.
 
 ### `websocket`