From 0f10251c00a2b79ca02f89920b5cd0b4da6643f4 Mon Sep 17 00:00:00 2001 From: Junbo Wang Date: Tue, 18 Aug 2026 22:42:23 +0800 Subject: [PATCH 1/3] [gateway] Validate configuration and redact secrets Add typed cluster, security, and request-limit configuration on top of the Gateway runtime foundation: validate native client options and cross-field constraints before any listener binds, require user identity mode to carry the request principal over SASL, keep Gateway-owned write guarantees and the per-request authorization identity out of static configuration, retain the warned legacy credential precedence, and redact every credential surface from diagnostics. Moves the tests to their own file, keeping both inside the 3000-line limit. Closes #3969. --- fluss-gateway/src/config.rs | 1802 ++++++++++++++++++++--------- fluss-gateway/src/config/tests.rs | 1375 ++++++++++++++++++++++ fluss-gateway/src/lifecycle.rs | 3 + fluss-gateway/tests/process.rs | 93 ++ 4 files changed, 2740 insertions(+), 533 deletions(-) create mode 100644 fluss-gateway/src/config/tests.rs diff --git a/fluss-gateway/src/config.rs b/fluss-gateway/src/config.rs index e1d66ff415..ecd2ff2871 100644 --- a/fluss-gateway/src/config.rs +++ b/fluss-gateway/src/config.rs @@ -22,10 +22,26 @@ //! ```yaml //! gateway.rest.listen: 0.0.0.0:8080 //! gateway.rest.write.max-request-bytes: 32MiB +//! gateway.clusters: default +//! gateway.cluster.default.bootstrap.servers: 127.0.0.1:9123 +//! gateway.cluster.default.connection.service.account: gateway_svc +//! gateway.cluster.default.client.writer.batch-size: 2MiB //! ``` //! //! Environment variable names are derived from these public keys; for example, -//! `gateway.rest.listen` becomes `FLUSS_GATEWAY__REST__LISTEN`. +//! `gateway.rest.listen` becomes `FLUSS_GATEWAY__REST__LISTEN`, and +//! `gateway.cluster.default.client.writer.batch-size` becomes +//! `FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__BATCH_SIZE`. +//! +//! Two rules make the configuration deterministic before anything binds or connects. `gateway.clusters` +//! is authoritative, and it is so whether or not it was written: an absent list means the single implicit +//! `default` cluster, so a mistyped cluster ID fails startup instead of creating a cluster nothing routes +//! to. And `gateway.cluster..client.*` is an allowlist, not a passthrough: the native-client options +//! that would weaken the write guarantees the gateway advertises, or pin the authorization identity that +//! user identity mode supplies per request, are rejected rather than honoured. +//! +//! Credentials are redacted in diagnostics: typed fields carry them as [`Secret`], and the open +//! `client.*` namespace declares per option which values are sensitive. use serde::Deserialize; use serde::de::{self, Deserializer}; @@ -36,9 +52,44 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::Path; use std::time::Duration; +#[cfg(test)] +mod tests; + /// Environment variable prefix for overrides. pub const ENV_PREFIX: &str = "FLUSS_GATEWAY__"; +/// Stand-in printed wherever a credential would otherwise be rendered. +/// +/// Spelled the same as `Password.HIDDEN_CONTENT` on the Java side, so one deployment's server and gateway +/// logs redact identically. +const REDACTED: &str = "******"; + +/// A configuration value that must never reach a log line, an error, or a debug dump. +/// +/// The wrapper is the whole mechanism: [`Debug`] is the only way diagnostics render configuration, so +/// redacting it here covers every present and future diagnostic without each call site remembering to. +#[derive(Clone, PartialEq, Eq, Deserialize)] +#[serde(transparent)] +pub struct Secret(String); + +impl Secret { + /// Wraps a credential that reached the gateway as plain text, so it redacts from here on. + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + /// Hands the credential to the component that authenticates with it. + pub fn expose(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for Secret { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(REDACTED) + } +} + /// A duration written as ``. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ConfigDuration(Duration); @@ -200,6 +251,13 @@ const REST_MAX_REQUEST_BYTES_KEY: &str = "gateway.rest.write.max-request-bytes"; const METRICS_ENABLED_KEY: &str = "gateway.metrics.enabled"; const METRICS_LISTEN_KEY: &str = "gateway.metrics.exporter.prometheus.listen"; const SHUTDOWN_DRAIN_TIMEOUT_KEY: &str = "gateway.shutdown.drain-timeout"; +const CLUSTERS_KEY: &str = "gateway.clusters"; +const CLUSTER_KEY_PREFIX: &str = "gateway.cluster."; +const CLIENT_OPTION_PREFIX: &str = "client."; +const SECURITY_AUTHENTICATION_KEY: &str = "gateway.security.authentication"; +const SECURITY_USERS_KEY: &str = "gateway.security.users"; +const SECURITY_TOKENS_KEY: &str = "gateway.security.tokens"; +const SECURITY_TRUSTED_HEADER_NAME_KEY: &str = "gateway.security.trusted-header.name"; const DEFAULT_REST_LISTEN: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080); const DEFAULT_REST_HEADER_READ_TIMEOUT: ConfigDuration = ConfigDuration::from_secs(10); @@ -209,44 +267,180 @@ const DEFAULT_METRICS_ENABLED: bool = true; const DEFAULT_METRICS_LISTEN: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9095); const DEFAULT_SHUTDOWN_DRAIN_TIMEOUT: ConfigDuration = ConfigDuration::from_secs(30); +/// The default cluster ID, so a single-cluster deployment configures no cluster list at all. +const DEFAULT_CLUSTER_ID: &str = "default"; +const DEFAULT_BOOTSTRAP_SERVERS: &str = "127.0.0.1:9123"; +const DEFAULT_TRUSTED_HEADER_NAME: &str = "x-forwarded-user"; + +/// How a source value is converted before it reaches Serde. +/// +/// Most options are strings whose typed parsing lives in a `Deserialize` impl, but a bool or an integer +/// has to arrive as the matching YAML scalar, and a server list accepts either a sequence or one +/// comma-separated string. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ValueKind { + /// Parsed downstream by the field's own `Deserialize` impl (address, duration, enum, secret, text). + Text, + Bool, + /// A non-negative integer, which Serde cannot read from a string. + Integer, + /// An integer count of bytes or a byte-size string such as `32MiB`. + Bytes, + /// A YAML sequence or one comma-separated string. + ServerList, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct ConfigEntry { key: &'static str, internal_path: &'static str, + kind: ValueKind, } const CONFIG_ENTRIES: &[ConfigEntry] = &[ ConfigEntry { key: INSTANCE_ID_KEY, internal_path: "server.instance_id", + kind: ValueKind::Text, }, ConfigEntry { key: REST_LISTEN_KEY, internal_path: "server.rest.bind_address", + kind: ValueKind::Text, }, ConfigEntry { key: REST_HEADER_READ_TIMEOUT_KEY, internal_path: "server.rest.header_read_timeout", + kind: ValueKind::Text, }, ConfigEntry { key: REST_REQUEST_TIMEOUT_KEY, internal_path: "server.rest.request_timeout", + kind: ValueKind::Text, }, ConfigEntry { key: REST_MAX_REQUEST_BYTES_KEY, internal_path: "server.rest.max_body_bytes", + kind: ValueKind::Bytes, + }, + ConfigEntry { + key: "gateway.rest.write.max-rows", + internal_path: "request_limits.write_max_rows", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.write.max-concurrent-requests", + internal_path: "request_limits.write_max_concurrent_requests", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.lookup.max-keys", + internal_path: "request_limits.lookup_max_keys", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.lookup.max-key-bytes", + internal_path: "request_limits.lookup_max_key_bytes", + kind: ValueKind::Bytes, + }, + ConfigEntry { + key: "gateway.rest.lookup.max-concurrent-requests", + internal_path: "request_limits.lookup_max_concurrent_requests", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.prefix-lookup.max-prefixes", + internal_path: "request_limits.prefix_lookup_max_prefixes", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.prefix-lookup.max-rows-per-prefix", + internal_path: "request_limits.prefix_lookup_max_rows_per_prefix", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "gateway.rest.prefix-lookup.max-concurrent-requests", + internal_path: "request_limits.prefix_lookup_max_concurrent_requests", + kind: ValueKind::Integer, }, ConfigEntry { key: METRICS_ENABLED_KEY, internal_path: "server.metrics.enabled", + kind: ValueKind::Bool, }, ConfigEntry { key: METRICS_LISTEN_KEY, internal_path: "server.metrics.bind_address", + kind: ValueKind::Text, }, ConfigEntry { key: SHUTDOWN_DRAIN_TIMEOUT_KEY, internal_path: "shutdown.drain_timeout", + kind: ValueKind::Text, + }, + ConfigEntry { + key: SECURITY_AUTHENTICATION_KEY, + internal_path: "security.authentication", + kind: ValueKind::Text, + }, + ConfigEntry { + key: SECURITY_USERS_KEY, + internal_path: "security.users", + kind: ValueKind::Text, + }, + ConfigEntry { + key: SECURITY_TOKENS_KEY, + internal_path: "security.tokens", + kind: ValueKind::Text, + }, + ConfigEntry { + key: SECURITY_TRUSTED_HEADER_NAME_KEY, + internal_path: "security.trusted_header_name", + kind: ValueKind::Text, + }, +]; + +/// The per-cluster options under `gateway.cluster..`, excluding the open `client.*` namespace. +const CLUSTER_ENTRIES: &[ConfigEntry] = &[ + ConfigEntry { + key: "bootstrap.servers", + internal_path: "bootstrap_servers", + kind: ValueKind::ServerList, + }, + ConfigEntry { + key: "connect-timeout", + internal_path: "connect_timeout", + kind: ValueKind::Text, + }, + ConfigEntry { + key: "request-timeout", + internal_path: "request_timeout", + kind: ValueKind::Text, + }, + ConfigEntry { + key: "connection.service.account", + internal_path: "service_account", + kind: ValueKind::Text, + }, + ConfigEntry { + key: "connection.service.secret", + internal_path: "service_secret", + kind: ValueKind::Text, + }, + ConfigEntry { + key: "connection.identity-mode", + internal_path: "identity_mode", + kind: ValueKind::Text, + }, + ConfigEntry { + key: "connection.max", + internal_path: "connection_max", + kind: ValueKind::Integer, + }, + ConfigEntry { + key: "connection.idle-timeout", + internal_path: "connection_idle_timeout", + kind: ValueKind::Text, }, ]; @@ -327,6 +521,520 @@ impl Default for MetricsServerConfig { } } +/// Deserializes a bootstrap list from either a YAML sequence or one comma-separated string. +fn deserialize_server_list<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + #[derive(Deserialize)] + #[serde(untagged)] + enum ListOrCsv { + List(Vec), + Csv(String), + } + + Ok(match ListOrCsv::deserialize(deserializer)? { + ListOrCsv::List(list) => list, + ListOrCsv::Csv(csv) => csv + .split(',') + .map(|entry| entry.trim().to_string()) + .collect(), + }) +} + +/// How the gateway derives the effective Fluss principal for one cluster. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum IdentityMode { + /// One shared connection authenticated as the configured service account. + #[default] + Service, + /// Authenticate as the service account and carry the request's principal as the authorization ID. + User, +} + +/// Connection and native-client settings for one Fluss cluster. +/// +/// The connections themselves arrive with the native backend; this task owns the typed, validated, and +/// redacted contract they will read. +#[derive(Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct ClusterConfig { + #[serde(deserialize_with = "deserialize_server_list")] + pub bootstrap_servers: Vec, + pub connect_timeout: ConfigDuration, + pub request_timeout: ConfigDuration, + /// Canonical account the gateway authenticates to Fluss with. An identity, not a credential, so it is + /// rendered in diagnostics: Fluss treats secrets and tokens as sensitive, not principal names. + pub service_account: Option, + /// Canonical credential for [`Self::service_account`]. + pub service_secret: Option, + pub identity_mode: IdentityMode, + /// Cap on pooled per-user connections. User identity mode only. + pub connection_max: Option, + /// Idle reclamation for pooled per-user connections. User identity mode only. + pub connection_idle_timeout: Option, + /// Validated `gateway.cluster..client.*` options, keyed without the `client.` prefix. + pub client_options: BTreeMap, +} + +impl Default for ClusterConfig { + fn default() -> Self { + Self { + bootstrap_servers: vec![DEFAULT_BOOTSTRAP_SERVERS.to_string()], + connect_timeout: ConfigDuration::from_secs(10), + request_timeout: ConfigDuration::from_secs(30), + service_account: None, + service_secret: None, + identity_mode: IdentityMode::Service, + connection_max: None, + connection_idle_timeout: None, + client_options: BTreeMap::new(), + } + } +} + +impl ClusterConfig { + /// Returns the account actually used, honouring the legacy last-wins override. + pub fn effective_service_account(&self) -> Option<&str> { + self.client_option(LEGACY_SERVICE_ACCOUNT_OPTION) + .or(self.service_account.as_deref()) + } + + /// Returns the credential actually used, honouring the legacy last-wins override. + pub fn effective_service_secret(&self) -> Option<&str> { + self.client_option(LEGACY_SERVICE_SECRET_OPTION) + .or_else(|| self.service_secret.as_ref().map(Secret::expose)) + } + + /// Reads one configured native-client option without its `client.` prefix. + pub fn client_option(&self, option: &str) -> Option<&str> { + self.client_options.get(option).map(String::as_str) + } +} + +impl fmt::Debug for ClusterConfig { + /// Renders the client options with each sensitive value replaced, rather than hiding all of them: the + /// configured tuning is what an operator needs to see, and only the credentials have to disappear. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let client_options: BTreeMap<&str, &str> = self + .client_options + .iter() + .map(|(option, value)| { + let rendered = if client_option_is_sensitive(option) { + REDACTED + } else { + value.as_str() + }; + (option.as_str(), rendered) + }) + .collect(); + f.debug_struct("ClusterConfig") + .field("bootstrap_servers", &self.bootstrap_servers) + .field("connect_timeout", &self.connect_timeout) + .field("request_timeout", &self.request_timeout) + .field("service_account", &self.service_account) + .field("service_secret", &self.service_secret) + .field("identity_mode", &self.identity_mode) + .field("connection_max", &self.connection_max) + .field("connection_idle_timeout", &self.connection_idle_timeout) + .field("client_options", &client_options) + .finish() + } +} + +/// A validated native-client option. The variant records the conversion the native client needs. +/// +/// A credential stays wrapped in [`Secret`] rather than becoming a `Text`, so the parsed value redacts +/// exactly like the configured one did and no caller can print it back out by accident. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ClientOptionValue { + Text(String), + Secret(Secret), + Boolean(bool), + Integer(u64), + Millis(u64), + Bytes(u64), +} + +/// Superseded by `connection.service.account`; still honoured, with a warning. +const LEGACY_SERVICE_ACCOUNT_OPTION: &str = "security.sasl.username"; +/// Superseded by `connection.service.secret`; still honoured, with a warning. +const LEGACY_SERVICE_SECRET_OPTION: &str = "security.sasl.password"; + +/// How one `client.*` value is parsed and bounded. +/// +/// A bound is a fact about the native field the value lands in, so it is declared per option rather than +/// applied uniformly: the writer sizes are `i32` there, while the buffer size and the lookup counts are +/// `usize` and may legitimately exceed `i32::MAX` on a 64-bit target. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ClientOptionKind { + /// Passed through verbatim; any further constraint is a cross-field rule. + Text, + /// Verbatim like [`Self::Text`], but a credential: redacted everywhere it is rendered. + Secret, + Boolean, + /// An integer count the native field accepts within `min..=max`. + Count { + min: u64, + max: u64, + }, + /// A byte size, with the ceiling and the default of the native field. + Size { + max: u64, + default: u64, + }, + /// A duration, handed to the native client as milliseconds. + Duration, +} + +/// Ceilings of the native field types the values are stored in. +const NATIVE_I32_MAX: u64 = i32::MAX as u64; +const NATIVE_USIZE_MAX: u64 = usize::MAX as u64; + +const KIB: u64 = 1024; +const MIB: u64 = 1024 * KIB; + +impl ClientOptionKind { + /// Sensitivity follows from the kind, so no option can be declared a credential and rendered as text + /// at the same time. The Java side has to infer this from the key name instead + /// (`ConfigurationUtils.SENSITIVE_KEY_PARTS`, plus an allowlist for the keys that over-match). + fn is_sensitive(self) -> bool { + matches!(self, Self::Secret) + } +} + +/// One native-client option the gateway accepts under `gateway.cluster..client.`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ClientOptionSpec { + option: &'static str, + kind: ClientOptionKind, +} + +/// The `client.*` allowlist. Anything absent is rejected, so a native-client option the gateway has not +/// considered cannot reach a connection. +const CLIENT_OPTIONS: &[ClientOptionSpec] = &[ + ClientOptionSpec { + option: "security.protocol", + kind: ClientOptionKind::Text, + }, + ClientOptionSpec { + option: "security.sasl.mechanism", + kind: ClientOptionKind::Text, + }, + ClientOptionSpec { + option: LEGACY_SERVICE_ACCOUNT_OPTION, + kind: ClientOptionKind::Text, + }, + ClientOptionSpec { + option: LEGACY_SERVICE_SECRET_OPTION, + kind: ClientOptionKind::Secret, + }, + ClientOptionSpec { + option: "connect.timeout", + kind: ClientOptionKind::Duration, + }, + ClientOptionSpec { + option: "request.timeout", + kind: ClientOptionKind::Duration, + }, + ClientOptionSpec { + option: "writer.batch-size", + kind: ClientOptionKind::Size { + max: NATIVE_I32_MAX, + default: 2 * MIB, + }, + }, + ClientOptionSpec { + option: "writer.request-max-size", + kind: ClientOptionKind::Size { + max: NATIVE_I32_MAX, + default: 10 * MIB, + }, + }, + ClientOptionSpec { + option: "writer.buffer.memory-size", + kind: ClientOptionKind::Size { + max: NATIVE_USIZE_MAX, + default: 64 * MIB, + }, + }, + ClientOptionSpec { + option: "writer.buffer.wait-timeout", + kind: ClientOptionKind::Duration, + }, + ClientOptionSpec { + option: "writer.batch-timeout", + kind: ClientOptionKind::Duration, + }, + ClientOptionSpec { + option: "writer.dynamic-batch-size.enabled", + kind: ClientOptionKind::Boolean, + }, + ClientOptionSpec { + option: "writer.dynamic-batch-size.min", + kind: ClientOptionKind::Size { + max: NATIVE_I32_MAX, + default: 256 * KIB, + }, + }, + ClientOptionSpec { + option: "writer.kv-backpressure.max-throttle", + kind: ClientOptionKind::Duration, + }, + ClientOptionSpec { + option: "lookup.queue-size", + kind: ClientOptionKind::Count { + min: 1, + max: NATIVE_USIZE_MAX, + }, + }, + ClientOptionSpec { + option: "lookup.max-batch-size", + kind: ClientOptionKind::Count { + min: 1, + max: NATIVE_USIZE_MAX, + }, + }, + ClientOptionSpec { + option: "lookup.max-inflight-requests", + kind: ClientOptionKind::Count { + min: 1, + max: NATIVE_USIZE_MAX, + }, + }, + ClientOptionSpec { + option: "lookup.max-retries", + kind: ClientOptionKind::Count { + min: 0, + max: NATIVE_I32_MAX, + }, + }, + ClientOptionSpec { + option: "lookup.batch-timeout", + kind: ClientOptionKind::Duration, + }, +]; + +/// The `client.*` options the gateway refuses outright, each with the reason an operator sees. +/// +/// The first three would silently weaken the write guarantees the gateway advertises. The fourth is never +/// static: user identity mode sets the authorization ID per connection from the authenticated principal. +const RESERVED_CLIENT_OPTIONS: &[(&str, &str)] = &[ + ( + "writer.acks", + "is owned by the Gateway and cannot be overridden", + ), + ( + "writer.retries", + "is owned by the Gateway and cannot be overridden", + ), + ( + "writer.enable-idempotence", + "is owned by the Gateway and cannot be overridden", + ), + ( + "security.sasl.authorization-id", + "is supplied per connection in user identity mode and cannot be configured statically", + ), +]; + +/// True when the option's value must not appear in diagnostics. Unknown options are assumed sensitive, so +/// a value that failed validation cannot leak through a debug dump. +fn client_option_is_sensitive(option: &str) -> bool { + client_option_spec(option).is_none_or(|spec| spec.kind.is_sensitive()) +} + +/// The size the native client will use for `option`: the configured value, or the native default when the +/// deployment left it alone. +/// +/// Returns `None` for an option that is not a size, or for a configured value that failed validation and +/// is already reported on its own account. +/// +/// TODO: source the default from the native `Config::default()` and delegate the relationships to +/// `Config::validate_writer` once the gateway takes fluss-rust as a dependency; the declared defaults +/// exist only because it cannot be called from here yet. +fn effective_size(option: &str, configured: Option<&str>) -> Option { + let ClientOptionKind::Size { default, .. } = client_option_spec(option)?.kind else { + return None; + }; + match configured { + Some(raw) => match parse_client_option(option, raw) { + Ok(ClientOptionValue::Bytes(bytes)) => Some(bytes), + _ => None, + }, + None => Some(default), + } +} + +fn client_option_spec(option: &str) -> Option<&'static ClientOptionSpec> { + CLIENT_OPTIONS.iter().find(|spec| spec.option == option) +} + +/// Parses and bounds one `client.*` option before any connection exists, so a bad native-client value +/// fails startup instead of the first write. +/// +/// Relationships *between* options are not checked here, because a value is validated on its own: see +/// `GatewayConfig::validate_client_size_pairs`. +pub fn parse_client_option(option: &str, raw: &str) -> Result { + if let Some((_, reason)) = RESERVED_CLIENT_OPTIONS + .iter() + .find(|(reserved, _)| *reserved == option) + { + return Err(format!("client.{option} {reason}")); + } + let Some(spec) = client_option_spec(option) else { + return Err(format!( + "client.{option} is not a supported native-client option" + )); + }; + // No message echoes the value: this namespace also carries the legacy SASL credentials. + let malformed = + |expected: &str, reason: String| format!("client.{option}: expected {expected}: {reason}"); + match spec.kind { + ClientOptionKind::Text => Ok(ClientOptionValue::Text(raw.to_string())), + ClientOptionKind::Secret => Ok(ClientOptionValue::Secret(Secret::new(raw))), + ClientOptionKind::Boolean => raw.parse().map(ClientOptionValue::Boolean).map_err( + |error: std::str::ParseBoolError| malformed("true or false", error.to_string()), + ), + ClientOptionKind::Count { min, max } => { + let parsed = raw + .parse::() + .map_err(|error| malformed("a non-negative integer", error.to_string()))?; + if !(min..=max).contains(&parsed) { + return Err(format!("client.{option}: must be between {min} and {max}")); + } + Ok(ClientOptionValue::Integer(parsed)) + } + ClientOptionKind::Size { max, .. } => { + let size = ByteSize::parse(raw).map_err(|error| malformed("a byte size", error))?; + if size.bytes() > max { + return Err(format!("client.{option}: must not exceed {max} bytes")); + } + Ok(ClientOptionValue::Bytes(size.bytes())) + } + ClientOptionKind::Duration => { + let duration = + ConfigDuration::parse(raw).map_err(|error| malformed("a duration", error))?; + u64::try_from(duration.get().as_millis()) + .map(ClientOptionValue::Millis) + .map_err(|_| format!("client.{option}: does not fit in milliseconds")) + } + } +} + +/// How the gateway authenticates its own HTTP callers. +/// +/// The authenticators arrive with the REST authentication task; this task owns the typed contract. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub enum AuthenticationMode { + /// Every caller is accepted and reported as an anonymous principal. + #[default] + Trust, + Password, + Token, + TrustedHeader, +} + +/// Client-to-gateway authentication settings. Every credential-bearing field is a [`Secret`]. +#[derive(Clone, PartialEq, Deserialize, Default)] +#[serde(deny_unknown_fields, default)] +pub struct SecurityConfig { + pub authentication: AuthenticationMode, + /// Password-mode user table; the entries embed password material. + pub users: Option, + /// Token-mode table; the entries are bearer tokens. + pub tokens: Option, + /// Trusted-header-mode header name, defaulting to `x-forwarded-user`. + pub trusted_header_name: Option, +} + +impl SecurityConfig { + /// Returns the header the trusted-header mode reads the principal from. + pub fn trusted_header_name(&self) -> &str { + self.trusted_header_name + .as_deref() + .unwrap_or(DEFAULT_TRUSTED_HEADER_NAME) + } +} + +impl fmt::Debug for SecurityConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SecurityConfig") + .field("authentication", &self.authentication) + .field("users", &self.users) + .field("tokens", &self.tokens) + .field("trusted_header_name", &self.trusted_header_name) + .finish() + } +} + +/// Admission limits for the data-plane APIs, whose handlers arrive in later tasks. +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct RequestLimitsConfig { + pub write_max_rows: u32, + pub write_max_concurrent_requests: u32, + pub lookup_max_keys: u32, + pub lookup_max_key_bytes: ByteSize, + pub lookup_max_concurrent_requests: u32, + pub prefix_lookup_max_prefixes: u32, + pub prefix_lookup_max_rows_per_prefix: u32, + pub prefix_lookup_max_concurrent_requests: u32, +} + +impl Default for RequestLimitsConfig { + fn default() -> Self { + Self { + write_max_rows: 10_000, + write_max_concurrent_requests: 64, + lookup_max_keys: 128, + lookup_max_key_bytes: ByteSize::new(1024 * 1024), + lookup_max_concurrent_requests: 64, + prefix_lookup_max_prefixes: 16, + prefix_lookup_max_rows_per_prefix: 1000, + prefix_lookup_max_concurrent_requests: 32, + } + } +} + +impl RequestLimitsConfig { + fn validate(&self, problems: &mut Vec) { + for (key, value) in [ + ("gateway.rest.write.max-rows", self.write_max_rows), + ( + "gateway.rest.write.max-concurrent-requests", + self.write_max_concurrent_requests, + ), + ("gateway.rest.lookup.max-keys", self.lookup_max_keys), + ( + "gateway.rest.lookup.max-concurrent-requests", + self.lookup_max_concurrent_requests, + ), + ( + "gateway.rest.prefix-lookup.max-prefixes", + self.prefix_lookup_max_prefixes, + ), + ( + "gateway.rest.prefix-lookup.max-rows-per-prefix", + self.prefix_lookup_max_rows_per_prefix, + ), + ( + "gateway.rest.prefix-lookup.max-concurrent-requests", + self.prefix_lookup_max_concurrent_requests, + ), + ] { + if value == 0 { + problems.push(format!("{key} must be greater than zero")); + } + } + if self.lookup_max_key_bytes.bytes() == 0 { + problems + .push("gateway.rest.lookup.max-key-bytes must be greater than zero".to_string()); + } + } +} + /// Graceful-shutdown configuration. #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(deny_unknown_fields, default)] @@ -355,13 +1063,30 @@ impl ShutdownConfig { } /// The validated gateway configuration: everything the process needs before it binds a listener. -#[derive(Debug, Clone, PartialEq, Deserialize, Default)] +#[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct GatewayConfig { pub server: ServerConfig, + /// Every declared Fluss cluster, keyed by the ID used in REST paths. + pub clusters: BTreeMap, + pub security: SecurityConfig, + pub request_limits: RequestLimitsConfig, pub shutdown: ShutdownConfig, } +impl Default for GatewayConfig { + /// Defaults to the single `default` cluster, so a local deployment needs no cluster list. + fn default() -> Self { + Self { + server: ServerConfig::default(), + clusters: BTreeMap::from([(DEFAULT_CLUSTER_ID.to_string(), ClusterConfig::default())]), + security: SecurityConfig::default(), + request_limits: RequestLimitsConfig::default(), + shutdown: ShutdownConfig::default(), + } + } +} + impl GatewayConfig { /// Checks invariants, including values supplied programmatically. pub fn validate(&self) -> Result<(), ConfigError> { @@ -369,6 +1094,9 @@ impl GatewayConfig { self.server.rest.validate(&mut problems); self.shutdown.validate(&mut problems); self.validate_identity(&mut problems); + self.validate_clusters(&mut problems); + self.validate_security(&mut problems); + self.request_limits.validate(&mut problems); if problems.is_empty() { Ok(()) } else { @@ -376,6 +1104,187 @@ impl GatewayConfig { } } + /// Rejects any cluster the gateway could not connect with, before a listener is bound. + fn validate_clusters(&self, problems: &mut Vec) { + if self.clusters.is_empty() { + problems.push(format!("{CLUSTERS_KEY} must declare at least one cluster")); + } + for (id, cluster) in &self.clusters { + if !valid_cluster_id(id) { + problems.push(format!( + "cluster ID {id:?} must start with a lowercase letter and contain only \ + lowercase letters, digits, or underscores" + )); + } + if cluster.bootstrap_servers.is_empty() + || cluster + .bootstrap_servers + .iter() + .any(|server| server.trim().is_empty()) + { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.bootstrap.servers must list at least one non-blank server" + )); + } + validate_duration( + &format!("{CLUSTER_KEY_PREFIX}{id}.connect-timeout"), + cluster.connect_timeout.get(), + problems, + ); + validate_duration( + &format!("{CLUSTER_KEY_PREFIX}{id}.request-timeout"), + cluster.request_timeout.get(), + problems, + ); + if let Some(idle_timeout) = cluster.connection_idle_timeout { + validate_duration( + &format!("{CLUSTER_KEY_PREFIX}{id}.connection.idle-timeout"), + idle_timeout.get(), + problems, + ); + } + if cluster.connection_max == Some(0) { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.max must be greater than zero" + )); + } + for (option, raw) in &cluster.client_options { + if let Err(problem) = parse_client_option(option, raw) { + problems.push(format!("{CLUSTER_KEY_PREFIX}{id}.{problem}")); + } + } + self.validate_credentials(id, cluster, problems); + self.validate_client_size_pairs(id, cluster, problems); + } + } + + /// Requires a service identity the gateway can actually authenticate with, wherever one is needed. + fn validate_credentials(&self, id: &str, cluster: &ClusterConfig, problems: &mut Vec) { + let account = cluster.effective_service_account(); + let secret = cluster.effective_service_secret(); + for (setting, value) in [("account", account), ("secret", secret)] { + if value.is_some_and(|value| value.trim().is_empty()) { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.service.{setting} must not be blank" + )); + } + } + // A blank value is not a credential, so "complete" means usable rather than merely present. + let usable = |value: Option<&str>| value.is_some_and(|value| !value.trim().is_empty()); + let credentials_usable = usable(account) && usable(secret); + if account.is_some() != secret.is_some() { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.service.account and connection.service.secret \ + must be set together" + )); + } + + let protocol = cluster.client_option("security.protocol"); + let sasl = protocol.is_some_and(|protocol| protocol.eq_ignore_ascii_case("sasl")); + if let Some(protocol) = protocol + && !sasl + && !protocol.eq_ignore_ascii_case("plaintext") + { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.client.security.protocol must be plaintext or sasl" + )); + } + if sasl && !credentials_usable { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.client.security.protocol sasl requires \ + connection.service.account and connection.service.secret" + )); + } + if let Some(mechanism) = cluster.client_option("security.sasl.mechanism") + && !mechanism.eq_ignore_ascii_case("plain") + { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.client.security.sasl.mechanism must be PLAIN" + )); + } + + if cluster.identity_mode == IdentityMode::User { + if !credentials_usable { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.identity-mode user requires \ + connection.service.account and connection.service.secret" + )); + } + // Act-as travels as the SASL/PLAIN authorization ID, so without SASL there is nowhere to put + // the request's principal: Fluss would authorize the gateway's own identity for every caller. + // Requiring it to be declared rather than inferring it keeps a security-relevant choice visible. + if !sasl { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.identity-mode user requires \ + client.security.protocol sasl, which carries the request principal as the SASL \ + authorization ID" + )); + } + } + } + + /// Rejects a writer size that cannot fit inside the one that has to hold it. + /// + /// Each side is the configured value or the native default, so overriding one size of a pair is caught + /// here rather than at the first write, which is the whole point of validating before a listener binds. + fn validate_client_size_pairs( + &self, + id: &str, + cluster: &ClusterConfig, + problems: &mut Vec, + ) { + let size = |option: &str| effective_size(option, cluster.client_option(option)); + for (smaller, larger) in [ + ("writer.batch-size", "writer.request-max-size"), + ("writer.batch-size", "writer.buffer.memory-size"), + ("writer.dynamic-batch-size.min", "writer.batch-size"), + ] { + if let (Some(smaller_bytes), Some(larger_bytes)) = (size(smaller), size(larger)) + && smaller_bytes > larger_bytes + { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.client.{smaller} must not exceed client.{larger}" + )); + } + } + } + + /// Requires the credential table the selected authentication mode reads. + fn validate_security(&self, problems: &mut Vec) { + let configured = |secret: &Option| { + secret + .as_ref() + .is_some_and(|value| !value.expose().trim().is_empty()) + }; + match self.security.authentication { + AuthenticationMode::Password if !configured(&self.security.users) => { + problems.push(format!( + "{SECURITY_USERS_KEY} must configure at least one user when \ + {SECURITY_AUTHENTICATION_KEY} is password" + )); + } + AuthenticationMode::Token if !configured(&self.security.tokens) => { + problems.push(format!( + "{SECURITY_TOKENS_KEY} must configure at least one token when \ + {SECURITY_AUTHENTICATION_KEY} is token" + )); + } + AuthenticationMode::TrustedHeader => { + let name = self.security.trusted_header_name(); + if name.is_empty() + || !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + problems.push(format!( + "{SECURITY_TRUSTED_HEADER_NAME_KEY} must be a legal HTTP header name" + )); + } + } + _ => {} + } + } + /// Rejects an unusable instance identity or a port clash between the two listeners. /// /// A non-loopback listener does **not** require an instance ID. Nothing the gateway returns is scoped to an @@ -414,8 +1323,47 @@ impl GatewayConfig { REST_LISTEN_KEY, self.server.rest.bind_address )); } + for (id, cluster) in &self.clusters { + let legacy: Vec<&str> = [LEGACY_SERVICE_ACCOUNT_OPTION, LEGACY_SERVICE_SECRET_OPTION] + .into_iter() + .filter(|option| cluster.client_options.contains_key(*option)) + .collect(); + // One warning per cluster, not per option: an operator migrates the pair together. + if !legacy.is_empty() { + warnings.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.client.{{{}}} is deprecated; use \ + connection.service.account and connection.service.secret. The legacy values \ + keep last-wins precedence", + legacy.join(",") + )); + } + if cluster.identity_mode == IdentityMode::Service + && (cluster.connection_max.is_some() || cluster.connection_idle_timeout.is_some()) + { + warnings.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.max and connection.idle-timeout are \ + ignored because connection.identity-mode is service" + )); + } + } warnings } + + /// Renders the whole configuration for diagnostics with every credential replaced. + /// + /// Redaction is a property of the types, so this is the plain [`Debug`] rendering; the method exists to + /// give callers something they can print without having to know that. + pub fn redacted_debug(&self) -> String { + format!("{self:?}") + } +} + +/// Cluster IDs appear in REST paths and in environment variable names, so they stay in the character set +/// that survives both: no hyphen, because the environment mapping already spends `_` on it. +fn valid_cluster_id(id: &str) -> bool { + let mut bytes = id.bytes(); + bytes.next().is_some_and(|first| first.is_ascii_lowercase()) + && bytes.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') } /// True when two listeners cannot both bind: the addresses are equal, or either is a wildcard @@ -503,10 +1451,7 @@ fn insert_path(table: &mut Mapping, path: &str, value: Value) { } /// Attributes a typed error to the override that supplied the failing option. -fn attribute( - message: String, - overrides: &[(&'static str, &'static str, String, Value)], -) -> ConfigError { +fn attribute(message: String, overrides: &[(String, String, String, Value)]) -> ConfigError { for (_, key, origin, _) in overrides.iter().rev() { if message.starts_with(key) { return ConfigError::Parse(format!("{origin}: {message}")); @@ -528,6 +1473,16 @@ fn publicize_error_path(message: String) -> String { return format!("{}{reason}", entry.key); } } + // A cluster path carries the ID, so the public key is rebuilt rather than looked up. + if let Some(rest) = message.strip_prefix("clusters.") + && let Some((id, rest)) = rest.split_once('.') + { + for entry in CLUSTER_ENTRIES { + if let Some(reason) = rest.strip_prefix(entry.internal_path) { + return format!("{CLUSTER_KEY_PREFIX}{id}.{}{reason}", entry.key); + } + } + } message } @@ -535,15 +1490,32 @@ fn config_entry(key: &str) -> Option<&'static ConfigEntry> { CONFIG_ENTRIES.iter().find(|entry| entry.key == key) } +/// Derives the environment variable name from a public key: drop the `gateway.` prefix, uppercase each +/// dotted segment with `-` folded to `_`, and join the segments with `__`. fn environment_variable(key: &str) -> String { - let suffix = key - .strip_prefix("gateway.") - .expect("configuration keys use the gateway prefix") + let suffix = environment_suffix( + key.strip_prefix("gateway.") + .expect("configuration keys use the gateway prefix"), + ); + format!("{ENV_PREFIX}{suffix}") +} + +fn environment_suffix(dotted: &str) -> String { + dotted .split('.') .map(|segment| segment.replace('-', "_").to_ascii_uppercase()) .collect::>() - .join("__"); - format!("{ENV_PREFIX}{suffix}") + .join("__") +} + +/// Inverts [`environment_suffix`] for the open `client.*` namespace, whose option names are not a fixed +/// list. The mapping round-trips because option names use `-` inside a segment and `.` between segments. +fn environment_suffix_to_option(suffix: &str) -> String { + suffix + .split("__") + .map(|segment| segment.to_ascii_lowercase().replace('_', "-")) + .collect::>() + .join(".") } fn environment_entry(variable: &str) -> Option<&'static ConfigEntry> { @@ -553,26 +1525,212 @@ fn environment_entry(variable: &str) -> Option<&'static ConfigEntry> { } fn convert_environment_value(entry: &ConfigEntry, raw: &str) -> Result { - match entry.key { - METRICS_ENABLED_KEY => raw + match entry.kind { + ValueKind::Bool => raw .parse::() .map(Value::Bool) .map_err(|_| "expected true or false".to_string()), - _ => Ok(Value::String(raw.to_string())), + ValueKind::Integer => raw + .parse::() + .map(|number| Value::Number(number.into())) + .map_err(|_| "expected a non-negative integer".to_string()), + ValueKind::Text | ValueKind::Bytes | ValueKind::ServerList => { + Ok(Value::String(raw.to_string())) + } } } fn convert_file_value(entry: &ConfigEntry, value: &Value) -> Result { - match entry.key { - METRICS_ENABLED_KEY => scalar(value).cloned(), - REST_MAX_REQUEST_BYTES_KEY => match scalar(value)? { + // A quoted scalar is accepted wherever a bare one is, so an operator can move the same value between + // the file and the environment, where everything arrives as a string. + match entry.kind { + ValueKind::Bool => match scalar(value)? { + Value::Bool(_) => Ok(value.clone()), + Value::String(text) => text + .parse::() + .map(Value::Bool) + .map_err(|_| "expected true or false".to_string()), + _ => Err("expected true or false".to_string()), + }, + ValueKind::Integer => match scalar(value)? { + Value::Number(_) => Ok(value.clone()), + Value::String(text) => text + .parse::() + .map(|number| Value::Number(number.into())) + .map_err(|_| "expected a non-negative integer".to_string()), + _ => Err("expected a non-negative integer".to_string()), + }, + ValueKind::Bytes => match scalar(value)? { Value::Number(_) | Value::String(_) => Ok(value.clone()), _ => Err("expected an integer or byte-size string".to_string()), }, - _ => scalar_text(value).map(Value::String), + // A sequence is the natural YAML spelling; one comma-separated string stays accepted so the + // same value can come from the environment. + ValueKind::ServerList => match value { + Value::Sequence(_) => Ok(value.clone()), + _ => scalar_text(value).map(Value::String), + }, + ValueKind::Text => scalar_text(value).map(Value::String), } } +/// One recognised flat configuration key. +enum ResolvedKey { + /// `gateway.clusters`, the authoritative list of configurable cluster IDs. + ClusterDeclaration, + /// A fixed key, resolved to its dotted internal path. + Fixed(&'static ConfigEntry), + /// A per-cluster key under `gateway.cluster..`. + Cluster { + id: String, + entry: &'static ConfigEntry, + }, + /// A native-client option under `gateway.cluster..client.`. + ClientOption { id: String, option: String }, +} + +/// Resolves one public key, naming the key exactly as the operator wrote it when it is not recognised. +fn resolve_key(key: &str) -> Result { + let unknown = || ConfigError::Parse(format!("unknown configuration key: {key}")); + if key == CLUSTERS_KEY { + return Ok(ResolvedKey::ClusterDeclaration); + } + if let Some(entry) = config_entry(key) { + return Ok(ResolvedKey::Fixed(entry)); + } + let Some((id, suffix)) = key + .strip_prefix(CLUSTER_KEY_PREFIX) + .and_then(|rest| rest.split_once('.')) + else { + return Err(unknown()); + }; + if !valid_cluster_id(id) { + return Err(ConfigError::Parse(format!( + "invalid cluster ID in configuration key: {key}" + ))); + } + if let Some(option) = suffix.strip_prefix(CLIENT_OPTION_PREFIX) { + if option.is_empty() { + return Err(unknown()); + } + return Ok(ResolvedKey::ClientOption { + id: id.to_string(), + option: option.to_string(), + }); + } + CLUSTER_ENTRIES + .iter() + .find(|entry| entry.key == suffix) + .map(|entry| ResolvedKey::Cluster { + id: id.to_string(), + entry, + }) + .ok_or_else(unknown) +} + +/// Resolves one `FLUSS_GATEWAY__*` variable, which uses the same vocabulary as the file keys. +fn resolve_environment_variable(variable: &str) -> Result { + let unknown = || ConfigError::UnknownEnvKey(variable.to_string()); + let Some(suffix) = variable.strip_prefix(ENV_PREFIX) else { + return Err(unknown()); + }; + if suffix == environment_suffix("clusters") { + return Ok(ResolvedKey::ClusterDeclaration); + } + if let Some(entry) = environment_entry(variable) { + return Ok(ResolvedKey::Fixed(entry)); + } + let Some((id, rest)) = suffix + .strip_prefix("CLUSTER__") + .and_then(|rest| rest.split_once("__")) + else { + return Err(unknown()); + }; + let id = id.to_ascii_lowercase(); + if !valid_cluster_id(&id) { + return Err(unknown()); + } + if let Some(option) = rest.strip_prefix("CLIENT__") { + let option = environment_suffix_to_option(option); + if option.is_empty() { + return Err(unknown()); + } + return Ok(ResolvedKey::ClientOption { id, option }); + } + CLUSTER_ENTRIES + .iter() + .find(|entry| environment_suffix(entry.key) == rest) + .map(|entry| ResolvedKey::Cluster { id, entry }) + .ok_or_else(unknown) +} + +/// Reads the cluster IDs the file may configure, from a comma-separated string or a YAML sequence. +fn declared_cluster_ids(value: &Value) -> Result, ConfigError> { + let ids: Vec = match value { + Value::String(csv) => csv.split(',').map(|id| id.trim().to_string()).collect(), + Value::Sequence(items) => items + .iter() + .map(|item| { + item.as_str().map(str::to_string).ok_or_else(|| { + ConfigError::Parse(format!("{CLUSTERS_KEY}: entries must be strings")) + }) + }) + .collect::>()?, + _ => { + return Err(ConfigError::Parse(format!( + "{CLUSTERS_KEY}: expected a comma-separated string or a list" + ))); + } + }; + for id in &ids { + if !valid_cluster_id(id) { + return Err(ConfigError::Parse(format!( + "invalid cluster ID in {CLUSTERS_KEY}: {id:?}" + ))); + } + } + Ok(ids) +} + +/// Returns the mapping holding one cluster's fields, creating it when the cluster is first mentioned. +fn cluster_mapping<'a>(table: &'a mut Mapping, id: &str) -> &'a mut Mapping { + let clusters = table + .entry(Value::String("clusters".to_string())) + .or_insert_with(|| Value::Mapping(Mapping::new())); + if !clusters.is_mapping() { + *clusters = Value::Mapping(Mapping::new()); + } + let clusters = clusters.as_mapping_mut().expect("mapping inserted above"); + let cluster = clusters + .entry(Value::String(id.to_string())) + .or_insert_with(|| Value::Mapping(Mapping::new())); + if !cluster.is_mapping() { + *cluster = Value::Mapping(Mapping::new()); + } + cluster.as_mapping_mut().expect("mapping inserted above") +} + +/// Writes one native-client option. The option name keeps its dots, so it cannot go through +/// [`insert_path`], which would read them as nesting. +fn insert_client_option(table: &mut Mapping, id: &str, option: &str, raw: String) { + let cluster = cluster_mapping(table, id); + let options = cluster + .entry(Value::String("client_options".to_string())) + .or_insert_with(|| Value::Mapping(Mapping::new())); + if !options.is_mapping() { + *options = Value::Mapping(Mapping::new()); + } + options + .as_mapping_mut() + .expect("mapping inserted above") + .insert(Value::String(option.to_string()), Value::String(raw)); +} + +/// Writes one per-cluster field. +fn insert_cluster_value(table: &mut Mapping, id: &str, field: &str, value: Value) { + cluster_mapping(table, id).insert(Value::String(field.to_string()), value); +} + fn scalar(value: &Value) -> Result<&Value, String> { match value { Value::Bool(_) | Value::Number(_) | Value::String(_) => Ok(value), @@ -592,13 +1750,14 @@ fn scalar_text(value: &Value) -> Result { } } -/// Parses the flat-key YAML file into the nested mapping deserialized by [`GatewayConfig`]. -fn read_config_file(contents: &str) -> Result { +/// Parses the flat-key YAML file into the nested mapping deserialized by [`GatewayConfig`], along with +/// the `gateway.clusters` declaration when the file carries one. +fn read_config_file(contents: &str) -> Result<(Mapping, Option>), ConfigError> { let document: Value = serde_yaml_ng::from_str(contents).map_err(|e| ConfigError::Parse(e.to_string()))?; let mut table = Mapping::new(); if document.is_null() { - return Ok(table); + return Ok((table, None)); } let mapping = document.as_mapping().ok_or_else(|| { ConfigError::Parse( @@ -606,17 +1765,61 @@ fn read_config_file(contents: &str) -> Result { ) })?; + let mut declared = None; for (key, value) in mapping { let key = key .as_str() .ok_or_else(|| ConfigError::Parse("configuration keys must be strings".to_string()))?; - let entry = config_entry(key) - .ok_or_else(|| ConfigError::Parse(format!("unknown configuration key: {key}")))?; - let value = convert_file_value(entry, value) - .map_err(|reason| ConfigError::Parse(format!("{key}: {reason}")))?; - insert_path(&mut table, entry.internal_path, value); + let reason = |reason: String| ConfigError::Parse(format!("{key}: {reason}")); + match resolve_key(key)? { + ResolvedKey::ClusterDeclaration => declared = Some(declared_cluster_ids(value)?), + ResolvedKey::Fixed(entry) => { + let value = convert_file_value(entry, value).map_err(reason)?; + insert_path(&mut table, entry.internal_path, value); + } + ResolvedKey::Cluster { id, entry } => { + let value = convert_file_value(entry, value).map_err(reason)?; + insert_cluster_value(&mut table, &id, entry.internal_path, value); + } + ResolvedKey::ClientOption { id, option } => { + let raw = scalar_text(value).map_err(reason)?; + insert_client_option(&mut table, &id, &option, raw); + } + } + } + Ok((table, declared)) +} + +/// Makes every declared cluster exist and rejects any cluster configured without being declared. +/// +/// The declaration is authoritative whether or not it was written: an absent `gateway.clusters` means the +/// single implicit `default` cluster, so a mistyped ID fails startup instead of quietly creating a second +/// cluster that nothing routes to. +fn reconcile_declared_clusters( + table: &mut Mapping, + declared: Option<&[String]>, +) -> Result<(), ConfigError> { + let implicit = [DEFAULT_CLUSTER_ID.to_string()]; + let declared = declared.unwrap_or(&implicit); + for id in declared { + cluster_mapping(table, id); + } + let configured: Vec = table + .get(Value::String("clusters".to_string())) + .and_then(Value::as_mapping) + .into_iter() + .flat_map(Mapping::keys) + .filter_map(Value::as_str) + .map(str::to_string) + .collect(); + for id in configured { + if !declared.contains(&id) { + return Err(ConfigError::Parse(format!( + "{CLUSTER_KEY_PREFIX}{id}.* is configured but {id} is not declared in {CLUSTERS_KEY}" + ))); + } } - Ok(table) + Ok(()) } /// Loads configuration from all sources with precedence CLI > env > file > defaults. @@ -628,34 +1831,56 @@ pub fn load( cli: &CliOverrides, ) -> Result { let mut table = Mapping::new(); + let mut declared = None; if let Some(path) = path { let contents = std::fs::read_to_string(path) .map_err(|e| ConfigError::Io(format!("{}: {e}", path.display())))?; - table = read_config_file(&contents)?; + (table, declared) = read_config_file(&contents)?; } // Each override is kept with the source that wrote it, so a failure names what the operator wrote. - let mut overrides: Vec<(&'static str, &'static str, String, Value)> = Vec::new(); - for (key, raw) in env { - if !key.starts_with(ENV_PREFIX) { + let mut overrides: Vec<(String, String, String, Value)> = Vec::new(); + for (variable, raw) in env { + if !variable.starts_with(ENV_PREFIX) { continue; } - let entry = - environment_entry(key).ok_or_else(|| ConfigError::UnknownEnvKey(key.clone()))?; - overrides.push(( - entry.internal_path, - entry.key, - key.clone(), - convert_environment_value(entry, raw) - .map_err(|reason| ConfigError::Parse(format!("{key}: {}: {reason}", entry.key)))?, - )); + match resolve_environment_variable(variable)? { + ResolvedKey::ClusterDeclaration => { + declared = Some(declared_cluster_ids(&Value::String(raw.clone()))?); + } + ResolvedKey::Fixed(entry) => overrides.push(( + entry.internal_path.to_string(), + entry.key.to_string(), + variable.clone(), + convert_environment_value(entry, raw).map_err(|reason| { + ConfigError::Parse(format!("{variable}: {}: {reason}", entry.key)) + })?, + )), + ResolvedKey::Cluster { id, entry } => { + let public_key = format!("{CLUSTER_KEY_PREFIX}{id}.{}", entry.key); + let value = convert_environment_value(entry, raw).map_err(|reason| { + ConfigError::Parse(format!("{variable}: {public_key}: {reason}")) + })?; + overrides.push(( + format!("clusters.{id}.{}", entry.internal_path), + public_key, + variable.clone(), + value, + )); + } + // Client options are written straight in: the whole namespace lands in one map, so there is + // no typed path an error could be attributed to. + ResolvedKey::ClientOption { id, option } => { + insert_client_option(&mut table, &id, &option, raw.clone()); + } + } } if let Some(value) = &cli.bind_address { let entry = config_entry(REST_LISTEN_KEY).expect("REST listen option is registered"); overrides.push(( - entry.internal_path, - entry.key, + entry.internal_path.to_string(), + entry.key.to_string(), "--bind-address".to_string(), Value::String(value.clone()), )); @@ -665,6 +1890,8 @@ pub fn load( insert_path(&mut table, path, value.clone()); } + reconcile_declared_clusters(&mut table, declared.as_deref())?; + let config = deserialize_config(Value::Mapping(table)).map_err(|error| { let ConfigError::Parse(message) = error else { unreachable!("deserialization only creates parse errors") @@ -675,494 +1902,3 @@ pub fn load( config.validate()?; Ok(config) } - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - - fn no_env() -> BTreeMap { - BTreeMap::new() - } - - fn write_temp_config(contents: &str) -> tempfile::NamedTempFile { - let mut file = tempfile::NamedTempFile::new().expect("temp file"); - file.write_all(contents.as_bytes()).expect("write"); - file - } - - fn load_file(contents: &str) -> Result { - let file = write_temp_config(contents); - load(Some(file.path()), &no_env(), &CliOverrides::default()) - } - - fn problems(error: ConfigError) -> Vec { - match error { - ConfigError::Invalid(problems) => problems, - other => panic!("expected Invalid, got: {other:?}"), - } - } - - #[test] - fn defaults_when_no_sources() { - let config = load(None, &no_env(), &CliOverrides::default()).unwrap(); - assert_eq!( - config.server.rest.bind_address, - "127.0.0.1:8080".parse().unwrap() - ); - assert_eq!(config.server.rest.max_body_bytes.bytes(), 32 * 1024 * 1024); - assert_eq!( - config.server.rest.request_timeout.get(), - Duration::from_secs(30) - ); - assert!(config.server.metrics.enabled); - assert_eq!( - config.server.metrics.bind_address, - "127.0.0.1:9095".parse().unwrap() - ); - assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(30)); - assert!(config.warnings().is_empty()); - } - - #[test] - fn public_yaml_options_are_loaded() { - let config = load_file( - r#" - gateway.instance-id: gateway-1 - gateway.rest.listen: 0.0.0.0:8080 - gateway.rest.write.max-request-bytes: 32MiB - gateway.rest.write.request-timeout: 30s - gateway.metrics.enabled: true - gateway.metrics.exporter.prometheus.listen: 0.0.0.0:9095 - gateway.shutdown.drain-timeout: 10s - "#, - ) - .unwrap(); - assert_eq!(config.server.instance_id.as_deref(), Some("gateway-1")); - assert_eq!( - config.server.rest.bind_address, - "0.0.0.0:8080".parse().unwrap() - ); - assert_eq!(config.server.rest.max_body_bytes.bytes(), 32 * 1024 * 1024); - assert_eq!( - config.server.rest.request_timeout.get(), - Duration::from_secs(30) - ); - assert!(config.server.metrics.enabled); - assert_eq!( - config.server.metrics.bind_address, - "0.0.0.0:9095".parse().unwrap() - ); - assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(10)); - } - - #[test] - fn unknown_file_keys_name_the_original_key() { - for contents in [ - "gateway.rest.listenn: 0.0.0.0:8080\n", - "rest.listen: 0.0.0.0:8080\n", - "gateway.rest.lookup.max-keyz: 5\n", - "gateway.scan.cursor-ttl: 1m\n", - "gateway.tls.cert: /etc/tls.pem\n", - ] { - let error = load_file(contents).unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); - let key = contents.split(':').next().unwrap(); - assert!(error.to_string().contains(key), "{key}: {error}"); - } - } - - #[test] - fn source_precedence_is_cli_then_env_then_file_then_defaults() { - let file = write_temp_config( - r#" - gateway.rest.listen: 127.0.0.1:18080 - gateway.metrics.enabled: true - "#, - ); - let mut env = no_env(); - env.insert( - "FLUSS_GATEWAY__REST__LISTEN".to_string(), - "127.0.0.1:28080".to_string(), - ); - env.insert( - "FLUSS_GATEWAY__METRICS__ENABLED".to_string(), - "false".to_string(), - ); - env.insert("PATH".to_string(), "/usr/bin".to_string()); - - let config = load( - Some(file.path()), - &env, - &CliOverrides { - bind_address: Some("127.0.0.1:38080".to_string()), - }, - ) - .unwrap(); - assert_eq!( - config.server.rest.bind_address, - "127.0.0.1:38080".parse().unwrap() - ); - assert!(!config.server.metrics.enabled); - } - - #[test] - fn missing_file_reported() { - let error = load( - Some(Path::new("/nonexistent/gateway.yaml")), - &no_env(), - &CliOverrides::default(), - ) - .unwrap_err(); - assert!(matches!(error, ConfigError::Io(_)), "got: {error:?}"); - } - - #[test] - fn malformed_file_reports_position() { - let error = load_file("gateway.rest.listen: [1\n").unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); - assert!(error.to_string().contains("line"), "got: {error}"); - } - - #[test] - fn duplicate_flat_key_rejected() { - let error = - load_file("gateway.rest.listen: 127.0.0.1:8080\ngateway.rest.listen: 127.0.0.1:8081\n") - .unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); - assert!(error.to_string().contains("duplicate"), "got: {error}"); - } - - #[test] - fn unknown_environment_variables_are_rejected() { - for key in [ - "FLUSS_GATEWAY__REST__LISTENN", - "FLUSS_GATEWAY__QUERY__ENABLED", - "FLUSS_GATEWAY__SERVER_REST__BIND_ADDRESS", - ] { - let mut env = no_env(); - env.insert(key.to_string(), "value".to_string()); - let error = load(None, &env, &CliOverrides::default()).unwrap_err(); - assert!( - matches!(error, ConfigError::UnknownEnvKey(_)), - "{key}: {error:?}" - ); - assert!(error.to_string().contains(key), "{key}: {error}"); - } - } - - #[test] - fn file_error_under_a_section_with_an_env_override_names_the_file() { - let file = write_temp_config("gateway.shutdown.drain-timeout: 0s\n"); - let mut env = no_env(); - env.insert( - "FLUSS_GATEWAY__REST__LISTEN".to_string(), - "127.0.0.1:28080".to_string(), - ); - let error = load(Some(file.path()), &env, &CliOverrides::default()).unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); - assert!( - error.to_string().contains("gateway.shutdown.drain-timeout"), - "got: {error}" - ); - assert!( - !error.to_string().contains("FLUSS_GATEWAY__"), - "file problem misattributed to the env override: {error}" - ); - } - - #[test] - fn public_environment_options_are_loaded_by_type() { - let env = BTreeMap::from([ - ("FLUSS_GATEWAY__INSTANCE_ID".to_string(), "123".to_string()), - ( - "FLUSS_GATEWAY__REST__LISTEN".to_string(), - "127.0.0.1:18080".to_string(), - ), - ( - "FLUSS_GATEWAY__REST__WRITE__REQUEST_TIMEOUT".to_string(), - "5s".to_string(), - ), - ( - "FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES".to_string(), - "2MiB".to_string(), - ), - ( - "FLUSS_GATEWAY__METRICS__ENABLED".to_string(), - "false".to_string(), - ), - ( - "FLUSS_GATEWAY__METRICS__EXPORTER__PROMETHEUS__LISTEN".to_string(), - "127.0.0.1:19095".to_string(), - ), - ( - "FLUSS_GATEWAY__SHUTDOWN__DRAIN_TIMEOUT".to_string(), - "10s".to_string(), - ), - ]); - - let config = load(None, &env, &CliOverrides::default()).unwrap(); - assert_eq!(config.server.instance_id.as_deref(), Some("123")); - assert_eq!( - config.server.rest.bind_address, - "127.0.0.1:18080".parse().unwrap() - ); - assert_eq!( - config.server.rest.request_timeout.get(), - Duration::from_secs(5) - ); - assert_eq!(config.server.rest.max_body_bytes.bytes(), 2 * 1024 * 1024); - assert!(!config.server.metrics.enabled); - assert_eq!( - config.server.metrics.bind_address, - "127.0.0.1:19095".parse().unwrap() - ); - assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(10)); - } - - #[test] - fn invalid_env_value_names_the_variable() { - let mut env = no_env(); - env.insert( - "FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES".to_string(), - "many".to_string(), - ); - let error = load(None, &env, &CliOverrides::default()).unwrap_err(); - assert!( - error - .to_string() - .contains("FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES"), - "got: {error}" - ); - } - - #[test] - fn invalid_cli_value_names_the_flag() { - let cli = CliOverrides { - bind_address: Some("not-an-address".to_string()), - }; - let error = load(None, &no_env(), &cli).unwrap_err(); - assert!(error.to_string().contains("--bind-address"), "got: {error}"); - } - - #[test] - fn invalid_duration_rejected() { - for bad in ["0ms", "60", "60 s", "6.5s", "s", "60d", "-1s"] { - let error = - load_file(&format!("gateway.shutdown.drain-timeout: \"{bad}\"\n")).unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "{bad}: {error:?}"); - assert!( - error.to_string().contains("gateway.shutdown.drain-timeout"), - "{bad}: {error}" - ); - } - } - - #[test] - fn overflowing_duration_is_rejected_rather_than_saturated() { - for bad in [ - "18446744073709551615ms", - "18446744073709551615s", - "18446744073709551615m", - "18446744073709551615h", - ] { - let error = ConfigDuration::parse(bad).unwrap_err(); - assert!(error.contains("must not exceed"), "{bad}: {error}"); - } - assert_eq!( - ConfigDuration::parse("31536000s").unwrap().get(), - MAX_CONFIG_DURATION - ); - assert!(ConfigDuration::parse("31536001s").is_err()); - } - - #[test] - fn programmatically_constructed_durations_are_validated() { - let mut config = GatewayConfig::default(); - config.server.rest.request_timeout = ConfigDuration::from_millis(0); - config.shutdown.drain_timeout = - ConfigDuration::from_secs(MAX_CONFIG_DURATION.as_secs() + 1); - - let errors = problems(config.validate().unwrap_err()); - assert!( - errors.iter().any(|error| { - error == "gateway.rest.write.request-timeout must be greater than zero" - }), - "got: {errors:?}" - ); - assert!( - errors.iter().any(|error| { - error == "gateway.shutdown.drain-timeout must not exceed 31536000 seconds" - }), - "got: {errors:?}" - ); - } - - #[test] - fn programmatically_constructed_zero_byte_limit_is_validated() { - let mut config = GatewayConfig::default(); - config.server.rest.max_body_bytes = ByteSize::new(0); - - let errors = problems(config.validate().unwrap_err()); - assert_eq!( - errors, - vec!["gateway.rest.write.max-request-bytes must be greater than zero"] - ); - } - - #[test] - fn invalid_byte_size_rejected() { - for bad in ["0", "\"4Mb\"", "\"MiB\"", "-1", "\"1.5MiB\""] { - let error = - load_file(&format!("gateway.rest.write.max-request-bytes: {bad}\n")).unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "{bad}: {error:?}"); - assert!( - error - .to_string() - .contains("gateway.rest.write.max-request-bytes"), - "{bad}: {error}" - ); - } - } - - #[test] - fn metrics_address_must_differ_from_rest_address() { - let error = load_file( - "gateway.rest.listen: 127.0.0.1:9095\ngateway.metrics.exporter.prometheus.listen: 127.0.0.1:9095\n", - ) - .unwrap_err(); - assert!(problems(error).iter().any(|problem| { - problem.contains( - "gateway.metrics.exporter.prometheus.listen (127.0.0.1:9095) must differ from \ - gateway.rest.listen (127.0.0.1:9095)", - ) - })); - } - - /// Overlap detection covers the wildcard and dual-stack pairs that differ textually but cannot - /// both bind, plus the pairs that coexist. - #[test] - fn listener_overlap_covers_wildcards_and_dual_stack() { - let clashes = [ - ("0.0.0.0:8080", "127.0.0.1:8080"), - ("127.0.0.1:8080", "0.0.0.0:8080"), - ("0.0.0.0:8080", "0.0.0.0:8080"), - ("[::]:8080", "[::1]:8080"), - ("[::]:8080", "0.0.0.0:8080"), - ("127.0.0.1:8080", "[::]:8080"), - ]; - for (rest, metrics) in clashes { - let rest: SocketAddr = rest.parse().unwrap(); - let metrics: SocketAddr = metrics.parse().unwrap(); - assert!( - addresses_overlap(rest, metrics), - "{rest} and {metrics} cannot both bind" - ); - } - - let coexist = [ - ("127.0.0.1:8080", "192.168.1.2:8080"), - ("127.0.0.1:8080", "[::1]:8080"), - ("0.0.0.0:8080", "127.0.0.1:9095"), - ("0.0.0.0:0", "0.0.0.0:0"), - ("127.0.0.1:0", "0.0.0.0:8080"), - ]; - for (rest, metrics) in coexist { - let rest: SocketAddr = rest.parse().unwrap(); - let metrics: SocketAddr = metrics.parse().unwrap(); - assert!( - !addresses_overlap(rest, metrics), - "{rest} and {metrics} can coexist" - ); - } - } - - /// Two ephemeral listeners are not a clash: the OS hands out a different port to each. - #[test] - fn both_listeners_may_ask_for_an_ephemeral_port() { - let config = load_file( - "gateway.rest.listen: 127.0.0.1:0\ngateway.metrics.exporter.prometheus.listen: 127.0.0.1:0\n", - ) - .unwrap(); - assert_eq!(config.server.rest.bind_address.port(), 0); - } - - #[test] - fn non_loopback_bind_is_accepted_without_an_instance_id_but_warns() { - let config = load_file("gateway.rest.listen: 0.0.0.0:8080\n").unwrap(); - assert!(config.server.instance_id.is_none()); - assert_eq!(config.warnings().len(), 1); - assert!(config.warnings()[0].contains("not loopback")); - assert!( - config.warnings()[0].contains("accepts unauthenticated requests"), - "{:?}", - config.warnings() - ); - } - - #[test] - fn malformed_instance_id_rejected() { - let error = load_file("gateway.instance-id: has space\n").unwrap_err(); - assert!( - problems(error) - .iter() - .any(|problem| problem.contains("gateway.instance-id must be 1-128 ASCII")) - ); - } - - #[test] - fn duration_units() { - assert_eq!( - ConfigDuration::parse("250ms").unwrap().get(), - Duration::from_millis(250) - ); - assert_eq!( - ConfigDuration::parse("15m").unwrap().get(), - Duration::from_secs(900) - ); - assert_eq!( - ConfigDuration::parse("2h").unwrap().get(), - Duration::from_secs(7200) - ); - assert!(ConfigDuration::parse("0s").is_err()); - } - - #[test] - fn byte_size_units() { - assert_eq!(ByteSize::parse("512").unwrap().bytes(), 512); - assert_eq!(ByteSize::parse("512B").unwrap().bytes(), 512); - assert_eq!(ByteSize::parse("4KB").unwrap().bytes(), 4000); - assert_eq!(ByteSize::parse("4KiB").unwrap().bytes(), 4096); - assert_eq!(ByteSize::parse("1GiB").unwrap().bytes(), 1024 * 1024 * 1024); - assert!(ByteSize::parse("4TB").is_err()); - assert!(ByteSize::parse("0").is_err()); - } - - #[test] - fn options_are_complete_and_unambiguous() { - let mut public_keys = std::collections::BTreeSet::new(); - let mut internal_paths = std::collections::BTreeSet::new(); - let mut environment_variables = std::collections::BTreeSet::new(); - - for entry in CONFIG_ENTRIES { - assert!(entry.key.starts_with("gateway."), "{entry:?}"); - assert!( - public_keys.insert(entry.key), - "duplicate key: {}", - entry.key - ); - assert!( - internal_paths.insert(entry.internal_path), - "duplicate path: {}", - entry.internal_path - ); - assert!( - environment_variables.insert(environment_variable(entry.key)), - "duplicate environment variable for {}", - entry.key - ); - } - - assert_eq!(CONFIG_ENTRIES.len(), 8); - } -} diff --git a/fluss-gateway/src/config/tests.rs b/fluss-gateway/src/config/tests.rs new file mode 100644 index 0000000000..c57695d02e --- /dev/null +++ b/fluss-gateway/src/config/tests.rs @@ -0,0 +1,1375 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Tests for the configuration vocabulary, its sources, and its validation. + +use super::*; +use std::io::Write; + +fn no_env() -> BTreeMap { + BTreeMap::new() +} + +fn write_temp_config(contents: &str) -> tempfile::NamedTempFile { + let mut file = tempfile::NamedTempFile::new().expect("temp file"); + file.write_all(contents.as_bytes()).expect("write"); + file +} + +fn load_file(contents: &str) -> Result { + let file = write_temp_config(contents); + load(Some(file.path()), &no_env(), &CliOverrides::default()) +} + +fn problems(error: ConfigError) -> Vec { + match error { + ConfigError::Invalid(problems) => problems, + other => panic!("expected Invalid, got: {other:?}"), + } +} + +#[test] +fn defaults_when_no_sources() { + let config = load(None, &no_env(), &CliOverrides::default()).unwrap(); + assert_eq!( + config.server.rest.bind_address, + "127.0.0.1:8080".parse().unwrap() + ); + assert_eq!(config.server.rest.max_body_bytes.bytes(), 32 * 1024 * 1024); + assert_eq!( + config.server.rest.request_timeout.get(), + Duration::from_secs(30) + ); + assert!(config.server.metrics.enabled); + assert_eq!( + config.server.metrics.bind_address, + "127.0.0.1:9095".parse().unwrap() + ); + assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(30)); + assert!(config.warnings().is_empty()); +} + +#[test] +fn public_yaml_options_are_loaded() { + let config = load_file( + r#" +gateway.instance-id: gateway-1 +gateway.rest.listen: 0.0.0.0:8080 +gateway.rest.write.max-request-bytes: 32MiB +gateway.rest.write.request-timeout: 30s +gateway.metrics.enabled: true +gateway.metrics.exporter.prometheus.listen: 0.0.0.0:9095 +gateway.shutdown.drain-timeout: 10s +"#, + ) + .unwrap(); + assert_eq!(config.server.instance_id.as_deref(), Some("gateway-1")); + assert_eq!( + config.server.rest.bind_address, + "0.0.0.0:8080".parse().unwrap() + ); + assert_eq!(config.server.rest.max_body_bytes.bytes(), 32 * 1024 * 1024); + assert_eq!( + config.server.rest.request_timeout.get(), + Duration::from_secs(30) + ); + assert!(config.server.metrics.enabled); + assert_eq!( + config.server.metrics.bind_address, + "0.0.0.0:9095".parse().unwrap() + ); + assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(10)); +} + +#[test] +fn unknown_file_keys_name_the_original_key() { + for contents in [ + "gateway.rest.listenn: 0.0.0.0:8080\n", + "rest.listen: 0.0.0.0:8080\n", + "gateway.rest.lookup.max-keyz: 5\n", + "gateway.scan.cursor-ttl: 1m\n", + "gateway.tls.cert: /etc/tls.pem\n", + ] { + let error = load_file(contents).unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); + let key = contents.split(':').next().unwrap(); + assert!(error.to_string().contains(key), "{key}: {error}"); + } +} + +#[test] +fn source_precedence_is_cli_then_env_then_file_then_defaults() { + let file = write_temp_config( + r#" +gateway.rest.listen: 127.0.0.1:18080 +gateway.metrics.enabled: true +"#, + ); + let mut env = no_env(); + env.insert( + "FLUSS_GATEWAY__REST__LISTEN".to_string(), + "127.0.0.1:28080".to_string(), + ); + env.insert( + "FLUSS_GATEWAY__METRICS__ENABLED".to_string(), + "false".to_string(), + ); + env.insert("PATH".to_string(), "/usr/bin".to_string()); + + let config = load( + Some(file.path()), + &env, + &CliOverrides { + bind_address: Some("127.0.0.1:38080".to_string()), + }, + ) + .unwrap(); + assert_eq!( + config.server.rest.bind_address, + "127.0.0.1:38080".parse().unwrap() + ); + assert!(!config.server.metrics.enabled); +} + +#[test] +fn missing_file_reported() { + let error = load( + Some(Path::new("/nonexistent/gateway.yaml")), + &no_env(), + &CliOverrides::default(), + ) + .unwrap_err(); + assert!(matches!(error, ConfigError::Io(_)), "got: {error:?}"); +} + +#[test] +fn malformed_file_reports_position() { + let error = load_file("gateway.rest.listen: [1\n").unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); + assert!(error.to_string().contains("line"), "got: {error}"); +} + +#[test] +fn duplicate_flat_key_rejected() { + let error = + load_file("gateway.rest.listen: 127.0.0.1:8080\ngateway.rest.listen: 127.0.0.1:8081\n") + .unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); + assert!(error.to_string().contains("duplicate"), "got: {error}"); +} + +#[test] +fn unknown_environment_variables_are_rejected() { + for key in [ + "FLUSS_GATEWAY__REST__LISTENN", + "FLUSS_GATEWAY__QUERY__ENABLED", + "FLUSS_GATEWAY__SERVER_REST__BIND_ADDRESS", + ] { + let mut env = no_env(); + env.insert(key.to_string(), "value".to_string()); + let error = load(None, &env, &CliOverrides::default()).unwrap_err(); + assert!( + matches!(error, ConfigError::UnknownEnvKey(_)), + "{key}: {error:?}" + ); + assert!(error.to_string().contains(key), "{key}: {error}"); + } +} + +#[test] +fn file_error_under_a_section_with_an_env_override_names_the_file() { + let file = write_temp_config("gateway.shutdown.drain-timeout: 0s\n"); + let mut env = no_env(); + env.insert( + "FLUSS_GATEWAY__REST__LISTEN".to_string(), + "127.0.0.1:28080".to_string(), + ); + let error = load(Some(file.path()), &env, &CliOverrides::default()).unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); + assert!( + error.to_string().contains("gateway.shutdown.drain-timeout"), + "got: {error}" + ); + assert!( + !error.to_string().contains("FLUSS_GATEWAY__"), + "file problem misattributed to the env override: {error}" + ); +} + +#[test] +fn public_environment_options_are_loaded_by_type() { + let env = BTreeMap::from([ + ("FLUSS_GATEWAY__INSTANCE_ID".to_string(), "123".to_string()), + ( + "FLUSS_GATEWAY__REST__LISTEN".to_string(), + "127.0.0.1:18080".to_string(), + ), + ( + "FLUSS_GATEWAY__REST__WRITE__REQUEST_TIMEOUT".to_string(), + "5s".to_string(), + ), + ( + "FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES".to_string(), + "2MiB".to_string(), + ), + ( + "FLUSS_GATEWAY__METRICS__ENABLED".to_string(), + "false".to_string(), + ), + ( + "FLUSS_GATEWAY__METRICS__EXPORTER__PROMETHEUS__LISTEN".to_string(), + "127.0.0.1:19095".to_string(), + ), + ( + "FLUSS_GATEWAY__SHUTDOWN__DRAIN_TIMEOUT".to_string(), + "10s".to_string(), + ), + ]); + + let config = load(None, &env, &CliOverrides::default()).unwrap(); + assert_eq!(config.server.instance_id.as_deref(), Some("123")); + assert_eq!( + config.server.rest.bind_address, + "127.0.0.1:18080".parse().unwrap() + ); + assert_eq!( + config.server.rest.request_timeout.get(), + Duration::from_secs(5) + ); + assert_eq!(config.server.rest.max_body_bytes.bytes(), 2 * 1024 * 1024); + assert!(!config.server.metrics.enabled); + assert_eq!( + config.server.metrics.bind_address, + "127.0.0.1:19095".parse().unwrap() + ); + assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(10)); +} + +#[test] +fn invalid_env_value_names_the_variable() { + let mut env = no_env(); + env.insert( + "FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES".to_string(), + "many".to_string(), + ); + let error = load(None, &env, &CliOverrides::default()).unwrap_err(); + assert!( + error + .to_string() + .contains("FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES"), + "got: {error}" + ); +} + +#[test] +fn invalid_cli_value_names_the_flag() { + let cli = CliOverrides { + bind_address: Some("not-an-address".to_string()), + }; + let error = load(None, &no_env(), &cli).unwrap_err(); + assert!(error.to_string().contains("--bind-address"), "got: {error}"); +} + +#[test] +fn invalid_duration_rejected() { + for bad in ["0ms", "60", "60 s", "6.5s", "s", "60d", "-1s"] { + let error = load_file(&format!("gateway.shutdown.drain-timeout: \"{bad}\"\n")).unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "{bad}: {error:?}"); + assert!( + error.to_string().contains("gateway.shutdown.drain-timeout"), + "{bad}: {error}" + ); + } +} + +#[test] +fn overflowing_duration_is_rejected_rather_than_saturated() { + for bad in [ + "18446744073709551615ms", + "18446744073709551615s", + "18446744073709551615m", + "18446744073709551615h", + ] { + let error = ConfigDuration::parse(bad).unwrap_err(); + assert!(error.contains("must not exceed"), "{bad}: {error}"); + } + assert_eq!( + ConfigDuration::parse("31536000s").unwrap().get(), + MAX_CONFIG_DURATION + ); + assert!(ConfigDuration::parse("31536001s").is_err()); +} + +#[test] +fn programmatically_constructed_durations_are_validated() { + let mut config = GatewayConfig::default(); + config.server.rest.request_timeout = ConfigDuration::from_millis(0); + config.shutdown.drain_timeout = ConfigDuration::from_secs(MAX_CONFIG_DURATION.as_secs() + 1); + + let errors = problems(config.validate().unwrap_err()); + assert!( + errors.iter().any(|error| { + error == "gateway.rest.write.request-timeout must be greater than zero" + }), + "got: {errors:?}" + ); + assert!( + errors.iter().any(|error| { + error == "gateway.shutdown.drain-timeout must not exceed 31536000 seconds" + }), + "got: {errors:?}" + ); +} + +#[test] +fn programmatically_constructed_zero_byte_limit_is_validated() { + let mut config = GatewayConfig::default(); + config.server.rest.max_body_bytes = ByteSize::new(0); + + let errors = problems(config.validate().unwrap_err()); + assert_eq!( + errors, + vec!["gateway.rest.write.max-request-bytes must be greater than zero"] + ); +} + +#[test] +fn invalid_byte_size_rejected() { + for bad in ["0", "\"4Mb\"", "\"MiB\"", "-1", "\"1.5MiB\""] { + let error = + load_file(&format!("gateway.rest.write.max-request-bytes: {bad}\n")).unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "{bad}: {error:?}"); + assert!( + error + .to_string() + .contains("gateway.rest.write.max-request-bytes"), + "{bad}: {error}" + ); + } +} + +#[test] +fn metrics_address_must_differ_from_rest_address() { + let error = load_file( + "gateway.rest.listen: 127.0.0.1:9095\ngateway.metrics.exporter.prometheus.listen: 127.0.0.1:9095\n", + ) + .unwrap_err(); + assert!(problems(error).iter().any(|problem| { + problem.contains( + "gateway.metrics.exporter.prometheus.listen (127.0.0.1:9095) must differ from \ + gateway.rest.listen (127.0.0.1:9095)", + ) + })); +} + +/// Overlap detection covers the wildcard and dual-stack pairs that differ textually but cannot +/// both bind, plus the pairs that coexist. +#[test] +fn listener_overlap_covers_wildcards_and_dual_stack() { + let clashes = [ + ("0.0.0.0:8080", "127.0.0.1:8080"), + ("127.0.0.1:8080", "0.0.0.0:8080"), + ("0.0.0.0:8080", "0.0.0.0:8080"), + ("[::]:8080", "[::1]:8080"), + ("[::]:8080", "0.0.0.0:8080"), + ("127.0.0.1:8080", "[::]:8080"), + ]; + for (rest, metrics) in clashes { + let rest: SocketAddr = rest.parse().unwrap(); + let metrics: SocketAddr = metrics.parse().unwrap(); + assert!( + addresses_overlap(rest, metrics), + "{rest} and {metrics} cannot both bind" + ); + } + + let coexist = [ + ("127.0.0.1:8080", "192.168.1.2:8080"), + ("127.0.0.1:8080", "[::1]:8080"), + ("0.0.0.0:8080", "127.0.0.1:9095"), + ("0.0.0.0:0", "0.0.0.0:0"), + ("127.0.0.1:0", "0.0.0.0:8080"), + ]; + for (rest, metrics) in coexist { + let rest: SocketAddr = rest.parse().unwrap(); + let metrics: SocketAddr = metrics.parse().unwrap(); + assert!( + !addresses_overlap(rest, metrics), + "{rest} and {metrics} can coexist" + ); + } +} + +/// Two ephemeral listeners are not a clash: the OS hands out a different port to each. +#[test] +fn both_listeners_may_ask_for_an_ephemeral_port() { + let config = load_file( + "gateway.rest.listen: 127.0.0.1:0\ngateway.metrics.exporter.prometheus.listen: 127.0.0.1:0\n", + ) + .unwrap(); + assert_eq!(config.server.rest.bind_address.port(), 0); +} + +#[test] +fn non_loopback_bind_is_accepted_without_an_instance_id_but_warns() { + let config = load_file("gateway.rest.listen: 0.0.0.0:8080\n").unwrap(); + assert!(config.server.instance_id.is_none()); + assert_eq!(config.warnings().len(), 1); + assert!(config.warnings()[0].contains("not loopback")); + assert!( + config.warnings()[0].contains("accepts unauthenticated requests"), + "{:?}", + config.warnings() + ); +} + +#[test] +fn malformed_instance_id_rejected() { + let error = load_file("gateway.instance-id: has space\n").unwrap_err(); + assert!( + problems(error) + .iter() + .any(|problem| problem.contains("gateway.instance-id must be 1-128 ASCII")) + ); +} + +#[test] +fn duration_units() { + assert_eq!( + ConfigDuration::parse("250ms").unwrap().get(), + Duration::from_millis(250) + ); + assert_eq!( + ConfigDuration::parse("15m").unwrap().get(), + Duration::from_secs(900) + ); + assert_eq!( + ConfigDuration::parse("2h").unwrap().get(), + Duration::from_secs(7200) + ); + assert!(ConfigDuration::parse("0s").is_err()); +} + +#[test] +fn byte_size_units() { + assert_eq!(ByteSize::parse("512").unwrap().bytes(), 512); + assert_eq!(ByteSize::parse("512B").unwrap().bytes(), 512); + assert_eq!(ByteSize::parse("4KB").unwrap().bytes(), 4000); + assert_eq!(ByteSize::parse("4KiB").unwrap().bytes(), 4096); + assert_eq!(ByteSize::parse("1GiB").unwrap().bytes(), 1024 * 1024 * 1024); + assert!(ByteSize::parse("4TB").is_err()); + assert!(ByteSize::parse("0").is_err()); +} + +fn cluster<'a>(config: &'a GatewayConfig, id: &str) -> &'a ClusterConfig { + config.clusters.get(id).expect("configured cluster") +} + +#[test] +fn a_single_default_cluster_needs_no_declaration() { + let config = load(None, &no_env(), &CliOverrides::default()).unwrap(); + assert_eq!(config.clusters.len(), 1); + assert_eq!( + cluster(&config, DEFAULT_CLUSTER_ID).bootstrap_servers, + [DEFAULT_BOOTSTRAP_SERVERS] + ); + assert_eq!( + cluster(&config, DEFAULT_CLUSTER_ID).identity_mode, + IdentityMode::Service + ); + assert_eq!(config.security.authentication, AuthenticationMode::Trust); + assert_eq!(config.request_limits, RequestLimitsConfig::default()); +} + +#[test] +fn typed_cluster_security_and_request_limit_options_are_loaded() { + let config = load_file( + "gateway.clusters: default, analytics\n\ + gateway.cluster.default.bootstrap.servers: [fluss-1:9123, fluss-2:9123]\n\ + gateway.cluster.default.connection.identity-mode: user\n\ + gateway.cluster.default.connection.service.account: gateway_svc\n\ + gateway.cluster.default.connection.service.secret: gw-pass\n\ + gateway.cluster.default.client.security.protocol: sasl\n\ + gateway.cluster.default.connection.max: 512\n\ + gateway.cluster.default.connection.idle-timeout: 10m\n\ + gateway.cluster.analytics.bootstrap.servers: analytics:9123,analytics-2:9123\n\ + gateway.cluster.analytics.connect-timeout: 5s\n\ + gateway.security.authentication: password\n\ + gateway.security.users: alice:secret\n\ + gateway.rest.write.max-rows: 500\n\ + gateway.rest.lookup.max-keys: 32\n\ + gateway.rest.lookup.max-key-bytes: 2MiB\n", + ) + .unwrap(); + + let default = cluster(&config, "default"); + assert_eq!(default.bootstrap_servers, ["fluss-1:9123", "fluss-2:9123"]); + assert_eq!(default.identity_mode, IdentityMode::User); + assert_eq!(default.effective_service_account(), Some("gateway_svc")); + assert_eq!(default.effective_service_secret(), Some("gw-pass")); + assert_eq!(default.connection_max, Some(512)); + assert_eq!( + default.connection_idle_timeout.map(ConfigDuration::get), + Some(Duration::from_secs(600)) + ); + // A comma-separated string is accepted so the same value can arrive from the environment. + assert_eq!( + cluster(&config, "analytics").bootstrap_servers, + ["analytics:9123", "analytics-2:9123"] + ); + assert_eq!( + cluster(&config, "analytics").connect_timeout.get(), + Duration::from_secs(5) + ); + assert_eq!(config.security.authentication, AuthenticationMode::Password); + assert_eq!(config.request_limits.write_max_rows, 500); + assert_eq!(config.request_limits.lookup_max_keys, 32); + assert_eq!( + config.request_limits.lookup_max_key_bytes.bytes(), + 2 * 1024 * 1024 + ); +} + +/// The declaration bounds which clusters may be configured, and it does so whether or not it was +/// written: with no `gateway.clusters`, the only configurable cluster is the implicit `default`. That is +/// what turns a mistyped cluster ID into a startup failure instead of an unreachable second cluster. +#[test] +fn declared_clusters_are_authoritative() { + for contents in [ + "gateway.clusters: default\n\ + gateway.cluster.analytics.bootstrap.servers: analytics:9123\n", + // No declaration at all: `analytics` is still not one of the allowed clusters. + "gateway.cluster.analytics.bootstrap.servers: analytics:9123\n", + "gateway.cluster.analytics.client.writer.batch-size: 2MiB\n", + ] { + let error = load_file(contents).unwrap_err(); + assert!( + error.to_string().contains("not declared"), + "{contents}: {error}" + ); + } + + // Declaring a cluster is enough to configure it; the rest of its settings default. + let config = load_file("gateway.clusters: default,analytics\n").unwrap(); + assert_eq!(config.clusters.len(), 2); + assert_eq!( + cluster(&config, "analytics").bootstrap_servers, + [DEFAULT_BOOTSTRAP_SERVERS] + ); + + // The implicit default needs no declaration, so a single-cluster deployment configures no list. + let config = load_file("gateway.cluster.default.bootstrap.servers: only:9123\n").unwrap(); + assert_eq!(config.clusters.keys().collect::>(), ["default"]); +} + +#[test] +fn malformed_cluster_ids_are_rejected() { + for contents in [ + "gateway.clusters: Default\n", + "gateway.clusters: 1st\n", + "gateway.clusters: eu-west\n", + "gateway.cluster.EU.bootstrap.servers: eu:9123\n", + ] { + let error = load_file(contents).unwrap_err(); + assert!( + error.to_string().contains("cluster ID"), + "{contents}: {error}" + ); + } +} + +#[test] +fn unknown_cluster_and_client_keys_name_the_original_key() { + for contents in [ + "gateway.cluster.default.bootstrap.serverz: fluss:9123\n", + "gateway.cluster.default.connection.identity: user\n", + "gateway.cluster.default.client.: 1\n", + "gateway.cluster.default: fluss:9123\n", + ] { + let error = load_file(contents).unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); + let key = contents.split(':').next().unwrap(); + assert!(error.to_string().contains(key), "{key}: {error}"); + } +} + +#[test] +fn native_client_options_are_parsed_into_their_native_types() { + assert_eq!( + parse_client_option("writer.batch-size", "2MiB").unwrap(), + ClientOptionValue::Bytes(2 * 1024 * 1024) + ); + assert_eq!( + parse_client_option("writer.batch-timeout", "50ms").unwrap(), + ClientOptionValue::Millis(50) + ); + assert_eq!( + parse_client_option("writer.dynamic-batch-size.enabled", "false").unwrap(), + ClientOptionValue::Boolean(false) + ); + assert_eq!( + parse_client_option("lookup.max-retries", "3").unwrap(), + ClientOptionValue::Integer(3) + ); + assert_eq!( + parse_client_option("security.protocol", "sasl").unwrap(), + ClientOptionValue::Text("sasl".to_string()) + ); + + let config = load_file( + "gateway.cluster.default.client.writer.batch-size: 2MiB\n\ + gateway.cluster.default.client.lookup.max-retries: 3\n", + ) + .unwrap(); + let default = cluster(&config, "default"); + assert_eq!(default.client_option("writer.batch-size"), Some("2MiB")); + assert_eq!(default.client_option("lookup.max-retries"), Some("3")); +} + +/// The gateway advertises the write guarantees, so the client options that would weaken them, and the +/// authorization ID that user mode supplies per request, are refused rather than silently honoured. +#[test] +fn client_options_cannot_override_gateway_owned_guarantees_or_the_identity() { + for (option, value, expected) in [ + ("writer.acks", "0", "owned by the Gateway"), + ("writer.retries", "0", "owned by the Gateway"), + ("writer.enable-idempotence", "false", "owned by the Gateway"), + ( + "security.sasl.authorization-id", + "alice", + "cannot be configured statically", + ), + ( + "writer.unknown-knob", + "1", + "is not a supported native-client option", + ), + ] { + let error = parse_client_option(option, value).unwrap_err(); + assert!(error.contains(expected), "{option}: {error}"); + + let problems = problems( + load_file(&format!( + "gateway.cluster.default.client.{option}: {value}\n" + )) + .unwrap_err(), + ); + assert!( + problems.iter().any(|problem| { + problem.starts_with("gateway.cluster.default.client.") && problem.contains(option) + }), + "{option}: {problems:?}" + ); + } +} + +#[test] +fn out_of_range_client_values_fail_before_startup() { + for (option, value) in [ + ("lookup.queue-size", "0"), + ("lookup.max-batch-size", "-1"), + ("lookup.max-retries", "-1"), + ("writer.request-max-size", "3GiB"), + ("writer.batch-timeout", "0s"), + ("writer.dynamic-batch-size.enabled", "maybe"), + ] { + let error = parse_client_option(option, value).unwrap_err(); + assert!(error.starts_with(&format!("client.{option}")), "{error}"); + assert!( + load_file(&format!( + "gateway.cluster.default.client.{option}: \"{value}\"\n" + )) + .is_err(), + "accepted client.{option} = {value}" + ); + } +} + +#[test] +fn cross_field_cluster_and_security_constraints_fail_before_startup() { + for contents in [ + // An account without its secret, and the reverse. + "gateway.cluster.default.connection.service.account: gateway_svc\n", + "gateway.cluster.default.connection.service.secret: gw-pass\n", + // User identity mode has no service account to authenticate the pool with. + "gateway.cluster.default.connection.identity-mode: user\n", + // SASL without credentials, and an unsupported protocol or mechanism. + "gateway.cluster.default.client.security.protocol: sasl\n", + "gateway.cluster.default.client.security.protocol: ssl\n", + "gateway.cluster.default.client.security.sasl.mechanism: SCRAM-SHA-256\n", + "gateway.cluster.default.connection.max: 0\n", + "gateway.cluster.default.bootstrap.servers: \" \"\n", + // The mode's credential table is missing. + "gateway.security.authentication: password\n", + "gateway.security.authentication: token\n", + "gateway.security.authentication: trusted-header\n\ + gateway.security.trusted-header.name: \"bad header\"\n", + "gateway.rest.lookup.max-keys: 0\n", + "gateway.rest.prefix-lookup.max-prefixes: 0\n", + ] { + assert!(load_file(contents).is_err(), "accepted: {contents}"); + } + + assert!( + load_file( + "gateway.cluster.default.connection.identity-mode: user\n\ + gateway.cluster.default.connection.service.account: gateway_svc\n\ + gateway.cluster.default.connection.service.secret: gw-pass\n\ + gateway.cluster.default.client.security.protocol: SASL\n\ + gateway.cluster.default.client.security.sasl.mechanism: PLAIN\n" + ) + .is_ok() + ); +} + +/// The legacy SASL options stay usable and keep winning, because silently changing which credential a +/// running deployment authenticates with would be worse than the deprecation. +#[test] +fn legacy_credentials_win_and_warn_once_per_cluster() { + let config = load_file( + "gateway.clusters: default,analytics\n\ + gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.username: legacy-user\n\ + gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ + gateway.cluster.analytics.client.security.sasl.username: other-user\n\ + gateway.cluster.analytics.client.security.sasl.password: other-secret\n", + ) + .unwrap(); + + let default = cluster(&config, "default"); + assert_eq!(default.effective_service_account(), Some("legacy-user")); + assert_eq!(default.effective_service_secret(), Some("legacy-secret")); + + let deprecations: Vec = config + .warnings() + .into_iter() + .filter(|warning| warning.contains("is deprecated")) + .collect(); + assert_eq!(deprecations.len(), 2, "one per cluster: {deprecations:?}"); + for warning in &deprecations { + for secret in [ + "canonical-user", + "canonical-secret", + "legacy-user", + "legacy-secret", + "other-user", + "other-secret", + ] { + assert!(!warning.contains(secret), "leaked {secret}: {warning}"); + } + } +} + +#[test] +fn pool_settings_warn_when_the_identity_mode_ignores_them() { + let config = load_file( + "gateway.cluster.default.connection.max: 8\n\ + gateway.cluster.default.connection.idle-timeout: 5m\n", + ) + .unwrap(); + assert!( + config + .warnings() + .iter() + .any(|warning| warning.contains("ignored because connection.identity-mode is service")), + "{:?}", + config.warnings() + ); +} + +#[test] +fn diagnostics_redact_every_credential_and_keep_the_identities() { + let config = load_file( + "gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.username: legacy-user\n\ + gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ + gateway.cluster.default.client.writer.batch-size: 4MiB\n\ + gateway.security.authentication: password\n\ + gateway.security.users: alice:user-secret\n\ + gateway.security.tokens: token-secret:alice\n", + ) + .unwrap(); + + for diagnostic in [config.redacted_debug(), format!("{config:?}")] { + for credential in [ + "canonical-secret", + "legacy-secret", + "user-secret", + "token-secret", + ] { + assert!( + !diagnostic.contains(credential), + "leaked {credential}: {diagnostic}" + ); + } + // The identities and the tuning stay readable: they are what an operator came to look at. + for readable in ["canonical-user", "legacy-user", "4MiB"] { + assert!(diagnostic.contains(readable), "{readable}: {diagnostic}"); + } + assert!(diagnostic.contains(REDACTED), "{diagnostic}"); + } + + // The credentials are still reachable by the components that authenticate with them. + assert_eq!( + config.security.users.as_ref().map(Secret::expose), + Some("alice:user-secret") + ); + assert_eq!( + cluster(&config, "default").effective_service_secret(), + Some("legacy-secret") + ); +} + +/// A configuration error is what an operator sees on stderr, so it must name the option without +/// quoting any credential the file happens to carry. +#[test] +fn configuration_errors_never_quote_a_credential() { + let file = write_temp_config( + "gateway.security.authentication: token\n\ + gateway.security.tokens: do-not-leak\n\ + gateway.cluster.default.connection.service.secret: also-secret\n\ + gateway.cluster.default.client.writer.acks: 0\n", + ); + let error = load(Some(file.path()), &no_env(), &CliOverrides::default()).unwrap_err(); + let rendered = error.to_string(); + assert!(rendered.contains("writer.acks"), "{rendered}"); + assert!(!rendered.contains("do-not-leak"), "{rendered}"); + assert!(!rendered.contains("also-secret"), "{rendered}"); +} + +/// The same precedence statement for the two dynamic namespaces, where the environment name is derived +/// rather than registered: for every per-cluster option and every allowed client option, setting the +/// environment variable must be indistinguishable from having written that value in the file. +#[test] +fn the_environment_overrides_the_file_for_every_cluster_and_client_option() { + // Keeps every variant loadable: user identity mode needs usable credentials over SASL. The key + // under test is removed from the base so the file never carries it twice. + let base = [ + ("connection.service.account", "base-account"), + ("connection.service.secret", "base-secret"), + ("client.security.protocol", "sasl"), + ]; + + let mut cases: Vec<(String, String, &str, &str)> = Vec::new(); + for entry in CLUSTER_ENTRIES { + let values = match entry.kind { + ValueKind::ServerList => ("file-host:9123", "env-host:9123"), + ValueKind::Integer => ("11", "22"), + ValueKind::Text if entry.key == "connection.identity-mode" => ("service", "user"), + ValueKind::Text if entry.key.ends_with("timeout") => ("11s", "22s"), + ValueKind::Text => ("file-value", "env-value"), + ValueKind::Bool | ValueKind::Bytes => { + unreachable!("no per-cluster option uses {:?}", entry.kind) + } + }; + cases.push(( + entry.key.to_string(), + environment_suffix(entry.key), + values.0, + values.1, + )); + } + for spec in CLIENT_OPTIONS { + let values = match spec.kind { + // The only legal value is PLAIN, so the two spellings differ only in case. + _ if spec.option == "security.sasl.mechanism" => ("PLAIN", "plain"), + _ if spec.option == "security.protocol" => ("plaintext", "sasl"), + ClientOptionKind::Text | ClientOptionKind::Secret => ("file-value", "env-value"), + ClientOptionKind::Boolean => ("true", "false"), + ClientOptionKind::Count { .. } => ("5", "6"), + // Both values have to keep the size relationships intact against the native defaults. + ClientOptionKind::Size { .. } if spec.option.ends_with("dynamic-batch-size.min") => { + ("1MiB", "2MiB") + } + ClientOptionKind::Size { .. } => ("4MiB", "8MiB"), + ClientOptionKind::Duration => ("11s", "22s"), + }; + cases.push(( + format!("{CLIENT_OPTION_PREFIX}{}", spec.option), + format!("CLIENT__{}", environment_suffix(spec.option)), + values.0, + values.1, + )); + } + + for (key, env_suffix, file_value, env_value) in cases { + let contents = |value: &str| { + base.iter() + .filter(|(base_key, _)| *base_key != key) + .map(|(base_key, base_value)| (*base_key, *base_value)) + .chain([(key.as_str(), value)]) + .map(|(key, value)| { + format!("{CLUSTER_KEY_PREFIX}{DEFAULT_CLUSTER_ID}.{key}: \"{value}\"\n") + }) + .collect::() + }; + let load_valid = |contents: &str, env: &BTreeMap| { + let file = write_temp_config(contents); + load(Some(file.path()), env, &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{key}: {error}\n{contents}")) + }; + + let env = BTreeMap::from([( + format!( + "{ENV_PREFIX}CLUSTER__{}__{env_suffix}", + DEFAULT_CLUSTER_ID.to_ascii_uppercase() + ), + env_value.to_string(), + )]); + let from_file = load_valid(&contents(file_value), &no_env()); + let overridden = load_valid(&contents(file_value), &env); + let as_written = load_valid(&contents(env_value), &no_env()); + + assert_ne!( + from_file, overridden, + "{key} ignores its environment variable" + ); + assert_eq!( + overridden, as_written, + "{key} from the environment differs from the same value in the file" + ); + } +} + +/// The reserved options are refused from the environment exactly as they are from the file, and the +/// fixed request limits are reachable there too. +#[test] +fn the_environment_is_held_to_the_same_client_option_rules_as_the_file() { + let mut env = BTreeMap::from([( + "FLUSS_GATEWAY__REST__LOOKUP__MAX_KEYS".to_string(), + "16".to_string(), + )]); + let config = load(None, &env, &CliOverrides::default()).unwrap(); + assert_eq!(config.request_limits.lookup_max_keys, 16); + + env.insert( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__ACKS".to_string(), + "0".to_string(), + ); + let error = load(None, &env, &CliOverrides::default()).unwrap_err(); + assert!(error.to_string().contains("writer.acks"), "got: {error}"); +} + +#[test] +fn the_environment_can_declare_clusters() { + let file = write_temp_config("gateway.cluster.analytics.bootstrap.servers: eu:9123\n"); + let mut env = no_env(); + env.insert( + "FLUSS_GATEWAY__CLUSTERS".to_string(), + "analytics".to_string(), + ); + let config = load(Some(file.path()), &env, &CliOverrides::default()).unwrap(); + assert_eq!(config.clusters.keys().collect::>(), ["analytics"]); + + env.insert("FLUSS_GATEWAY__CLUSTERS".to_string(), "default".to_string()); + let error = load(Some(file.path()), &env, &CliOverrides::default()).unwrap_err(); + assert!(error.to_string().contains("not declared"), "got: {error}"); +} + +#[test] +fn unknown_cluster_environment_variables_are_rejected() { + for variable in [ + "FLUSS_GATEWAY__CLUSTER__DEFAULT__BOOTSTRAP__SERVERZ", + "FLUSS_GATEWAY__CLUSTER__DEFAULT", + "FLUSS_GATEWAY__CLUSTER__1ST__BOOTSTRAP__SERVERS", + ] { + let mut env = no_env(); + env.insert(variable.to_string(), "value".to_string()); + let error = load(None, &env, &CliOverrides::default()).unwrap_err(); + assert!( + matches!(error, ConfigError::UnknownEnvKey(_)), + "{variable}: {error:?}" + ); + assert!(error.to_string().contains(variable), "{variable}: {error}"); + } +} + +/// A bad per-cluster value is reported with the public key rather than the internal Serde path, and an +/// environment override additionally names the variable the operator set. +#[test] +fn a_bad_cluster_value_names_the_public_key_and_its_source() { + let error = load_file("gateway.cluster.default.request-timeout: 0s\n").unwrap_err(); + assert!( + error + .to_string() + .contains("gateway.cluster.default.request-timeout"), + "got: {error}" + ); + + let env = BTreeMap::from([( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT".to_string(), + "soon".to_string(), + )]); + let rendered = load(None, &env, &CliOverrides::default()) + .unwrap_err() + .to_string(); + assert!( + rendered.contains("FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT"), + "{rendered}" + ); + assert!( + rendered.contains("gateway.cluster.default.connect-timeout"), + "{rendered}" + ); +} + +#[test] +fn programmatically_constructed_clusters_are_validated() { + let mut config = GatewayConfig::default(); + config.clusters.clear(); + let errors = problems(config.validate().unwrap_err()); + assert!( + errors + .iter() + .any(|error| error == "gateway.clusters must declare at least one cluster"), + "got: {errors:?}" + ); + + let mut config = GatewayConfig::default(); + config + .clusters + .get_mut(DEFAULT_CLUSTER_ID) + .expect("default cluster") + .identity_mode = IdentityMode::User; + let errors = problems(config.validate().unwrap_err()); + assert!( + errors + .iter() + .any(|error| error.contains("identity-mode user requires")), + "got: {errors:?}" + ); +} + +/// The `client.*` namespace is open, so its environment mapping cannot be checked key by key like the +/// fixed vocabulary. It round-trips only while option names keep `-` inside a segment and `.` between +/// segments: an option name containing `_` would come back from the environment as a different name and +/// silently configure the wrong option. Exhaustive over the allowlist, so adding such a name fails here. +#[test] +fn every_client_option_round_trips_through_the_environment() { + for spec in CLIENT_OPTIONS { + let suffix = environment_suffix(spec.option); + assert_eq!( + environment_suffix_to_option(&suffix), + spec.option, + "{} does not survive the environment mapping", + spec.option + ); + assert!( + !spec.option.contains('_'), + "{} must spell words with '-', which the environment mapping reserves '_' for", + spec.option + ); + let file_key = format!( + "{CLUSTER_KEY_PREFIX}{DEFAULT_CLUSTER_ID}.{CLIENT_OPTION_PREFIX}{}", + spec.option + ); + assert!( + matches!(resolve_key(&file_key), Ok(ResolvedKey::ClientOption { .. })), + "{file_key} is not reachable as a file key" + ); + } +} + +/// Sensitivity is declared per option, so the declaration is what has to be right. Fluss decides it on +/// the Java side from these same substrings, which is the cross-check: any option whose name looks like +/// a credential must be marked, and the allowlist and the refusals must not overlap. +#[test] +fn client_option_sensitivity_and_refusals_are_declared_consistently() { + for spec in CLIENT_OPTIONS { + let looks_sensitive = ["password", "secret", "token"] + .iter() + .any(|part| spec.option.contains(part)); + assert_eq!( + spec.kind.is_sensitive(), + looks_sensitive, + "{}: the kind and the name disagree about being a credential", + spec.option + ); + assert!( + !RESERVED_CLIENT_OPTIONS + .iter() + .any(|(reserved, _)| *reserved == spec.option), + "{} is both allowed and refused", + spec.option + ); + } + // An option the gateway never validated must not be rendered on the chance that it holds a secret. + assert!(client_option_is_sensitive("some.unknown.option")); + assert!(client_option_is_sensitive(LEGACY_SERVICE_SECRET_OPTION)); + assert!(!client_option_is_sensitive(LEGACY_SERVICE_ACCOUNT_OPTION)); + + // A parsed credential stays wrapped, so printing the parse result cannot leak it either. + let parsed = parse_client_option(LEGACY_SERVICE_SECRET_OPTION, "legacy-secret").unwrap(); + assert_eq!( + parsed, + ClientOptionValue::Secret(Secret::new("legacy-secret")) + ); + assert!(!format!("{parsed:?}").contains("legacy-secret")); + assert!(format!("{parsed:?}").contains(REDACTED)); +} + +/// Precedence is stated once for the whole vocabulary rather than sampled on one key, so a per-kind +/// conversion that only reads one source cannot hide: every option is driven from the file, then +/// overridden from the environment, and the environment value has to win in the loaded config. +#[test] +fn the_environment_overrides_the_file_for_every_option() { + for entry in CONFIG_ENTRIES { + let (file_value, env_value) = match entry.kind { + ValueKind::Bool => ("true", "false"), + ValueKind::Integer => ("11", "22"), + ValueKind::Bytes => ("1MiB", "2MiB"), + ValueKind::ServerList => ("file-host:9123", "env-host:9123"), + ValueKind::Text => match entry.key { + REST_LISTEN_KEY => ("127.0.0.1:11111", "127.0.0.1:22222"), + METRICS_LISTEN_KEY => ("127.0.0.1:11112", "127.0.0.1:22223"), + REST_HEADER_READ_TIMEOUT_KEY + | REST_REQUEST_TIMEOUT_KEY + | SHUTDOWN_DRAIN_TIMEOUT_KEY => ("11s", "22s"), + // Both modes must be valid on their own: the file value is loaded without the + // environment override, and password and token modes need a credential table. + SECURITY_AUTHENTICATION_KEY => ("trusted-header", "trust"), + _ => ("file-value", "env-value"), + }, + }; + + let file = write_temp_config(&format!("{}: \"{file_value}\"\n", entry.key)); + let from_file = load(Some(file.path()), &no_env(), &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); + let env = BTreeMap::from([(environment_variable(entry.key), env_value.to_string())]); + let from_env = load(Some(file.path()), &env, &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); + + assert_ne!( + from_file, from_env, + "{} ignores its environment variable", + entry.key + ); + let only_env = load(None, &env, &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); + assert_eq!( + from_env, only_env, + "{} lets the file value survive the environment override", + entry.key + ); + } +} + +/// User identity mode is only safe when the connection can actually carry the request's principal, so +/// the credentials must be usable *and* SASL must be selected. Both were previously satisfied by a +/// `Some("")` credential over the default PLAINTEXT, which authorizes the gateway's own identity for +/// every caller instead of failing. +#[test] +fn user_identity_mode_requires_usable_credentials_over_sasl() { + let user_mode = "gateway.cluster.default.connection.identity-mode: user\n"; + let credentials = "gateway.cluster.default.connection.service.account: gateway_svc\n\ + gateway.cluster.default.connection.service.secret: gw-pass\n"; + let sasl = "gateway.cluster.default.client.security.protocol: sasl\n"; + + for (contents, expected) in [ + ( + format!("{user_mode}{credentials}"), + "requires client.security.protocol sasl", + ), + ( + format!("{user_mode}{sasl}"), + "requires connection.service.account", + ), + ( + format!( + "{user_mode}{sasl}gateway.cluster.default.connection.service.account: \"\"\n\ + gateway.cluster.default.connection.service.secret: \" \"\n" + ), + "must not be blank", + ), + ( + format!( + "{user_mode}{sasl}{credentials}\ + gateway.cluster.default.client.security.sasl.mechanism: SCRAM-SHA-256\n" + ), + "must be PLAIN", + ), + ] { + let problems = problems(load_file(&contents).unwrap_err()); + assert!( + problems.iter().any(|problem| problem.contains(expected)), + "expected {expected:?} for:\n{contents}got: {problems:?}" + ); + } + + // The complete, coherent form is accepted. + assert!(load_file(&format!("{user_mode}{sasl}{credentials}")).is_ok()); + // Service mode needs no SASL: it authenticates as itself, with no principal to propagate. + assert!(load_file("gateway.cluster.default.connection.identity-mode: service\n").is_ok()); +} + +/// A size that cannot fit inside the size holding it fails before a listener binds, whether the operator +/// set both sides or only one: leaving the other at its native default is the common way to break a pair. +#[test] +fn writer_size_pairs_must_fit_including_against_the_native_defaults() { + for (contents, rejected) in [ + // Both sides configured. + ( + "gateway.cluster.default.client.writer.batch-size: 2MiB\n\ + gateway.cluster.default.client.writer.request-max-size: 1MiB\n", + true, + ), + ( + "gateway.cluster.default.client.writer.dynamic-batch-size.min: 4MiB\n\ + gateway.cluster.default.client.writer.batch-size: 2MiB\n", + true, + ), + // One side only: the other is the native default, 10MiB request-max and 2MiB batch-size. + ( + "gateway.cluster.default.client.writer.batch-size: 128MiB\n", + true, + ), + ( + "gateway.cluster.default.client.writer.dynamic-batch-size.min: 3MiB\n", + true, + ), + ( + "gateway.cluster.default.client.writer.request-max-size: 1MiB\n", + true, + ), + // Coherent against the defaults, and coherent as a pair. + ( + "gateway.cluster.default.client.writer.batch-size: 4MiB\n", + false, + ), + ( + "gateway.cluster.default.client.writer.batch-size: 32MiB\n\ + gateway.cluster.default.client.writer.request-max-size: 64MiB\n", + false, + ), + ] { + let result = load_file(contents); + assert_eq!( + result.is_err(), + rejected, + "unexpected outcome for:\n{contents}" + ); + if rejected { + assert!( + problems(result.unwrap_err()) + .iter() + .any(|problem| problem.contains("must not exceed client.")), + "{contents}" + ); + } + } +} + +/// A value is bounded by the native field it lands in, not by one blanket ceiling: the writer sizes are +/// `i32` there, while the buffer size and the lookup counts are `usize` and may exceed `i32::MAX`. +#[test] +fn client_option_bounds_follow_the_native_field_type() { + let over_i32 = u64::from(i32::MAX as u32) + 1; + + for option in ["writer.batch-size", "writer.request-max-size"] { + let error = parse_client_option(option, &format!("{over_i32}")).unwrap_err(); + assert!(error.contains("must not exceed"), "{option}: {error}"); + } + assert_eq!( + parse_client_option("writer.buffer.memory-size", "4GiB").unwrap(), + ClientOptionValue::Bytes(4 * 1024 * 1024 * 1024) + ); + for option in [ + "lookup.queue-size", + "lookup.max-batch-size", + "lookup.max-inflight-requests", + ] { + assert_eq!( + parse_client_option(option, &format!("{over_i32}")).unwrap(), + ClientOptionValue::Integer(over_i32), + "{option} is stored as usize and must accept this" + ); + } + let error = parse_client_option("lookup.max-retries", &format!("{over_i32}")).unwrap_err(); + assert!(error.contains("must be between 0 and"), "{error}"); +} + +/// The declared native defaults are a copy of the client's, so they must at least satisfy the +/// relationships the client enforces; a mistyped copy shows up here and not as a rejected valid file. +#[test] +fn the_declared_native_size_defaults_are_coherent() { + let default = |option| effective_size(option, None).expect("a declared size default"); + let batch = default("writer.batch-size"); + assert!(batch <= default("writer.request-max-size")); + assert!(batch <= default("writer.buffer.memory-size")); + assert!(default("writer.dynamic-batch-size.min") <= batch); +} + +#[test] +fn options_are_complete_and_unambiguous() { + let mut public_keys = std::collections::BTreeSet::new(); + let mut internal_paths = std::collections::BTreeSet::new(); + let mut environment_variables = std::collections::BTreeSet::new(); + + for entry in CONFIG_ENTRIES { + assert!(entry.key.starts_with("gateway."), "{entry:?}"); + assert!( + public_keys.insert(entry.key), + "duplicate key: {}", + entry.key + ); + assert!( + internal_paths.insert(entry.internal_path), + "duplicate path: {}", + entry.internal_path + ); + assert!( + environment_variables.insert(environment_variable(entry.key)), + "duplicate environment variable for {}", + entry.key + ); + } + + assert_eq!(CONFIG_ENTRIES.len(), 20); +} + +/// The per-cluster vocabulary shares the environment namespace with `client.*`, so its keys have to +/// stay distinct from each other and unreachable through the client prefix. +#[test] +fn cluster_options_are_complete_and_unambiguous() { + let mut keys = std::collections::BTreeSet::new(); + let mut fields = std::collections::BTreeSet::new(); + let mut suffixes = std::collections::BTreeSet::new(); + + for entry in CLUSTER_ENTRIES { + assert!(!entry.key.starts_with("gateway."), "{entry:?}"); + assert!( + !entry.key.starts_with(CLIENT_OPTION_PREFIX), + "{} collides with the client namespace", + entry.key + ); + assert!(keys.insert(entry.key), "duplicate key: {}", entry.key); + assert!( + fields.insert(entry.internal_path), + "duplicate field: {}", + entry.internal_path + ); + assert!( + suffixes.insert(environment_suffix(entry.key)), + "duplicate environment suffix for {}", + entry.key + ); + } + + assert_eq!(CLUSTER_ENTRIES.len(), 8); +} diff --git a/fluss-gateway/src/lifecycle.rs b/fluss-gateway/src/lifecycle.rs index 03a304e1c3..4d810304e8 100644 --- a/fluss-gateway/src/lifecycle.rs +++ b/fluss-gateway/src/lifecycle.rs @@ -245,6 +245,9 @@ pub async fn start(config: GatewayConfig) -> Result { /// Binds the listeners, installs the router, and spawns every process-owned task. async fn start_internal(config: GatewayConfig) -> Result { + // The effective configuration is what an operator needs when a deployment misbehaves, and the + // redacted rendering is the only form of it that is safe to write to a log. + log::debug!("effective configuration: {}", config.redacted_debug()); for warning in config.warnings() { log::warn!("{warning}"); } diff --git a/fluss-gateway/tests/process.rs b/fluss-gateway/tests/process.rs index 80ca730d99..346346ad5b 100644 --- a/fluss-gateway/tests/process.rs +++ b/fluss-gateway/tests/process.rs @@ -23,6 +23,7 @@ mod support; +use std::io::Read; use std::time::Duration; use support::{ChildGuard, await_http_ok, binary, free_port, write_config}; @@ -40,6 +41,98 @@ async fn an_invalid_configuration_fails_before_binding_with_exit_code_2() { ); } +/// A rejected `client.*` option is what an operator hits when migrating a native-client configuration, and +/// the file that carries it usually carries credentials too: the process must name the option on stderr +/// while leaking none of them. +#[tokio::test] +async fn a_restricted_client_option_fails_before_binding_without_leaking_credentials() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("gateway.yaml"); + std::fs::write( + &path, + "gateway.cluster.default.connection.service.account: gateway-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ + gateway.security.authentication: token\n\ + gateway.security.tokens: token-secret:alice\n\ + gateway.cluster.default.client.writer.acks: 0\n", + ) + .expect("write"); + + let output = binary().arg("--config").arg(&path).output().expect("run"); + assert_eq!(output.status.code(), Some(2)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("writer.acks"), "{stderr}"); + for credential in ["canonical-secret", "legacy-secret", "token-secret"] { + assert!( + !stderr.contains(credential), + "stderr leaked {credential}: {stderr}" + ); + } +} + +/// The deprecated SASL options keep working, so the only signal an operator gets is the startup warning: +/// it has to reach stderr, once per cluster, with the credentials redacted. Debug logging is on, so the +/// effective-configuration dump is on stderr too and is held to the same rule. +#[tokio::test] +async fn legacy_credentials_emit_one_redacted_warning_per_cluster() { + let dir = tempfile::tempdir().expect("tempdir"); + let port = free_port(); + let path = dir.path().join("gateway.yaml"); + std::fs::write( + &path, + format!( + "gateway.rest.listen: 127.0.0.1:{port}\n\ + gateway.metrics.enabled: false\n\ + gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.username: legacy-user\n\ + gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ + gateway.security.authentication: token\n\ + gateway.security.tokens: token-secret:alice\n" + ), + ) + .expect("write"); + + // Only the few startup log lines are piped, so the pipe buffer cannot fill and stall the child. + let child = binary() + .arg("--config") + .arg(&path) + .env("RUST_LOG", "debug") + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn"); + let mut guard = ChildGuard(child); + assert!( + await_http_ok( + &format!("http://127.0.0.1:{port}/health"), + Duration::from_secs(15) + ) + .await, + "health" + ); + guard.send_sigterm(); + let status = guard.wait_for_exit(Duration::from_secs(35)).await; + assert_eq!(status.code(), Some(0)); + + let mut stderr = String::new(); + guard + .0 + .stderr + .take() + .expect("piped stderr") + .read_to_string(&mut stderr) + .expect("read stderr"); + assert_eq!(stderr.matches("is deprecated").count(), 1, "{stderr}"); + assert!(stderr.contains("effective configuration"), "{stderr}"); + for credential in ["canonical-secret", "legacy-secret", "token-secret"] { + assert!( + !stderr.contains(credential), + "stderr leaked {credential}: {stderr}" + ); + } +} + #[tokio::test] async fn the_binary_starts_serves_health_and_drains_on_sigterm_with_exit_code_0() { let dir = tempfile::tempdir().expect("tempdir"); From ccdf2679aae0969c4f65c6c3d6b3c835319adbe7 Mon Sep 17 00:00:00 2001 From: Junbo Wang Date: Wed, 19 Aug 2026 16:53:57 +0800 Subject: [PATCH 2/3] [gateway] Keep configuration tests inline Move the configuration tests back into config.rs to keep the PR diff smaller. --- fluss-gateway/src/config.rs | 1369 +++++++++++++++++++++++++++- fluss-gateway/src/config/tests.rs | 1375 ----------------------------- 2 files changed, 1366 insertions(+), 1378 deletions(-) delete mode 100644 fluss-gateway/src/config/tests.rs diff --git a/fluss-gateway/src/config.rs b/fluss-gateway/src/config.rs index ecd2ff2871..bf0f6febcc 100644 --- a/fluss-gateway/src/config.rs +++ b/fluss-gateway/src/config.rs @@ -52,9 +52,6 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::Path; use std::time::Duration; -#[cfg(test)] -mod tests; - /// Environment variable prefix for overrides. pub const ENV_PREFIX: &str = "FLUSS_GATEWAY__"; @@ -1902,3 +1899,1369 @@ pub fn load( config.validate()?; Ok(config) } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn no_env() -> BTreeMap { + BTreeMap::new() + } + + fn write_temp_config(contents: &str) -> tempfile::NamedTempFile { + let mut file = tempfile::NamedTempFile::new().expect("temp file"); + file.write_all(contents.as_bytes()).expect("write"); + file + } + + fn load_file(contents: &str) -> Result { + let file = write_temp_config(contents); + load(Some(file.path()), &no_env(), &CliOverrides::default()) + } + + fn problems(error: ConfigError) -> Vec { + match error { + ConfigError::Invalid(problems) => problems, + other => panic!("expected Invalid, got: {other:?}"), + } + } + + #[test] + fn defaults_when_no_sources() { + let config = load(None, &no_env(), &CliOverrides::default()).unwrap(); + assert_eq!( + config.server.rest.bind_address, + "127.0.0.1:8080".parse().unwrap() + ); + assert_eq!(config.server.rest.max_body_bytes.bytes(), 32 * 1024 * 1024); + assert_eq!( + config.server.rest.request_timeout.get(), + Duration::from_secs(30) + ); + assert!(config.server.metrics.enabled); + assert_eq!( + config.server.metrics.bind_address, + "127.0.0.1:9095".parse().unwrap() + ); + assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(30)); + assert!(config.warnings().is_empty()); + } + + #[test] + fn public_yaml_options_are_loaded() { + let config = load_file( + r#" + gateway.instance-id: gateway-1 + gateway.rest.listen: 0.0.0.0:8080 + gateway.rest.write.max-request-bytes: 32MiB + gateway.rest.write.request-timeout: 30s + gateway.metrics.enabled: true + gateway.metrics.exporter.prometheus.listen: 0.0.0.0:9095 + gateway.shutdown.drain-timeout: 10s + "#, + ) + .unwrap(); + assert_eq!(config.server.instance_id.as_deref(), Some("gateway-1")); + assert_eq!( + config.server.rest.bind_address, + "0.0.0.0:8080".parse().unwrap() + ); + assert_eq!(config.server.rest.max_body_bytes.bytes(), 32 * 1024 * 1024); + assert_eq!( + config.server.rest.request_timeout.get(), + Duration::from_secs(30) + ); + assert!(config.server.metrics.enabled); + assert_eq!( + config.server.metrics.bind_address, + "0.0.0.0:9095".parse().unwrap() + ); + assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(10)); + } + + #[test] + fn unknown_file_keys_name_the_original_key() { + for contents in [ + "gateway.rest.listenn: 0.0.0.0:8080\n", + "rest.listen: 0.0.0.0:8080\n", + "gateway.rest.lookup.max-keyz: 5\n", + "gateway.scan.cursor-ttl: 1m\n", + "gateway.tls.cert: /etc/tls.pem\n", + ] { + let error = load_file(contents).unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); + let key = contents.split(':').next().unwrap(); + assert!(error.to_string().contains(key), "{key}: {error}"); + } + } + + #[test] + fn source_precedence_is_cli_then_env_then_file_then_defaults() { + let file = write_temp_config( + r#" + gateway.rest.listen: 127.0.0.1:18080 + gateway.metrics.enabled: true + "#, + ); + let mut env = no_env(); + env.insert( + "FLUSS_GATEWAY__REST__LISTEN".to_string(), + "127.0.0.1:28080".to_string(), + ); + env.insert( + "FLUSS_GATEWAY__METRICS__ENABLED".to_string(), + "false".to_string(), + ); + env.insert("PATH".to_string(), "/usr/bin".to_string()); + + let config = load( + Some(file.path()), + &env, + &CliOverrides { + bind_address: Some("127.0.0.1:38080".to_string()), + }, + ) + .unwrap(); + assert_eq!( + config.server.rest.bind_address, + "127.0.0.1:38080".parse().unwrap() + ); + assert!(!config.server.metrics.enabled); + } + + #[test] + fn missing_file_reported() { + let error = load( + Some(Path::new("/nonexistent/gateway.yaml")), + &no_env(), + &CliOverrides::default(), + ) + .unwrap_err(); + assert!(matches!(error, ConfigError::Io(_)), "got: {error:?}"); + } + + #[test] + fn malformed_file_reports_position() { + let error = load_file("gateway.rest.listen: [1\n").unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); + assert!(error.to_string().contains("line"), "got: {error}"); + } + + #[test] + fn duplicate_flat_key_rejected() { + let error = + load_file("gateway.rest.listen: 127.0.0.1:8080\ngateway.rest.listen: 127.0.0.1:8081\n") + .unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); + assert!(error.to_string().contains("duplicate"), "got: {error}"); + } + + #[test] + fn unknown_environment_variables_are_rejected() { + for key in [ + "FLUSS_GATEWAY__REST__LISTENN", + "FLUSS_GATEWAY__QUERY__ENABLED", + "FLUSS_GATEWAY__SERVER_REST__BIND_ADDRESS", + ] { + let mut env = no_env(); + env.insert(key.to_string(), "value".to_string()); + let error = load(None, &env, &CliOverrides::default()).unwrap_err(); + assert!( + matches!(error, ConfigError::UnknownEnvKey(_)), + "{key}: {error:?}" + ); + assert!(error.to_string().contains(key), "{key}: {error}"); + } + } + + #[test] + fn file_error_under_a_section_with_an_env_override_names_the_file() { + let file = write_temp_config("gateway.shutdown.drain-timeout: 0s\n"); + let mut env = no_env(); + env.insert( + "FLUSS_GATEWAY__REST__LISTEN".to_string(), + "127.0.0.1:28080".to_string(), + ); + let error = load(Some(file.path()), &env, &CliOverrides::default()).unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); + assert!( + error.to_string().contains("gateway.shutdown.drain-timeout"), + "got: {error}" + ); + assert!( + !error.to_string().contains("FLUSS_GATEWAY__"), + "file problem misattributed to the env override: {error}" + ); + } + + #[test] + fn public_environment_options_are_loaded_by_type() { + let env = BTreeMap::from([ + ("FLUSS_GATEWAY__INSTANCE_ID".to_string(), "123".to_string()), + ( + "FLUSS_GATEWAY__REST__LISTEN".to_string(), + "127.0.0.1:18080".to_string(), + ), + ( + "FLUSS_GATEWAY__REST__WRITE__REQUEST_TIMEOUT".to_string(), + "5s".to_string(), + ), + ( + "FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES".to_string(), + "2MiB".to_string(), + ), + ( + "FLUSS_GATEWAY__METRICS__ENABLED".to_string(), + "false".to_string(), + ), + ( + "FLUSS_GATEWAY__METRICS__EXPORTER__PROMETHEUS__LISTEN".to_string(), + "127.0.0.1:19095".to_string(), + ), + ( + "FLUSS_GATEWAY__SHUTDOWN__DRAIN_TIMEOUT".to_string(), + "10s".to_string(), + ), + ]); + + let config = load(None, &env, &CliOverrides::default()).unwrap(); + assert_eq!(config.server.instance_id.as_deref(), Some("123")); + assert_eq!( + config.server.rest.bind_address, + "127.0.0.1:18080".parse().unwrap() + ); + assert_eq!( + config.server.rest.request_timeout.get(), + Duration::from_secs(5) + ); + assert_eq!(config.server.rest.max_body_bytes.bytes(), 2 * 1024 * 1024); + assert!(!config.server.metrics.enabled); + assert_eq!( + config.server.metrics.bind_address, + "127.0.0.1:19095".parse().unwrap() + ); + assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(10)); + } + + #[test] + fn invalid_env_value_names_the_variable() { + let mut env = no_env(); + env.insert( + "FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES".to_string(), + "many".to_string(), + ); + let error = load(None, &env, &CliOverrides::default()).unwrap_err(); + assert!( + error + .to_string() + .contains("FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES"), + "got: {error}" + ); + } + + #[test] + fn invalid_cli_value_names_the_flag() { + let cli = CliOverrides { + bind_address: Some("not-an-address".to_string()), + }; + let error = load(None, &no_env(), &cli).unwrap_err(); + assert!(error.to_string().contains("--bind-address"), "got: {error}"); + } + + #[test] + fn invalid_duration_rejected() { + for bad in ["0ms", "60", "60 s", "6.5s", "s", "60d", "-1s"] { + let error = + load_file(&format!("gateway.shutdown.drain-timeout: \"{bad}\"\n")).unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "{bad}: {error:?}"); + assert!( + error.to_string().contains("gateway.shutdown.drain-timeout"), + "{bad}: {error}" + ); + } + } + + #[test] + fn overflowing_duration_is_rejected_rather_than_saturated() { + for bad in [ + "18446744073709551615ms", + "18446744073709551615s", + "18446744073709551615m", + "18446744073709551615h", + ] { + let error = ConfigDuration::parse(bad).unwrap_err(); + assert!(error.contains("must not exceed"), "{bad}: {error}"); + } + assert_eq!( + ConfigDuration::parse("31536000s").unwrap().get(), + MAX_CONFIG_DURATION + ); + assert!(ConfigDuration::parse("31536001s").is_err()); + } + + #[test] + fn programmatically_constructed_durations_are_validated() { + let mut config = GatewayConfig::default(); + config.server.rest.request_timeout = ConfigDuration::from_millis(0); + config.shutdown.drain_timeout = + ConfigDuration::from_secs(MAX_CONFIG_DURATION.as_secs() + 1); + + let errors = problems(config.validate().unwrap_err()); + assert!( + errors.iter().any(|error| { + error == "gateway.rest.write.request-timeout must be greater than zero" + }), + "got: {errors:?}" + ); + assert!( + errors.iter().any(|error| { + error == "gateway.shutdown.drain-timeout must not exceed 31536000 seconds" + }), + "got: {errors:?}" + ); + } + + #[test] + fn programmatically_constructed_zero_byte_limit_is_validated() { + let mut config = GatewayConfig::default(); + config.server.rest.max_body_bytes = ByteSize::new(0); + + let errors = problems(config.validate().unwrap_err()); + assert_eq!( + errors, + vec!["gateway.rest.write.max-request-bytes must be greater than zero"] + ); + } + + #[test] + fn invalid_byte_size_rejected() { + for bad in ["0", "\"4Mb\"", "\"MiB\"", "-1", "\"1.5MiB\""] { + let error = + load_file(&format!("gateway.rest.write.max-request-bytes: {bad}\n")).unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "{bad}: {error:?}"); + assert!( + error + .to_string() + .contains("gateway.rest.write.max-request-bytes"), + "{bad}: {error}" + ); + } + } + + #[test] + fn metrics_address_must_differ_from_rest_address() { + let error = load_file( + "gateway.rest.listen: 127.0.0.1:9095\ngateway.metrics.exporter.prometheus.listen: 127.0.0.1:9095\n", + ) + .unwrap_err(); + assert!(problems(error).iter().any(|problem| { + problem.contains( + "gateway.metrics.exporter.prometheus.listen (127.0.0.1:9095) must differ from \ + gateway.rest.listen (127.0.0.1:9095)", + ) + })); + } + + /// Overlap detection covers the wildcard and dual-stack pairs that differ textually but cannot + /// both bind, plus the pairs that coexist. + #[test] + fn listener_overlap_covers_wildcards_and_dual_stack() { + let clashes = [ + ("0.0.0.0:8080", "127.0.0.1:8080"), + ("127.0.0.1:8080", "0.0.0.0:8080"), + ("0.0.0.0:8080", "0.0.0.0:8080"), + ("[::]:8080", "[::1]:8080"), + ("[::]:8080", "0.0.0.0:8080"), + ("127.0.0.1:8080", "[::]:8080"), + ]; + for (rest, metrics) in clashes { + let rest: SocketAddr = rest.parse().unwrap(); + let metrics: SocketAddr = metrics.parse().unwrap(); + assert!( + addresses_overlap(rest, metrics), + "{rest} and {metrics} cannot both bind" + ); + } + + let coexist = [ + ("127.0.0.1:8080", "192.168.1.2:8080"), + ("127.0.0.1:8080", "[::1]:8080"), + ("0.0.0.0:8080", "127.0.0.1:9095"), + ("0.0.0.0:0", "0.0.0.0:0"), + ("127.0.0.1:0", "0.0.0.0:8080"), + ]; + for (rest, metrics) in coexist { + let rest: SocketAddr = rest.parse().unwrap(); + let metrics: SocketAddr = metrics.parse().unwrap(); + assert!( + !addresses_overlap(rest, metrics), + "{rest} and {metrics} can coexist" + ); + } + } + + /// Two ephemeral listeners are not a clash: the OS hands out a different port to each. + #[test] + fn both_listeners_may_ask_for_an_ephemeral_port() { + let config = load_file( + "gateway.rest.listen: 127.0.0.1:0\ngateway.metrics.exporter.prometheus.listen: 127.0.0.1:0\n", + ) + .unwrap(); + assert_eq!(config.server.rest.bind_address.port(), 0); + } + + #[test] + fn non_loopback_bind_is_accepted_without_an_instance_id_but_warns() { + let config = load_file("gateway.rest.listen: 0.0.0.0:8080\n").unwrap(); + assert!(config.server.instance_id.is_none()); + assert_eq!(config.warnings().len(), 1); + assert!(config.warnings()[0].contains("not loopback")); + assert!( + config.warnings()[0].contains("accepts unauthenticated requests"), + "{:?}", + config.warnings() + ); + } + + #[test] + fn malformed_instance_id_rejected() { + let error = load_file("gateway.instance-id: has space\n").unwrap_err(); + assert!( + problems(error) + .iter() + .any(|problem| problem.contains("gateway.instance-id must be 1-128 ASCII")) + ); + } + + #[test] + fn duration_units() { + assert_eq!( + ConfigDuration::parse("250ms").unwrap().get(), + Duration::from_millis(250) + ); + assert_eq!( + ConfigDuration::parse("15m").unwrap().get(), + Duration::from_secs(900) + ); + assert_eq!( + ConfigDuration::parse("2h").unwrap().get(), + Duration::from_secs(7200) + ); + assert!(ConfigDuration::parse("0s").is_err()); + } + + #[test] + fn byte_size_units() { + assert_eq!(ByteSize::parse("512").unwrap().bytes(), 512); + assert_eq!(ByteSize::parse("512B").unwrap().bytes(), 512); + assert_eq!(ByteSize::parse("4KB").unwrap().bytes(), 4000); + assert_eq!(ByteSize::parse("4KiB").unwrap().bytes(), 4096); + assert_eq!(ByteSize::parse("1GiB").unwrap().bytes(), 1024 * 1024 * 1024); + assert!(ByteSize::parse("4TB").is_err()); + assert!(ByteSize::parse("0").is_err()); + } + + fn cluster<'a>(config: &'a GatewayConfig, id: &str) -> &'a ClusterConfig { + config.clusters.get(id).expect("configured cluster") + } + + #[test] + fn a_single_default_cluster_needs_no_declaration() { + let config = load(None, &no_env(), &CliOverrides::default()).unwrap(); + assert_eq!(config.clusters.len(), 1); + assert_eq!( + cluster(&config, DEFAULT_CLUSTER_ID).bootstrap_servers, + [DEFAULT_BOOTSTRAP_SERVERS] + ); + assert_eq!( + cluster(&config, DEFAULT_CLUSTER_ID).identity_mode, + IdentityMode::Service + ); + assert_eq!(config.security.authentication, AuthenticationMode::Trust); + assert_eq!(config.request_limits, RequestLimitsConfig::default()); + } + + #[test] + fn typed_cluster_security_and_request_limit_options_are_loaded() { + let config = load_file( + "gateway.clusters: default, analytics\n\ + gateway.cluster.default.bootstrap.servers: [fluss-1:9123, fluss-2:9123]\n\ + gateway.cluster.default.connection.identity-mode: user\n\ + gateway.cluster.default.connection.service.account: gateway_svc\n\ + gateway.cluster.default.connection.service.secret: gw-pass\n\ + gateway.cluster.default.client.security.protocol: sasl\n\ + gateway.cluster.default.connection.max: 512\n\ + gateway.cluster.default.connection.idle-timeout: 10m\n\ + gateway.cluster.analytics.bootstrap.servers: analytics:9123,analytics-2:9123\n\ + gateway.cluster.analytics.connect-timeout: 5s\n\ + gateway.security.authentication: password\n\ + gateway.security.users: alice:secret\n\ + gateway.rest.write.max-rows: 500\n\ + gateway.rest.lookup.max-keys: 32\n\ + gateway.rest.lookup.max-key-bytes: 2MiB\n", + ) + .unwrap(); + + let default = cluster(&config, "default"); + assert_eq!(default.bootstrap_servers, ["fluss-1:9123", "fluss-2:9123"]); + assert_eq!(default.identity_mode, IdentityMode::User); + assert_eq!(default.effective_service_account(), Some("gateway_svc")); + assert_eq!(default.effective_service_secret(), Some("gw-pass")); + assert_eq!(default.connection_max, Some(512)); + assert_eq!( + default.connection_idle_timeout.map(ConfigDuration::get), + Some(Duration::from_secs(600)) + ); + // A comma-separated string is accepted so the same value can arrive from the environment. + assert_eq!( + cluster(&config, "analytics").bootstrap_servers, + ["analytics:9123", "analytics-2:9123"] + ); + assert_eq!( + cluster(&config, "analytics").connect_timeout.get(), + Duration::from_secs(5) + ); + assert_eq!(config.security.authentication, AuthenticationMode::Password); + assert_eq!(config.request_limits.write_max_rows, 500); + assert_eq!(config.request_limits.lookup_max_keys, 32); + assert_eq!( + config.request_limits.lookup_max_key_bytes.bytes(), + 2 * 1024 * 1024 + ); + } + + /// The declaration bounds which clusters may be configured, and it does so whether or not it was + /// written: with no `gateway.clusters`, the only configurable cluster is the implicit `default`. That is + /// what turns a mistyped cluster ID into a startup failure instead of an unreachable second cluster. + #[test] + fn declared_clusters_are_authoritative() { + for contents in [ + "gateway.clusters: default\n\ + gateway.cluster.analytics.bootstrap.servers: analytics:9123\n", + // No declaration at all: `analytics` is still not one of the allowed clusters. + "gateway.cluster.analytics.bootstrap.servers: analytics:9123\n", + "gateway.cluster.analytics.client.writer.batch-size: 2MiB\n", + ] { + let error = load_file(contents).unwrap_err(); + assert!( + error.to_string().contains("not declared"), + "{contents}: {error}" + ); + } + + // Declaring a cluster is enough to configure it; the rest of its settings default. + let config = load_file("gateway.clusters: default,analytics\n").unwrap(); + assert_eq!(config.clusters.len(), 2); + assert_eq!( + cluster(&config, "analytics").bootstrap_servers, + [DEFAULT_BOOTSTRAP_SERVERS] + ); + + // The implicit default needs no declaration, so a single-cluster deployment configures no list. + let config = load_file("gateway.cluster.default.bootstrap.servers: only:9123\n").unwrap(); + assert_eq!(config.clusters.keys().collect::>(), ["default"]); + } + + #[test] + fn malformed_cluster_ids_are_rejected() { + for contents in [ + "gateway.clusters: Default\n", + "gateway.clusters: 1st\n", + "gateway.clusters: eu-west\n", + "gateway.cluster.EU.bootstrap.servers: eu:9123\n", + ] { + let error = load_file(contents).unwrap_err(); + assert!( + error.to_string().contains("cluster ID"), + "{contents}: {error}" + ); + } + } + + #[test] + fn unknown_cluster_and_client_keys_name_the_original_key() { + for contents in [ + "gateway.cluster.default.bootstrap.serverz: fluss:9123\n", + "gateway.cluster.default.connection.identity: user\n", + "gateway.cluster.default.client.: 1\n", + "gateway.cluster.default: fluss:9123\n", + ] { + let error = load_file(contents).unwrap_err(); + assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); + let key = contents.split(':').next().unwrap(); + assert!(error.to_string().contains(key), "{key}: {error}"); + } + } + + #[test] + fn native_client_options_are_parsed_into_their_native_types() { + assert_eq!( + parse_client_option("writer.batch-size", "2MiB").unwrap(), + ClientOptionValue::Bytes(2 * 1024 * 1024) + ); + assert_eq!( + parse_client_option("writer.batch-timeout", "50ms").unwrap(), + ClientOptionValue::Millis(50) + ); + assert_eq!( + parse_client_option("writer.dynamic-batch-size.enabled", "false").unwrap(), + ClientOptionValue::Boolean(false) + ); + assert_eq!( + parse_client_option("lookup.max-retries", "3").unwrap(), + ClientOptionValue::Integer(3) + ); + assert_eq!( + parse_client_option("security.protocol", "sasl").unwrap(), + ClientOptionValue::Text("sasl".to_string()) + ); + + let config = load_file( + "gateway.cluster.default.client.writer.batch-size: 2MiB\n\ + gateway.cluster.default.client.lookup.max-retries: 3\n", + ) + .unwrap(); + let default = cluster(&config, "default"); + assert_eq!(default.client_option("writer.batch-size"), Some("2MiB")); + assert_eq!(default.client_option("lookup.max-retries"), Some("3")); + } + + /// The gateway advertises the write guarantees, so the client options that would weaken them, and the + /// authorization ID that user mode supplies per request, are refused rather than silently honoured. + #[test] + fn client_options_cannot_override_gateway_owned_guarantees_or_the_identity() { + for (option, value, expected) in [ + ("writer.acks", "0", "owned by the Gateway"), + ("writer.retries", "0", "owned by the Gateway"), + ("writer.enable-idempotence", "false", "owned by the Gateway"), + ( + "security.sasl.authorization-id", + "alice", + "cannot be configured statically", + ), + ( + "writer.unknown-knob", + "1", + "is not a supported native-client option", + ), + ] { + let error = parse_client_option(option, value).unwrap_err(); + assert!(error.contains(expected), "{option}: {error}"); + + let problems = problems( + load_file(&format!( + "gateway.cluster.default.client.{option}: {value}\n" + )) + .unwrap_err(), + ); + assert!( + problems.iter().any(|problem| { + problem.starts_with("gateway.cluster.default.client.") + && problem.contains(option) + }), + "{option}: {problems:?}" + ); + } + } + + #[test] + fn out_of_range_client_values_fail_before_startup() { + for (option, value) in [ + ("lookup.queue-size", "0"), + ("lookup.max-batch-size", "-1"), + ("lookup.max-retries", "-1"), + ("writer.request-max-size", "3GiB"), + ("writer.batch-timeout", "0s"), + ("writer.dynamic-batch-size.enabled", "maybe"), + ] { + let error = parse_client_option(option, value).unwrap_err(); + assert!(error.starts_with(&format!("client.{option}")), "{error}"); + assert!( + load_file(&format!( + "gateway.cluster.default.client.{option}: \"{value}\"\n" + )) + .is_err(), + "accepted client.{option} = {value}" + ); + } + } + + #[test] + fn cross_field_cluster_and_security_constraints_fail_before_startup() { + for contents in [ + // An account without its secret, and the reverse. + "gateway.cluster.default.connection.service.account: gateway_svc\n", + "gateway.cluster.default.connection.service.secret: gw-pass\n", + // User identity mode has no service account to authenticate the pool with. + "gateway.cluster.default.connection.identity-mode: user\n", + // SASL without credentials, and an unsupported protocol or mechanism. + "gateway.cluster.default.client.security.protocol: sasl\n", + "gateway.cluster.default.client.security.protocol: ssl\n", + "gateway.cluster.default.client.security.sasl.mechanism: SCRAM-SHA-256\n", + "gateway.cluster.default.connection.max: 0\n", + "gateway.cluster.default.bootstrap.servers: \" \"\n", + // The mode's credential table is missing. + "gateway.security.authentication: password\n", + "gateway.security.authentication: token\n", + "gateway.security.authentication: trusted-header\n\ + gateway.security.trusted-header.name: \"bad header\"\n", + "gateway.rest.lookup.max-keys: 0\n", + "gateway.rest.prefix-lookup.max-prefixes: 0\n", + ] { + assert!(load_file(contents).is_err(), "accepted: {contents}"); + } + + assert!( + load_file( + "gateway.cluster.default.connection.identity-mode: user\n\ + gateway.cluster.default.connection.service.account: gateway_svc\n\ + gateway.cluster.default.connection.service.secret: gw-pass\n\ + gateway.cluster.default.client.security.protocol: SASL\n\ + gateway.cluster.default.client.security.sasl.mechanism: PLAIN\n" + ) + .is_ok() + ); + } + + /// The legacy SASL options stay usable and keep winning, because silently changing which credential a + /// running deployment authenticates with would be worse than the deprecation. + #[test] + fn legacy_credentials_win_and_warn_once_per_cluster() { + let config = load_file( + "gateway.clusters: default,analytics\n\ + gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.username: legacy-user\n\ + gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ + gateway.cluster.analytics.client.security.sasl.username: other-user\n\ + gateway.cluster.analytics.client.security.sasl.password: other-secret\n", + ) + .unwrap(); + + let default = cluster(&config, "default"); + assert_eq!(default.effective_service_account(), Some("legacy-user")); + assert_eq!(default.effective_service_secret(), Some("legacy-secret")); + + let deprecations: Vec = config + .warnings() + .into_iter() + .filter(|warning| warning.contains("is deprecated")) + .collect(); + assert_eq!(deprecations.len(), 2, "one per cluster: {deprecations:?}"); + for warning in &deprecations { + for secret in [ + "canonical-user", + "canonical-secret", + "legacy-user", + "legacy-secret", + "other-user", + "other-secret", + ] { + assert!(!warning.contains(secret), "leaked {secret}: {warning}"); + } + } + } + + #[test] + fn pool_settings_warn_when_the_identity_mode_ignores_them() { + let config = load_file( + "gateway.cluster.default.connection.max: 8\n\ + gateway.cluster.default.connection.idle-timeout: 5m\n", + ) + .unwrap(); + assert!( + config + .warnings() + .iter() + .any(|warning| warning + .contains("ignored because connection.identity-mode is service")), + "{:?}", + config.warnings() + ); + } + + #[test] + fn diagnostics_redact_every_credential_and_keep_the_identities() { + let config = load_file( + "gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.username: legacy-user\n\ + gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ + gateway.cluster.default.client.writer.batch-size: 4MiB\n\ + gateway.security.authentication: password\n\ + gateway.security.users: alice:user-secret\n\ + gateway.security.tokens: token-secret:alice\n", + ) + .unwrap(); + + for diagnostic in [config.redacted_debug(), format!("{config:?}")] { + for credential in [ + "canonical-secret", + "legacy-secret", + "user-secret", + "token-secret", + ] { + assert!( + !diagnostic.contains(credential), + "leaked {credential}: {diagnostic}" + ); + } + // The identities and the tuning stay readable: they are what an operator came to look at. + for readable in ["canonical-user", "legacy-user", "4MiB"] { + assert!(diagnostic.contains(readable), "{readable}: {diagnostic}"); + } + assert!(diagnostic.contains(REDACTED), "{diagnostic}"); + } + + // The credentials are still reachable by the components that authenticate with them. + assert_eq!( + config.security.users.as_ref().map(Secret::expose), + Some("alice:user-secret") + ); + assert_eq!( + cluster(&config, "default").effective_service_secret(), + Some("legacy-secret") + ); + } + + /// A configuration error is what an operator sees on stderr, so it must name the option without + /// quoting any credential the file happens to carry. + #[test] + fn configuration_errors_never_quote_a_credential() { + let file = write_temp_config( + "gateway.security.authentication: token\n\ + gateway.security.tokens: do-not-leak\n\ + gateway.cluster.default.connection.service.secret: also-secret\n\ + gateway.cluster.default.client.writer.acks: 0\n", + ); + let error = load(Some(file.path()), &no_env(), &CliOverrides::default()).unwrap_err(); + let rendered = error.to_string(); + assert!(rendered.contains("writer.acks"), "{rendered}"); + assert!(!rendered.contains("do-not-leak"), "{rendered}"); + assert!(!rendered.contains("also-secret"), "{rendered}"); + } + + /// The same precedence statement for the two dynamic namespaces, where the environment name is derived + /// rather than registered: for every per-cluster option and every allowed client option, setting the + /// environment variable must be indistinguishable from having written that value in the file. + #[test] + fn the_environment_overrides_the_file_for_every_cluster_and_client_option() { + // Keeps every variant loadable: user identity mode needs usable credentials over SASL. The key + // under test is removed from the base so the file never carries it twice. + let base = [ + ("connection.service.account", "base-account"), + ("connection.service.secret", "base-secret"), + ("client.security.protocol", "sasl"), + ]; + + let mut cases: Vec<(String, String, &str, &str)> = Vec::new(); + for entry in CLUSTER_ENTRIES { + let values = match entry.kind { + ValueKind::ServerList => ("file-host:9123", "env-host:9123"), + ValueKind::Integer => ("11", "22"), + ValueKind::Text if entry.key == "connection.identity-mode" => ("service", "user"), + ValueKind::Text if entry.key.ends_with("timeout") => ("11s", "22s"), + ValueKind::Text => ("file-value", "env-value"), + ValueKind::Bool | ValueKind::Bytes => { + unreachable!("no per-cluster option uses {:?}", entry.kind) + } + }; + cases.push(( + entry.key.to_string(), + environment_suffix(entry.key), + values.0, + values.1, + )); + } + for spec in CLIENT_OPTIONS { + let values = match spec.kind { + // The only legal value is PLAIN, so the two spellings differ only in case. + _ if spec.option == "security.sasl.mechanism" => ("PLAIN", "plain"), + _ if spec.option == "security.protocol" => ("plaintext", "sasl"), + ClientOptionKind::Text | ClientOptionKind::Secret => ("file-value", "env-value"), + ClientOptionKind::Boolean => ("true", "false"), + ClientOptionKind::Count { .. } => ("5", "6"), + // Both values have to keep the size relationships intact against the native defaults. + ClientOptionKind::Size { .. } + if spec.option.ends_with("dynamic-batch-size.min") => + { + ("1MiB", "2MiB") + } + ClientOptionKind::Size { .. } => ("4MiB", "8MiB"), + ClientOptionKind::Duration => ("11s", "22s"), + }; + cases.push(( + format!("{CLIENT_OPTION_PREFIX}{}", spec.option), + format!("CLIENT__{}", environment_suffix(spec.option)), + values.0, + values.1, + )); + } + + for (key, env_suffix, file_value, env_value) in cases { + let contents = |value: &str| { + base.iter() + .filter(|(base_key, _)| *base_key != key) + .map(|(base_key, base_value)| (*base_key, *base_value)) + .chain([(key.as_str(), value)]) + .map(|(key, value)| { + format!("{CLUSTER_KEY_PREFIX}{DEFAULT_CLUSTER_ID}.{key}: \"{value}\"\n") + }) + .collect::() + }; + let load_valid = |contents: &str, env: &BTreeMap| { + let file = write_temp_config(contents); + load(Some(file.path()), env, &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{key}: {error}\n{contents}")) + }; + + let env = BTreeMap::from([( + format!( + "{ENV_PREFIX}CLUSTER__{}__{env_suffix}", + DEFAULT_CLUSTER_ID.to_ascii_uppercase() + ), + env_value.to_string(), + )]); + let from_file = load_valid(&contents(file_value), &no_env()); + let overridden = load_valid(&contents(file_value), &env); + let as_written = load_valid(&contents(env_value), &no_env()); + + assert_ne!( + from_file, overridden, + "{key} ignores its environment variable" + ); + assert_eq!( + overridden, as_written, + "{key} from the environment differs from the same value in the file" + ); + } + } + + /// The reserved options are refused from the environment exactly as they are from the file, and the + /// fixed request limits are reachable there too. + #[test] + fn the_environment_is_held_to_the_same_client_option_rules_as_the_file() { + let mut env = BTreeMap::from([( + "FLUSS_GATEWAY__REST__LOOKUP__MAX_KEYS".to_string(), + "16".to_string(), + )]); + let config = load(None, &env, &CliOverrides::default()).unwrap(); + assert_eq!(config.request_limits.lookup_max_keys, 16); + + env.insert( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__ACKS".to_string(), + "0".to_string(), + ); + let error = load(None, &env, &CliOverrides::default()).unwrap_err(); + assert!(error.to_string().contains("writer.acks"), "got: {error}"); + } + + #[test] + fn the_environment_can_declare_clusters() { + let file = write_temp_config("gateway.cluster.analytics.bootstrap.servers: eu:9123\n"); + let mut env = no_env(); + env.insert( + "FLUSS_GATEWAY__CLUSTERS".to_string(), + "analytics".to_string(), + ); + let config = load(Some(file.path()), &env, &CliOverrides::default()).unwrap(); + assert_eq!(config.clusters.keys().collect::>(), ["analytics"]); + + env.insert("FLUSS_GATEWAY__CLUSTERS".to_string(), "default".to_string()); + let error = load(Some(file.path()), &env, &CliOverrides::default()).unwrap_err(); + assert!(error.to_string().contains("not declared"), "got: {error}"); + } + + #[test] + fn unknown_cluster_environment_variables_are_rejected() { + for variable in [ + "FLUSS_GATEWAY__CLUSTER__DEFAULT__BOOTSTRAP__SERVERZ", + "FLUSS_GATEWAY__CLUSTER__DEFAULT", + "FLUSS_GATEWAY__CLUSTER__1ST__BOOTSTRAP__SERVERS", + ] { + let mut env = no_env(); + env.insert(variable.to_string(), "value".to_string()); + let error = load(None, &env, &CliOverrides::default()).unwrap_err(); + assert!( + matches!(error, ConfigError::UnknownEnvKey(_)), + "{variable}: {error:?}" + ); + assert!(error.to_string().contains(variable), "{variable}: {error}"); + } + } + + /// A bad per-cluster value is reported with the public key rather than the internal Serde path, and an + /// environment override additionally names the variable the operator set. + #[test] + fn a_bad_cluster_value_names_the_public_key_and_its_source() { + let error = load_file("gateway.cluster.default.request-timeout: 0s\n").unwrap_err(); + assert!( + error + .to_string() + .contains("gateway.cluster.default.request-timeout"), + "got: {error}" + ); + + let env = BTreeMap::from([( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT".to_string(), + "soon".to_string(), + )]); + let rendered = load(None, &env, &CliOverrides::default()) + .unwrap_err() + .to_string(); + assert!( + rendered.contains("FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT"), + "{rendered}" + ); + assert!( + rendered.contains("gateway.cluster.default.connect-timeout"), + "{rendered}" + ); + } + + #[test] + fn programmatically_constructed_clusters_are_validated() { + let mut config = GatewayConfig::default(); + config.clusters.clear(); + let errors = problems(config.validate().unwrap_err()); + assert!( + errors + .iter() + .any(|error| error == "gateway.clusters must declare at least one cluster"), + "got: {errors:?}" + ); + + let mut config = GatewayConfig::default(); + config + .clusters + .get_mut(DEFAULT_CLUSTER_ID) + .expect("default cluster") + .identity_mode = IdentityMode::User; + let errors = problems(config.validate().unwrap_err()); + assert!( + errors + .iter() + .any(|error| error.contains("identity-mode user requires")), + "got: {errors:?}" + ); + } + + /// The `client.*` namespace is open, so its environment mapping cannot be checked key by key like the + /// fixed vocabulary. It round-trips only while option names keep `-` inside a segment and `.` between + /// segments: an option name containing `_` would come back from the environment as a different name and + /// silently configure the wrong option. Exhaustive over the allowlist, so adding such a name fails here. + #[test] + fn every_client_option_round_trips_through_the_environment() { + for spec in CLIENT_OPTIONS { + let suffix = environment_suffix(spec.option); + assert_eq!( + environment_suffix_to_option(&suffix), + spec.option, + "{} does not survive the environment mapping", + spec.option + ); + assert!( + !spec.option.contains('_'), + "{} must spell words with '-', which the environment mapping reserves '_' for", + spec.option + ); + let file_key = format!( + "{CLUSTER_KEY_PREFIX}{DEFAULT_CLUSTER_ID}.{CLIENT_OPTION_PREFIX}{}", + spec.option + ); + assert!( + matches!(resolve_key(&file_key), Ok(ResolvedKey::ClientOption { .. })), + "{file_key} is not reachable as a file key" + ); + } + } + + /// Sensitivity is declared per option, so the declaration is what has to be right. Fluss decides it on + /// the Java side from these same substrings, which is the cross-check: any option whose name looks like + /// a credential must be marked, and the allowlist and the refusals must not overlap. + #[test] + fn client_option_sensitivity_and_refusals_are_declared_consistently() { + for spec in CLIENT_OPTIONS { + let looks_sensitive = ["password", "secret", "token"] + .iter() + .any(|part| spec.option.contains(part)); + assert_eq!( + spec.kind.is_sensitive(), + looks_sensitive, + "{}: the kind and the name disagree about being a credential", + spec.option + ); + assert!( + !RESERVED_CLIENT_OPTIONS + .iter() + .any(|(reserved, _)| *reserved == spec.option), + "{} is both allowed and refused", + spec.option + ); + } + // An option the gateway never validated must not be rendered on the chance that it holds a secret. + assert!(client_option_is_sensitive("some.unknown.option")); + assert!(client_option_is_sensitive(LEGACY_SERVICE_SECRET_OPTION)); + assert!(!client_option_is_sensitive(LEGACY_SERVICE_ACCOUNT_OPTION)); + + // A parsed credential stays wrapped, so printing the parse result cannot leak it either. + let parsed = parse_client_option(LEGACY_SERVICE_SECRET_OPTION, "legacy-secret").unwrap(); + assert_eq!( + parsed, + ClientOptionValue::Secret(Secret::new("legacy-secret")) + ); + assert!(!format!("{parsed:?}").contains("legacy-secret")); + assert!(format!("{parsed:?}").contains(REDACTED)); + } + + /// Precedence is stated once for the whole vocabulary rather than sampled on one key, so a per-kind + /// conversion that only reads one source cannot hide: every option is driven from the file, then + /// overridden from the environment, and the environment value has to win in the loaded config. + #[test] + fn the_environment_overrides_the_file_for_every_option() { + for entry in CONFIG_ENTRIES { + let (file_value, env_value) = match entry.kind { + ValueKind::Bool => ("true", "false"), + ValueKind::Integer => ("11", "22"), + ValueKind::Bytes => ("1MiB", "2MiB"), + ValueKind::ServerList => ("file-host:9123", "env-host:9123"), + ValueKind::Text => match entry.key { + REST_LISTEN_KEY => ("127.0.0.1:11111", "127.0.0.1:22222"), + METRICS_LISTEN_KEY => ("127.0.0.1:11112", "127.0.0.1:22223"), + REST_HEADER_READ_TIMEOUT_KEY + | REST_REQUEST_TIMEOUT_KEY + | SHUTDOWN_DRAIN_TIMEOUT_KEY => ("11s", "22s"), + // Both modes must be valid on their own: the file value is loaded without the + // environment override, and password and token modes need a credential table. + SECURITY_AUTHENTICATION_KEY => ("trusted-header", "trust"), + _ => ("file-value", "env-value"), + }, + }; + + let file = write_temp_config(&format!("{}: \"{file_value}\"\n", entry.key)); + let from_file = load(Some(file.path()), &no_env(), &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); + let env = BTreeMap::from([(environment_variable(entry.key), env_value.to_string())]); + let from_env = load(Some(file.path()), &env, &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); + + assert_ne!( + from_file, from_env, + "{} ignores its environment variable", + entry.key + ); + let only_env = load(None, &env, &CliOverrides::default()) + .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); + assert_eq!( + from_env, only_env, + "{} lets the file value survive the environment override", + entry.key + ); + } + } + + /// User identity mode is only safe when the connection can actually carry the request's principal, so + /// the credentials must be usable *and* SASL must be selected. Both were previously satisfied by a + /// `Some("")` credential over the default PLAINTEXT, which authorizes the gateway's own identity for + /// every caller instead of failing. + #[test] + fn user_identity_mode_requires_usable_credentials_over_sasl() { + let user_mode = "gateway.cluster.default.connection.identity-mode: user\n"; + let credentials = "gateway.cluster.default.connection.service.account: gateway_svc\n\ + gateway.cluster.default.connection.service.secret: gw-pass\n"; + let sasl = "gateway.cluster.default.client.security.protocol: sasl\n"; + + for (contents, expected) in [ + ( + format!("{user_mode}{credentials}"), + "requires client.security.protocol sasl", + ), + ( + format!("{user_mode}{sasl}"), + "requires connection.service.account", + ), + ( + format!( + "{user_mode}{sasl}gateway.cluster.default.connection.service.account: \"\"\n\ + gateway.cluster.default.connection.service.secret: \" \"\n" + ), + "must not be blank", + ), + ( + format!( + "{user_mode}{sasl}{credentials}\ + gateway.cluster.default.client.security.sasl.mechanism: SCRAM-SHA-256\n" + ), + "must be PLAIN", + ), + ] { + let problems = problems(load_file(&contents).unwrap_err()); + assert!( + problems.iter().any(|problem| problem.contains(expected)), + "expected {expected:?} for:\n{contents}got: {problems:?}" + ); + } + + // The complete, coherent form is accepted. + assert!(load_file(&format!("{user_mode}{sasl}{credentials}")).is_ok()); + // Service mode needs no SASL: it authenticates as itself, with no principal to propagate. + assert!(load_file("gateway.cluster.default.connection.identity-mode: service\n").is_ok()); + } + + /// A size that cannot fit inside the size holding it fails before a listener binds, whether the operator + /// set both sides or only one: leaving the other at its native default is the common way to break a pair. + #[test] + fn writer_size_pairs_must_fit_including_against_the_native_defaults() { + for (contents, rejected) in [ + // Both sides configured. + ( + "gateway.cluster.default.client.writer.batch-size: 2MiB\n\ + gateway.cluster.default.client.writer.request-max-size: 1MiB\n", + true, + ), + ( + "gateway.cluster.default.client.writer.dynamic-batch-size.min: 4MiB\n\ + gateway.cluster.default.client.writer.batch-size: 2MiB\n", + true, + ), + // One side only: the other is the native default, 10MiB request-max and 2MiB batch-size. + ( + "gateway.cluster.default.client.writer.batch-size: 128MiB\n", + true, + ), + ( + "gateway.cluster.default.client.writer.dynamic-batch-size.min: 3MiB\n", + true, + ), + ( + "gateway.cluster.default.client.writer.request-max-size: 1MiB\n", + true, + ), + // Coherent against the defaults, and coherent as a pair. + ( + "gateway.cluster.default.client.writer.batch-size: 4MiB\n", + false, + ), + ( + "gateway.cluster.default.client.writer.batch-size: 32MiB\n\ + gateway.cluster.default.client.writer.request-max-size: 64MiB\n", + false, + ), + ] { + let result = load_file(contents); + assert_eq!( + result.is_err(), + rejected, + "unexpected outcome for:\n{contents}" + ); + if rejected { + assert!( + problems(result.unwrap_err()) + .iter() + .any(|problem| problem.contains("must not exceed client.")), + "{contents}" + ); + } + } + } + + /// A value is bounded by the native field it lands in, not by one blanket ceiling: the writer sizes are + /// `i32` there, while the buffer size and the lookup counts are `usize` and may exceed `i32::MAX`. + #[test] + fn client_option_bounds_follow_the_native_field_type() { + let over_i32 = u64::from(i32::MAX as u32) + 1; + + for option in ["writer.batch-size", "writer.request-max-size"] { + let error = parse_client_option(option, &format!("{over_i32}")).unwrap_err(); + assert!(error.contains("must not exceed"), "{option}: {error}"); + } + assert_eq!( + parse_client_option("writer.buffer.memory-size", "4GiB").unwrap(), + ClientOptionValue::Bytes(4 * 1024 * 1024 * 1024) + ); + for option in [ + "lookup.queue-size", + "lookup.max-batch-size", + "lookup.max-inflight-requests", + ] { + assert_eq!( + parse_client_option(option, &format!("{over_i32}")).unwrap(), + ClientOptionValue::Integer(over_i32), + "{option} is stored as usize and must accept this" + ); + } + let error = parse_client_option("lookup.max-retries", &format!("{over_i32}")).unwrap_err(); + assert!(error.contains("must be between 0 and"), "{error}"); + } + + /// The declared native defaults are a copy of the client's, so they must at least satisfy the + /// relationships the client enforces; a mistyped copy shows up here and not as a rejected valid file. + #[test] + fn the_declared_native_size_defaults_are_coherent() { + let default = |option| effective_size(option, None).expect("a declared size default"); + let batch = default("writer.batch-size"); + assert!(batch <= default("writer.request-max-size")); + assert!(batch <= default("writer.buffer.memory-size")); + assert!(default("writer.dynamic-batch-size.min") <= batch); + } + + #[test] + fn options_are_complete_and_unambiguous() { + let mut public_keys = std::collections::BTreeSet::new(); + let mut internal_paths = std::collections::BTreeSet::new(); + let mut environment_variables = std::collections::BTreeSet::new(); + + for entry in CONFIG_ENTRIES { + assert!(entry.key.starts_with("gateway."), "{entry:?}"); + assert!( + public_keys.insert(entry.key), + "duplicate key: {}", + entry.key + ); + assert!( + internal_paths.insert(entry.internal_path), + "duplicate path: {}", + entry.internal_path + ); + assert!( + environment_variables.insert(environment_variable(entry.key)), + "duplicate environment variable for {}", + entry.key + ); + } + + assert_eq!(CONFIG_ENTRIES.len(), 20); + } + + /// The per-cluster vocabulary shares the environment namespace with `client.*`, so its keys have to + /// stay distinct from each other and unreachable through the client prefix. + #[test] + fn cluster_options_are_complete_and_unambiguous() { + let mut keys = std::collections::BTreeSet::new(); + let mut fields = std::collections::BTreeSet::new(); + let mut suffixes = std::collections::BTreeSet::new(); + + for entry in CLUSTER_ENTRIES { + assert!(!entry.key.starts_with("gateway."), "{entry:?}"); + assert!( + !entry.key.starts_with(CLIENT_OPTION_PREFIX), + "{} collides with the client namespace", + entry.key + ); + assert!(keys.insert(entry.key), "duplicate key: {}", entry.key); + assert!( + fields.insert(entry.internal_path), + "duplicate field: {}", + entry.internal_path + ); + assert!( + suffixes.insert(environment_suffix(entry.key)), + "duplicate environment suffix for {}", + entry.key + ); + } + + assert_eq!(CLUSTER_ENTRIES.len(), 8); + } +} diff --git a/fluss-gateway/src/config/tests.rs b/fluss-gateway/src/config/tests.rs deleted file mode 100644 index c57695d02e..0000000000 --- a/fluss-gateway/src/config/tests.rs +++ /dev/null @@ -1,1375 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Tests for the configuration vocabulary, its sources, and its validation. - -use super::*; -use std::io::Write; - -fn no_env() -> BTreeMap { - BTreeMap::new() -} - -fn write_temp_config(contents: &str) -> tempfile::NamedTempFile { - let mut file = tempfile::NamedTempFile::new().expect("temp file"); - file.write_all(contents.as_bytes()).expect("write"); - file -} - -fn load_file(contents: &str) -> Result { - let file = write_temp_config(contents); - load(Some(file.path()), &no_env(), &CliOverrides::default()) -} - -fn problems(error: ConfigError) -> Vec { - match error { - ConfigError::Invalid(problems) => problems, - other => panic!("expected Invalid, got: {other:?}"), - } -} - -#[test] -fn defaults_when_no_sources() { - let config = load(None, &no_env(), &CliOverrides::default()).unwrap(); - assert_eq!( - config.server.rest.bind_address, - "127.0.0.1:8080".parse().unwrap() - ); - assert_eq!(config.server.rest.max_body_bytes.bytes(), 32 * 1024 * 1024); - assert_eq!( - config.server.rest.request_timeout.get(), - Duration::from_secs(30) - ); - assert!(config.server.metrics.enabled); - assert_eq!( - config.server.metrics.bind_address, - "127.0.0.1:9095".parse().unwrap() - ); - assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(30)); - assert!(config.warnings().is_empty()); -} - -#[test] -fn public_yaml_options_are_loaded() { - let config = load_file( - r#" -gateway.instance-id: gateway-1 -gateway.rest.listen: 0.0.0.0:8080 -gateway.rest.write.max-request-bytes: 32MiB -gateway.rest.write.request-timeout: 30s -gateway.metrics.enabled: true -gateway.metrics.exporter.prometheus.listen: 0.0.0.0:9095 -gateway.shutdown.drain-timeout: 10s -"#, - ) - .unwrap(); - assert_eq!(config.server.instance_id.as_deref(), Some("gateway-1")); - assert_eq!( - config.server.rest.bind_address, - "0.0.0.0:8080".parse().unwrap() - ); - assert_eq!(config.server.rest.max_body_bytes.bytes(), 32 * 1024 * 1024); - assert_eq!( - config.server.rest.request_timeout.get(), - Duration::from_secs(30) - ); - assert!(config.server.metrics.enabled); - assert_eq!( - config.server.metrics.bind_address, - "0.0.0.0:9095".parse().unwrap() - ); - assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(10)); -} - -#[test] -fn unknown_file_keys_name_the_original_key() { - for contents in [ - "gateway.rest.listenn: 0.0.0.0:8080\n", - "rest.listen: 0.0.0.0:8080\n", - "gateway.rest.lookup.max-keyz: 5\n", - "gateway.scan.cursor-ttl: 1m\n", - "gateway.tls.cert: /etc/tls.pem\n", - ] { - let error = load_file(contents).unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); - let key = contents.split(':').next().unwrap(); - assert!(error.to_string().contains(key), "{key}: {error}"); - } -} - -#[test] -fn source_precedence_is_cli_then_env_then_file_then_defaults() { - let file = write_temp_config( - r#" -gateway.rest.listen: 127.0.0.1:18080 -gateway.metrics.enabled: true -"#, - ); - let mut env = no_env(); - env.insert( - "FLUSS_GATEWAY__REST__LISTEN".to_string(), - "127.0.0.1:28080".to_string(), - ); - env.insert( - "FLUSS_GATEWAY__METRICS__ENABLED".to_string(), - "false".to_string(), - ); - env.insert("PATH".to_string(), "/usr/bin".to_string()); - - let config = load( - Some(file.path()), - &env, - &CliOverrides { - bind_address: Some("127.0.0.1:38080".to_string()), - }, - ) - .unwrap(); - assert_eq!( - config.server.rest.bind_address, - "127.0.0.1:38080".parse().unwrap() - ); - assert!(!config.server.metrics.enabled); -} - -#[test] -fn missing_file_reported() { - let error = load( - Some(Path::new("/nonexistent/gateway.yaml")), - &no_env(), - &CliOverrides::default(), - ) - .unwrap_err(); - assert!(matches!(error, ConfigError::Io(_)), "got: {error:?}"); -} - -#[test] -fn malformed_file_reports_position() { - let error = load_file("gateway.rest.listen: [1\n").unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); - assert!(error.to_string().contains("line"), "got: {error}"); -} - -#[test] -fn duplicate_flat_key_rejected() { - let error = - load_file("gateway.rest.listen: 127.0.0.1:8080\ngateway.rest.listen: 127.0.0.1:8081\n") - .unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); - assert!(error.to_string().contains("duplicate"), "got: {error}"); -} - -#[test] -fn unknown_environment_variables_are_rejected() { - for key in [ - "FLUSS_GATEWAY__REST__LISTENN", - "FLUSS_GATEWAY__QUERY__ENABLED", - "FLUSS_GATEWAY__SERVER_REST__BIND_ADDRESS", - ] { - let mut env = no_env(); - env.insert(key.to_string(), "value".to_string()); - let error = load(None, &env, &CliOverrides::default()).unwrap_err(); - assert!( - matches!(error, ConfigError::UnknownEnvKey(_)), - "{key}: {error:?}" - ); - assert!(error.to_string().contains(key), "{key}: {error}"); - } -} - -#[test] -fn file_error_under_a_section_with_an_env_override_names_the_file() { - let file = write_temp_config("gateway.shutdown.drain-timeout: 0s\n"); - let mut env = no_env(); - env.insert( - "FLUSS_GATEWAY__REST__LISTEN".to_string(), - "127.0.0.1:28080".to_string(), - ); - let error = load(Some(file.path()), &env, &CliOverrides::default()).unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); - assert!( - error.to_string().contains("gateway.shutdown.drain-timeout"), - "got: {error}" - ); - assert!( - !error.to_string().contains("FLUSS_GATEWAY__"), - "file problem misattributed to the env override: {error}" - ); -} - -#[test] -fn public_environment_options_are_loaded_by_type() { - let env = BTreeMap::from([ - ("FLUSS_GATEWAY__INSTANCE_ID".to_string(), "123".to_string()), - ( - "FLUSS_GATEWAY__REST__LISTEN".to_string(), - "127.0.0.1:18080".to_string(), - ), - ( - "FLUSS_GATEWAY__REST__WRITE__REQUEST_TIMEOUT".to_string(), - "5s".to_string(), - ), - ( - "FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES".to_string(), - "2MiB".to_string(), - ), - ( - "FLUSS_GATEWAY__METRICS__ENABLED".to_string(), - "false".to_string(), - ), - ( - "FLUSS_GATEWAY__METRICS__EXPORTER__PROMETHEUS__LISTEN".to_string(), - "127.0.0.1:19095".to_string(), - ), - ( - "FLUSS_GATEWAY__SHUTDOWN__DRAIN_TIMEOUT".to_string(), - "10s".to_string(), - ), - ]); - - let config = load(None, &env, &CliOverrides::default()).unwrap(); - assert_eq!(config.server.instance_id.as_deref(), Some("123")); - assert_eq!( - config.server.rest.bind_address, - "127.0.0.1:18080".parse().unwrap() - ); - assert_eq!( - config.server.rest.request_timeout.get(), - Duration::from_secs(5) - ); - assert_eq!(config.server.rest.max_body_bytes.bytes(), 2 * 1024 * 1024); - assert!(!config.server.metrics.enabled); - assert_eq!( - config.server.metrics.bind_address, - "127.0.0.1:19095".parse().unwrap() - ); - assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(10)); -} - -#[test] -fn invalid_env_value_names_the_variable() { - let mut env = no_env(); - env.insert( - "FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES".to_string(), - "many".to_string(), - ); - let error = load(None, &env, &CliOverrides::default()).unwrap_err(); - assert!( - error - .to_string() - .contains("FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES"), - "got: {error}" - ); -} - -#[test] -fn invalid_cli_value_names_the_flag() { - let cli = CliOverrides { - bind_address: Some("not-an-address".to_string()), - }; - let error = load(None, &no_env(), &cli).unwrap_err(); - assert!(error.to_string().contains("--bind-address"), "got: {error}"); -} - -#[test] -fn invalid_duration_rejected() { - for bad in ["0ms", "60", "60 s", "6.5s", "s", "60d", "-1s"] { - let error = load_file(&format!("gateway.shutdown.drain-timeout: \"{bad}\"\n")).unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "{bad}: {error:?}"); - assert!( - error.to_string().contains("gateway.shutdown.drain-timeout"), - "{bad}: {error}" - ); - } -} - -#[test] -fn overflowing_duration_is_rejected_rather_than_saturated() { - for bad in [ - "18446744073709551615ms", - "18446744073709551615s", - "18446744073709551615m", - "18446744073709551615h", - ] { - let error = ConfigDuration::parse(bad).unwrap_err(); - assert!(error.contains("must not exceed"), "{bad}: {error}"); - } - assert_eq!( - ConfigDuration::parse("31536000s").unwrap().get(), - MAX_CONFIG_DURATION - ); - assert!(ConfigDuration::parse("31536001s").is_err()); -} - -#[test] -fn programmatically_constructed_durations_are_validated() { - let mut config = GatewayConfig::default(); - config.server.rest.request_timeout = ConfigDuration::from_millis(0); - config.shutdown.drain_timeout = ConfigDuration::from_secs(MAX_CONFIG_DURATION.as_secs() + 1); - - let errors = problems(config.validate().unwrap_err()); - assert!( - errors.iter().any(|error| { - error == "gateway.rest.write.request-timeout must be greater than zero" - }), - "got: {errors:?}" - ); - assert!( - errors.iter().any(|error| { - error == "gateway.shutdown.drain-timeout must not exceed 31536000 seconds" - }), - "got: {errors:?}" - ); -} - -#[test] -fn programmatically_constructed_zero_byte_limit_is_validated() { - let mut config = GatewayConfig::default(); - config.server.rest.max_body_bytes = ByteSize::new(0); - - let errors = problems(config.validate().unwrap_err()); - assert_eq!( - errors, - vec!["gateway.rest.write.max-request-bytes must be greater than zero"] - ); -} - -#[test] -fn invalid_byte_size_rejected() { - for bad in ["0", "\"4Mb\"", "\"MiB\"", "-1", "\"1.5MiB\""] { - let error = - load_file(&format!("gateway.rest.write.max-request-bytes: {bad}\n")).unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "{bad}: {error:?}"); - assert!( - error - .to_string() - .contains("gateway.rest.write.max-request-bytes"), - "{bad}: {error}" - ); - } -} - -#[test] -fn metrics_address_must_differ_from_rest_address() { - let error = load_file( - "gateway.rest.listen: 127.0.0.1:9095\ngateway.metrics.exporter.prometheus.listen: 127.0.0.1:9095\n", - ) - .unwrap_err(); - assert!(problems(error).iter().any(|problem| { - problem.contains( - "gateway.metrics.exporter.prometheus.listen (127.0.0.1:9095) must differ from \ - gateway.rest.listen (127.0.0.1:9095)", - ) - })); -} - -/// Overlap detection covers the wildcard and dual-stack pairs that differ textually but cannot -/// both bind, plus the pairs that coexist. -#[test] -fn listener_overlap_covers_wildcards_and_dual_stack() { - let clashes = [ - ("0.0.0.0:8080", "127.0.0.1:8080"), - ("127.0.0.1:8080", "0.0.0.0:8080"), - ("0.0.0.0:8080", "0.0.0.0:8080"), - ("[::]:8080", "[::1]:8080"), - ("[::]:8080", "0.0.0.0:8080"), - ("127.0.0.1:8080", "[::]:8080"), - ]; - for (rest, metrics) in clashes { - let rest: SocketAddr = rest.parse().unwrap(); - let metrics: SocketAddr = metrics.parse().unwrap(); - assert!( - addresses_overlap(rest, metrics), - "{rest} and {metrics} cannot both bind" - ); - } - - let coexist = [ - ("127.0.0.1:8080", "192.168.1.2:8080"), - ("127.0.0.1:8080", "[::1]:8080"), - ("0.0.0.0:8080", "127.0.0.1:9095"), - ("0.0.0.0:0", "0.0.0.0:0"), - ("127.0.0.1:0", "0.0.0.0:8080"), - ]; - for (rest, metrics) in coexist { - let rest: SocketAddr = rest.parse().unwrap(); - let metrics: SocketAddr = metrics.parse().unwrap(); - assert!( - !addresses_overlap(rest, metrics), - "{rest} and {metrics} can coexist" - ); - } -} - -/// Two ephemeral listeners are not a clash: the OS hands out a different port to each. -#[test] -fn both_listeners_may_ask_for_an_ephemeral_port() { - let config = load_file( - "gateway.rest.listen: 127.0.0.1:0\ngateway.metrics.exporter.prometheus.listen: 127.0.0.1:0\n", - ) - .unwrap(); - assert_eq!(config.server.rest.bind_address.port(), 0); -} - -#[test] -fn non_loopback_bind_is_accepted_without_an_instance_id_but_warns() { - let config = load_file("gateway.rest.listen: 0.0.0.0:8080\n").unwrap(); - assert!(config.server.instance_id.is_none()); - assert_eq!(config.warnings().len(), 1); - assert!(config.warnings()[0].contains("not loopback")); - assert!( - config.warnings()[0].contains("accepts unauthenticated requests"), - "{:?}", - config.warnings() - ); -} - -#[test] -fn malformed_instance_id_rejected() { - let error = load_file("gateway.instance-id: has space\n").unwrap_err(); - assert!( - problems(error) - .iter() - .any(|problem| problem.contains("gateway.instance-id must be 1-128 ASCII")) - ); -} - -#[test] -fn duration_units() { - assert_eq!( - ConfigDuration::parse("250ms").unwrap().get(), - Duration::from_millis(250) - ); - assert_eq!( - ConfigDuration::parse("15m").unwrap().get(), - Duration::from_secs(900) - ); - assert_eq!( - ConfigDuration::parse("2h").unwrap().get(), - Duration::from_secs(7200) - ); - assert!(ConfigDuration::parse("0s").is_err()); -} - -#[test] -fn byte_size_units() { - assert_eq!(ByteSize::parse("512").unwrap().bytes(), 512); - assert_eq!(ByteSize::parse("512B").unwrap().bytes(), 512); - assert_eq!(ByteSize::parse("4KB").unwrap().bytes(), 4000); - assert_eq!(ByteSize::parse("4KiB").unwrap().bytes(), 4096); - assert_eq!(ByteSize::parse("1GiB").unwrap().bytes(), 1024 * 1024 * 1024); - assert!(ByteSize::parse("4TB").is_err()); - assert!(ByteSize::parse("0").is_err()); -} - -fn cluster<'a>(config: &'a GatewayConfig, id: &str) -> &'a ClusterConfig { - config.clusters.get(id).expect("configured cluster") -} - -#[test] -fn a_single_default_cluster_needs_no_declaration() { - let config = load(None, &no_env(), &CliOverrides::default()).unwrap(); - assert_eq!(config.clusters.len(), 1); - assert_eq!( - cluster(&config, DEFAULT_CLUSTER_ID).bootstrap_servers, - [DEFAULT_BOOTSTRAP_SERVERS] - ); - assert_eq!( - cluster(&config, DEFAULT_CLUSTER_ID).identity_mode, - IdentityMode::Service - ); - assert_eq!(config.security.authentication, AuthenticationMode::Trust); - assert_eq!(config.request_limits, RequestLimitsConfig::default()); -} - -#[test] -fn typed_cluster_security_and_request_limit_options_are_loaded() { - let config = load_file( - "gateway.clusters: default, analytics\n\ - gateway.cluster.default.bootstrap.servers: [fluss-1:9123, fluss-2:9123]\n\ - gateway.cluster.default.connection.identity-mode: user\n\ - gateway.cluster.default.connection.service.account: gateway_svc\n\ - gateway.cluster.default.connection.service.secret: gw-pass\n\ - gateway.cluster.default.client.security.protocol: sasl\n\ - gateway.cluster.default.connection.max: 512\n\ - gateway.cluster.default.connection.idle-timeout: 10m\n\ - gateway.cluster.analytics.bootstrap.servers: analytics:9123,analytics-2:9123\n\ - gateway.cluster.analytics.connect-timeout: 5s\n\ - gateway.security.authentication: password\n\ - gateway.security.users: alice:secret\n\ - gateway.rest.write.max-rows: 500\n\ - gateway.rest.lookup.max-keys: 32\n\ - gateway.rest.lookup.max-key-bytes: 2MiB\n", - ) - .unwrap(); - - let default = cluster(&config, "default"); - assert_eq!(default.bootstrap_servers, ["fluss-1:9123", "fluss-2:9123"]); - assert_eq!(default.identity_mode, IdentityMode::User); - assert_eq!(default.effective_service_account(), Some("gateway_svc")); - assert_eq!(default.effective_service_secret(), Some("gw-pass")); - assert_eq!(default.connection_max, Some(512)); - assert_eq!( - default.connection_idle_timeout.map(ConfigDuration::get), - Some(Duration::from_secs(600)) - ); - // A comma-separated string is accepted so the same value can arrive from the environment. - assert_eq!( - cluster(&config, "analytics").bootstrap_servers, - ["analytics:9123", "analytics-2:9123"] - ); - assert_eq!( - cluster(&config, "analytics").connect_timeout.get(), - Duration::from_secs(5) - ); - assert_eq!(config.security.authentication, AuthenticationMode::Password); - assert_eq!(config.request_limits.write_max_rows, 500); - assert_eq!(config.request_limits.lookup_max_keys, 32); - assert_eq!( - config.request_limits.lookup_max_key_bytes.bytes(), - 2 * 1024 * 1024 - ); -} - -/// The declaration bounds which clusters may be configured, and it does so whether or not it was -/// written: with no `gateway.clusters`, the only configurable cluster is the implicit `default`. That is -/// what turns a mistyped cluster ID into a startup failure instead of an unreachable second cluster. -#[test] -fn declared_clusters_are_authoritative() { - for contents in [ - "gateway.clusters: default\n\ - gateway.cluster.analytics.bootstrap.servers: analytics:9123\n", - // No declaration at all: `analytics` is still not one of the allowed clusters. - "gateway.cluster.analytics.bootstrap.servers: analytics:9123\n", - "gateway.cluster.analytics.client.writer.batch-size: 2MiB\n", - ] { - let error = load_file(contents).unwrap_err(); - assert!( - error.to_string().contains("not declared"), - "{contents}: {error}" - ); - } - - // Declaring a cluster is enough to configure it; the rest of its settings default. - let config = load_file("gateway.clusters: default,analytics\n").unwrap(); - assert_eq!(config.clusters.len(), 2); - assert_eq!( - cluster(&config, "analytics").bootstrap_servers, - [DEFAULT_BOOTSTRAP_SERVERS] - ); - - // The implicit default needs no declaration, so a single-cluster deployment configures no list. - let config = load_file("gateway.cluster.default.bootstrap.servers: only:9123\n").unwrap(); - assert_eq!(config.clusters.keys().collect::>(), ["default"]); -} - -#[test] -fn malformed_cluster_ids_are_rejected() { - for contents in [ - "gateway.clusters: Default\n", - "gateway.clusters: 1st\n", - "gateway.clusters: eu-west\n", - "gateway.cluster.EU.bootstrap.servers: eu:9123\n", - ] { - let error = load_file(contents).unwrap_err(); - assert!( - error.to_string().contains("cluster ID"), - "{contents}: {error}" - ); - } -} - -#[test] -fn unknown_cluster_and_client_keys_name_the_original_key() { - for contents in [ - "gateway.cluster.default.bootstrap.serverz: fluss:9123\n", - "gateway.cluster.default.connection.identity: user\n", - "gateway.cluster.default.client.: 1\n", - "gateway.cluster.default: fluss:9123\n", - ] { - let error = load_file(contents).unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); - let key = contents.split(':').next().unwrap(); - assert!(error.to_string().contains(key), "{key}: {error}"); - } -} - -#[test] -fn native_client_options_are_parsed_into_their_native_types() { - assert_eq!( - parse_client_option("writer.batch-size", "2MiB").unwrap(), - ClientOptionValue::Bytes(2 * 1024 * 1024) - ); - assert_eq!( - parse_client_option("writer.batch-timeout", "50ms").unwrap(), - ClientOptionValue::Millis(50) - ); - assert_eq!( - parse_client_option("writer.dynamic-batch-size.enabled", "false").unwrap(), - ClientOptionValue::Boolean(false) - ); - assert_eq!( - parse_client_option("lookup.max-retries", "3").unwrap(), - ClientOptionValue::Integer(3) - ); - assert_eq!( - parse_client_option("security.protocol", "sasl").unwrap(), - ClientOptionValue::Text("sasl".to_string()) - ); - - let config = load_file( - "gateway.cluster.default.client.writer.batch-size: 2MiB\n\ - gateway.cluster.default.client.lookup.max-retries: 3\n", - ) - .unwrap(); - let default = cluster(&config, "default"); - assert_eq!(default.client_option("writer.batch-size"), Some("2MiB")); - assert_eq!(default.client_option("lookup.max-retries"), Some("3")); -} - -/// The gateway advertises the write guarantees, so the client options that would weaken them, and the -/// authorization ID that user mode supplies per request, are refused rather than silently honoured. -#[test] -fn client_options_cannot_override_gateway_owned_guarantees_or_the_identity() { - for (option, value, expected) in [ - ("writer.acks", "0", "owned by the Gateway"), - ("writer.retries", "0", "owned by the Gateway"), - ("writer.enable-idempotence", "false", "owned by the Gateway"), - ( - "security.sasl.authorization-id", - "alice", - "cannot be configured statically", - ), - ( - "writer.unknown-knob", - "1", - "is not a supported native-client option", - ), - ] { - let error = parse_client_option(option, value).unwrap_err(); - assert!(error.contains(expected), "{option}: {error}"); - - let problems = problems( - load_file(&format!( - "gateway.cluster.default.client.{option}: {value}\n" - )) - .unwrap_err(), - ); - assert!( - problems.iter().any(|problem| { - problem.starts_with("gateway.cluster.default.client.") && problem.contains(option) - }), - "{option}: {problems:?}" - ); - } -} - -#[test] -fn out_of_range_client_values_fail_before_startup() { - for (option, value) in [ - ("lookup.queue-size", "0"), - ("lookup.max-batch-size", "-1"), - ("lookup.max-retries", "-1"), - ("writer.request-max-size", "3GiB"), - ("writer.batch-timeout", "0s"), - ("writer.dynamic-batch-size.enabled", "maybe"), - ] { - let error = parse_client_option(option, value).unwrap_err(); - assert!(error.starts_with(&format!("client.{option}")), "{error}"); - assert!( - load_file(&format!( - "gateway.cluster.default.client.{option}: \"{value}\"\n" - )) - .is_err(), - "accepted client.{option} = {value}" - ); - } -} - -#[test] -fn cross_field_cluster_and_security_constraints_fail_before_startup() { - for contents in [ - // An account without its secret, and the reverse. - "gateway.cluster.default.connection.service.account: gateway_svc\n", - "gateway.cluster.default.connection.service.secret: gw-pass\n", - // User identity mode has no service account to authenticate the pool with. - "gateway.cluster.default.connection.identity-mode: user\n", - // SASL without credentials, and an unsupported protocol or mechanism. - "gateway.cluster.default.client.security.protocol: sasl\n", - "gateway.cluster.default.client.security.protocol: ssl\n", - "gateway.cluster.default.client.security.sasl.mechanism: SCRAM-SHA-256\n", - "gateway.cluster.default.connection.max: 0\n", - "gateway.cluster.default.bootstrap.servers: \" \"\n", - // The mode's credential table is missing. - "gateway.security.authentication: password\n", - "gateway.security.authentication: token\n", - "gateway.security.authentication: trusted-header\n\ - gateway.security.trusted-header.name: \"bad header\"\n", - "gateway.rest.lookup.max-keys: 0\n", - "gateway.rest.prefix-lookup.max-prefixes: 0\n", - ] { - assert!(load_file(contents).is_err(), "accepted: {contents}"); - } - - assert!( - load_file( - "gateway.cluster.default.connection.identity-mode: user\n\ - gateway.cluster.default.connection.service.account: gateway_svc\n\ - gateway.cluster.default.connection.service.secret: gw-pass\n\ - gateway.cluster.default.client.security.protocol: SASL\n\ - gateway.cluster.default.client.security.sasl.mechanism: PLAIN\n" - ) - .is_ok() - ); -} - -/// The legacy SASL options stay usable and keep winning, because silently changing which credential a -/// running deployment authenticates with would be worse than the deprecation. -#[test] -fn legacy_credentials_win_and_warn_once_per_cluster() { - let config = load_file( - "gateway.clusters: default,analytics\n\ - gateway.cluster.default.connection.service.account: canonical-user\n\ - gateway.cluster.default.connection.service.secret: canonical-secret\n\ - gateway.cluster.default.client.security.sasl.username: legacy-user\n\ - gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ - gateway.cluster.analytics.client.security.sasl.username: other-user\n\ - gateway.cluster.analytics.client.security.sasl.password: other-secret\n", - ) - .unwrap(); - - let default = cluster(&config, "default"); - assert_eq!(default.effective_service_account(), Some("legacy-user")); - assert_eq!(default.effective_service_secret(), Some("legacy-secret")); - - let deprecations: Vec = config - .warnings() - .into_iter() - .filter(|warning| warning.contains("is deprecated")) - .collect(); - assert_eq!(deprecations.len(), 2, "one per cluster: {deprecations:?}"); - for warning in &deprecations { - for secret in [ - "canonical-user", - "canonical-secret", - "legacy-user", - "legacy-secret", - "other-user", - "other-secret", - ] { - assert!(!warning.contains(secret), "leaked {secret}: {warning}"); - } - } -} - -#[test] -fn pool_settings_warn_when_the_identity_mode_ignores_them() { - let config = load_file( - "gateway.cluster.default.connection.max: 8\n\ - gateway.cluster.default.connection.idle-timeout: 5m\n", - ) - .unwrap(); - assert!( - config - .warnings() - .iter() - .any(|warning| warning.contains("ignored because connection.identity-mode is service")), - "{:?}", - config.warnings() - ); -} - -#[test] -fn diagnostics_redact_every_credential_and_keep_the_identities() { - let config = load_file( - "gateway.cluster.default.connection.service.account: canonical-user\n\ - gateway.cluster.default.connection.service.secret: canonical-secret\n\ - gateway.cluster.default.client.security.sasl.username: legacy-user\n\ - gateway.cluster.default.client.security.sasl.password: legacy-secret\n\ - gateway.cluster.default.client.writer.batch-size: 4MiB\n\ - gateway.security.authentication: password\n\ - gateway.security.users: alice:user-secret\n\ - gateway.security.tokens: token-secret:alice\n", - ) - .unwrap(); - - for diagnostic in [config.redacted_debug(), format!("{config:?}")] { - for credential in [ - "canonical-secret", - "legacy-secret", - "user-secret", - "token-secret", - ] { - assert!( - !diagnostic.contains(credential), - "leaked {credential}: {diagnostic}" - ); - } - // The identities and the tuning stay readable: they are what an operator came to look at. - for readable in ["canonical-user", "legacy-user", "4MiB"] { - assert!(diagnostic.contains(readable), "{readable}: {diagnostic}"); - } - assert!(diagnostic.contains(REDACTED), "{diagnostic}"); - } - - // The credentials are still reachable by the components that authenticate with them. - assert_eq!( - config.security.users.as_ref().map(Secret::expose), - Some("alice:user-secret") - ); - assert_eq!( - cluster(&config, "default").effective_service_secret(), - Some("legacy-secret") - ); -} - -/// A configuration error is what an operator sees on stderr, so it must name the option without -/// quoting any credential the file happens to carry. -#[test] -fn configuration_errors_never_quote_a_credential() { - let file = write_temp_config( - "gateway.security.authentication: token\n\ - gateway.security.tokens: do-not-leak\n\ - gateway.cluster.default.connection.service.secret: also-secret\n\ - gateway.cluster.default.client.writer.acks: 0\n", - ); - let error = load(Some(file.path()), &no_env(), &CliOverrides::default()).unwrap_err(); - let rendered = error.to_string(); - assert!(rendered.contains("writer.acks"), "{rendered}"); - assert!(!rendered.contains("do-not-leak"), "{rendered}"); - assert!(!rendered.contains("also-secret"), "{rendered}"); -} - -/// The same precedence statement for the two dynamic namespaces, where the environment name is derived -/// rather than registered: for every per-cluster option and every allowed client option, setting the -/// environment variable must be indistinguishable from having written that value in the file. -#[test] -fn the_environment_overrides_the_file_for_every_cluster_and_client_option() { - // Keeps every variant loadable: user identity mode needs usable credentials over SASL. The key - // under test is removed from the base so the file never carries it twice. - let base = [ - ("connection.service.account", "base-account"), - ("connection.service.secret", "base-secret"), - ("client.security.protocol", "sasl"), - ]; - - let mut cases: Vec<(String, String, &str, &str)> = Vec::new(); - for entry in CLUSTER_ENTRIES { - let values = match entry.kind { - ValueKind::ServerList => ("file-host:9123", "env-host:9123"), - ValueKind::Integer => ("11", "22"), - ValueKind::Text if entry.key == "connection.identity-mode" => ("service", "user"), - ValueKind::Text if entry.key.ends_with("timeout") => ("11s", "22s"), - ValueKind::Text => ("file-value", "env-value"), - ValueKind::Bool | ValueKind::Bytes => { - unreachable!("no per-cluster option uses {:?}", entry.kind) - } - }; - cases.push(( - entry.key.to_string(), - environment_suffix(entry.key), - values.0, - values.1, - )); - } - for spec in CLIENT_OPTIONS { - let values = match spec.kind { - // The only legal value is PLAIN, so the two spellings differ only in case. - _ if spec.option == "security.sasl.mechanism" => ("PLAIN", "plain"), - _ if spec.option == "security.protocol" => ("plaintext", "sasl"), - ClientOptionKind::Text | ClientOptionKind::Secret => ("file-value", "env-value"), - ClientOptionKind::Boolean => ("true", "false"), - ClientOptionKind::Count { .. } => ("5", "6"), - // Both values have to keep the size relationships intact against the native defaults. - ClientOptionKind::Size { .. } if spec.option.ends_with("dynamic-batch-size.min") => { - ("1MiB", "2MiB") - } - ClientOptionKind::Size { .. } => ("4MiB", "8MiB"), - ClientOptionKind::Duration => ("11s", "22s"), - }; - cases.push(( - format!("{CLIENT_OPTION_PREFIX}{}", spec.option), - format!("CLIENT__{}", environment_suffix(spec.option)), - values.0, - values.1, - )); - } - - for (key, env_suffix, file_value, env_value) in cases { - let contents = |value: &str| { - base.iter() - .filter(|(base_key, _)| *base_key != key) - .map(|(base_key, base_value)| (*base_key, *base_value)) - .chain([(key.as_str(), value)]) - .map(|(key, value)| { - format!("{CLUSTER_KEY_PREFIX}{DEFAULT_CLUSTER_ID}.{key}: \"{value}\"\n") - }) - .collect::() - }; - let load_valid = |contents: &str, env: &BTreeMap| { - let file = write_temp_config(contents); - load(Some(file.path()), env, &CliOverrides::default()) - .unwrap_or_else(|error| panic!("{key}: {error}\n{contents}")) - }; - - let env = BTreeMap::from([( - format!( - "{ENV_PREFIX}CLUSTER__{}__{env_suffix}", - DEFAULT_CLUSTER_ID.to_ascii_uppercase() - ), - env_value.to_string(), - )]); - let from_file = load_valid(&contents(file_value), &no_env()); - let overridden = load_valid(&contents(file_value), &env); - let as_written = load_valid(&contents(env_value), &no_env()); - - assert_ne!( - from_file, overridden, - "{key} ignores its environment variable" - ); - assert_eq!( - overridden, as_written, - "{key} from the environment differs from the same value in the file" - ); - } -} - -/// The reserved options are refused from the environment exactly as they are from the file, and the -/// fixed request limits are reachable there too. -#[test] -fn the_environment_is_held_to_the_same_client_option_rules_as_the_file() { - let mut env = BTreeMap::from([( - "FLUSS_GATEWAY__REST__LOOKUP__MAX_KEYS".to_string(), - "16".to_string(), - )]); - let config = load(None, &env, &CliOverrides::default()).unwrap(); - assert_eq!(config.request_limits.lookup_max_keys, 16); - - env.insert( - "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__ACKS".to_string(), - "0".to_string(), - ); - let error = load(None, &env, &CliOverrides::default()).unwrap_err(); - assert!(error.to_string().contains("writer.acks"), "got: {error}"); -} - -#[test] -fn the_environment_can_declare_clusters() { - let file = write_temp_config("gateway.cluster.analytics.bootstrap.servers: eu:9123\n"); - let mut env = no_env(); - env.insert( - "FLUSS_GATEWAY__CLUSTERS".to_string(), - "analytics".to_string(), - ); - let config = load(Some(file.path()), &env, &CliOverrides::default()).unwrap(); - assert_eq!(config.clusters.keys().collect::>(), ["analytics"]); - - env.insert("FLUSS_GATEWAY__CLUSTERS".to_string(), "default".to_string()); - let error = load(Some(file.path()), &env, &CliOverrides::default()).unwrap_err(); - assert!(error.to_string().contains("not declared"), "got: {error}"); -} - -#[test] -fn unknown_cluster_environment_variables_are_rejected() { - for variable in [ - "FLUSS_GATEWAY__CLUSTER__DEFAULT__BOOTSTRAP__SERVERZ", - "FLUSS_GATEWAY__CLUSTER__DEFAULT", - "FLUSS_GATEWAY__CLUSTER__1ST__BOOTSTRAP__SERVERS", - ] { - let mut env = no_env(); - env.insert(variable.to_string(), "value".to_string()); - let error = load(None, &env, &CliOverrides::default()).unwrap_err(); - assert!( - matches!(error, ConfigError::UnknownEnvKey(_)), - "{variable}: {error:?}" - ); - assert!(error.to_string().contains(variable), "{variable}: {error}"); - } -} - -/// A bad per-cluster value is reported with the public key rather than the internal Serde path, and an -/// environment override additionally names the variable the operator set. -#[test] -fn a_bad_cluster_value_names_the_public_key_and_its_source() { - let error = load_file("gateway.cluster.default.request-timeout: 0s\n").unwrap_err(); - assert!( - error - .to_string() - .contains("gateway.cluster.default.request-timeout"), - "got: {error}" - ); - - let env = BTreeMap::from([( - "FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT".to_string(), - "soon".to_string(), - )]); - let rendered = load(None, &env, &CliOverrides::default()) - .unwrap_err() - .to_string(); - assert!( - rendered.contains("FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT"), - "{rendered}" - ); - assert!( - rendered.contains("gateway.cluster.default.connect-timeout"), - "{rendered}" - ); -} - -#[test] -fn programmatically_constructed_clusters_are_validated() { - let mut config = GatewayConfig::default(); - config.clusters.clear(); - let errors = problems(config.validate().unwrap_err()); - assert!( - errors - .iter() - .any(|error| error == "gateway.clusters must declare at least one cluster"), - "got: {errors:?}" - ); - - let mut config = GatewayConfig::default(); - config - .clusters - .get_mut(DEFAULT_CLUSTER_ID) - .expect("default cluster") - .identity_mode = IdentityMode::User; - let errors = problems(config.validate().unwrap_err()); - assert!( - errors - .iter() - .any(|error| error.contains("identity-mode user requires")), - "got: {errors:?}" - ); -} - -/// The `client.*` namespace is open, so its environment mapping cannot be checked key by key like the -/// fixed vocabulary. It round-trips only while option names keep `-` inside a segment and `.` between -/// segments: an option name containing `_` would come back from the environment as a different name and -/// silently configure the wrong option. Exhaustive over the allowlist, so adding such a name fails here. -#[test] -fn every_client_option_round_trips_through_the_environment() { - for spec in CLIENT_OPTIONS { - let suffix = environment_suffix(spec.option); - assert_eq!( - environment_suffix_to_option(&suffix), - spec.option, - "{} does not survive the environment mapping", - spec.option - ); - assert!( - !spec.option.contains('_'), - "{} must spell words with '-', which the environment mapping reserves '_' for", - spec.option - ); - let file_key = format!( - "{CLUSTER_KEY_PREFIX}{DEFAULT_CLUSTER_ID}.{CLIENT_OPTION_PREFIX}{}", - spec.option - ); - assert!( - matches!(resolve_key(&file_key), Ok(ResolvedKey::ClientOption { .. })), - "{file_key} is not reachable as a file key" - ); - } -} - -/// Sensitivity is declared per option, so the declaration is what has to be right. Fluss decides it on -/// the Java side from these same substrings, which is the cross-check: any option whose name looks like -/// a credential must be marked, and the allowlist and the refusals must not overlap. -#[test] -fn client_option_sensitivity_and_refusals_are_declared_consistently() { - for spec in CLIENT_OPTIONS { - let looks_sensitive = ["password", "secret", "token"] - .iter() - .any(|part| spec.option.contains(part)); - assert_eq!( - spec.kind.is_sensitive(), - looks_sensitive, - "{}: the kind and the name disagree about being a credential", - spec.option - ); - assert!( - !RESERVED_CLIENT_OPTIONS - .iter() - .any(|(reserved, _)| *reserved == spec.option), - "{} is both allowed and refused", - spec.option - ); - } - // An option the gateway never validated must not be rendered on the chance that it holds a secret. - assert!(client_option_is_sensitive("some.unknown.option")); - assert!(client_option_is_sensitive(LEGACY_SERVICE_SECRET_OPTION)); - assert!(!client_option_is_sensitive(LEGACY_SERVICE_ACCOUNT_OPTION)); - - // A parsed credential stays wrapped, so printing the parse result cannot leak it either. - let parsed = parse_client_option(LEGACY_SERVICE_SECRET_OPTION, "legacy-secret").unwrap(); - assert_eq!( - parsed, - ClientOptionValue::Secret(Secret::new("legacy-secret")) - ); - assert!(!format!("{parsed:?}").contains("legacy-secret")); - assert!(format!("{parsed:?}").contains(REDACTED)); -} - -/// Precedence is stated once for the whole vocabulary rather than sampled on one key, so a per-kind -/// conversion that only reads one source cannot hide: every option is driven from the file, then -/// overridden from the environment, and the environment value has to win in the loaded config. -#[test] -fn the_environment_overrides_the_file_for_every_option() { - for entry in CONFIG_ENTRIES { - let (file_value, env_value) = match entry.kind { - ValueKind::Bool => ("true", "false"), - ValueKind::Integer => ("11", "22"), - ValueKind::Bytes => ("1MiB", "2MiB"), - ValueKind::ServerList => ("file-host:9123", "env-host:9123"), - ValueKind::Text => match entry.key { - REST_LISTEN_KEY => ("127.0.0.1:11111", "127.0.0.1:22222"), - METRICS_LISTEN_KEY => ("127.0.0.1:11112", "127.0.0.1:22223"), - REST_HEADER_READ_TIMEOUT_KEY - | REST_REQUEST_TIMEOUT_KEY - | SHUTDOWN_DRAIN_TIMEOUT_KEY => ("11s", "22s"), - // Both modes must be valid on their own: the file value is loaded without the - // environment override, and password and token modes need a credential table. - SECURITY_AUTHENTICATION_KEY => ("trusted-header", "trust"), - _ => ("file-value", "env-value"), - }, - }; - - let file = write_temp_config(&format!("{}: \"{file_value}\"\n", entry.key)); - let from_file = load(Some(file.path()), &no_env(), &CliOverrides::default()) - .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); - let env = BTreeMap::from([(environment_variable(entry.key), env_value.to_string())]); - let from_env = load(Some(file.path()), &env, &CliOverrides::default()) - .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); - - assert_ne!( - from_file, from_env, - "{} ignores its environment variable", - entry.key - ); - let only_env = load(None, &env, &CliOverrides::default()) - .unwrap_or_else(|error| panic!("{}: {error}", entry.key)); - assert_eq!( - from_env, only_env, - "{} lets the file value survive the environment override", - entry.key - ); - } -} - -/// User identity mode is only safe when the connection can actually carry the request's principal, so -/// the credentials must be usable *and* SASL must be selected. Both were previously satisfied by a -/// `Some("")` credential over the default PLAINTEXT, which authorizes the gateway's own identity for -/// every caller instead of failing. -#[test] -fn user_identity_mode_requires_usable_credentials_over_sasl() { - let user_mode = "gateway.cluster.default.connection.identity-mode: user\n"; - let credentials = "gateway.cluster.default.connection.service.account: gateway_svc\n\ - gateway.cluster.default.connection.service.secret: gw-pass\n"; - let sasl = "gateway.cluster.default.client.security.protocol: sasl\n"; - - for (contents, expected) in [ - ( - format!("{user_mode}{credentials}"), - "requires client.security.protocol sasl", - ), - ( - format!("{user_mode}{sasl}"), - "requires connection.service.account", - ), - ( - format!( - "{user_mode}{sasl}gateway.cluster.default.connection.service.account: \"\"\n\ - gateway.cluster.default.connection.service.secret: \" \"\n" - ), - "must not be blank", - ), - ( - format!( - "{user_mode}{sasl}{credentials}\ - gateway.cluster.default.client.security.sasl.mechanism: SCRAM-SHA-256\n" - ), - "must be PLAIN", - ), - ] { - let problems = problems(load_file(&contents).unwrap_err()); - assert!( - problems.iter().any(|problem| problem.contains(expected)), - "expected {expected:?} for:\n{contents}got: {problems:?}" - ); - } - - // The complete, coherent form is accepted. - assert!(load_file(&format!("{user_mode}{sasl}{credentials}")).is_ok()); - // Service mode needs no SASL: it authenticates as itself, with no principal to propagate. - assert!(load_file("gateway.cluster.default.connection.identity-mode: service\n").is_ok()); -} - -/// A size that cannot fit inside the size holding it fails before a listener binds, whether the operator -/// set both sides or only one: leaving the other at its native default is the common way to break a pair. -#[test] -fn writer_size_pairs_must_fit_including_against_the_native_defaults() { - for (contents, rejected) in [ - // Both sides configured. - ( - "gateway.cluster.default.client.writer.batch-size: 2MiB\n\ - gateway.cluster.default.client.writer.request-max-size: 1MiB\n", - true, - ), - ( - "gateway.cluster.default.client.writer.dynamic-batch-size.min: 4MiB\n\ - gateway.cluster.default.client.writer.batch-size: 2MiB\n", - true, - ), - // One side only: the other is the native default, 10MiB request-max and 2MiB batch-size. - ( - "gateway.cluster.default.client.writer.batch-size: 128MiB\n", - true, - ), - ( - "gateway.cluster.default.client.writer.dynamic-batch-size.min: 3MiB\n", - true, - ), - ( - "gateway.cluster.default.client.writer.request-max-size: 1MiB\n", - true, - ), - // Coherent against the defaults, and coherent as a pair. - ( - "gateway.cluster.default.client.writer.batch-size: 4MiB\n", - false, - ), - ( - "gateway.cluster.default.client.writer.batch-size: 32MiB\n\ - gateway.cluster.default.client.writer.request-max-size: 64MiB\n", - false, - ), - ] { - let result = load_file(contents); - assert_eq!( - result.is_err(), - rejected, - "unexpected outcome for:\n{contents}" - ); - if rejected { - assert!( - problems(result.unwrap_err()) - .iter() - .any(|problem| problem.contains("must not exceed client.")), - "{contents}" - ); - } - } -} - -/// A value is bounded by the native field it lands in, not by one blanket ceiling: the writer sizes are -/// `i32` there, while the buffer size and the lookup counts are `usize` and may exceed `i32::MAX`. -#[test] -fn client_option_bounds_follow_the_native_field_type() { - let over_i32 = u64::from(i32::MAX as u32) + 1; - - for option in ["writer.batch-size", "writer.request-max-size"] { - let error = parse_client_option(option, &format!("{over_i32}")).unwrap_err(); - assert!(error.contains("must not exceed"), "{option}: {error}"); - } - assert_eq!( - parse_client_option("writer.buffer.memory-size", "4GiB").unwrap(), - ClientOptionValue::Bytes(4 * 1024 * 1024 * 1024) - ); - for option in [ - "lookup.queue-size", - "lookup.max-batch-size", - "lookup.max-inflight-requests", - ] { - assert_eq!( - parse_client_option(option, &format!("{over_i32}")).unwrap(), - ClientOptionValue::Integer(over_i32), - "{option} is stored as usize and must accept this" - ); - } - let error = parse_client_option("lookup.max-retries", &format!("{over_i32}")).unwrap_err(); - assert!(error.contains("must be between 0 and"), "{error}"); -} - -/// The declared native defaults are a copy of the client's, so they must at least satisfy the -/// relationships the client enforces; a mistyped copy shows up here and not as a rejected valid file. -#[test] -fn the_declared_native_size_defaults_are_coherent() { - let default = |option| effective_size(option, None).expect("a declared size default"); - let batch = default("writer.batch-size"); - assert!(batch <= default("writer.request-max-size")); - assert!(batch <= default("writer.buffer.memory-size")); - assert!(default("writer.dynamic-batch-size.min") <= batch); -} - -#[test] -fn options_are_complete_and_unambiguous() { - let mut public_keys = std::collections::BTreeSet::new(); - let mut internal_paths = std::collections::BTreeSet::new(); - let mut environment_variables = std::collections::BTreeSet::new(); - - for entry in CONFIG_ENTRIES { - assert!(entry.key.starts_with("gateway."), "{entry:?}"); - assert!( - public_keys.insert(entry.key), - "duplicate key: {}", - entry.key - ); - assert!( - internal_paths.insert(entry.internal_path), - "duplicate path: {}", - entry.internal_path - ); - assert!( - environment_variables.insert(environment_variable(entry.key)), - "duplicate environment variable for {}", - entry.key - ); - } - - assert_eq!(CONFIG_ENTRIES.len(), 20); -} - -/// The per-cluster vocabulary shares the environment namespace with `client.*`, so its keys have to -/// stay distinct from each other and unreachable through the client prefix. -#[test] -fn cluster_options_are_complete_and_unambiguous() { - let mut keys = std::collections::BTreeSet::new(); - let mut fields = std::collections::BTreeSet::new(); - let mut suffixes = std::collections::BTreeSet::new(); - - for entry in CLUSTER_ENTRIES { - assert!(!entry.key.starts_with("gateway."), "{entry:?}"); - assert!( - !entry.key.starts_with(CLIENT_OPTION_PREFIX), - "{} collides with the client namespace", - entry.key - ); - assert!(keys.insert(entry.key), "duplicate key: {}", entry.key); - assert!( - fields.insert(entry.internal_path), - "duplicate field: {}", - entry.internal_path - ); - assert!( - suffixes.insert(environment_suffix(entry.key)), - "duplicate environment suffix for {}", - entry.key - ); - } - - assert_eq!(CLUSTER_ENTRIES.len(), 8); -} From 74b9853e52b73101234ca6444667496a55fccce1 Mon Sep 17 00:00:00 2001 From: Junbo Wang Date: Wed, 19 Aug 2026 17:09:28 +0800 Subject: [PATCH 3/3] [gateway] Consolidate configuration tests Merge overlapping configuration test cases while preserving coverage of validation, precedence, and redaction scenarios. --- .github/workflows/gateway-ci.yml | 22 +- fluss-gateway/Cargo.lock | 2254 ++++++++++++++++++++++++--- fluss-gateway/Cargo.toml | 2 + fluss-gateway/DEPENDENCIES.rust.tsv | 440 ++++-- fluss-gateway/justfile | 2 +- fluss-gateway/src/config.rs | 1085 +++++++------ 6 files changed, 2971 insertions(+), 834 deletions(-) diff --git a/.github/workflows/gateway-ci.yml b/.github/workflows/gateway-ci.yml index b43f7656ef..c43f14f1ce 100644 --- a/.github/workflows/gateway-ci.yml +++ b/.github/workflows/gateway-ci.yml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -# Gateway-only CI: gateway changes never build the fluss-rust workspace and vice versa. +# Gateway CI also follows the native Rust client sources used by its configuration adapter. # `uses:` step inputs are relative to the repository root. name: Gateway CI @@ -25,12 +25,16 @@ on: - main paths: - 'fluss-gateway/**' + - 'fluss-rust/crates/fluss/**' + - 'fluss-rpc/src/main/proto/**' - '.github/workflows/gateway-ci.yml' pull_request: branches: - main paths: - 'fluss-gateway/**' + - 'fluss-rust/crates/fluss/**' + - 'fluss-rpc/src/main/proto/**' - '.github/workflows/gateway-ci.yml' workflow_dispatch: @@ -55,6 +59,14 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Install protobuf compiler + run: brew install protobuf + if: runner.os == 'macOS' + + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + if: runner.os == 'Linux' + - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -91,6 +103,9 @@ jobs: - name: Install the MSRV toolchain run: rustup toolchain install 1.88.0 --profile minimal + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -115,6 +130,9 @@ jobs: with: tool: cargo-deny@0.14.22 + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + - name: Check dependency licenses (Apache-compatible) run: cargo deny --locked check licenses @@ -124,7 +142,7 @@ jobs: # The inventory ships with the source release, so a stale one is worse than none. - name: Dependency inventory drift check run: | - cargo deny --locked list -f tsv -t 0.6 > DEPENDENCIES.rust.tsv + cargo deny --locked list -f tsv -t 0.6 | sed 's/[[:space:]]*$//' > DEPENDENCIES.rust.tsv git diff --exit-code DEPENDENCIES.rust.tsv - name: Rust Cache diff --git a/fluss-gateway/Cargo.lock b/fluss-gateway/Cargo.lock index 0e57278e5e..4c08f394a8 100644 --- a/fluss-gateway/Cargo.lock +++ b/fluss-gateway/Cargo.lock @@ -2,6 +2,20 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -11,6 +25,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -47,7 +70,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -58,7 +81,244 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrow" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "libc", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" +dependencies = [ + "bytes", + "half", + "num-bigint 0.5.1", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.23.1", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", + "zstd", +] + +[[package]] +name = "arrow-json" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "arrow-select" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] [[package]] @@ -67,6 +327,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "axum" version = "0.8.9" @@ -113,36 +379,140 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint 0.4.8", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cc" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + [[package]] name = "clap" version = "4.6.6" @@ -190,118 +560,389 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] -name = "crossbeam-epoch" -version = "0.9.20" +name = "const-random" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ - "crossbeam-utils", + "const-random-macro", ] [[package]] -name = "crossbeam-utils" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - -[[package]] -name = "displaydoc" -version = "0.2.7" +name = "const-random-macro" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", ] [[package]] -name = "equivalent" -version = "1.0.2" +name = "core-foundation-sys" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "errno" -version = "0.3.14" +name = "cpufeatures" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", - "windows-sys", ] [[package]] -name = "fastrand" -version = "2.5.0" +name = "crc32c" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] [[package]] -name = "fluss-gateway" -version = "1.0.0" +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ - "axum", - "clap", - "futures-util", - "http-body-util", - "hyper", - "hyper-util", - "libc", - "log", - "metrics", - "metrics-exporter-prometheus", - "reqwest", - "serde", - "serde_json", - "serde_path_to_error", - "serde_yaml_ng", - "tempfile", - "tokio", - "tokio-util", - "tower", - "utoipa", - "utoipa-axum", - "uuid", + "crossbeam-utils", ] [[package]] -name = "foldhash" -version = "0.2.0" +name = "crossbeam-utils" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] -name = "form_urlencoded" -version = "1.2.2" +name = "crunchy" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "percent-encoding", + "generic-array", + "typenum", ] [[package]] -name = "futures-channel" -version = "0.3.34" +name = "csv" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" dependencies = [ - "futures-core", + "csv-core", + "itoa", + "ryu", + "serde_core", ] [[package]] -name = "futures-core" -version = "0.3.34" +name = "csv-core" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] [[package]] -name = "futures-macro" -version = "0.3.34" +name = "dashmap" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ - "proc-macro2", + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.20", +] + +[[package]] +name = "delegate" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.13.1", + "rustc_version", +] + +[[package]] +name = "fluss-gateway" +version = "1.0.0" +dependencies = [ + "axum", + "clap", + "fluss-rs", + "futures-util", + "http-body-util", + "hyper", + "hyper-util", + "libc", + "log", + "metrics", + "metrics-exporter-prometheus", + "reqwest", + "serde", + "serde_json", + "serde_path_to_error", + "serde_yaml_ng", + "tempfile", + "tokio", + "tokio-util", + "tower", + "utoipa", + "utoipa-axum", + "uuid", +] + +[[package]] +name = "fluss-rs" +version = "1.0.0" +dependencies = [ + "arrow", + "arrow-schema", + "bigdecimal", + "bitvec", + "byteorder", + "bytes", + "clap", + "crc32c", + "dashmap", + "delegate", + "futures", + "jiff", + "linked-hash-map", + "log", + "metrics", + "opendal", + "ordered-float", + "parking_lot", + "parse-display", + "prost", + "prost-build", + "rand 0.9.5", + "scopeguard", + "serde", + "serde_json", + "snafu", + "strum", + "strum_macros", + "tempfile", + "thiserror 1.0.69", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", "quote", "syn 3.0.3", ] @@ -324,13 +965,40 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -350,8 +1018,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", ] [[package]] @@ -360,7 +1070,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -441,13 +1151,29 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + [[package]] name = "hyper-util" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -464,6 +1190,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.1.1" @@ -546,65 +1296,196 @@ dependencies = [ ] [[package]] -name = "idna" -version = "1.1.0" +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "js-sys", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", + "getrandom 0.4.3", + "libc", ] [[package]] -name = "idna_adapter" -version = "1.2.1" +name = "js-sys" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ - "icu_normalizer", - "icu_properties", + "cfg-if", + "futures-util", + "wasm-bindgen", ] [[package]] -name = "indexmap" -version = "2.14.0" +name = "lexical-core" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" dependencies = [ - "equivalent", - "hashbrown 0.17.1", - "serde", - "serde_core", + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", ] [[package]] -name = "ipnet" -version = "2.12.1" +name = "lexical-parse-float" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] [[package]] -name = "is_terminal_polyfill" -version = "1.70.2" +name = "lexical-parse-integer" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] [[package]] -name = "itoa" -version = "1.0.18" +name = "lexical-util" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] -name = "js-sys" -version = "0.3.104" +name = "lexical-write-float" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", ] [[package]] @@ -613,6 +1494,18 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -625,11 +1518,38 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +dependencies = [ + "value-bag", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] [[package]] name = "matchit" @@ -637,6 +1557,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" @@ -659,12 +1589,12 @@ version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b166dea96003ee2531cf14833efedced545751d800f03535801d833313f8c15" dependencies = [ - "base64", + "base64 0.22.1", "indexmap", "metrics", "metrics-util", "quanta", - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -678,7 +1608,7 @@ dependencies = [ "hashbrown 0.16.1", "metrics", "quanta", - "rand", + "rand 0.9.5", "rand_xoshiro", "rapidhash", "sketches-ddsketch", @@ -698,7 +1628,61 @@ checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", ] [[package]] @@ -713,6 +1697,92 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opendal" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a" +dependencies = [ + "anyhow", + "backon", + "base64 0.22.1", + "bytes", + "futures", + "getrandom 0.2.17", + "http", + "http-body", + "jiff", + "log", + "md-5", + "percent-encoding", + "quick-xml", + "reqwest", + "serde", + "serde_json", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", + "rand 0.8.7", + "serde", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parse-display" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287d8d3ebdce117b8539f59411e4ed9ec226e0a4153c7f55495c6070d68e6f72" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc048687be30d79502dea2f623d052f3a074012c6eac41726b7ab17213616b1" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn 2.0.119", +] + [[package]] name = "paste" version = "1.0.15" @@ -725,18 +1795,44 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "portable-atomic" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -756,27 +1852,154 @@ dependencies = [ ] [[package]] -name = "proc-macro2" -version = "1.0.107" +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ - "unicode-ident", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", ] [[package]] -name = "quanta" -version = "0.12.6" +name = "quinn-udp" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ - "crossbeam-utils", + "cfg_aliases", "libc", "once_cell", - "raw-cpuid", - "wasi", - "web-sys", - "winapi", + "socket2", + "tracing", + "windows-sys 0.61.2", ] [[package]] @@ -800,6 +2023,22 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "rand_core 0.6.4", + "serde", +] + [[package]] name = "rand" version = "0.9.5" @@ -807,7 +2046,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -817,7 +2067,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "serde", ] [[package]] @@ -829,13 +2088,28 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xoshiro" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" dependencies = [ - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -853,7 +2127,16 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags", + "bitflags 2.13.1", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", ] [[package]] @@ -891,30 +2174,68 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", + "futures-util", "http", "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", ] [[package]] @@ -923,11 +2244,46 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -942,6 +2298,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.229" @@ -973,103 +2341,272 @@ dependencies = [ ] [[package]] -name = "serde_json" -version = "1.0.151" +name = "serde_fmt" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e497af288b3b95d067a23a4f749f2861121ffcb2f6d8379310dcda040c345ed" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml_ng" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.119", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "sval" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "ec4a2a7d92fa86fcc6222e4c3845f8486cff899d9db32480b26c91a5dbf2e22d" + +[[package]] +name = "sval_buffer" +version = "2.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4324db9ac500c609d659b752edf9c8abbf2233f8afd61a503fd6f88ed625032" dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", + "sval", + "sval_ref", + "zerocopy", ] [[package]] -name = "serde_path_to_error" -version = "0.1.20" +name = "sval_dynamic" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +checksum = "4046add0eecf55e680b9e207edf5fc7737b18a1d950db363d97e7f1b2d7c629c" dependencies = [ - "itoa", - "serde", - "serde_core", + "sval", ] [[package]] -name = "serde_urlencoded" -version = "0.7.1" +name = "sval_fmt" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +checksum = "911a3486b5984a0a4f25edefcf2c2dba23654c29f63e75493b671d338bf24243" dependencies = [ - "form_urlencoded", "itoa", "ryu", - "serde", + "sval", ] [[package]] -name = "serde_yaml_ng" -version = "0.10.0" +name = "sval_json" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" +checksum = "da53aae7c737b5b5f1be4bcb0ff20e057bf6b2ee4e9d025560075c5830d09f95" dependencies = [ - "indexmap", "itoa", "ryu", - "serde", - "unsafe-libyaml", + "sval", ] [[package]] -name = "signal-hook-registry" -version = "1.4.8" +name = "sval_nested" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +checksum = "df24df43cbdc4bb8c9f5ed19d0d57dc8f60a1a4259cdce52d597fe774ad3a71f" dependencies = [ - "errno", - "libc", + "sval", + "sval_buffer", + "sval_ref", ] [[package]] -name = "sketches-ddsketch" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" - -[[package]] -name = "socket2" -version = "0.6.5" +name = "sval_ref" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +checksum = "2bebc17f0f1fad060e57b778728d41ef87627e9111a6365d7463472cb58fc1b3" dependencies = [ - "libc", - "windows-sys", + "sval", ] [[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "strsim" -version = "0.11.1" +name = "sval_serde" +version = "2.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +checksum = "9f26fe3f6a68b40e6c8d654ea48c00e4316272fddf68c80493714c1b034ae70b" +dependencies = [ + "serde_core", + "sval", + "sval_nested", +] [[package]] name = "syn" @@ -1113,6 +2650,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.27.0" @@ -1123,7 +2666,16 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", ] [[package]] @@ -1132,7 +2684,18 @@ version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -1146,6 +2709,15 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -1156,6 +2728,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.53.1" @@ -1165,11 +2752,12 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1183,6 +2771,16 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -1217,7 +2815,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.1", "bytes", "futures-util", "http", @@ -1266,6 +2864,24 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1278,6 +2894,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -1347,9 +2969,52 @@ checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", + "serde_core", "wasm-bindgen", ] +[[package]] +name = "value-bag" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068e763e8279de7ab94b6afebded2cb701678af094feb1c12ccb061b4783c1be" +dependencies = [ + "value-bag-serde1", + "value-bag-sval2", +] + +[[package]] +name = "value-bag-serde1" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417d6197dd0ee696783d6be4276ac6ea74b985e00024c85ccfb37aff4f2bed82" +dependencies = [ + "erased-serde", + "serde_core", + "serde_fmt", +] + +[[package]] +name = "value-bag-sval2" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61f7251ecde2c9ed431bbe0659853e7991753447447bbf1ae59d8b31c578d4e" +dependencies = [ + "sval", + "sval_buffer", + "sval_dynamic", + "sval_fmt", + "sval_json", + "sval_ref", + "sval_serde", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "want" version = "0.3.1" @@ -1429,6 +3094,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.104" @@ -1439,6 +3117,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1461,12 +3158,74 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1476,6 +3235,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.46.0" @@ -1488,6 +3311,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "yoke" version = "0.8.3" @@ -1552,6 +3384,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.4" @@ -1590,3 +3428,31 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/fluss-gateway/Cargo.toml b/fluss-gateway/Cargo.toml index b2df3ba10b..f5457f0003 100644 --- a/fluss-gateway/Cargo.toml +++ b/fluss-gateway/Cargo.toml @@ -51,6 +51,8 @@ axum = { version = "0.8", default-features = false, features = ["http1", "matche clap = { version = "4.5.37", features = ["derive"] } # Only FutureExt/join_all are used; avoid the larger futures facade crate. futures-util = "0.3" +# Reuse the native client configuration defaults and validation rather than copying them here. +fluss = { package = "fluss-rs", path = "../fluss-rust/crates/fluss" } # The listeners follow axum's official serve-with-hyper example: axum::serve does not expose the # hyper builder that the header read timeout lives on. The plain http1 builder (not auto) has no # HTTP/2 preface sniffing, so the header timer covers a connection from its first byte. diff --git a/fluss-gateway/DEPENDENCIES.rust.tsv b/fluss-gateway/DEPENDENCIES.rust.tsv index 5d3a4da26b..ec9bc96c26 100644 --- a/fluss-gateway/DEPENDENCIES.rust.tsv +++ b/fluss-gateway/DEPENDENCIES.rust.tsv @@ -1,130 +1,310 @@ -crate Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -aho-corasick@1.1.5 X X -anstream@1.0.0 X X -anstyle@1.0.14 X X -anstyle-parse@1.0.0 X X -anstyle-query@1.1.5 X X -anstyle-wincon@3.0.11 X X -atomic-waker@1.1.2 X X -axum@0.8.9 X -axum-core@0.5.6 X -base64@0.22.1 X X -bitflags@2.13.1 X X -bumpalo@3.20.3 X X -bytes@1.12.1 X -cfg-if@1.0.4 X X -clap@4.6.6 X X -clap_builder@4.6.6 X X -clap_derive@4.6.4 X X -clap_lex@1.1.0 X X -colorchoice@1.0.5 X X -crossbeam-epoch@0.9.20 X X -crossbeam-utils@0.8.22 X X -equivalent@1.0.2 X X -errno@0.3.14 X X -fluss-gateway@1.0.0 X -foldhash@0.2.0 X -futures-channel@0.3.34 X X -futures-core@0.3.34 X X -futures-macro@0.3.34 X X -futures-sink@0.3.34 X X -futures-task@0.3.34 X X -futures-util@0.3.34 X X -getrandom@0.3.4 X X -getrandom@0.4.3 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -http@1.5.0 X X -http-body@1.1.0 X -http-body-util@0.1.5 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hyper@1.11.0 X -hyper-util@0.1.20 X -indexmap@2.14.0 X X -ipnet@2.12.1 X X -is_terminal_polyfill@1.70.2 X X -itoa@1.0.18 X X -js-sys@0.3.104 X X -libc@0.2.189 X X -log@0.4.33 X X -matchit@0.8.4 X X -memchr@2.8.3 X X -metrics@0.24.6 X -metrics-exporter-prometheus@0.17.2 X -metrics-util@0.20.4 X -mime@0.3.17 X X -mio@1.2.2 X -once_cell@1.21.4 X X -once_cell_polyfill@1.70.2 X X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -portable-atomic@1.15.0 X X -ppv-lite86@0.2.21 X X -proc-macro2@1.0.107 X X -quanta@0.12.6 X -quote@1.0.47 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.5 X X -rand_chacha@0.9.0 X X -rand_core@0.9.5 X X -rand_xoshiro@0.7.0 X X -rapidhash@4.5.1 X X -raw-cpuid@11.6.0 X -regex@1.13.1 X X -regex-automata@0.4.18 X X -regex-syntax@0.8.11 X X -rustversion@1.0.23 X X -ryu@1.0.23 X X -serde@1.0.229 X X -serde_core@1.0.229 X X -serde_derive@1.0.229 X X -serde_json@1.0.151 X X -serde_path_to_error@0.1.20 X X -serde_yaml_ng@0.10.0 X -signal-hook-registry@1.4.8 X X -sketches-ddsketch@0.3.1 X -slab@0.4.12 X -smallvec@1.15.2 X X -socket2@0.6.5 X X -strsim@0.11.1 X -syn@2.0.119 X X -syn@3.0.3 X X -sync_wrapper@1.0.2 X -thiserror@2.0.20 X X -thiserror-impl@2.0.20 X X -tokio@1.53.1 X -tokio-macros@2.7.2 X -tokio-util@0.7.19 X -tower@0.5.3 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-core@0.1.36 X -try-lock@0.2.5 X -unicode-ident@1.0.24 X X X -unsafe-libyaml@0.2.11 X -utf8parse@0.2.2 X X -utoipa@5.5.0 X X -utoipa-axum@0.2.0 X X -utoipa-gen@5.5.0 X X -uuid@1.24.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.1+wasi-0.2.4 X X X -wasm-bindgen@0.2.127 X X -wasm-bindgen-macro@0.2.127 X X -wasm-bindgen-macro-support@0.2.127 X X -wasm-bindgen-shared@0.2.127 X X -web-sys@0.3.104 X X -winapi@0.3.9 X X -winapi-i686-pc-windows-gnu@0.4.0 X X -winapi-x86_64-pc-windows-gnu@0.4.0 X X -windows-link@0.2.1 X X -windows-sys@0.61.2 X X -wit-bindgen@0.46.0 X X X -zerocopy@0.8.56 X X X -zmij@1.0.23 X +crate Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib +ahash@0.8.12 X X +aho-corasick@1.1.5 X X +android_system_properties@0.1.6 X X +anstream@1.0.0 X X +anstyle@1.0.14 X X +anstyle-parse@1.0.0 X X +anstyle-query@1.1.5 X X +anstyle-wincon@3.0.11 X X +anyhow@1.0.104 X X +arrow@59.2.0 X +arrow-arith@59.2.0 X +arrow-array@59.2.0 X X +arrow-buffer@59.2.0 X +arrow-cast@59.2.0 X +arrow-csv@59.2.0 X +arrow-data@59.2.0 X +arrow-ipc@59.2.0 X +arrow-json@59.2.0 X +arrow-ord@59.2.0 X +arrow-row@59.2.0 X +arrow-schema@59.2.0 X +arrow-select@59.2.0 X +arrow-string@59.2.0 X +atoi@2.0.0 X +atomic-waker@1.1.2 X X +autocfg@1.5.1 X X +axum@0.8.9 X +axum-core@0.5.6 X +backon@1.6.0 X +base64@0.22.1 X X +base64@0.23.1 X X +bigdecimal@0.4.10 X X +bitflags@2.13.1 X X +bitvec@1.1.1 X +block-buffer@0.10.4 X X +bumpalo@3.20.3 X X +byteorder@1.5.0 X X +bytes@1.12.1 X +cc@1.4.3 X X +cfg-if@1.0.4 X X +chrono@0.4.45 X X +clap@4.6.6 X X +clap_builder@4.6.6 X X +clap_derive@4.6.4 X X +clap_lex@1.1.0 X X +colorchoice@1.0.5 X X +const-random@0.1.18 X X +const-random-macro@0.1.16 X X +core-foundation-sys@0.8.7 X X +crc32c@0.6.8 X X +crossbeam-epoch@0.9.20 X X +crossbeam-utils@0.8.22 X X +crunchy@0.2.4 X +crypto-common@0.1.7 X X +csv@1.4.0 X X +csv-core@0.1.13 X X +dashmap@6.2.1 X +delegate@0.13.5 X X +digest@0.10.7 X X +displaydoc@0.2.7 X X +either@1.17.0 X X +equivalent@1.0.2 X X +errno@0.3.14 X X +fastrand@2.5.0 X X +find-msvc-tools@0.1.11 X X +fixedbitset@0.5.7 X X +flatbuffers@25.12.19 X +fluss-gateway@1.0.0 X +fluss-rs@1.0.0 X +foldhash@0.1.5 X +foldhash@0.2.0 X +form_urlencoded@1.2.2 X X +funty@2.0.0 X +futures@0.3.34 X X +futures-channel@0.3.34 X X +futures-core@0.3.34 X X +futures-executor@0.3.34 X X +futures-io@0.3.34 X X +futures-macro@0.3.34 X X +futures-sink@0.3.34 X X +futures-task@0.3.34 X X +futures-util@0.3.34 X X +generic-array@0.14.7 X +getrandom@0.2.17 X X +getrandom@0.3.4 X X +getrandom@0.4.3 X X +gloo-timers@0.3.0 X X +half@2.7.1 X X +hashbrown@0.14.5 X X +hashbrown@0.15.5 X X +hashbrown@0.16.1 X X +hashbrown@0.17.1 X X +heck@0.5.0 X X +http@1.5.0 X X +http-body@1.1.0 X +http-body-util@0.1.5 X +httparse@1.10.1 X X +httpdate@1.0.3 X X +hyper@1.11.0 X +hyper-rustls@0.27.9 X X X +hyper-util@0.1.20 X +iana-time-zone@0.1.65 X X +iana-time-zone-haiku@0.1.2 X X +icu_collections@2.1.1 X +icu_locale_core@2.1.1 X +icu_normalizer@2.1.1 X +icu_normalizer_data@2.1.1 X +icu_properties@2.1.2 X +icu_properties_data@2.1.2 X +icu_provider@2.1.1 X +idna@1.1.0 X X +idna_adapter@1.2.1 X X +indexmap@2.14.0 X X +ipnet@2.12.1 X X +is_terminal_polyfill@1.70.2 X X +itertools@0.14.0 X X +itoa@1.0.18 X X +jiff@0.2.35 X X +jiff-core@0.1.0 X X +jiff-tzdb@0.1.8 X X +jiff-tzdb-platform@0.1.3 X X +jobserver@0.1.35 X X +js-sys@0.3.104 X X +lexical-core@1.0.6 X X +lexical-parse-float@1.0.6 X X +lexical-parse-integer@1.0.6 X X +lexical-util@1.0.7 X X +lexical-write-float@1.0.6 X X +lexical-write-integer@1.0.6 X X +libc@0.2.189 X X +libm@0.2.16 X +linked-hash-map@0.5.6 X X +linux-raw-sys@0.12.1 X X X +litemap@0.8.2 X +lock_api@0.4.14 X X +log@0.4.33 X X +lz4_flex@0.14.0 X +matchit@0.8.4 X X +md-5@0.10.6 X X +memchr@2.8.3 X X +metrics@0.24.6 X +metrics-exporter-prometheus@0.17.2 X +metrics-util@0.20.4 X +mime@0.3.17 X X +mio@1.2.2 X +multimap@0.10.1 X X +num-bigint@0.4.8 X X +num-bigint@0.5.1 X X +num-complex@0.4.6 X X +num-integer@0.1.47 X X +num-traits@0.2.19 X X +once_cell@1.21.4 X X +once_cell_polyfill@1.70.2 X X +opendal@0.55.0 X +ordered-float@5.3.0 X +parking_lot@0.12.5 X X +parking_lot_core@0.9.12 X X +parse-display@0.10.0 X X +parse-display-derive@0.10.0 X X +paste@1.0.15 X X +percent-encoding@2.3.2 X X +petgraph@0.8.3 X X +pin-project-lite@0.2.17 X X +pkg-config@0.3.34 X X +portable-atomic@1.15.0 X X +portable-atomic-util@0.2.7 X X +potential_utf@0.1.5 X +ppv-lite86@0.2.21 X X +prettyplease@0.2.37 X X +proc-macro2@1.0.107 X X +prost@0.14.4 X +prost-build@0.14.4 X +prost-derive@0.14.4 X +prost-types@0.14.4 X +quanta@0.12.6 X +quick-xml@0.38.4 X +quote@1.0.47 X X +r-efi@5.3.0 X X X +r-efi@6.0.0 X X X +radium@0.7.0 X +rand@0.9.5 X X +rand_chacha@0.9.0 X X +rand_core@0.9.5 X X +rand_xoshiro@0.7.0 X X +rapidhash@4.5.1 X X +raw-cpuid@11.6.0 X +redox_syscall@0.5.18 X +regex@1.13.1 X X +regex-automata@0.4.18 X X +regex-syntax@0.8.11 X X +reqwest@0.12.28 X X +ring@0.17.14 X X +rustc_version@0.4.1 X X +rustix@1.1.4 X X X +rustls@0.23.43 X X X +rustls-pki-types@1.15.1 X X +rustls-webpki@0.103.14 X +rustversion@1.0.23 X X +ryu@1.0.23 X X +scopeguard@1.2.0 X X +semver@1.0.28 X X +serde@1.0.229 X X +serde_core@1.0.229 X X +serde_derive@1.0.229 X X +serde_json@1.0.151 X X +serde_path_to_error@0.1.20 X X +serde_urlencoded@0.7.1 X X +serde_yaml_ng@0.10.0 X +shlex@2.0.1 X X +signal-hook-registry@1.4.8 X X +simdutf8@0.1.5 X X +sketches-ddsketch@0.3.1 X +slab@0.4.12 X +smallvec@1.15.2 X X +snafu@0.8.9 X X +snafu-derive@0.8.9 X X +socket2@0.6.5 X X +stable_deref_trait@1.2.1 X X +strsim@0.11.1 X +structmeta@0.3.0 X X +structmeta-derive@0.3.0 X X +strum@0.26.3 X +strum_macros@0.26.4 X +subtle@2.6.1 X +syn@2.0.119 X X +syn@3.0.3 X X +sync_wrapper@1.0.2 X +synstructure@0.13.2 X +tap@1.0.1 X +tempfile@3.27.0 X X +thiserror@1.0.69 X X +thiserror@2.0.20 X X +thiserror-impl@1.0.69 X X +thiserror-impl@2.0.20 X X +tiny-keccak@2.0.2 X +tinystr@0.8.3 X +tokio@1.53.1 X +tokio-macros@2.7.2 X +tokio-rustls@0.26.4 X X +tokio-util@0.7.19 X +tower@0.5.3 X +tower-http@0.6.11 X +tower-layer@0.3.3 X +tower-service@0.3.3 X +tracing@0.1.44 X +tracing-core@0.1.36 X +try-lock@0.2.5 X +twox-hash@2.1.3 X +typenum@1.20.1 X X +unicode-ident@1.0.24 X X X +unsafe-libyaml@0.2.11 X +untrusted@0.9.0 X +url@2.5.8 X X +utf8_iter@1.0.4 X X +utf8parse@0.2.2 X X +utoipa@5.5.0 X X +utoipa-axum@0.2.0 X X +utoipa-gen@5.5.0 X X +uuid@1.24.0 X X +value-bag@1.13.2 X X +version_check@0.9.5 X X +want@0.3.1 X +wasi@0.11.1+wasi-snapshot-preview1 X X X +wasip2@1.0.1+wasi-0.2.4 X X X +wasm-bindgen@0.2.127 X X +wasm-bindgen-futures@0.4.77 X X +wasm-bindgen-macro@0.2.127 X X +wasm-bindgen-macro-support@0.2.127 X X +wasm-bindgen-shared@0.2.127 X X +wasm-streams@0.4.2 X X +web-sys@0.3.104 X X +webpki-roots@1.0.9 X +winapi@0.3.9 X X +winapi-i686-pc-windows-gnu@0.4.0 X X +winapi-x86_64-pc-windows-gnu@0.4.0 X X +windows-core@0.62.2 X X +windows-implement@0.60.2 X X +windows-interface@0.59.3 X X +windows-link@0.2.1 X X +windows-result@0.4.1 X X +windows-strings@0.5.1 X X +windows-sys@0.52.0 X X +windows-sys@0.61.2 X X +windows-targets@0.52.6 X X +windows_aarch64_gnullvm@0.52.6 X X +windows_aarch64_msvc@0.52.6 X X +windows_i686_gnu@0.52.6 X X +windows_i686_gnullvm@0.52.6 X X +windows_i686_msvc@0.52.6 X X +windows_x86_64_gnu@0.52.6 X X +windows_x86_64_gnullvm@0.52.6 X X +windows_x86_64_msvc@0.52.6 X X +wit-bindgen@0.46.0 X X X +writeable@0.6.3 X +wyz@0.5.1 X +yoke@0.8.3 X +yoke-derive@0.8.2 X +zerocopy@0.8.56 X X X +zerocopy-derive@0.8.56 X X X +zerofrom@0.1.8 X +zerofrom-derive@0.1.7 X +zeroize@1.9.0 X X +zerotrie@0.2.4 X +zerovec@0.11.6 X +zerovec-derive@0.11.3 X +zmij@1.0.23 X +zstd@0.13.3 X +zstd-safe@7.2.4 X X +zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/fluss-gateway/justfile b/fluss-gateway/justfile index 968b59b363..9172a6c1c4 100644 --- a/fluss-gateway/justfile +++ b/fluss-gateway/justfile @@ -63,4 +63,4 @@ licenses: # Regenerate the checked-in dependency license inventory. deps: - cargo deny --locked list -f tsv -t 0.6 > DEPENDENCIES.rust.tsv + cargo deny --locked list -f tsv -t 0.6 | sed 's/[[:space:]]*$//' > DEPENDENCIES.rust.tsv diff --git a/fluss-gateway/src/config.rs b/fluss-gateway/src/config.rs index bf0f6febcc..0f7023a3a5 100644 --- a/fluss-gateway/src/config.rs +++ b/fluss-gateway/src/config.rs @@ -33,16 +33,13 @@ //! `gateway.cluster.default.client.writer.batch-size` becomes //! `FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__BATCH_SIZE`. //! -//! Two rules make the configuration deterministic before anything binds or connects. `gateway.clusters` -//! is authoritative, and it is so whether or not it was written: an absent list means the single implicit -//! `default` cluster, so a mistyped cluster ID fails startup instead of creating a cluster nothing routes -//! to. And `gateway.cluster..client.*` is an allowlist, not a passthrough: the native-client options -//! that would weaken the write guarantees the gateway advertises, or pin the authorization identity that -//! user identity mode supplies per request, are rejected rather than honoured. +//! `gateway.clusters` is authoritative (with an implicit single `default` cluster), and +//! `gateway.cluster..client.*` accepts only validated options that preserve Gateway guarantees. //! //! Credentials are redacted in diagnostics: typed fields carry them as [`Secret`], and the open //! `client.*` namespace declares per option which values are sensitive. +use fluss::config::Config as NativeClientConfig; use serde::Deserialize; use serde::de::{self, Deserializer}; use serde_yaml_ng::{Mapping, Value}; @@ -591,13 +588,13 @@ impl Default for ClusterConfig { } impl ClusterConfig { - /// Returns the account actually used, honouring the legacy last-wins override. + /// Returns the account actually used; the deprecated alias takes precedence when present. pub fn effective_service_account(&self) -> Option<&str> { self.client_option(LEGACY_SERVICE_ACCOUNT_OPTION) .or(self.service_account.as_deref()) } - /// Returns the credential actually used, honouring the legacy last-wins override. + /// Returns the credential actually used; the deprecated alias takes precedence when present. pub fn effective_service_secret(&self) -> Option<&str> { self.client_option(LEGACY_SERVICE_SECRET_OPTION) .or_else(|| self.service_secret.as_ref().map(Secret::expose)) @@ -653,162 +650,176 @@ pub enum ClientOptionValue { Bytes(u64), } +impl ClientOptionValue { + fn text(&self) -> &str { + match self { + Self::Text(value) => value, + _ => unreachable!("option kind produces text"), + } + } + + fn secret(&self) -> &str { + match self { + Self::Secret(value) => value.expose(), + _ => unreachable!("option kind produces a secret"), + } + } + + fn boolean(&self) -> bool { + match self { + Self::Boolean(value) => *value, + _ => unreachable!("option kind produces a boolean"), + } + } + + fn integer(&self) -> u64 { + match self { + Self::Integer(value) => *value, + _ => unreachable!("option kind produces an integer"), + } + } + + fn millis(&self) -> u64 { + match self { + Self::Millis(value) => *value, + _ => unreachable!("option kind produces milliseconds"), + } + } + + fn bytes(&self) -> u64 { + match self { + Self::Bytes(value) => *value, + _ => unreachable!("option kind produces bytes"), + } + } +} + /// Superseded by `connection.service.account`; still honoured, with a warning. const LEGACY_SERVICE_ACCOUNT_OPTION: &str = "security.sasl.username"; /// Superseded by `connection.service.secret`; still honoured, with a warning. const LEGACY_SERVICE_SECRET_OPTION: &str = "security.sasl.password"; -/// How one `client.*` value is parsed and bounded. -/// -/// A bound is a fact about the native field the value lands in, so it is declared per option rather than -/// applied uniformly: the writer sizes are `i32` there, while the buffer size and the lookup counts are -/// `usize` and may legitimately exceed `i32::MAX` on a 64-bit target. +/// The native field type targeted by one public `client.*` option. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ClientOptionKind { - /// Passed through verbatim; any further constraint is a cross-field rule. Text, - /// Verbatim like [`Self::Text`], but a credential: redacted everywhere it is rendered. Secret, Boolean, - /// An integer count the native field accepts within `min..=max`. - Count { - min: u64, - max: u64, - }, - /// A byte size, with the ceiling and the default of the native field. - Size { - max: u64, - default: u64, - }, - /// A duration, handed to the native client as milliseconds. - Duration, + I32Count { min: u64 }, + UsizeCount { min: u64 }, + I32Bytes, + UsizeBytes, + I64Millis, + U64Millis, } -/// Ceilings of the native field types the values are stored in. -const NATIVE_I32_MAX: u64 = i32::MAX as u64; -const NATIVE_USIZE_MAX: u64 = usize::MAX as u64; - -const KIB: u64 = 1024; -const MIB: u64 = 1024 * KIB; - impl ClientOptionKind { - /// Sensitivity follows from the kind, so no option can be declared a credential and rendered as text - /// at the same time. The Java side has to infer this from the key name instead - /// (`ConfigurationUtils.SENSITIVE_KEY_PARTS`, plus an allowlist for the keys that over-match). fn is_sensitive(self) -> bool { matches!(self, Self::Secret) } } -/// One native-client option the gateway accepts under `gateway.cluster..client.`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +type ApplyClientOption = fn(&mut NativeClientConfig, &ClientOptionValue); + +#[derive(Clone, Copy)] struct ClientOptionSpec { option: &'static str, kind: ClientOptionKind, + apply: ApplyClientOption, } -/// The `client.*` allowlist. Anything absent is rejected, so a native-client option the gateway has not -/// considered cannot reach a connection. +/// Public FIP names mapped to fields on [`NativeClientConfig`]. Native defaults and cross-field +/// validation come from that type rather than being copied into the gateway. const CLIENT_OPTIONS: &[ClientOptionSpec] = &[ ClientOptionSpec { option: "security.protocol", kind: ClientOptionKind::Text, + apply: |config, value| config.security_protocol = value.text().to_string(), }, ClientOptionSpec { option: "security.sasl.mechanism", kind: ClientOptionKind::Text, + apply: |config, value| config.security_sasl_mechanism = value.text().to_string(), }, ClientOptionSpec { option: LEGACY_SERVICE_ACCOUNT_OPTION, kind: ClientOptionKind::Text, + apply: |config, value| config.security_sasl_username = value.text().to_string(), }, ClientOptionSpec { option: LEGACY_SERVICE_SECRET_OPTION, kind: ClientOptionKind::Secret, + apply: |config, value| config.security_sasl_password = value.secret().to_string(), }, ClientOptionSpec { - option: "connect.timeout", - kind: ClientOptionKind::Duration, - }, - ClientOptionSpec { - option: "request.timeout", - kind: ClientOptionKind::Duration, + option: "connect-timeout", + kind: ClientOptionKind::U64Millis, + apply: |config, value| config.connect_timeout_ms = value.millis(), }, ClientOptionSpec { option: "writer.batch-size", - kind: ClientOptionKind::Size { - max: NATIVE_I32_MAX, - default: 2 * MIB, - }, + kind: ClientOptionKind::I32Bytes, + apply: |config, value| config.writer_batch_size = value.bytes() as i32, }, ClientOptionSpec { option: "writer.request-max-size", - kind: ClientOptionKind::Size { - max: NATIVE_I32_MAX, - default: 10 * MIB, - }, + kind: ClientOptionKind::I32Bytes, + apply: |config, value| config.writer_request_max_size = value.bytes() as i32, }, ClientOptionSpec { option: "writer.buffer.memory-size", - kind: ClientOptionKind::Size { - max: NATIVE_USIZE_MAX, - default: 64 * MIB, - }, + kind: ClientOptionKind::UsizeBytes, + apply: |config, value| config.writer_buffer_memory_size = value.bytes() as usize, }, ClientOptionSpec { option: "writer.buffer.wait-timeout", - kind: ClientOptionKind::Duration, + kind: ClientOptionKind::U64Millis, + apply: |config, value| config.writer_buffer_wait_timeout_ms = value.millis(), }, ClientOptionSpec { option: "writer.batch-timeout", - kind: ClientOptionKind::Duration, + kind: ClientOptionKind::I64Millis, + apply: |config, value| config.writer_batch_timeout_ms = value.millis() as i64, }, ClientOptionSpec { option: "writer.dynamic-batch-size.enabled", kind: ClientOptionKind::Boolean, + apply: |config, value| config.writer_dynamic_batch_size_enabled = value.boolean(), }, ClientOptionSpec { option: "writer.dynamic-batch-size.min", - kind: ClientOptionKind::Size { - max: NATIVE_I32_MAX, - default: 256 * KIB, - }, + kind: ClientOptionKind::I32Bytes, + apply: |config, value| config.writer_dynamic_batch_size_min = value.bytes() as i32, }, ClientOptionSpec { option: "writer.kv-backpressure.max-throttle", - kind: ClientOptionKind::Duration, + kind: ClientOptionKind::U64Millis, + apply: |config, value| config.writer_kv_backpressure_max_throttle_ms = value.millis(), }, ClientOptionSpec { option: "lookup.queue-size", - kind: ClientOptionKind::Count { - min: 1, - max: NATIVE_USIZE_MAX, - }, + kind: ClientOptionKind::UsizeCount { min: 1 }, + apply: |config, value| config.lookup_queue_size = value.integer() as usize, }, ClientOptionSpec { option: "lookup.max-batch-size", - kind: ClientOptionKind::Count { - min: 1, - max: NATIVE_USIZE_MAX, - }, + kind: ClientOptionKind::UsizeCount { min: 1 }, + apply: |config, value| config.lookup_max_batch_size = value.integer() as usize, }, ClientOptionSpec { option: "lookup.max-inflight-requests", - kind: ClientOptionKind::Count { - min: 1, - max: NATIVE_USIZE_MAX, - }, + kind: ClientOptionKind::UsizeCount { min: 1 }, + apply: |config, value| config.lookup_max_inflight_requests = value.integer() as usize, }, ClientOptionSpec { option: "lookup.max-retries", - kind: ClientOptionKind::Count { - min: 0, - max: NATIVE_I32_MAX, - }, + kind: ClientOptionKind::I32Count { min: 0 }, + apply: |config, value| config.lookup_max_retries = value.integer() as i32, }, ClientOptionSpec { option: "lookup.batch-timeout", - kind: ClientOptionKind::Duration, + kind: ClientOptionKind::U64Millis, + apply: |config, value| config.lookup_batch_timeout_ms = value.millis(), }, ]; @@ -841,37 +852,11 @@ fn client_option_is_sensitive(option: &str) -> bool { client_option_spec(option).is_none_or(|spec| spec.kind.is_sensitive()) } -/// The size the native client will use for `option`: the configured value, or the native default when the -/// deployment left it alone. -/// -/// Returns `None` for an option that is not a size, or for a configured value that failed validation and -/// is already reported on its own account. -/// -/// TODO: source the default from the native `Config::default()` and delegate the relationships to -/// `Config::validate_writer` once the gateway takes fluss-rust as a dependency; the declared defaults -/// exist only because it cannot be called from here yet. -fn effective_size(option: &str, configured: Option<&str>) -> Option { - let ClientOptionKind::Size { default, .. } = client_option_spec(option)?.kind else { - return None; - }; - match configured { - Some(raw) => match parse_client_option(option, raw) { - Ok(ClientOptionValue::Bytes(bytes)) => Some(bytes), - _ => None, - }, - None => Some(default), - } -} - fn client_option_spec(option: &str) -> Option<&'static ClientOptionSpec> { CLIENT_OPTIONS.iter().find(|spec| spec.option == option) } -/// Parses and bounds one `client.*` option before any connection exists, so a bad native-client value -/// fails startup instead of the first write. -/// -/// Relationships *between* options are not checked here, because a value is validated on its own: see -/// `GatewayConfig::validate_client_size_pairs`. +/// Parses one option using the type and range of its destination in [`NativeClientConfig`]. pub fn parse_client_option(option: &str, raw: &str) -> Result { if let Some((_, reason)) = RESERVED_CLIENT_OPTIONS .iter() @@ -893,30 +878,75 @@ pub fn parse_client_option(option: &str, raw: &str) -> Result raw.parse().map(ClientOptionValue::Boolean).map_err( |error: std::str::ParseBoolError| malformed("true or false", error.to_string()), ), - ClientOptionKind::Count { min, max } => { + ClientOptionKind::I32Count { min } => { let parsed = raw - .parse::() + .parse::() .map_err(|error| malformed("a non-negative integer", error.to_string()))?; - if !(min..=max).contains(&parsed) { - return Err(format!("client.{option}: must be between {min} and {max}")); + if parsed < min as i32 { + return Err(format!("client.{option}: must be at least {min}")); } - Ok(ClientOptionValue::Integer(parsed)) + Ok(ClientOptionValue::Integer(parsed as u64)) } - ClientOptionKind::Size { max, .. } => { - let size = ByteSize::parse(raw).map_err(|error| malformed("a byte size", error))?; - if size.bytes() > max { - return Err(format!("client.{option}: must not exceed {max} bytes")); + ClientOptionKind::UsizeCount { min } => { + let parsed = raw + .parse::() + .map_err(|error| malformed("a non-negative integer", error.to_string()))?; + if parsed < min as usize { + return Err(format!("client.{option}: must be at least {min}")); } - Ok(ClientOptionValue::Bytes(size.bytes())) + Ok(ClientOptionValue::Integer(parsed as u64)) + } + ClientOptionKind::I32Bytes => { + let bytes = ByteSize::parse(raw) + .map_err(|error| malformed("a byte size", error))? + .bytes(); + i32::try_from(bytes) + .map(|_| ClientOptionValue::Bytes(bytes)) + .map_err(|_| format!("client.{option}: must not exceed {} bytes", i32::MAX)) + } + ClientOptionKind::UsizeBytes => { + let bytes = ByteSize::parse(raw) + .map_err(|error| malformed("a byte size", error))? + .bytes(); + usize::try_from(bytes) + .map(|_| ClientOptionValue::Bytes(bytes)) + .map_err(|_| format!("client.{option}: does not fit the native field")) + } + ClientOptionKind::I64Millis => parse_client_millis(option, raw, true) + .and_then(|millis| { + i64::try_from(millis) + .map_err(|_| format!("client.{option}: does not fit the native field")) + }) + .map(|millis| ClientOptionValue::Millis(millis as u64)), + ClientOptionKind::U64Millis => { + parse_client_millis(option, raw, true).map(ClientOptionValue::Millis) } - ClientOptionKind::Duration => { - let duration = - ConfigDuration::parse(raw).map_err(|error| malformed("a duration", error))?; - u64::try_from(duration.get().as_millis()) - .map(ClientOptionValue::Millis) - .map_err(|_| format!("client.{option}: does not fit in milliseconds")) + } +} + +fn parse_client_millis(option: &str, raw: &str, allow_zero: bool) -> Result { + let (digits, unit) = split_number_and_unit(raw); + let value = digits + .parse::() + .map_err(|error| format!("client.{option}: expected a duration: {error}"))?; + let multiplier = match unit { + "ms" => 1, + "s" => 1_000, + "m" => 60_000, + "h" => 3_600_000, + _ => { + return Err(format!( + "client.{option}: expected a duration with unit ms, s, m, or h" + )); } + }; + let millis = value + .checked_mul(multiplier) + .ok_or_else(|| format!("client.{option}: duration is too large"))?; + if !allow_zero && millis == 0 { + return Err(format!("client.{option}: must be greater than zero")); } + Ok(millis) } /// How the gateway authenticates its own HTTP callers. @@ -1145,13 +1175,28 @@ impl GatewayConfig { "{CLUSTER_KEY_PREFIX}{id}.connection.max must be greater than zero" )); } + let mut native = NativeClientConfig::default(); for (option, raw) in &cluster.client_options { - if let Err(problem) = parse_client_option(option, raw) { - problems.push(format!("{CLUSTER_KEY_PREFIX}{id}.{problem}")); + match parse_client_option(option, raw) { + Ok(value) => (client_option_spec(option) + .expect("parsed option has a registered native destination") + .apply)(&mut native, &value), + Err(problem) => problems.push(format!("{CLUSTER_KEY_PREFIX}{id}.{problem}")), } } + if let Some(account) = cluster.effective_service_account() { + native.security_sasl_username = account.to_string(); + } + if let Some(secret) = cluster.effective_service_secret() { + native.security_sasl_password = secret.to_string(); + } + if let Err(problem) = native.validate_writer() { + problems.push(format!("{CLUSTER_KEY_PREFIX}{id}.client: {problem}")); + } + if let Err(problem) = native.validate_security() { + problems.push(format!("{CLUSTER_KEY_PREFIX}{id}.client: {problem}")); + } self.validate_credentials(id, cluster, problems); - self.validate_client_size_pairs(id, cluster, problems); } } @@ -1201,6 +1246,11 @@ impl GatewayConfig { } if cluster.identity_mode == IdentityMode::User { + if self.security.authentication == AuthenticationMode::Trust { + problems.push(format!( + "{CLUSTER_KEY_PREFIX}{id}.connection.identity-mode user requires verified client identities; set {SECURITY_AUTHENTICATION_KEY} to password, token, or trusted-header" + )); + } if !credentials_usable { problems.push(format!( "{CLUSTER_KEY_PREFIX}{id}.connection.identity-mode user requires \ @@ -1220,32 +1270,6 @@ impl GatewayConfig { } } - /// Rejects a writer size that cannot fit inside the one that has to hold it. - /// - /// Each side is the configured value or the native default, so overriding one size of a pair is caught - /// here rather than at the first write, which is the whole point of validating before a listener binds. - fn validate_client_size_pairs( - &self, - id: &str, - cluster: &ClusterConfig, - problems: &mut Vec, - ) { - let size = |option: &str| effective_size(option, cluster.client_option(option)); - for (smaller, larger) in [ - ("writer.batch-size", "writer.request-max-size"), - ("writer.batch-size", "writer.buffer.memory-size"), - ("writer.dynamic-batch-size.min", "writer.batch-size"), - ] { - if let (Some(smaller_bytes), Some(larger_bytes)) = (size(smaller), size(larger)) - && smaller_bytes > larger_bytes - { - problems.push(format!( - "{CLUSTER_KEY_PREFIX}{id}.client.{smaller} must not exceed client.{larger}" - )); - } - } - } - /// Requires the credential table the selected authentication mode reads. fn validate_security(&self, problems: &mut Vec) { let configured = |secret: &Option| { @@ -1330,7 +1354,7 @@ impl GatewayConfig { warnings.push(format!( "{CLUSTER_KEY_PREFIX}{id}.client.{{{}}} is deprecated; use \ connection.service.account and connection.service.secret. The legacy values \ - keep last-wins precedence", + take precedence over the canonical fields", legacy.join(",") )); } @@ -1647,9 +1671,13 @@ fn resolve_environment_variable(variable: &str) -> Result Result, ConfigError> { ))); } }; + let mut unique = std::collections::BTreeSet::new(); for id in &ids { if !valid_cluster_id(id) { return Err(ConfigError::Parse(format!( "invalid cluster ID in {CLUSTERS_KEY}: {id:?}" ))); } + if !unique.insert(id) { + return Err(ConfigError::Parse(format!( + "duplicate cluster ID in {CLUSTERS_KEY}: {id:?}" + ))); + } } Ok(ids) } @@ -1780,6 +1814,13 @@ fn read_config_file(contents: &str) -> Result<(Mapping, Option>), Co } ResolvedKey::ClientOption { id, option } => { let raw = scalar_text(value).map_err(reason)?; + if matches!( + option.as_str(), + LEGACY_SERVICE_ACCOUNT_OPTION | LEGACY_SERVICE_SECRET_OPTION + ) && raw.trim().is_empty() + { + return Err(ConfigError::Parse(format!("{key}: must not be blank"))); + } insert_client_option(&mut table, &id, &option, raw); } } @@ -1865,9 +1906,21 @@ pub fn load( value, )); } - // Client options are written straight in: the whole namespace lands in one map, so there is - // no typed path an error could be attributed to. ResolvedKey::ClientOption { id, option } => { + parse_client_option(&option, raw).map_err(|reason| { + ConfigError::Parse(format!( + "{variable}: {CLUSTER_KEY_PREFIX}{id}.client.{option}: {reason}" + )) + })?; + if matches!( + option.as_str(), + LEGACY_SERVICE_ACCOUNT_OPTION | LEGACY_SERVICE_SECRET_OPTION + ) && raw.trim().is_empty() + { + return Err(ConfigError::Parse(format!( + "{variable}: {CLUSTER_KEY_PREFIX}{id}.client.{option} must not be blank" + ))); + } insert_client_option(&mut table, &id, &option, raw.clone()); } } @@ -1945,6 +1998,17 @@ mod tests { "127.0.0.1:9095".parse().unwrap() ); assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(30)); + assert_eq!(config.clusters.len(), 1); + assert_eq!( + cluster(&config, DEFAULT_CLUSTER_ID).bootstrap_servers, + [DEFAULT_BOOTSTRAP_SERVERS] + ); + assert_eq!( + cluster(&config, DEFAULT_CLUSTER_ID).identity_mode, + IdentityMode::Service + ); + assert_eq!(config.security.authentication, AuthenticationMode::Trust); + assert_eq!(config.request_limits, RequestLimitsConfig::default()); assert!(config.warnings().is_empty()); } @@ -1988,12 +2052,24 @@ mod tests { "gateway.rest.lookup.max-keyz: 5\n", "gateway.scan.cursor-ttl: 1m\n", "gateway.tls.cert: /etc/tls.pem\n", + "gateway.cluster.default.bootstrap.serverz: fluss:9123\n", + "gateway.cluster.default.connection.identity: user\n", + "gateway.cluster.default.client.: 1\n", + "gateway.cluster.default: fluss:9123\n", ] { let error = load_file(contents).unwrap_err(); assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); let key = contents.split(':').next().unwrap(); assert!(error.to_string().contains(key), "{key}: {error}"); } + + let error = load_file("gateway.cluster.default.request-timeout: 0s\n").unwrap_err(); + assert!( + error + .to_string() + .contains("gateway.cluster.default.request-timeout"), + "got: {error}" + ); } #[test] @@ -2063,6 +2139,10 @@ mod tests { "FLUSS_GATEWAY__REST__LISTENN", "FLUSS_GATEWAY__QUERY__ENABLED", "FLUSS_GATEWAY__SERVER_REST__BIND_ADDRESS", + "FLUSS_GATEWAY__CLUSTER__DEFAULT__BOOTSTRAP__SERVERZ", + "FLUSS_GATEWAY__CLUSTER__DEFAULT", + "FLUSS_GATEWAY__CLUSTER__1ST__BOOTSTRAP__SERVERS", + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER_BATCH_SIZE", ] { let mut env = no_env(); env.insert(key.to_string(), "value".to_string()); @@ -2123,6 +2203,10 @@ mod tests { "FLUSS_GATEWAY__SHUTDOWN__DRAIN_TIMEOUT".to_string(), "10s".to_string(), ), + ( + "FLUSS_GATEWAY__REST__LOOKUP__MAX_KEYS".to_string(), + "16".to_string(), + ), ]); let config = load(None, &env, &CliOverrides::default()).unwrap(); @@ -2142,6 +2226,7 @@ mod tests { "127.0.0.1:19095".parse().unwrap() ); assert_eq!(config.shutdown.drain_timeout.get(), Duration::from_secs(10)); + assert_eq!(config.request_limits.lookup_max_keys, 16); } #[test] @@ -2158,6 +2243,35 @@ mod tests { .contains("FLUSS_GATEWAY__REST__WRITE__MAX_REQUEST_BYTES"), "got: {error}" ); + + let env = BTreeMap::from([( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT".to_string(), + "soon".to_string(), + )]); + let rendered = load(None, &env, &CliOverrides::default()) + .unwrap_err() + .to_string(); + assert!( + rendered.contains("FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT"), + "{rendered}" + ); + assert!( + rendered.contains("gateway.cluster.default.connect-timeout"), + "{rendered}" + ); + + let env = BTreeMap::from([( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__BATCH_SIZE".to_string(), + "many".to_string(), + )]); + let rendered = load(None, &env, &CliOverrides::default()) + .unwrap_err() + .to_string(); + assert!( + rendered.contains("FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__BATCH_SIZE"), + "{rendered}" + ); + assert!(rendered.contains("writer.batch-size"), "{rendered}"); } #[test] @@ -2366,22 +2480,6 @@ mod tests { config.clusters.get(id).expect("configured cluster") } - #[test] - fn a_single_default_cluster_needs_no_declaration() { - let config = load(None, &no_env(), &CliOverrides::default()).unwrap(); - assert_eq!(config.clusters.len(), 1); - assert_eq!( - cluster(&config, DEFAULT_CLUSTER_ID).bootstrap_servers, - [DEFAULT_BOOTSTRAP_SERVERS] - ); - assert_eq!( - cluster(&config, DEFAULT_CLUSTER_ID).identity_mode, - IdentityMode::Service - ); - assert_eq!(config.security.authentication, AuthenticationMode::Trust); - assert_eq!(config.request_limits, RequestLimitsConfig::default()); - } - #[test] fn typed_cluster_security_and_request_limit_options_are_loaded() { let config = load_file( @@ -2435,7 +2533,7 @@ mod tests { /// written: with no `gateway.clusters`, the only configurable cluster is the implicit `default`. That is /// what turns a mistyped cluster ID into a startup failure instead of an unreachable second cluster. #[test] - fn declared_clusters_are_authoritative() { + fn cluster_declarations_are_authoritative_and_validated() { for contents in [ "gateway.clusters: default\n\ gateway.cluster.analytics.bootstrap.servers: analytics:9123\n", @@ -2457,45 +2555,56 @@ mod tests { cluster(&config, "analytics").bootstrap_servers, [DEFAULT_BOOTSTRAP_SERVERS] ); + for declaration in [ + "gateway.clusters: default,default\n", + "gateway.clusters: [default, default]\n", + ] { + assert!( + load_file(declaration) + .unwrap_err() + .to_string() + .contains("duplicate cluster ID"), + "{declaration}" + ); + } // The implicit default needs no declaration, so a single-cluster deployment configures no list. let config = load_file("gateway.cluster.default.bootstrap.servers: only:9123\n").unwrap(); assert_eq!(config.clusters.keys().collect::>(), ["default"]); - } - #[test] - fn malformed_cluster_ids_are_rejected() { - for contents in [ - "gateway.clusters: Default\n", - "gateway.clusters: 1st\n", - "gateway.clusters: eu-west\n", - "gateway.cluster.EU.bootstrap.servers: eu:9123\n", - ] { - let error = load_file(contents).unwrap_err(); - assert!( - error.to_string().contains("cluster ID"), - "{contents}: {error}" - ); + { + for contents in [ + "gateway.clusters: Default\n", + "gateway.clusters: 1st\n", + "gateway.clusters: eu-west\n", + "gateway.cluster.EU.bootstrap.servers: eu:9123\n", + ] { + let error = load_file(contents).unwrap_err(); + assert!( + error.to_string().contains("cluster ID"), + "{contents}: {error}" + ); + } } - } - #[test] - fn unknown_cluster_and_client_keys_name_the_original_key() { - for contents in [ - "gateway.cluster.default.bootstrap.serverz: fluss:9123\n", - "gateway.cluster.default.connection.identity: user\n", - "gateway.cluster.default.client.: 1\n", - "gateway.cluster.default: fluss:9123\n", - ] { - let error = load_file(contents).unwrap_err(); - assert!(matches!(error, ConfigError::Parse(_)), "got: {error:?}"); - let key = contents.split(':').next().unwrap(); - assert!(error.to_string().contains(key), "{key}: {error}"); + { + let file = write_temp_config("gateway.cluster.analytics.bootstrap.servers: eu:9123\n"); + let mut env = no_env(); + env.insert( + "FLUSS_GATEWAY__CLUSTERS".to_string(), + "analytics".to_string(), + ); + let config = load(Some(file.path()), &env, &CliOverrides::default()).unwrap(); + assert_eq!(config.clusters.keys().collect::>(), ["analytics"]); + + env.insert("FLUSS_GATEWAY__CLUSTERS".to_string(), "default".to_string()); + let error = load(Some(file.path()), &env, &CliOverrides::default()).unwrap_err(); + assert!(error.to_string().contains("not declared"), "got: {error}"); } } #[test] - fn native_client_options_are_parsed_into_their_native_types() { + fn native_client_options_are_parsed_and_bounded_by_native_types() { assert_eq!( parse_client_option("writer.batch-size", "2MiB").unwrap(), ClientOptionValue::Bytes(2 * 1024 * 1024) @@ -2504,6 +2613,12 @@ mod tests { parse_client_option("writer.batch-timeout", "50ms").unwrap(), ClientOptionValue::Millis(50) ); + assert_eq!( + parse_client_option("connect-timeout", "2s").unwrap(), + ClientOptionValue::Millis(2_000) + ); + assert!(parse_client_option("connect.timeout", "2s").is_err()); + assert!(parse_client_option("request-timeout", "30s").is_err()); assert_eq!( parse_client_option("writer.dynamic-batch-size.enabled", "false").unwrap(), ClientOptionValue::Boolean(false) @@ -2525,6 +2640,61 @@ mod tests { let default = cluster(&config, "default"); assert_eq!(default.client_option("writer.batch-size"), Some("2MiB")); assert_eq!(default.client_option("lookup.max-retries"), Some("3")); + + { + for (option, value) in [ + ("lookup.queue-size", "0"), + ("lookup.max-batch-size", "-1"), + ("lookup.max-retries", "-1"), + ("writer.request-max-size", "3GiB"), + ("writer.dynamic-batch-size.enabled", "maybe"), + ] { + let error = parse_client_option(option, value).unwrap_err(); + assert!(error.starts_with(&format!("client.{option}")), "{error}"); + assert!( + load_file(&format!( + "gateway.cluster.default.client.{option}: \"{value}\"\n" + )) + .is_err(), + "accepted client.{option} = {value}" + ); + } + } + + { + let over_i32 = u64::from(i32::MAX as u32) + 1; + + for option in ["writer.batch-size", "writer.request-max-size"] { + let error = parse_client_option(option, &format!("{over_i32}")).unwrap_err(); + assert!(error.contains("must not exceed"), "{option}: {error}"); + } + assert_eq!( + parse_client_option("writer.buffer.memory-size", "4GiB").unwrap(), + ClientOptionValue::Bytes(4 * 1024 * 1024 * 1024) + ); + for option in [ + "lookup.queue-size", + "lookup.max-batch-size", + "lookup.max-inflight-requests", + ] { + assert_eq!( + parse_client_option(option, &format!("{over_i32}")).unwrap(), + ClientOptionValue::Integer(over_i32), + "{option} is stored as usize and must accept this" + ); + } + let error = + parse_client_option("lookup.max-retries", &format!("{over_i32}")).unwrap_err(); + assert!(error.contains("expected a non-negative integer"), "{error}"); + assert_eq!( + parse_client_option("writer.batch-timeout", "0ms").unwrap(), + ClientOptionValue::Millis(0) + ); + assert_eq!( + parse_client_option("lookup.batch-timeout", "0ms").unwrap(), + ClientOptionValue::Millis(0) + ); + } } /// The gateway advertises the write guarantees, so the client options that would weaken them, and the @@ -2563,28 +2733,13 @@ mod tests { "{option}: {problems:?}" ); } - } - #[test] - fn out_of_range_client_values_fail_before_startup() { - for (option, value) in [ - ("lookup.queue-size", "0"), - ("lookup.max-batch-size", "-1"), - ("lookup.max-retries", "-1"), - ("writer.request-max-size", "3GiB"), - ("writer.batch-timeout", "0s"), - ("writer.dynamic-batch-size.enabled", "maybe"), - ] { - let error = parse_client_option(option, value).unwrap_err(); - assert!(error.starts_with(&format!("client.{option}")), "{error}"); - assert!( - load_file(&format!( - "gateway.cluster.default.client.{option}: \"{value}\"\n" - )) - .is_err(), - "accepted client.{option} = {value}" - ); - } + let env = BTreeMap::from([( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__ACKS".to_string(), + "0".to_string(), + )]); + let error = load(None, &env, &CliOverrides::default()).unwrap_err(); + assert!(error.to_string().contains("writer.acks"), "got: {error}"); } #[test] @@ -2593,12 +2748,6 @@ mod tests { // An account without its secret, and the reverse. "gateway.cluster.default.connection.service.account: gateway_svc\n", "gateway.cluster.default.connection.service.secret: gw-pass\n", - // User identity mode has no service account to authenticate the pool with. - "gateway.cluster.default.connection.identity-mode: user\n", - // SASL without credentials, and an unsupported protocol or mechanism. - "gateway.cluster.default.client.security.protocol: sasl\n", - "gateway.cluster.default.client.security.protocol: ssl\n", - "gateway.cluster.default.client.security.sasl.mechanism: SCRAM-SHA-256\n", "gateway.cluster.default.connection.max: 0\n", "gateway.cluster.default.bootstrap.servers: \" \"\n", // The mode's credential table is missing. @@ -2612,22 +2761,32 @@ mod tests { assert!(load_file(contents).is_err(), "accepted: {contents}"); } + // Directly constructed configurations are subject to the same validation. + let mut config = GatewayConfig::default(); + config.clusters.clear(); assert!( - load_file( - "gateway.cluster.default.connection.identity-mode: user\n\ - gateway.cluster.default.connection.service.account: gateway_svc\n\ - gateway.cluster.default.connection.service.secret: gw-pass\n\ - gateway.cluster.default.client.security.protocol: SASL\n\ - gateway.cluster.default.client.security.sasl.mechanism: PLAIN\n" - ) - .is_ok() + problems(config.validate().unwrap_err()) + .iter() + .any(|error| error == "gateway.clusters must declare at least one cluster") + ); + + let mut config = GatewayConfig::default(); + config + .clusters + .get_mut(DEFAULT_CLUSTER_ID) + .expect("default cluster") + .identity_mode = IdentityMode::User; + assert!( + problems(config.validate().unwrap_err()) + .iter() + .any(|error| error.contains("identity-mode user requires")) ); } /// The legacy SASL options stay usable and keep winning, because silently changing which credential a /// running deployment authenticates with would be worse than the deprecation. #[test] - fn legacy_credentials_win_and_warn_once_per_cluster() { + fn warnings_cover_legacy_credentials_and_ignored_pool_settings() { let config = load_file( "gateway.clusters: default,analytics\n\ gateway.cluster.default.connection.service.account: canonical-user\n\ @@ -2643,6 +2802,74 @@ mod tests { assert_eq!(default.effective_service_account(), Some("legacy-user")); assert_eq!(default.effective_service_secret(), Some("legacy-secret")); + for (legacy, expected_account, expected_secret) in [ + ( + "gateway.cluster.default.client.security.sasl.username: legacy-user\n", + "legacy-user", + "canonical-secret", + ), + ( + "gateway.cluster.default.client.security.sasl.password: legacy-secret\n", + "canonical-user", + "legacy-secret", + ), + ] { + let config = load_file(&format!( + "gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n{legacy}" + )) + .unwrap(); + let cluster = cluster(&config, "default"); + assert_eq!(cluster.effective_service_account(), Some(expected_account)); + assert_eq!(cluster.effective_service_secret(), Some(expected_secret)); + assert_eq!( + config + .warnings() + .iter() + .filter(|warning| warning.contains("is deprecated")) + .count(), + 1 + ); + } + + for option in [LEGACY_SERVICE_ACCOUNT_OPTION, LEGACY_SERVICE_SECRET_OPTION] { + let error = load_file(&format!( + "gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.{option}: \" \"\n" + )) + .unwrap_err(); + assert!(error.to_string().contains(option), "{error}"); + assert!(error.to_string().contains("must not be blank"), "{error}"); + } + + let file = write_temp_config( + "gateway.cluster.default.connection.service.account: canonical-user\n\ + gateway.cluster.default.connection.service.secret: canonical-secret\n\ + gateway.cluster.default.client.security.sasl.username: file-user\n", + ); + let env = BTreeMap::from([( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__SECURITY__SASL__PASSWORD".to_string(), + "env-secret".to_string(), + )]); + let mixed = load(Some(file.path()), &env, &CliOverrides::default()).unwrap(); + let cluster = cluster(&mixed, "default"); + assert_eq!(cluster.effective_service_account(), Some("file-user")); + assert_eq!(cluster.effective_service_secret(), Some("env-secret")); + + let mut blank_env = env; + blank_env.insert( + "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__SECURITY__SASL__PASSWORD".to_string(), + " ".to_string(), + ); + let error = load(Some(file.path()), &blank_env, &CliOverrides::default()).unwrap_err(); + assert!( + error + .to_string() + .contains("FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__SECURITY__SASL__PASSWORD"), + "{error}" + ); + let deprecations: Vec = config .warnings() .into_iter() @@ -2661,24 +2888,20 @@ mod tests { assert!(!warning.contains(secret), "leaked {secret}: {warning}"); } } - } - #[test] - fn pool_settings_warn_when_the_identity_mode_ignores_them() { - let config = load_file( - "gateway.cluster.default.connection.max: 8\n\ + { + let config = load_file( + "gateway.cluster.default.connection.max: 8\n\ gateway.cluster.default.connection.idle-timeout: 5m\n", - ) - .unwrap(); - assert!( - config - .warnings() - .iter() - .any(|warning| warning + ) + .unwrap(); + assert!( + config.warnings().iter().any(|warning| warning .contains("ignored because connection.identity-mode is service")), - "{:?}", - config.warnings() - ); + "{:?}", + config.warnings() + ); + } } #[test] @@ -2723,28 +2946,22 @@ mod tests { cluster(&config, "default").effective_service_secret(), Some("legacy-secret") ); - } - /// A configuration error is what an operator sees on stderr, so it must name the option without - /// quoting any credential the file happens to carry. - #[test] - fn configuration_errors_never_quote_a_credential() { - let file = write_temp_config( + // Startup errors name the bad option without quoting credentials from the same input. + let error = load_file( "gateway.security.authentication: token\n\ gateway.security.tokens: do-not-leak\n\ gateway.cluster.default.connection.service.secret: also-secret\n\ gateway.cluster.default.client.writer.acks: 0\n", - ); - let error = load(Some(file.path()), &no_env(), &CliOverrides::default()).unwrap_err(); - let rendered = error.to_string(); - assert!(rendered.contains("writer.acks"), "{rendered}"); - assert!(!rendered.contains("do-not-leak"), "{rendered}"); - assert!(!rendered.contains("also-secret"), "{rendered}"); + ) + .unwrap_err() + .to_string(); + assert!(error.contains("writer.acks"), "{error}"); + assert!(!error.contains("do-not-leak"), "{error}"); + assert!(!error.contains("also-secret"), "{error}"); } - /// The same precedence statement for the two dynamic namespaces, where the environment name is derived - /// rather than registered: for every per-cluster option and every allowed client option, setting the - /// environment variable must be indistinguishable from having written that value in the file. + /// Every dynamic cluster/client option follows environment-over-file precedence. #[test] fn the_environment_overrides_the_file_for_every_cluster_and_client_option() { // Keeps every variant loadable: user identity mode needs usable credentials over SASL. The key @@ -2754,6 +2971,8 @@ mod tests { ("connection.service.secret", "base-secret"), ("client.security.protocol", "sasl"), ]; + let security = "gateway.security.authentication: password\n\ + gateway.security.users: alice:secret\n"; let mut cases: Vec<(String, String, &str, &str)> = Vec::new(); for entry in CLUSTER_ENTRIES { @@ -2781,15 +3000,14 @@ mod tests { _ if spec.option == "security.protocol" => ("plaintext", "sasl"), ClientOptionKind::Text | ClientOptionKind::Secret => ("file-value", "env-value"), ClientOptionKind::Boolean => ("true", "false"), - ClientOptionKind::Count { .. } => ("5", "6"), - // Both values have to keep the size relationships intact against the native defaults. - ClientOptionKind::Size { .. } - if spec.option.ends_with("dynamic-batch-size.min") => - { + ClientOptionKind::I32Count { .. } | ClientOptionKind::UsizeCount { .. } => { + ("5", "6") + } + ClientOptionKind::I32Bytes if spec.option.ends_with("dynamic-batch-size.min") => { ("1MiB", "2MiB") } - ClientOptionKind::Size { .. } => ("4MiB", "8MiB"), - ClientOptionKind::Duration => ("11s", "22s"), + ClientOptionKind::I32Bytes | ClientOptionKind::UsizeBytes => ("4MiB", "8MiB"), + ClientOptionKind::I64Millis | ClientOptionKind::U64Millis => ("11s", "22s"), }; cases.push(( format!("{CLIENT_OPTION_PREFIX}{}", spec.option), @@ -2811,7 +3029,7 @@ mod tests { .collect::() }; let load_valid = |contents: &str, env: &BTreeMap| { - let file = write_temp_config(contents); + let file = write_temp_config(&format!("{security}{contents}")); load(Some(file.path()), env, &CliOverrides::default()) .unwrap_or_else(|error| panic!("{key}: {error}\n{contents}")) }; @@ -2838,121 +3056,9 @@ mod tests { } } - /// The reserved options are refused from the environment exactly as they are from the file, and the - /// fixed request limits are reachable there too. - #[test] - fn the_environment_is_held_to_the_same_client_option_rules_as_the_file() { - let mut env = BTreeMap::from([( - "FLUSS_GATEWAY__REST__LOOKUP__MAX_KEYS".to_string(), - "16".to_string(), - )]); - let config = load(None, &env, &CliOverrides::default()).unwrap(); - assert_eq!(config.request_limits.lookup_max_keys, 16); - - env.insert( - "FLUSS_GATEWAY__CLUSTER__DEFAULT__CLIENT__WRITER__ACKS".to_string(), - "0".to_string(), - ); - let error = load(None, &env, &CliOverrides::default()).unwrap_err(); - assert!(error.to_string().contains("writer.acks"), "got: {error}"); - } - - #[test] - fn the_environment_can_declare_clusters() { - let file = write_temp_config("gateway.cluster.analytics.bootstrap.servers: eu:9123\n"); - let mut env = no_env(); - env.insert( - "FLUSS_GATEWAY__CLUSTERS".to_string(), - "analytics".to_string(), - ); - let config = load(Some(file.path()), &env, &CliOverrides::default()).unwrap(); - assert_eq!(config.clusters.keys().collect::>(), ["analytics"]); - - env.insert("FLUSS_GATEWAY__CLUSTERS".to_string(), "default".to_string()); - let error = load(Some(file.path()), &env, &CliOverrides::default()).unwrap_err(); - assert!(error.to_string().contains("not declared"), "got: {error}"); - } - - #[test] - fn unknown_cluster_environment_variables_are_rejected() { - for variable in [ - "FLUSS_GATEWAY__CLUSTER__DEFAULT__BOOTSTRAP__SERVERZ", - "FLUSS_GATEWAY__CLUSTER__DEFAULT", - "FLUSS_GATEWAY__CLUSTER__1ST__BOOTSTRAP__SERVERS", - ] { - let mut env = no_env(); - env.insert(variable.to_string(), "value".to_string()); - let error = load(None, &env, &CliOverrides::default()).unwrap_err(); - assert!( - matches!(error, ConfigError::UnknownEnvKey(_)), - "{variable}: {error:?}" - ); - assert!(error.to_string().contains(variable), "{variable}: {error}"); - } - } - - /// A bad per-cluster value is reported with the public key rather than the internal Serde path, and an - /// environment override additionally names the variable the operator set. + /// Registry invariants keep public keys unique, non-overlapping, and environment-compatible. #[test] - fn a_bad_cluster_value_names_the_public_key_and_its_source() { - let error = load_file("gateway.cluster.default.request-timeout: 0s\n").unwrap_err(); - assert!( - error - .to_string() - .contains("gateway.cluster.default.request-timeout"), - "got: {error}" - ); - - let env = BTreeMap::from([( - "FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT".to_string(), - "soon".to_string(), - )]); - let rendered = load(None, &env, &CliOverrides::default()) - .unwrap_err() - .to_string(); - assert!( - rendered.contains("FLUSS_GATEWAY__CLUSTER__DEFAULT__CONNECT_TIMEOUT"), - "{rendered}" - ); - assert!( - rendered.contains("gateway.cluster.default.connect-timeout"), - "{rendered}" - ); - } - - #[test] - fn programmatically_constructed_clusters_are_validated() { - let mut config = GatewayConfig::default(); - config.clusters.clear(); - let errors = problems(config.validate().unwrap_err()); - assert!( - errors - .iter() - .any(|error| error == "gateway.clusters must declare at least one cluster"), - "got: {errors:?}" - ); - - let mut config = GatewayConfig::default(); - config - .clusters - .get_mut(DEFAULT_CLUSTER_ID) - .expect("default cluster") - .identity_mode = IdentityMode::User; - let errors = problems(config.validate().unwrap_err()); - assert!( - errors - .iter() - .any(|error| error.contains("identity-mode user requires")), - "got: {errors:?}" - ); - } - - /// The `client.*` namespace is open, so its environment mapping cannot be checked key by key like the - /// fixed vocabulary. It round-trips only while option names keep `-` inside a segment and `.` between - /// segments: an option name containing `_` would come back from the environment as a different name and - /// silently configure the wrong option. Exhaustive over the allowlist, so adding such a name fails here. - #[test] - fn every_client_option_round_trips_through_the_environment() { + fn option_registries_are_safe_and_environment_compatible() { for spec in CLIENT_OPTIONS { let suffix = environment_suffix(spec.option); assert_eq!( @@ -2975,44 +3081,69 @@ mod tests { "{file_key} is not reachable as a file key" ); } - } - /// Sensitivity is declared per option, so the declaration is what has to be right. Fluss decides it on - /// the Java side from these same substrings, which is the cross-check: any option whose name looks like - /// a credential must be marked, and the allowlist and the refusals must not overlap. - #[test] - fn client_option_sensitivity_and_refusals_are_declared_consistently() { - for spec in CLIENT_OPTIONS { - let looks_sensitive = ["password", "secret", "token"] - .iter() - .any(|part| spec.option.contains(part)); - assert_eq!( - spec.kind.is_sensitive(), - looks_sensitive, - "{}: the kind and the name disagree about being a credential", - spec.option - ); - assert!( - !RESERVED_CLIENT_OPTIONS + { + for spec in CLIENT_OPTIONS { + let looks_sensitive = ["password", "secret", "token"] .iter() - .any(|(reserved, _)| *reserved == spec.option), - "{} is both allowed and refused", - spec.option + .any(|part| spec.option.contains(part)); + assert_eq!( + spec.kind.is_sensitive(), + looks_sensitive, + "{}: the kind and the name disagree about being a credential", + spec.option + ); + assert!( + !RESERVED_CLIENT_OPTIONS + .iter() + .any(|(reserved, _)| *reserved == spec.option), + "{} is both allowed and refused", + spec.option + ); + } + // An option the gateway never validated must not be rendered on the chance that it holds a secret. + assert!(client_option_is_sensitive("some.unknown.option")); + assert!(client_option_is_sensitive(LEGACY_SERVICE_SECRET_OPTION)); + assert!(!client_option_is_sensitive(LEGACY_SERVICE_ACCOUNT_OPTION)); + + // A parsed credential stays wrapped, so printing the parse result cannot leak it either. + let parsed = + parse_client_option(LEGACY_SERVICE_SECRET_OPTION, "legacy-secret").unwrap(); + assert_eq!( + parsed, + ClientOptionValue::Secret(Secret::new("legacy-secret")) ); + assert!(!format!("{parsed:?}").contains("legacy-secret")); + assert!(format!("{parsed:?}").contains(REDACTED)); } - // An option the gateway never validated must not be rendered on the chance that it holds a secret. - assert!(client_option_is_sensitive("some.unknown.option")); - assert!(client_option_is_sensitive(LEGACY_SERVICE_SECRET_OPTION)); - assert!(!client_option_is_sensitive(LEGACY_SERVICE_ACCOUNT_OPTION)); - // A parsed credential stays wrapped, so printing the parse result cannot leak it either. - let parsed = parse_client_option(LEGACY_SERVICE_SECRET_OPTION, "legacy-secret").unwrap(); - assert_eq!( - parsed, - ClientOptionValue::Secret(Secret::new("legacy-secret")) - ); - assert!(!format!("{parsed:?}").contains("legacy-secret")); - assert!(format!("{parsed:?}").contains(REDACTED)); + { + let mut keys = std::collections::BTreeSet::new(); + let mut fields = std::collections::BTreeSet::new(); + let mut suffixes = std::collections::BTreeSet::new(); + + for entry in CLUSTER_ENTRIES { + assert!(!entry.key.starts_with("gateway."), "{entry:?}"); + assert!( + !entry.key.starts_with(CLIENT_OPTION_PREFIX), + "{} collides with the client namespace", + entry.key + ); + assert!(keys.insert(entry.key), "duplicate key: {}", entry.key); + assert!( + fields.insert(entry.internal_path), + "duplicate field: {}", + entry.internal_path + ); + assert!( + suffixes.insert(environment_suffix(entry.key)), + "duplicate environment suffix for {}", + entry.key + ); + } + + assert_eq!(CLUSTER_ENTRIES.len(), 8); + } } /// Precedence is stated once for the whole vocabulary rather than sampled on one key, so a per-kind @@ -3061,10 +3192,7 @@ mod tests { } } - /// User identity mode is only safe when the connection can actually carry the request's principal, so - /// the credentials must be usable *and* SASL must be selected. Both were previously satisfied by a - /// `Some("")` credential over the default PLAINTEXT, which authorizes the gateway's own identity for - /// every caller instead of failing. + /// User identity requires verified callers and usable service credentials over SASL. #[test] fn user_identity_mode_requires_usable_credentials_over_sasl() { let user_mode = "gateway.cluster.default.connection.identity-mode: user\n"; @@ -3103,14 +3231,23 @@ mod tests { ); } - // The complete, coherent form is accepted. - assert!(load_file(&format!("{user_mode}{sasl}{credentials}")).is_ok()); + assert!( + problems(load_file(&format!("{user_mode}{sasl}{credentials}")).unwrap_err()) + .iter() + .any(|problem| problem.contains("verified client identities")) + ); + assert!( + load_file(&format!( + "gateway.security.authentication: password\n\ + gateway.security.users: alice:secret\n{user_mode}{sasl}{credentials}" + )) + .is_ok() + ); // Service mode needs no SASL: it authenticates as itself, with no principal to propagate. assert!(load_file("gateway.cluster.default.connection.identity-mode: service\n").is_ok()); } - /// A size that cannot fit inside the size holding it fails before a listener binds, whether the operator - /// set both sides or only one: leaving the other at its native default is the common way to break a pair. + /// Native writer size relationships apply to configured values and native defaults. #[test] fn writer_size_pairs_must_fit_including_against_the_native_defaults() { for (contents, rejected) in [ @@ -3120,6 +3257,11 @@ mod tests { gateway.cluster.default.client.writer.request-max-size: 1MiB\n", true, ), + ( + "gateway.cluster.default.client.writer.batch-size: 65MiB\n\ + gateway.cluster.default.client.writer.buffer.memory-size: 64MiB\n", + true, + ), ( "gateway.cluster.default.client.writer.dynamic-batch-size.min: 4MiB\n\ gateway.cluster.default.client.writer.batch-size: 2MiB\n", @@ -3159,53 +3301,13 @@ mod tests { assert!( problems(result.unwrap_err()) .iter() - .any(|problem| problem.contains("must not exceed client.")), + .any(|problem| problem.contains("must be <=")), "{contents}" ); } } } - /// A value is bounded by the native field it lands in, not by one blanket ceiling: the writer sizes are - /// `i32` there, while the buffer size and the lookup counts are `usize` and may exceed `i32::MAX`. - #[test] - fn client_option_bounds_follow_the_native_field_type() { - let over_i32 = u64::from(i32::MAX as u32) + 1; - - for option in ["writer.batch-size", "writer.request-max-size"] { - let error = parse_client_option(option, &format!("{over_i32}")).unwrap_err(); - assert!(error.contains("must not exceed"), "{option}: {error}"); - } - assert_eq!( - parse_client_option("writer.buffer.memory-size", "4GiB").unwrap(), - ClientOptionValue::Bytes(4 * 1024 * 1024 * 1024) - ); - for option in [ - "lookup.queue-size", - "lookup.max-batch-size", - "lookup.max-inflight-requests", - ] { - assert_eq!( - parse_client_option(option, &format!("{over_i32}")).unwrap(), - ClientOptionValue::Integer(over_i32), - "{option} is stored as usize and must accept this" - ); - } - let error = parse_client_option("lookup.max-retries", &format!("{over_i32}")).unwrap_err(); - assert!(error.contains("must be between 0 and"), "{error}"); - } - - /// The declared native defaults are a copy of the client's, so they must at least satisfy the - /// relationships the client enforces; a mistyped copy shows up here and not as a rejected valid file. - #[test] - fn the_declared_native_size_defaults_are_coherent() { - let default = |option| effective_size(option, None).expect("a declared size default"); - let batch = default("writer.batch-size"); - assert!(batch <= default("writer.request-max-size")); - assert!(batch <= default("writer.buffer.memory-size")); - assert!(default("writer.dynamic-batch-size.min") <= batch); - } - #[test] fn options_are_complete_and_unambiguous() { let mut public_keys = std::collections::BTreeSet::new(); @@ -3233,35 +3335,4 @@ mod tests { assert_eq!(CONFIG_ENTRIES.len(), 20); } - - /// The per-cluster vocabulary shares the environment namespace with `client.*`, so its keys have to - /// stay distinct from each other and unreachable through the client prefix. - #[test] - fn cluster_options_are_complete_and_unambiguous() { - let mut keys = std::collections::BTreeSet::new(); - let mut fields = std::collections::BTreeSet::new(); - let mut suffixes = std::collections::BTreeSet::new(); - - for entry in CLUSTER_ENTRIES { - assert!(!entry.key.starts_with("gateway."), "{entry:?}"); - assert!( - !entry.key.starts_with(CLIENT_OPTION_PREFIX), - "{} collides with the client namespace", - entry.key - ); - assert!(keys.insert(entry.key), "duplicate key: {}", entry.key); - assert!( - fields.insert(entry.internal_path), - "duplicate field: {}", - entry.internal_path - ); - assert!( - suffixes.insert(environment_suffix(entry.key)), - "duplicate environment suffix for {}", - entry.key - ); - } - - assert_eq!(CLUSTER_ENTRIES.len(), 8); - } }