From 2ad8d0189c850da8556e12f61a87282d8fe673a1 Mon Sep 17 00:00:00 2001 From: Francois Massot Date: Sat, 11 Jul 2026 16:50:00 +0200 Subject: [PATCH 1/2] Tune PostgreSQL metastore in-flight requests --- config/quickwit.yaml | 2 + docs/configuration/metastore-config.md | 29 ++++++++++ docs/configuration/node-config.md | 12 +++-- docs/reference/metrics.md | 8 +++ .../quickwit-config/src/metastore_config.rs | 1 + quickwit/quickwit-serve/src/lib.rs | 54 +++++++++++++++---- 6 files changed, 90 insertions(+), 16 deletions(-) diff --git a/config/quickwit.yaml b/config/quickwit.yaml index 6ad7cd69323..9a6af1989c9 100644 --- a/config/quickwit.yaml +++ b/config/quickwit.yaml @@ -152,6 +152,8 @@ version: 0.8 # # metastore: # postgres: +# # Applies per node running the `metastore` service. Make sure the cluster-wide +# # total stays below your PostgreSQL or RDS connection limit. # min_connections: 0 # max_connections: 10 # acquire_connection_timeout: 10s diff --git a/docs/configuration/metastore-config.md b/docs/configuration/metastore-config.md index 05ed0a6b2fd..21ba764fb46 100644 --- a/docs/configuration/metastore-config.md +++ b/docs/configuration/metastore-config.md @@ -43,6 +43,35 @@ On its first execution, Quickwit will transparently create the necessary tables. Likewise, if you upgrade Quickwit to a version that includes some changes in the PostgreSQL schema, Quickwit will transparently operate the migration startup. +## PostgreSQL connection pool + +Each Quickwit node running the `metastore` service maintains a PostgreSQL connection pool. Nodes that do not run the `metastore` service use gRPC to reach those metastore nodes and do not open their own PostgreSQL pool. + +You can configure the pool in the `metastore.postgres` section of the [node configuration file](./node-config.md): + +```yaml +metastore: + postgres: + min_connections: 0 + max_connections: 10 + acquire_connection_timeout: 10s + idle_connection_timeout: 10min + max_connection_lifetime: 30min +``` + +The default `max_connections` is `10` per metastore node. Database-backed metastore nodes admit at most `2 * max_connections` in-flight metastore requests, so the default is `20` in-flight requests per metastore node. For production deployments backed by PostgreSQL or Amazon RDS, size this value together with the database connection limit: + +```text +metastore_nodes * metastore.postgres.max_connections +``` + +Keep that total below the PostgreSQL or RDS connection limit with headroom for migrations, administration, monitoring, and other applications sharing the database. If Quickwit logs errors such as `pool timed out while waiting for an open connection`, the local PostgreSQL pool on a metastore node was exhausted for at least `acquire_connection_timeout`. In that case: + +- Check `quickwit_metastore_active_connections`, `quickwit_metastore_idle_connections`, and `quickwit_metastore_acquire_connections`. +- If active connections stay close to `max_connections` and the database has spare capacity, increase `metastore.postgres.max_connections`. +- If active connections are high and PostgreSQL or RDS is already saturated, investigate slow metastore queries, locks, database CPU or I/O pressure, and RDS `DatabaseConnections` before increasing the pool. +- Increase `acquire_connection_timeout` only to tolerate short bursts. It does not add database capacity. + # File-backed metastore For convenience, Quickwit also makes it possible to store its metadata in files using a file-backed metastore. In that case, Quickwit will write one file per index. diff --git a/docs/configuration/node-config.md b/docs/configuration/node-config.md index 66da698665b..8c59baa48d6 100644 --- a/docs/configuration/node-config.md +++ b/docs/configuration/node-config.md @@ -216,16 +216,18 @@ File-backed metastore doesn't have any node level configuration. You can configu | `idle_connection_timeout` | Maximum idle duration before closing individual connections. | `10min` | | `max_connection_lifetime` | Maximum lifetime of individual connections. | `30min` | +`max_connections` applies per Quickwit node running the `metastore` service. When using PostgreSQL or Amazon RDS, keep `metastore_nodes * max_connections` below the database connection limit with operational headroom. + Example of a metastore configuration for PostgreSQL in YAML format: ```yaml metastore: postgres: - min_connections: 10 - max_connections: 50 - acquire_connection_timeout: 30s - idle_connection_timeout: 1h - max_connection_lifetime: 1d + min_connections: 0 + max_connections: 10 + acquire_connection_timeout: 10s + idle_connection_timeout: 10min + max_connection_lifetime: 30min ``` ## Indexer configuration diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md index 7d6b69fa7e7..03993b1c9d4 100644 --- a/docs/reference/metrics.md +++ b/docs/reference/metrics.md @@ -60,6 +60,14 @@ All metastore methods are monitored by the 3 metrics: Examples of operation names: `create_index`, `index_metadata`, `delete_index`, `stage_splits`, `publish_splits`, `list_splits`, `add_source`, ... +PostgreSQL-backed metastores also expose connection pool gauges: + +| Namespace | Metric Name | Description | Type | +| --------- | ----------- | ----------- | ---- | +| `quickwit_metastore` | `active_connections` | Number of active PostgreSQL pool connections, including used and idle connections | `gauge` | +| `quickwit_metastore` | `idle_connections` | Number of idle PostgreSQL pool connections | `gauge` | +| `quickwit_metastore` | `acquire_connections` | Number of requests currently waiting to acquire a PostgreSQL pool connection | `gauge` | + ## Rest API Metrics | Namespace | Metric Name | Description | Type | diff --git a/quickwit/quickwit-config/src/metastore_config.rs b/quickwit/quickwit-config/src/metastore_config.rs index 47a5ef5f917..9180d4f1391 100644 --- a/quickwit/quickwit-config/src/metastore_config.rs +++ b/quickwit/quickwit-config/src/metastore_config.rs @@ -337,6 +337,7 @@ mod tests { let pg_metastore_config: PostgresMetastoreConfig = serde_yaml::from_str(pg_metastore_config_yaml).unwrap(); assert_eq!(pg_metastore_config, PostgresMetastoreConfig::default()); + assert_eq!(pg_metastore_config.max_connections.get(), 10); } { let pg_metastore_config_yaml = r#" diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index e502363aaa3..0d4828011b1 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -80,7 +80,7 @@ use quickwit_compaction::{ wait_for_compactor_decommission, }; use quickwit_config::service::QuickwitService; -use quickwit_config::{ClusterConfig, IngestApiConfig, NodeConfig}; +use quickwit_config::{ClusterConfig, IngestApiConfig, NodeConfig, PostgresMetastoreConfig}; use quickwit_control_plane::control_plane::{ControlPlane, ControlPlaneEventSubscriber}; use quickwit_control_plane::{IndexerNodeInfo, IndexerPool}; use quickwit_index_management::{IndexService as IndexManager, IndexServiceError}; @@ -166,6 +166,22 @@ fn get_metastore_client_max_concurrency() -> usize { ) } +fn db_metastore_max_in_flight_requests(max_connections: NonZeroUsize) -> usize { + max_connections.get().saturating_mul(2) +} + +fn get_metastore_server_max_in_flight_requests(node_config: &NodeConfig) -> usize { + if !node_config.metastore_uri.protocol().is_database() { + return 100; + } + let max_connections = node_config + .metastore_configs + .find_postgres() + .map(|config| config.max_connections) + .unwrap_or_else(PostgresMetastoreConfig::default_max_connections); + db_metastore_max_in_flight_requests(max_connections) +} + static CP_GRPC_CLIENT_METRICS_LAYER: LazyLock = LazyLock::new(|| GrpcMetricsLayer::new("control_plane", "client")); static CP_GRPC_SERVER_METRICS_LAYER: LazyLock = @@ -586,16 +602,7 @@ pub async fn serve_quickwit( node_config.metastore_uri ) })?; - let max_in_flight_requests = if node_config.metastore_uri.protocol().is_database() { - node_config - .metastore_configs - .find_postgres() - .map(|config| config.max_connections.get() * 2) - .unwrap_or_default() - .max(100) - } else { - 100 - }; + let max_in_flight_requests = get_metastore_server_max_in_flight_requests(&node_config); // These layers apply to all the RPCs of the metastore. let shared_layer = ServiceBuilder::new() .layer(METASTORE_GRPC_SERVER_METRICS_LAYER.clone()) @@ -1763,6 +1770,31 @@ mod tests { use super::*; + #[test] + fn test_db_metastore_max_in_flight_requests() { + let max_connections = NonZeroUsize::new(10).unwrap(); + assert_eq!(db_metastore_max_in_flight_requests(max_connections), 20); + + let max_connections = NonZeroUsize::new(50).unwrap(); + assert_eq!(db_metastore_max_in_flight_requests(max_connections), 100); + } + + #[test] + fn test_get_metastore_server_max_in_flight_requests() { + let node_config = NodeConfig::for_test(); + assert_eq!( + get_metastore_server_max_in_flight_requests(&node_config), + 100 + ); + + let mut node_config = NodeConfig::for_test(); + node_config.metastore_uri = Uri::for_test("postgresql://quickwit@localhost/quickwit"); + assert_eq!( + get_metastore_server_max_in_flight_requests(&node_config), + 20 + ); + } + #[tokio::test] async fn test_check_cluster_configuration() { let services = HashSet::from_iter([QuickwitService::Metastore]); From 6e9154cfe443c49c3560284b4a62d5e03e226cf4 Mon Sep 17 00:00:00 2001 From: Francois Massot Date: Sat, 11 Jul 2026 17:16:59 +0200 Subject: [PATCH 2/2] Simplify metastore in-flight sizing change --- config/quickwit.yaml | 2 - docs/configuration/metastore-config.md | 27 +--------- docs/configuration/node-config.md | 12 ++--- .../quickwit-config/src/metastore_config.rs | 1 - quickwit/quickwit-serve/src/lib.rs | 52 ++++--------------- 5 files changed, 18 insertions(+), 76 deletions(-) diff --git a/config/quickwit.yaml b/config/quickwit.yaml index 9a6af1989c9..6ad7cd69323 100644 --- a/config/quickwit.yaml +++ b/config/quickwit.yaml @@ -152,8 +152,6 @@ version: 0.8 # # metastore: # postgres: -# # Applies per node running the `metastore` service. Make sure the cluster-wide -# # total stays below your PostgreSQL or RDS connection limit. # min_connections: 0 # max_connections: 10 # acquire_connection_timeout: 10s diff --git a/docs/configuration/metastore-config.md b/docs/configuration/metastore-config.md index 21ba764fb46..0b03a7f1e0b 100644 --- a/docs/configuration/metastore-config.md +++ b/docs/configuration/metastore-config.md @@ -45,32 +45,9 @@ Likewise, if you upgrade Quickwit to a version that includes some changes in the ## PostgreSQL connection pool -Each Quickwit node running the `metastore` service maintains a PostgreSQL connection pool. Nodes that do not run the `metastore` service use gRPC to reach those metastore nodes and do not open their own PostgreSQL pool. - -You can configure the pool in the `metastore.postgres` section of the [node configuration file](./node-config.md): - -```yaml -metastore: - postgres: - min_connections: 0 - max_connections: 10 - acquire_connection_timeout: 10s - idle_connection_timeout: 10min - max_connection_lifetime: 30min -``` - -The default `max_connections` is `10` per metastore node. Database-backed metastore nodes admit at most `2 * max_connections` in-flight metastore requests, so the default is `20` in-flight requests per metastore node. For production deployments backed by PostgreSQL or Amazon RDS, size this value together with the database connection limit: - -```text -metastore_nodes * metastore.postgres.max_connections -``` - -Keep that total below the PostgreSQL or RDS connection limit with headroom for migrations, administration, monitoring, and other applications sharing the database. If Quickwit logs errors such as `pool timed out while waiting for an open connection`, the local PostgreSQL pool on a metastore node was exhausted for at least `acquire_connection_timeout`. In that case: +Each Quickwit node running the `metastore` service maintains its own PostgreSQL connection pool. Database-backed metastore nodes admit at most `2 * metastore.postgres.max_connections` in-flight requests, so the default is `20`. -- Check `quickwit_metastore_active_connections`, `quickwit_metastore_idle_connections`, and `quickwit_metastore_acquire_connections`. -- If active connections stay close to `max_connections` and the database has spare capacity, increase `metastore.postgres.max_connections`. -- If active connections are high and PostgreSQL or RDS is already saturated, investigate slow metastore queries, locks, database CPU or I/O pressure, and RDS `DatabaseConnections` before increasing the pool. -- Increase `acquire_connection_timeout` only to tolerate short bursts. It does not add database capacity. +When sizing the pool, keep `metastore_nodes * metastore.postgres.max_connections` below the PostgreSQL connection limit with operational headroom. If Quickwit logs `pool timed out while waiting for an open connection`, check `quickwit_metastore_active_connections`, `quickwit_metastore_idle_connections`, and `quickwit_metastore_acquire_connections`. # File-backed metastore diff --git a/docs/configuration/node-config.md b/docs/configuration/node-config.md index 8c59baa48d6..fdcd26054d3 100644 --- a/docs/configuration/node-config.md +++ b/docs/configuration/node-config.md @@ -216,18 +216,18 @@ File-backed metastore doesn't have any node level configuration. You can configu | `idle_connection_timeout` | Maximum idle duration before closing individual connections. | `10min` | | `max_connection_lifetime` | Maximum lifetime of individual connections. | `30min` | -`max_connections` applies per Quickwit node running the `metastore` service. When using PostgreSQL or Amazon RDS, keep `metastore_nodes * max_connections` below the database connection limit with operational headroom. +`max_connections` applies per Quickwit node running the `metastore` service. Database-backed metastore nodes admit at most `2 * max_connections` in-flight requests. Example of a metastore configuration for PostgreSQL in YAML format: ```yaml metastore: postgres: - min_connections: 0 - max_connections: 10 - acquire_connection_timeout: 10s - idle_connection_timeout: 10min - max_connection_lifetime: 30min + min_connections: 10 + max_connections: 50 + acquire_connection_timeout: 30s + idle_connection_timeout: 1h + max_connection_lifetime: 1d ``` ## Indexer configuration diff --git a/quickwit/quickwit-config/src/metastore_config.rs b/quickwit/quickwit-config/src/metastore_config.rs index 9180d4f1391..47a5ef5f917 100644 --- a/quickwit/quickwit-config/src/metastore_config.rs +++ b/quickwit/quickwit-config/src/metastore_config.rs @@ -337,7 +337,6 @@ mod tests { let pg_metastore_config: PostgresMetastoreConfig = serde_yaml::from_str(pg_metastore_config_yaml).unwrap(); assert_eq!(pg_metastore_config, PostgresMetastoreConfig::default()); - assert_eq!(pg_metastore_config.max_connections.get(), 10); } { let pg_metastore_config_yaml = r#" diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index 0d4828011b1..a93033acde3 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -166,22 +166,6 @@ fn get_metastore_client_max_concurrency() -> usize { ) } -fn db_metastore_max_in_flight_requests(max_connections: NonZeroUsize) -> usize { - max_connections.get().saturating_mul(2) -} - -fn get_metastore_server_max_in_flight_requests(node_config: &NodeConfig) -> usize { - if !node_config.metastore_uri.protocol().is_database() { - return 100; - } - let max_connections = node_config - .metastore_configs - .find_postgres() - .map(|config| config.max_connections) - .unwrap_or_else(PostgresMetastoreConfig::default_max_connections); - db_metastore_max_in_flight_requests(max_connections) -} - static CP_GRPC_CLIENT_METRICS_LAYER: LazyLock = LazyLock::new(|| GrpcMetricsLayer::new("control_plane", "client")); static CP_GRPC_SERVER_METRICS_LAYER: LazyLock = @@ -602,7 +586,16 @@ pub async fn serve_quickwit( node_config.metastore_uri ) })?; - let max_in_flight_requests = get_metastore_server_max_in_flight_requests(&node_config); + let max_in_flight_requests = if node_config.metastore_uri.protocol().is_database() { + let max_connections = node_config + .metastore_configs + .find_postgres() + .map(|config| config.max_connections) + .unwrap_or_else(PostgresMetastoreConfig::default_max_connections); + max_connections.get().saturating_mul(2) + } else { + 100 + }; // These layers apply to all the RPCs of the metastore. let shared_layer = ServiceBuilder::new() .layer(METASTORE_GRPC_SERVER_METRICS_LAYER.clone()) @@ -1770,31 +1763,6 @@ mod tests { use super::*; - #[test] - fn test_db_metastore_max_in_flight_requests() { - let max_connections = NonZeroUsize::new(10).unwrap(); - assert_eq!(db_metastore_max_in_flight_requests(max_connections), 20); - - let max_connections = NonZeroUsize::new(50).unwrap(); - assert_eq!(db_metastore_max_in_flight_requests(max_connections), 100); - } - - #[test] - fn test_get_metastore_server_max_in_flight_requests() { - let node_config = NodeConfig::for_test(); - assert_eq!( - get_metastore_server_max_in_flight_requests(&node_config), - 100 - ); - - let mut node_config = NodeConfig::for_test(); - node_config.metastore_uri = Uri::for_test("postgresql://quickwit@localhost/quickwit"); - assert_eq!( - get_metastore_server_max_in_flight_requests(&node_config), - 20 - ); - } - #[tokio::test] async fn test_check_cluster_configuration() { let services = HashSet::from_iter([QuickwitService::Metastore]);