From 8cbb2e130b18cae40c73c5f197328e3df4737a02 Mon Sep 17 00:00:00 2001 From: Harsh Dev Pathak Date: Sun, 26 Oct 2025 07:13:41 +0530 Subject: [PATCH 1/2] feat: Added in-memory storage for testing purposes --- .../workflows/build-and-test-in-memory.yml | 70 ++++ .github/workflows/ldk-node-integration.yml | 53 +++ rust/README.md | 5 + rust/impls/src/in_memory_store.rs | 347 ++++++++++++++++++ rust/impls/src/lib.rs | 40 +- rust/impls/src/postgres_store.rs | 24 +- rust/server/src/main.rs | 96 +++-- rust/server/src/util/config.rs | 72 +++- rust/server/vss-server-config.toml | 1 + 9 files changed, 631 insertions(+), 77 deletions(-) create mode 100644 .github/workflows/build-and-test-in-memory.yml create mode 100644 rust/impls/src/in_memory_store.rs diff --git a/.github/workflows/build-and-test-in-memory.yml b/.github/workflows/build-and-test-in-memory.yml new file mode 100644 index 0000000..10c51f8 --- /dev/null +++ b/.github/workflows/build-and-test-in-memory.yml @@ -0,0 +1,70 @@ +name: In-Memory VSS Server CI + +on: + push: + branches: [ main ] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test-in-memory: + runs-on: ubuntu-latest + timeout-minutes: 6 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create in-memory config + run: | + mkdir -p rust/server + cat > rust/server/vss-server-config.toml < server.log 2>&1 & + echo "Server PID: $!" + + - name: Wait for server + run: | + for i in {1..15}; do + if curl -s http://127.0.0.1:8080 > /dev/null; then + echo "Server is up!" + exit 0 + fi + sleep 1 + done + echo "Server failed. Dumping log:" + cat rust/server.log + exit 1 + + - name: HTTP Smoke Test + run: | + sudo apt-get update && sudo apt-get install -y xxd + + curl -f \ + -H "Authorization: Bearer test_user" \ + --data-binary @<(echo "0A04746573741A150A026B3110FFFFFFFFFFFFFFFFFF011A046B317631" | xxd -r -p) \ + http://127.0.0.1:8080/vss/putObjects + + RESPONSE=$(curl -f \ + -H "Authorization: Bearer test_user" \ + --data-binary @<(echo "0A047465737412026B31" | xxd -r -p) \ + http://127.0.0.1:8080/vss/getObject) + + - name: Run In-Memory unit tests + working-directory: rust + run: cargo test --package impls --lib -- in_memory_store::tests --nocapture \ No newline at end of file diff --git a/.github/workflows/ldk-node-integration.yml b/.github/workflows/ldk-node-integration.yml index 0c68cb7..6be4fb2 100644 --- a/.github/workflows/ldk-node-integration.yml +++ b/.github/workflows/ldk-node-integration.yml @@ -52,3 +52,56 @@ jobs: export TEST_VSS_BASE_URL="http://localhost:8080/vss" RUSTFLAGS="--cfg vss_test" cargo test io::vss_store -- --test-threads=1 RUSTFLAGS="--cfg vss_test" cargo test --test integration_tests_vss -- --test-threads=1 + + test-in-memory: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + path: vss-server + + - name: Checkout LDK Node + uses: actions/checkout@v3 + with: + repository: lightningdevkit/ldk-node + path: ldk-node + + - name: Create In-Memory config + run: | + mkdir -p vss-server/rust/server + cat > vss-server/rust/server/vss-server-config.toml < server.log 2>&1 & + echo "Server PID: $!" + + - name: Wait for VSS + run: | + for i in {1..30}; do + if curl -s http://127.0.0.1:8080/vss > /dev/null; then + echo "VSS ready" + exit 0 + fi + sleep 1 + done + echo "VSS failed:" + cat vss-server/rust/vss.log + exit 1 + + - name: Run LDK Node Integration tests + working-directory: ldk-node + run: | + export TEST_VSS_BASE_URL="http://127.0.0.1:8080/vss" + RUSTFLAGS="--cfg vss_test" cargo test io::vss_store + RUSTFLAGS="--cfg vss_test" cargo test --test integration_tests_vss \ No newline at end of file diff --git a/rust/README.md b/rust/README.md index 73cf9c2..6858be4 100644 --- a/rust/README.md +++ b/rust/README.md @@ -26,6 +26,11 @@ cargo build --release ``` cargo run server/vss-server-config.toml ``` + + **Note:** For testing purposes, you can pass `--in-memory` to use in-memory instead of PostgreSQL + ``` + cargo run -- server/vss-server-config.toml --in-memory + ``` 4. VSS endpoint should be reachable at `http://localhost:8080/vss`. ### Configuration diff --git a/rust/impls/src/in_memory_store.rs b/rust/impls/src/in_memory_store.rs new file mode 100644 index 0000000..eceb313 --- /dev/null +++ b/rust/impls/src/in_memory_store.rs @@ -0,0 +1,347 @@ +use crate::{VssDbRecord, LIST_KEY_VERSIONS_MAX_PAGE_SIZE, MAX_PUT_REQUEST_ITEM_COUNT}; +use api::error::VssError; +use api::kv_store::{KvStore, GLOBAL_VERSION_KEY, INITIAL_RECORD_VERSION}; +use api::types::{ + DeleteObjectRequest, DeleteObjectResponse, GetObjectRequest, GetObjectResponse, KeyValue, + ListKeyVersionsRequest, ListKeyVersionsResponse, PutObjectRequest, PutObjectResponse, +}; +use async_trait::async_trait; +use bytes::Bytes; +use chrono::prelude::Utc; +use std::collections::BTreeMap; +use tokio::sync::Mutex; + +fn build_storage_key(user_token: &str, store_id: &str, key: &str) -> String { + format!("{}#{}#{}", user_token, store_id, key) +} + +/// In-memory implementation of the VSS Store. +pub struct InMemoryBackendImpl { + store: Mutex>, +} + +impl InMemoryBackendImpl { + /// Creates an in-memory instance. + pub fn new() -> Self { + Self { store: Mutex::new(BTreeMap::new()) } + } + + fn get_current_global_version( + &self, guard: &BTreeMap, user_token: &str, store_id: &str, + ) -> i64 { + let global_key = build_storage_key(user_token, store_id, GLOBAL_VERSION_KEY); + guard.get(&global_key).map(|r| r.version).unwrap_or(0) + } +} + +fn validate_put_operation( + store: &BTreeMap, user_token: &str, store_id: &str, key_value: &KeyValue, +) -> Result<(), VssError> { + let key = build_storage_key(user_token, store_id, &key_value.key); + + if key_value.version == -1 { + Ok(()) + } else if key_value.version == 0 { + if store.contains_key(&key) { + Err(VssError::ConflictError(format!( + "Key {} already exists for conditional insert", + key_value.key + ))) + } else { + Ok(()) + } + } else { + if let Some(existing) = store.get(&key) { + if existing.version == key_value.version { + Ok(()) + } else { + Err(VssError::ConflictError(format!( + "Version mismatch for key {}: expected {}, found {}", + key_value.key, key_value.version, existing.version + ))) + } + } else { + Err(VssError::ConflictError(format!( + "Key {} does not exist for conditional update", + key_value.key + ))) + } + } +} + +fn validate_delete_operation( + store: &BTreeMap, user_token: &str, store_id: &str, key_value: &KeyValue, +) -> Result<(), VssError> { + let key = build_storage_key(user_token, store_id, &key_value.key); + + if key_value.version == -1 { + Ok(()) + } else { + if let Some(existing) = store.get(&key) { + if existing.version == key_value.version { + Ok(()) + } else { + Err(VssError::ConflictError(format!( + "Version mismatch for delete key {}: expected {}, found {}", + key_value.key, key_value.version, existing.version + ))) + } + } else { + Err(VssError::ConflictError(format!( + "Key {} does not exist for conditional delete", + key_value.key + ))) + } + } +} + +fn execute_put_object( + store: &mut BTreeMap, user_token: &str, store_id: &str, + key_value: KeyValue, +) { + let key = build_storage_key(user_token, store_id, &key_value.key); + let now = Utc::now(); + + match store.entry(key) { + std::collections::btree_map::Entry::Occupied(mut occ) => { + let existing = occ.get_mut(); + existing.version = if key_value.version == -1 { + INITIAL_RECORD_VERSION as i64 + } else { + existing.version.saturating_add(1) + }; + existing.value = key_value.value.to_vec(); + existing.last_updated_at = now; + }, + std::collections::btree_map::Entry::Vacant(vac) => { + let new_record = VssDbRecord { + user_token: user_token.to_string(), + store_id: store_id.to_string(), + key: key_value.key, + value: key_value.value.to_vec(), + version: INITIAL_RECORD_VERSION as i64, + created_at: now, + last_updated_at: now, + }; + vac.insert(new_record); + }, + } +} + +fn execute_delete_object( + store: &mut BTreeMap, user_token: &str, store_id: &str, + key_value: &KeyValue, +) { + let key = build_storage_key(user_token, store_id, &key_value.key); + store.remove(&key); +} + +#[async_trait] +impl KvStore for InMemoryBackendImpl { + async fn get( + &self, user_token: String, request: GetObjectRequest, + ) -> Result { + if request.key == GLOBAL_VERSION_KEY { + let guard = self.store.lock().await; + let global_version = + self.get_current_global_version(&guard, &user_token, &request.store_id); + return Ok(GetObjectResponse { + value: Some(KeyValue { + key: GLOBAL_VERSION_KEY.to_string(), + value: Bytes::new(), + version: global_version, + }), + }); + } + + let key = build_storage_key(&user_token, &request.store_id, &request.key); + let guard = self.store.lock().await; + + if let Some(record) = guard.get(&key) { + Ok(GetObjectResponse { + value: Some(KeyValue { + key: record.key.clone(), + value: Bytes::from(record.value.clone()), + version: record.version, + }), + }) + } else { + Err(VssError::NoSuchKeyError("Requested key not found.".to_string())) + } + } + + async fn put( + &self, user_token: String, request: PutObjectRequest, + ) -> Result { + if request.transaction_items.len() + request.delete_items.len() > MAX_PUT_REQUEST_ITEM_COUNT + { + return Err(VssError::InvalidRequestError(format!( + "Number of write items per request should be less than equal to {}", + MAX_PUT_REQUEST_ITEM_COUNT + ))); + } + + let store_id = request.store_id; + let mut guard = self.store.lock().await; + + if let Some(version) = request.global_version { + validate_put_operation( + &guard, + &user_token, + &store_id, + &KeyValue { key: GLOBAL_VERSION_KEY.to_string(), value: Bytes::new(), version }, + )?; + } + + for key_value in &request.transaction_items { + validate_put_operation(&guard, &user_token, &store_id, key_value)?; + } + + for key_value in &request.delete_items { + validate_delete_operation(&guard, &user_token, &store_id, key_value)?; + } + + for key_value in request.transaction_items { + execute_put_object(&mut guard, &user_token, &store_id, key_value); + } + + for key_value in &request.delete_items { + execute_delete_object(&mut guard, &user_token, &store_id, key_value); + } + + if let Some(version) = request.global_version { + execute_put_object( + &mut guard, + &user_token, + &store_id, + KeyValue { key: GLOBAL_VERSION_KEY.to_string(), value: Bytes::new(), version }, + ); + } + + Ok(PutObjectResponse {}) + } + + async fn delete( + &self, user_token: String, request: DeleteObjectRequest, + ) -> Result { + let key_value = request.key_value.ok_or_else(|| { + VssError::InvalidRequestError("key_value missing in DeleteObjectRequest".to_string()) + })?; + + let store_id = request.store_id; + let mut guard = self.store.lock().await; + + execute_delete_object(&mut guard, &user_token, &store_id, &key_value); + + Ok(DeleteObjectResponse {}) + } + + async fn list_key_versions( + &self, user_token: String, request: ListKeyVersionsRequest, + ) -> Result { + let store_id = request.store_id; + let key_prefix = request.key_prefix.clone().unwrap_or("".to_string()); + let page_size = request.page_size.unwrap_or(LIST_KEY_VERSIONS_MAX_PAGE_SIZE); + let limit = std::cmp::min(page_size, LIST_KEY_VERSIONS_MAX_PAGE_SIZE) as usize; + + let offset: usize = + request.page_token.as_ref().and_then(|s| s.parse::().ok()).unwrap_or(0); + + let guard = self.store.lock().await; + + let global_version = if offset == 0 { + Some(self.get_current_global_version(&guard, &user_token, &store_id)) + } else { + None + }; + + let storage_prefix = build_storage_key(&user_token, &store_id, &key_prefix); + let global_key = build_storage_key(&user_token, &store_id, GLOBAL_VERSION_KEY); + let prefix_len = format!("{}#{}#", user_token, store_id).len(); + let page_items: Vec = guard + .iter() + .filter(|(storage_key, _)| { + storage_key.starts_with(&storage_prefix) && *storage_key != &global_key + }) + .skip(offset) + .take(limit) + .map(|(storage_key, record)| { + let key = &storage_key[prefix_len..]; + KeyValue { key: key.to_string(), value: Bytes::new(), version: record.version } + }) + .collect(); + + let next_offset = offset + page_items.len(); + let next_page_token = if page_items.is_empty() { + Some("".to_string()) + } else { + Some(next_offset.to_string()) + }; + + Ok(ListKeyVersionsResponse { key_versions: page_items, next_page_token, global_version }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use api::define_kv_store_tests; + use api::types::{GetObjectRequest, KeyValue, PutObjectRequest}; + use bytes::Bytes; + use tokio::test; + + define_kv_store_tests!(InMemoryKvStoreTest, InMemoryBackendImpl, InMemoryBackendImpl::new()); + + #[test] + async fn test_in_memory_crud() { + let store = InMemoryBackendImpl::new(); + let user_token = "test_user".to_string(); + let store_id = "test_store".to_string(); + + let put_request = PutObjectRequest { + store_id: store_id.clone(), + transaction_items: vec![KeyValue { + key: "key1".to_string(), + value: Bytes::from("value1"), + version: 0, + }], + delete_items: vec![], + global_version: None, + }; + store.put(user_token.clone(), put_request).await.unwrap(); + + let get_request = GetObjectRequest { store_id: store_id.clone(), key: "key1".to_string() }; + let response = store.get(user_token.clone(), get_request).await.unwrap(); + let key_value = response.value.unwrap(); + assert_eq!(key_value.value, Bytes::from("value1")); + assert_eq!(key_value.version, 1, "Expected version 1 after put"); + + let list_request = ListKeyVersionsRequest { + store_id: store_id.clone(), + key_prefix: None, + page_size: Some(1), + page_token: None, + }; + let response = store.list_key_versions(user_token.clone(), list_request).await.unwrap(); + assert_eq!(response.key_versions.len(), 1); + assert_eq!(response.key_versions[0].key, "key1"); + assert_eq!(response.key_versions[0].version, 1); + + let delete_request = DeleteObjectRequest { + store_id: store_id.clone(), + key_value: Some(KeyValue { key: "key1".to_string(), value: Bytes::new(), version: 1 }), + }; + store.delete(user_token.clone(), delete_request).await.unwrap(); + + let get_request = GetObjectRequest { store_id: store_id.clone(), key: "key1".to_string() }; + assert!(matches!( + store.get(user_token.clone(), get_request).await, + Err(VssError::NoSuchKeyError(_)) + )); + + let global_request = + GetObjectRequest { store_id: store_id.clone(), key: GLOBAL_VERSION_KEY.to_string() }; + let response = store.get(user_token.clone(), global_request).await.unwrap(); + assert_eq!(response.value.unwrap().version, 0, "Expected global_version=0"); + } +} diff --git a/rust/impls/src/lib.rs b/rust/impls/src/lib.rs index d58b84e..2a8e0c6 100644 --- a/rust/impls/src/lib.rs +++ b/rust/impls/src/lib.rs @@ -11,8 +11,46 @@ #![deny(rustdoc::private_intra_doc_links)] #![deny(missing_docs)] -mod migrations; +use chrono::Utc; + +/// Contains in-memory backend implementation for VSS, for testing purposes only. +pub mod in_memory_store; /// Contains [PostgreSQL](https://www.postgresql.org/) based backend implementation for VSS. pub mod postgres_store; +/// A record stored in the VSS database. +struct VssDbRecord { + /// Token uniquely identifying the user that owns this record. + user_token: String, + /// Identifier for the store this record belongs to. + store_id: String, + /// Key under which the value is stored. + key: String, + /// Stored value as raw bytes. + value: Vec, + /// Version number for optimistic concurrency control. + version: i64, + /// Timestamp when the record was created (UTC). + created_at: chrono::DateTime, + /// Timestamp when the record was last updated (UTC). + last_updated_at: chrono::DateTime, +} + +/// The maximum number of key versions that can be returned in a single page. +/// +/// This constant helps control memory and bandwidth usage for list operations, +/// preventing overly large payloads. If the number of results exceeds this limit, +/// the response will be paginated. +const LIST_KEY_VERSIONS_MAX_PAGE_SIZE: i32 = 100; + +/// The maximum number of items allowed in a single `PutObjectRequest`. +/// +/// Setting an upper bound on the number of items helps ensure that +/// each request stays within acceptable memory and performance limits. +/// Exceeding this value will result in request rejection through [`VssError::InvalidRequestError`]. +const MAX_PUT_REQUEST_ITEM_COUNT: usize = 1000; + +mod migrations; + +#[macro_use] extern crate api; diff --git a/rust/impls/src/postgres_store.rs b/rust/impls/src/postgres_store.rs index daaf437..929045f 100644 --- a/rust/impls/src/postgres_store.rs +++ b/rust/impls/src/postgres_store.rs @@ -1,5 +1,6 @@ use crate::migrations::*; +use crate::{VssDbRecord, LIST_KEY_VERSIONS_MAX_PAGE_SIZE, MAX_PUT_REQUEST_ITEM_COUNT}; use api::error::VssError; use api::kv_store::{KvStore, GLOBAL_VERSION_KEY, INITIAL_RECORD_VERSION}; use api::types::{ @@ -21,33 +22,10 @@ use log::{debug, info, warn}; pub use native_tls::Certificate; -pub(crate) struct VssDbRecord { - pub(crate) user_token: String, - pub(crate) store_id: String, - pub(crate) key: String, - pub(crate) value: Vec, - pub(crate) version: i64, - pub(crate) created_at: chrono::DateTime, - pub(crate) last_updated_at: chrono::DateTime, -} const KEY_COLUMN: &str = "key"; const VALUE_COLUMN: &str = "value"; const VERSION_COLUMN: &str = "version"; -/// The maximum number of key versions that can be returned in a single page. -/// -/// This constant helps control memory and bandwidth usage for list operations, -/// preventing overly large payloads. If the number of results exceeds this limit, -/// the response will be paginated. -pub const LIST_KEY_VERSIONS_MAX_PAGE_SIZE: i32 = 100; - -/// The maximum number of items allowed in a single `PutObjectRequest`. -/// -/// Setting an upper bound on the number of items helps ensure that -/// each request stays within acceptable memory and performance limits. -/// Exceeding this value will result in request rejection through [`VssError::InvalidRequestError`]. -pub const MAX_PUT_REQUEST_ITEM_COUNT: usize = 1000; - const POOL_SIZE: usize = 10; struct SmallPool { diff --git a/rust/server/src/main.rs b/rust/server/src/main.rs index 6da71af..4ba6f43 100644 --- a/rust/server/src/main.rs +++ b/rust/server/src/main.rs @@ -27,6 +27,7 @@ use api::kv_store::KvStore; use auth_impls::jwt::JWTAuthorizer; #[cfg(feature = "sigs")] use auth_impls::signature::SignatureValidatingAuthorizer; +use impls::in_memory_store::InMemoryBackendImpl; use impls::postgres_store::{PostgresPlaintextBackend, PostgresTlsBackend}; use util::logger::ServerLogger; use vss_service::{VssService, VssServiceConfig}; @@ -36,12 +37,24 @@ mod vss_service; fn main() { let args: Vec = std::env::args().collect(); + if args.len() < 2 { + eprintln!("Usage: {} [--in-memory]", args[0]); + std::process::exit(1); + } - let config = - util::config::load_configuration(args.get(1).map(|s| s.as_str())).unwrap_or_else(|e| { + let use_in_memory = args.contains(&"--in-memory".to_string()); + + let mut config = util::config::load_configuration(args.get(1).map(|s| s.as_str())) + .unwrap_or_else(|e| { eprintln!("Failed to load configuration: {}", e); std::process::exit(-1); }); + + // Override config if --in-memory flag is present + if use_in_memory { + config.store_type = Some(util::config::StoreType::InMemory); + } + let vss_service_config = match config.max_request_body_size { Some(size) => match VssServiceConfig::new(size) { Ok(config) => config, @@ -125,39 +138,50 @@ fn main() { std::process::exit(-1); }); - let store: Arc = if let Some(crt_pem) = config.tls_config { - let postgres_tls_backend = PostgresTlsBackend::new( - &config.postgresql_prefix, - &config.default_db, - &config.vss_db, - crt_pem.as_deref(), - ) - .await - .unwrap_or_else(|e| { - error!("Failed to start postgres TLS backend: {}", e); - std::process::exit(-1); - }); - info!( - "Connected to PostgreSQL TLS backend with DSN: {}/{}", - config.postgresql_prefix, config.vss_db - ); - Arc::new(postgres_tls_backend) - } else { - let postgres_plaintext_backend = PostgresPlaintextBackend::new( - &config.postgresql_prefix, - &config.default_db, - &config.vss_db, - ) - .await - .unwrap_or_else(|e| { - error!("Failed to start postgres plaintext backend: {}", e); - std::process::exit(-1); - }); - info!( - "Connected to PostgreSQL plaintext backend with DSN: {}/{}", - config.postgresql_prefix, config.vss_db - ); - Arc::new(postgres_plaintext_backend) + // Default to Postgres if no store_type is specified in config + let store_type = config.store_type.unwrap_or(util::config::StoreType::Postgres); + + let store: Arc = match store_type { + util::config::StoreType::InMemory => { + info!("Using in-memory backend for testing"); + Arc::new(InMemoryBackendImpl::new()) + }, + util::config::StoreType::Postgres => { + if let Some(crt_pem) = config.tls_config { + let postgres_tls_backend = PostgresTlsBackend::new( + &config.postgresql_prefix, + &config.default_db, + &config.vss_db, + crt_pem.as_deref(), + ) + .await + .unwrap_or_else(|e| { + error!("Failed to start postgres TLS backend: {}", e); + std::process::exit(-1); + }); + info!( + "Connected to PostgreSQL TLS backend with DSN: {}/{}", + config.postgresql_prefix, config.vss_db + ); + Arc::new(postgres_tls_backend) + } else { + let postgres_plaintext_backend = PostgresPlaintextBackend::new( + &config.postgresql_prefix, + &config.default_db, + &config.vss_db, + ) + .await + .unwrap_or_else(|e| { + error!("Failed to start postgres plaintext backend: {}", e); + std::process::exit(-1); + }); + info!( + "Connected to PostgreSQL plaintext backend with DSN: {}/{}", + config.postgresql_prefix, config.vss_db + ); + Arc::new(postgres_plaintext_backend) + } + } }; let rest_svc_listener = TcpListener::bind(&config.bind_address).await.unwrap_or_else(|e| { @@ -198,4 +222,4 @@ fn main() { } } }); -} +} \ No newline at end of file diff --git a/rust/server/src/util/config.rs b/rust/server/src/util/config.rs index 6560334..203d144 100644 --- a/rust/server/src/util/config.rs +++ b/rust/server/src/util/config.rs @@ -25,10 +25,19 @@ struct TomlConfig { postgresql_config: Option, } +#[derive(Deserialize, Clone)] +pub(crate) enum StoreType { + #[serde(rename = "postgres")] + Postgres, + #[serde(rename = "in-memory")] + InMemory, +} + #[derive(Deserialize)] struct ServerConfig { bind_address: Option, max_request_body_size: Option, + store_type: Option, } #[derive(Deserialize)] @@ -66,6 +75,7 @@ pub(crate) struct Configuration { pub(crate) default_db: String, pub(crate) vss_db: String, pub(crate) tls_config: Option>, + pub(crate) store_type: Option, pub(crate) log_file: PathBuf, pub(crate) log_level: LevelFilter, } @@ -101,9 +111,10 @@ pub(crate) fn load_configuration(config_file_path: Option<&str>) -> Result TomlConfig::default(), // All fields are set to `None` }; - let (bind_address_config, max_request_body_size_config) = match server_config { - Some(c) => (c.bind_address, c.max_request_body_size), - None => (None, None), + let (bind_address_config, max_request_body_size_config, store_type_config) = match server_config + { + Some(c) => (c.bind_address, c.max_request_body_size, c.store_type), + None => (None, None, None), }; let bind_address_env = read_env(BIND_ADDR_VAR)?; @@ -154,6 +165,9 @@ pub(crate) fn load_configuration(config_file_path: Option<&str>) -> Result = read_env(PSQL_ADDR_VAR)?; @@ -181,20 +195,43 @@ pub(crate) fn load_configuration(config_file_path: Option<&str>) -> Result (None, None, None, None, None, None), }; - let username = - read_config(username_env, username_config, "PostgreSQL database username", PSQL_USER_VAR)?; - let password = - read_config(password_env, password_config, "PostgreSQL database password", PSQL_PASS_VAR)?; - let address = - read_config(address_env, address_config, "PostgreSQL service address", PSQL_ADDR_VAR)?; - let default_db = read_config( - default_db_env, - default_db_config, - "PostgreSQL default database name", - PSQL_DB_VAR, - )?; - let vss_db = - read_config(vss_db_env, vss_db_config, "PostgreSQL vss database name", PSQL_VSS_DB_VAR)?; + let (username, password, address, default_db, vss_db) = if using_in_memory { + ( + username_env.or(username_config).unwrap_or_else(|| "dummy".to_string()), + password_env.or(password_config).unwrap_or_else(|| "dummy".to_string()), + address_env.or(address_config).unwrap_or_else(|| "localhost:5432".to_string()), + default_db_env.or(default_db_config).unwrap_or_else(|| "postgres".to_string()), + vss_db_env.or(vss_db_config).unwrap_or_else(|| "vss".to_string()), + ) + } else { + ( + read_config( + username_env, + username_config, + "PostgreSQL database username", + PSQL_USER_VAR, + )?, + read_config( + password_env, + password_config, + "PostgreSQL database password", + PSQL_PASS_VAR, + )?, + read_config(address_env, address_config, "PostgreSQL service address", PSQL_ADDR_VAR)?, + read_config( + default_db_env, + default_db_config, + "PostgreSQL default database name", + PSQL_DB_VAR, + )?, + read_config( + vss_db_env, + vss_db_config, + "PostgreSQL vss database name", + PSQL_VSS_DB_VAR, + )?, + ) + }; let tls_config = crt_pem_env.map(|pem| Some(pem)).or(tls_config_env.map(|_| None)).or(tls_config); @@ -211,5 +248,6 @@ pub(crate) fn load_configuration(config_file_path: Option<&str>) -> Result Date: Wed, 12 Nov 2025 09:29:07 +0530 Subject: [PATCH 2/2] feat: Added CI for in_memory_testing --- .../workflows/build-and-test-in-memory.yml | 9 ++- .github/workflows/ldk-node-integration.yml | 70 +++++++++++++------ rust/server/src/main.rs | 2 +- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-and-test-in-memory.yml b/.github/workflows/build-and-test-in-memory.yml index 10c51f8..4192167 100644 --- a/.github/workflows/build-and-test-in-memory.yml +++ b/.github/workflows/build-and-test-in-memory.yml @@ -23,25 +23,24 @@ jobs: mkdir -p rust/server cat > rust/server/vss-server-config.toml < server.log 2>&1 & + ./target/release/vss-server ./server/vss-server-config.toml > server.log 2>&1 & echo "Server PID: $!" - name: Wait for server run: | for i in {1..15}; do - if curl -s http://127.0.0.1:8080 > /dev/null; then + if curl -s http://127.0.0.1:8080/vss > /dev/null; then echo "Server is up!" exit 0 fi diff --git a/.github/workflows/ldk-node-integration.yml b/.github/workflows/ldk-node-integration.yml index 6be4fb2..5617a90 100644 --- a/.github/workflows/ldk-node-integration.yml +++ b/.github/workflows/ldk-node-integration.yml @@ -7,19 +7,14 @@ concurrency: cancel-in-progress: true jobs: - build-and-test: - strategy: - fail-fast: false - matrix: - platform: [ ubuntu-latest ] - toolchain: [ stable, 1.85.0 ] # 1.85.0 is the MSRV - runs-on: ${{ matrix.platform }} + test-postgres: + runs-on: ubuntu-latest + timeout-minutes: 30 services: postgres: image: postgres:latest - ports: - - 5432:5432 + ports: [5432:5432] env: POSTGRES_DB: postgres POSTGRES_USER: postgres @@ -35,21 +30,53 @@ jobs: uses: actions/checkout@v3 with: path: vss-server + - name: Checkout LDK Node uses: actions/checkout@v3 with: repository: lightningdevkit/ldk-node path: ldk-node - - name: Build and Deploy VSS Server + - name: Create Postgres config + run: | + mkdir -p vss-server/rust/server + cat > vss-server/rust/server/vss-server-config.toml < server.log 2>&1 & + echo "Server PID: $!" + + - name: Wait for VSS run: | - cd vss-server/rust - RUSTFLAGS="--cfg noop_authorizer" cargo build --no-default-features - ./target/debug/vss-server server/vss-server-config.toml& + for i in {1..30}; do + if curl -s http://127.0.0.1:8080/vss > /dev/null; then + echo "VSS ready" + exit 0 + fi + sleep 2 + done + echo "VSS failed:" + cat vss-server/rust/server.log + exit 1 + - name: Run LDK Node Integration tests + working-directory: ldk-node run: | - cd ldk-node - export TEST_VSS_BASE_URL="http://localhost:8080/vss" + export TEST_VSS_BASE_URL="http://127.0.0.1:8080/vss" RUSTFLAGS="--cfg vss_test" cargo test io::vss_store -- --test-threads=1 RUSTFLAGS="--cfg vss_test" cargo test --test integration_tests_vss -- --test-threads=1 @@ -74,16 +101,15 @@ jobs: mkdir -p vss-server/rust/server cat > vss-server/rust/server/vss-server-config.toml < server.log 2>&1 & + RUSTFLAGS="--cfg noop_authorizer" cargo build --release --no-default-features --bin vss-server + ./target/release/vss-server ./server/vss-server-config.toml > server.log 2>&1 & echo "Server PID: $!" - name: Wait for VSS @@ -96,12 +122,12 @@ jobs: sleep 1 done echo "VSS failed:" - cat vss-server/rust/vss.log + cat vss-server/rust/server.log exit 1 - name: Run LDK Node Integration tests working-directory: ldk-node run: | export TEST_VSS_BASE_URL="http://127.0.0.1:8080/vss" - RUSTFLAGS="--cfg vss_test" cargo test io::vss_store - RUSTFLAGS="--cfg vss_test" cargo test --test integration_tests_vss \ No newline at end of file + RUSTFLAGS="--cfg vss_test" cargo test io::vss_store -- --test-threads=1 + RUSTFLAGS="--cfg vss_test" cargo test --test integration_tests_vss -- --test-threads=1 \ No newline at end of file diff --git a/rust/server/src/main.rs b/rust/server/src/main.rs index 4ba6f43..e730541 100644 --- a/rust/server/src/main.rs +++ b/rust/server/src/main.rs @@ -222,4 +222,4 @@ fn main() { } } }); -} \ No newline at end of file +}