Merge pull request #2 from moneydevkit/austin_mdk-672_vss-support

Wire up VSS storage and standalone test harness

Author: amackillop

.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
         run: nix flake check --print-build-logs
 
       - name: Run integration tests
-        run: nix build .#integration-test --print-build-logs
+        run: nix run .#integration-test --print-build-logs
 
   build:
     name: Build Static (${{ matrix.arch }})

flake.lock

@@ -10,6 +10,10 @@
     };
     crane.url = "github:ipetkov/crane";
     nixpkgs-unstable.url = "github:nixos/nixpkgs/e6f23dc08d3624daab7094b701aa3954923c6bbb";
+    vss-server = {
+      url = "github:lightningdevkit/vss-server";
+      flake = false;
+    };
   };
 
   outputs =
@@ -20,6 +24,7 @@
       fenix,
       crane,
       nixpkgs-unstable,
+      vss-server,
     }:
     flake-utils.lib.eachDefaultSystem (
       system:
@@ -107,6 +112,35 @@
             Entrypoint = [ "/bin/mdk-server" ];
           };
         };
+
+        # VSS server (lightningdevkit/vss-server) for integration tests.
+        # Built with noop_authorizer so no JWT/sig config is needed.
+        vssSrc = craneLib.cleanCargoSource "${vss-server}/rust";
+        vssArgs = {
+          src = vssSrc;
+          pname = "vss-server";
+          version = "0.1.0";
+          strictDeps = true;
+          nativeBuildInputs = [
+            pkgs.protobuf
+            pkgs.pkg-config
+            pkgs.autoPatchelfHook
+          ];
+          buildInputs = [
+            pkgs.openssl
+            pkgs.stdenv.cc.cc.lib
+          ];
+          cargoExtraArgs = "--no-default-features";
+          CARGO_BUILD_RUSTFLAGS = "--cfg noop_authorizer";
+        };
+        vssCargoArtifacts = craneLib.buildDepsOnly vssArgs;
+        vss = craneLib.buildPackage (
+          vssArgs
+          // {
+            cargoArtifacts = vssCargoArtifacts;
+            doCheck = false;
+          }
+        );
       in
       {
         packages = {
@@ -117,13 +151,28 @@
               doCheck = false; # Tests run in checks.test
             }
           );
-          integration-test = craneLib.cargoNextest (
-            commonArgs
-            // {
-              inherit cargoArtifacts;
-              cargoNextestExtraArgs = "--test integration";
-            }
-          );
+          # Wrapper script that runs `just integration-test` with all
+          # dependencies available.  Intended for CI (`nix run .#integration-test`).
+          integration-test = pkgs.writeShellApplication {
+            name = "integration-test";
+            runtimeInputs = [
+              buildToolchain
+              pkgs.cargo-nextest
+              pkgs.just
+              pkgs.postgresql_16
+              pkgs.curl
+              pkgs.protobuf
+              pkgsUnstable.bitcoind
+              vss
+            ];
+            text = ''
+              export BITCOIND_EXE="${pkgsUnstable.bitcoind}/bin/bitcoind"
+              export VSS_EXE="${vss}/bin/vss-server"
+              export NIX_SYSTEM="${system}"
+              just integration-test "$@"
+            '';
+          };
+          inherit vss;
         }
         // lib.optionalAttrs isLinux {
           static = staticBin;
@@ -161,10 +210,12 @@
             jq
             unixtools.xxd
             microsocks
+            postgresql_16
           ];
 
           env = {
             BITCOIND_EXE = "${pkgsUnstable.bitcoind}/bin/bitcoind";
+            VSS_EXE = "${vss}/bin/vss-server";
             NIX_SYSTEM = system;
           };
 

flake.nix

@@ -23,17 +23,80 @@ clippy:
 unit-test:
     nix build .#checks.{{system}}.test
 
-# Run integration tests
+# Run integration tests (starts ephemeral PostgreSQL + VSS, tears down after)
 integration-test *args:
-    cargo nextest run --test integration {{args}}
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    : "${VSS_EXE:?VSS_EXE not set (use nix develop)}"
+
+    # -- Ephemeral PostgreSQL -----------------------------------------------
+    pg_dir=$(mktemp -d)
+    pg_port=$(shuf -i 10000-60000 -n 1)
+    pg_log="$pg_dir/pg.log"
+
+    cleanup() {
+      echo "==> Tearing down..."
+      [ -n "${VSS_PID:-}" ] && kill "$VSS_PID" 2>/dev/null && wait "$VSS_PID" 2>/dev/null || true
+      pg_ctl -D "$pg_dir/data" -m immediate stop 2>/dev/null || true
+      rm -rf "$pg_dir"
+    }
+    trap cleanup EXIT
+
+    echo "==> Starting ephemeral PostgreSQL on port $pg_port..."
+    initdb -D "$pg_dir/data" --no-locale --encoding=UTF8 -A trust >/dev/null
+    pg_ctl -D "$pg_dir/data" -l "$pg_log" -o "-p $pg_port -k $pg_dir -h 127.0.0.1" start >/dev/null
+    for i in $(seq 1 30); do
+      pg_isready -h 127.0.0.1 -p "$pg_port" -q && break
+      sleep 0.1
+    done
+
+    # -- Start VSS ----------------------------------------------------------
+    vss_port=$(shuf -i 10000-60000 -n 1)
+    vss_config="$pg_dir/vss-config.toml"
+    cat > "$vss_config" << TOML
+    [server_config]
+    bind_address = "127.0.0.1:$vss_port"
+
+    [postgresql_config]
+    username = "$USER"
+    password = ""
+    address = "127.0.0.1:$pg_port"
+    default_database = "postgres"
+    vss_database = "vss_test"
+    TOML
+
+    vss_log="$pg_dir/vss.log"
+    echo "==> Starting VSS on port $vss_port..."
+    "$VSS_EXE" "$vss_config" >"$vss_log" 2>&1 &
+    VSS_PID=$!
+
+    for i in $(seq 1 30); do
+      # VSS has no health endpoint; probe putObjects (returns 400 on empty body = alive)
+      code=$(curl -sf -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:$vss_port/vss/putObjects" 2>/dev/null || true)
+      [ "$code" = "400" ] && break
+      sleep 0.2
+    done
+    echo "==> VSS ready at http://127.0.0.1:$vss_port/vss"
+
+    # -- Run tests ----------------------------------------------------------
+    if MDK_VSS_URL="http://127.0.0.1:$vss_port/vss" \
+      cargo nextest run --test integration {{args}}; then
+      :
+    else
+      echo ""
+      echo "==> VSS server log (on failure):"
+      # cat "$vss_log" skip this for now
+      exit 1
+    fi
 
 # Auto-fix lint issues
 fix:
     cargo clippy --all-targets --fix --allow-dirty --allow-staged
 
-# Run tests (cargo, all)
+# Run tests (unit + doc; use `just integration-test` for integration tests)
 test *args:
-    cargo nextest run {{args}}
+    cargo nextest run --bin mdk-server {{args}}
 
 # Run the server
 run *args:
@@ -517,6 +580,7 @@ dev-config:
     MDK_LSP_NODE_ID=${n1_pubkey}
     MDK_LSP_ADDRESS=127.0.0.1:${n1_p2p}
     MDK_API_BASE_URL=${MDK_API_BASE_URL:-http://localhost:3900/rpc}
+    MDK_VSS_URL=${MDK_VSS_URL:-http://localhost:9999/vss}
     MDK_WEBHOOK_SECRET=${MDK_WEBHOOK_SECRET:-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}
     MDK_HTTP_PASSWORD_FULL=${MDK_HTTP_PASSWORD_FULL:-dev_full_password}
     MDK_HTTP_PASSWORD_READ_ONLY=${MDK_HTTP_PASSWORD_READ_ONLY:-dev_readonly_password}

justfile

@@ -145,6 +145,7 @@ pub struct LspInfra {
     pub lsp_node_id: &'static str,
     pub lsp_address: &'static str,
     pub mdk_api_base_url: &'static str,
+    pub vss_url: &'static str,
 }
 
 impl LspInfra {
@@ -155,12 +156,14 @@ impl LspInfra {
                 lsp_node_id: "02a63339cc6b913b6330bd61b2f469af8785a6011a6305bb102298a8e76697473b",
                 lsp_address: "lsp.moneydevkit.com:9735",
                 mdk_api_base_url: "https://moneydevkit.com/rpc",
+                vss_url: "https://vss.moneydevkit.com/vss",
             }),
             Network::Signet => Some(LspInfra {
                 chain_source: ChainSource::Esplora("https://mutinynet.com/api"),
                 lsp_node_id: "03fd9a377576df94cc7e458471c43c400630655083dee89df66c6ad38d1b7acffd",
                 lsp_address: "lsp.staging.moneydevkit.com:9735",
                 mdk_api_base_url: "https://staging.moneydevkit.com/rpc",
+                vss_url: "https://vss.staging.moneydevkit.com/vss",
             }),
             _ => None,
         }
@@ -184,6 +187,7 @@ pub enum NetworkInfra {
         lsp_node_id: String,
         lsp_address: String,
         mdk_api_base_url: String,
+        vss_url: String,
     },
 }
 
@@ -209,6 +213,7 @@ impl NetworkInfra {
                     lsp_node_id: env_required("MDK_LSP_NODE_ID")?,
                     lsp_address: env_required("MDK_LSP_ADDRESS")?,
                     mdk_api_base_url: env_required("MDK_API_BASE_URL")?,
+                    vss_url: env_required("MDK_VSS_URL")?,
                 })
             }
         }
@@ -223,26 +228,33 @@ impl NetworkInfra {
 
     pub fn lsp_node_id(&self) -> &str {
         match self {
-            NetworkInfra::Production(i) => i.lsp_node_id,
+            NetworkInfra::Production(lsp_infra) => lsp_infra.lsp_node_id,
             NetworkInfra::Regtest { lsp_node_id, .. } => lsp_node_id,
         }
     }
 
     pub fn lsp_address(&self) -> &str {
         match self {
-            NetworkInfra::Production(i) => i.lsp_address,
+            NetworkInfra::Production(lsp_infra) => lsp_infra.lsp_address,
             NetworkInfra::Regtest { lsp_address, .. } => lsp_address,
         }
     }
 
     pub fn mdk_api_base_url(&self) -> &str {
         match self {
-            NetworkInfra::Production(i) => i.mdk_api_base_url,
+            NetworkInfra::Production(lsp_infra) => lsp_infra.mdk_api_base_url,
             NetworkInfra::Regtest {
                 mdk_api_base_url, ..
             } => mdk_api_base_url,
         }
     }
+
+    pub fn vss_url(&self) -> &str {
+        match self {
+            NetworkInfra::Production(lsp_infra) => lsp_infra.vss_url,
+            NetworkInfra::Regtest { vss_url, .. } => vss_url,
+        }
+    }
 }
 
 fn env_required(name: &str) -> io::Result<String> {

src/config.rs

@@ -10,6 +10,7 @@ mod time;
 mod types;
 mod webhook;
 
+use std::collections::HashMap;
 use std::net::ToSocketAddrs;
 use std::path::PathBuf;
 use std::str::FromStr;
@@ -18,6 +19,8 @@ use std::sync::Arc;
 use clap::Parser;
 use hex::FromHex;
 use ldk_node::bip39::Mnemonic;
+use ldk_node::bitcoin::hashes::sha256;
+use ldk_node::bitcoin::hashes::Hash;
 use ldk_node::bitcoin::secp256k1::PublicKey;
 use ldk_node::config::Config as LdkNodeConfig;
 use ldk_node::lightning::ln::msgs::SocketAddress;
@@ -113,6 +116,13 @@ fn main() {
     };
 
     let network_dir: PathBuf = storage_dir.join(format!("{}", config_file.network));
+    if let Err(e) = std::fs::create_dir_all(&network_dir) {
+        eprintln!(
+            "Failed to create data directory {}: {e}",
+            network_dir.display()
+        );
+        std::process::exit(1);
+    }
 
     logger::init(config_file.log_level);
 
@@ -213,18 +223,28 @@ fn main() {
         error!("Invalid MDK_MNEMONIC: {e}");
         std::process::exit(1);
     });
-    builder.set_entropy_bip39_mnemonic(mnemonic, None);
-    info!("Wallet seed derived from MDK_MNEMONIC");
-
-    let node = match builder.build() {
+    builder.set_entropy_bip39_mnemonic(mnemonic.clone(), None);
+
+    let store_id = derive_vss_identifier(&mnemonic);
+    info!(
+        "VSS store: {} (store_id={}...)",
+        infra.vss_url(),
+        &store_id[..16]
+    );
+
+    let node = match builder.build_with_vss_store_and_fixed_headers(
+        infra.vss_url().to_string(),
+        store_id,
+        HashMap::new(),
+    ) {
         Ok(node) => Arc::new(node),
         Err(e) => {
             error!("Failed to build LDK Node: {e}");
             std::process::exit(1);
         }
     };
 
-    let db_path = network_dir.join("ldk_server_data.sqlite");
+    let db_path = network_dir.join("mdkd.sqlite");
     let metadata_store = match InvoiceMetadataStore::new(&db_path) {
         Ok(store) => Arc::new(store),
         Err(e) => {
@@ -359,3 +379,8 @@ fn main() {
     node.stop().expect("Shutdown should always succeed.");
     info!("Shutdown complete.");
 }
+
+fn derive_vss_identifier(mnemonic: &Mnemonic) -> String {
+    let mnemonic_phrase = mnemonic.to_string();
+    sha256::Hash::hash(mnemonic_phrase.as_bytes()).to_string()
+}