diff --git a/README.md b/README.md index 2e0b6941d..1a88899d0 100644 --- a/README.md +++ b/README.md @@ -67,10 +67,18 @@ In addition to electrs's original configuration options, a few new options are a - `--cors ` - origins allowed to make cross-site request (optional, defaults to none). - `--address-search` - enables the by-prefix address search index. - `--index-unspendables` - enables indexing of provably unspendable outputs. +- `--enable-mining-rest` - enables cached mining-related HTTP endpoints. - `--utxos-limit ` - maximum number of utxos to return per address. - `--electrum-txs-limit ` - maximum number of txs to return per address in the electrum server (does not apply for the http api). - `--electrum-banner ` - welcome banner text for electrum server. +### Mining-related HTTP endpoints + +`GET /block-template` is available only with `--enable-mining-rest`. It proxies +the daemon's `getblocktemplate` template-mode response and caches successful +responses for 15 seconds, invalidating early when electrs indexes a new tip. +Callers that require fresher templates should account for this cache behavior. + Additional options with the `liquid` feature: - `--parent-network ` - the parent network this chain is pegged to. diff --git a/src/config.rs b/src/config.rs index ba21be7fc..85c591e59 100644 --- a/src/config.rs +++ b/src/config.rs @@ -38,6 +38,7 @@ pub struct Config { pub light_mode: bool, pub address_search: bool, pub index_unspendables: bool, + pub enable_mining_rest: bool, pub cors: Option, pub precache_scripts: Option, pub utxos_limit: usize, @@ -212,6 +213,11 @@ impl Config { .long("index-unspendables") .help("Enable indexing of provably unspendable outputs") ) + .arg( + Arg::with_name("enable_mining_rest") + .long("enable-mining-rest") + .help("Enable cached mining-related HTTP endpoints") + ) .arg( Arg::with_name("cors") .long("cors") @@ -531,6 +537,7 @@ impl Config { light_mode: m.is_present("light_mode"), address_search: m.is_present("address_search"), index_unspendables: m.is_present("index_unspendables"), + enable_mining_rest: m.is_present("enable_mining_rest"), cors: m.value_of("cors").map(|s| s.to_string()), precache_scripts: m.value_of("precache_scripts").map(|s| s.to_string()), db_block_cache_mb: value_t_or_exit!(m, "db_block_cache_mb", usize), diff --git a/src/daemon.rs b/src/daemon.rs index ee802b6a7..cb2cc2553 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -108,11 +108,7 @@ fn parse_jsonrpc_reply(mut reply: Value, method: &str, expected_id: u64) -> Resu let msg = err["message"] .as_str() .map_or_else(|| err.to_string(), |s| s.to_string()); - match code { - // RPC_IN_WARMUP -> retry by later reconnection - -28 => bail!(ErrorKind::Connection(err.to_string())), - code => bail!(ErrorKind::RpcError(code, msg, method.to_string())), - } + bail!(ErrorKind::RpcError(code, msg, method.to_string())) } } } @@ -733,6 +729,13 @@ impl Daemon { *conn = conn.reconnect()?; continue; } + Err(e @ Error(ErrorKind::RpcError(-28, _, _), _)) => { + warn!("bitcoind is warming up: {}", e.display_chain()); + self.signal.wait(Duration::from_secs(3), false)?; + let mut conn = self.conn.lock().unwrap(); + *conn = conn.reconnect()?; + continue; + } result => return result, } } @@ -743,6 +746,11 @@ impl Daemon { self.retry_request(method, ¶ms) } + #[trace] + fn request_no_retry(&self, method: &str, params: Value) -> Result { + self.handle_request(method, ¶ms) + } + #[trace] fn retry_reconnect(&self) -> Daemon { // XXX add a max reconnection attempts limit? @@ -930,6 +938,11 @@ impl Daemon { Ok(serde_json::from_value(res).chain_err(|| "invalid getrawmempool reply")?) } + #[trace] + pub fn getblocktemplate(&self, rules: &[&str]) -> Result { + self.request_no_retry("getblocktemplate", json!([{ "rules": rules }])) + } + #[trace] pub fn broadcast(&self, tx: &Transaction) -> Result { self.broadcast_raw(&serialize_hex(tx)) @@ -1080,7 +1093,9 @@ impl Daemon { #[cfg(test)] mod tests { - use super::recycle_due; + use super::{parse_jsonrpc_reply, recycle_due}; + use crate::errors::{Error, ErrorKind}; + use serde_json::json; use std::time::Duration; const COOLDOWN: Duration = Duration::from_secs(30); @@ -1117,4 +1132,20 @@ mod tests { fn expired_after_cooldown_retries() { assert!(recycle_due(secs(600), MAX_AGE, Some(secs(31)), COOLDOWN)); } + + #[test] + fn warmup_error_parses_as_rpc_error() { + let reply = json!({ + "result": null, + "error": { "code": -28, "message": "warming up" }, + "id": 1 + }); + match parse_jsonrpc_reply(reply, "getblocktemplate", 1) { + Err(Error(ErrorKind::RpcError(-28, message, method), _)) => { + assert_eq!(message, "warming up"); + assert_eq!(method, "getblocktemplate"); + } + other => panic!("unexpected getblocktemplate warmup result: {:?}", other), + } + } } diff --git a/src/new_index/mod.rs b/src/new_index/mod.rs index f82291e55..723dd27ab 100644 --- a/src/new_index/mod.rs +++ b/src/new_index/mod.rs @@ -10,7 +10,7 @@ pub mod zmq; pub use self::db::{DBRow, DB}; pub use self::fetch::{BlockEntry, FetchFrom}; pub use self::mempool::Mempool; -pub use self::query::Query; +pub use self::query::{Query, GETBLOCKTEMPLATE_TTL}; pub use self::schema::{ compute_script_hash, parse_hash, ChainQuery, FundingInfo, GetAmountVal, Indexer, ScriptStats, SpendingInfo, SpendingInput, Store, TxHistoryInfo, TxHistoryKey, TxHistoryRow, Utxo, diff --git a/src/new_index/query.rs b/src/new_index/query.rs index ae1fda7c2..20cc0969c 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -1,5 +1,5 @@ use std::collections::{BTreeSet, HashMap}; -use std::sync::{Arc, RwLock, RwLockReadGuard}; +use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard}; use std::time::{Duration, Instant}; use crate::chain::{Network, OutPoint, Transaction, TxOut, Txid}; @@ -10,6 +10,9 @@ use crate::new_index::{ChainQuery, Mempool, ScriptStats, SpendingInput, Utxo}; use crate::util::{is_spendable, BlockId, Bytes, TransactionStatus}; use electrs_macros::trace; +use hyper::body::Bytes as BodyBytes; +use serde_json::Value; +use std::str::FromStr; #[cfg(feature = "liquid")] use crate::{ @@ -18,6 +21,7 @@ use crate::{ }; const FEE_ESTIMATES_TTL: u64 = 60; // seconds +pub const GETBLOCKTEMPLATE_TTL: u64 = 15; // seconds const CONF_TARGETS: [u16; 28] = [ 1u16, 2u16, 3u16, 4u16, 5u16, 6u16, 7u16, 8u16, 9u16, 10u16, 11u16, 12u16, 13u16, 14u16, 15u16, @@ -31,10 +35,17 @@ pub struct Query { config: Arc, cached_estimates: RwLock<(HashMap, Option)>, cached_relayfee: RwLock>, + cached_block_template: Mutex>, #[cfg(feature = "liquid")] asset_db: Option>>, } +struct CachedBlockTemplate { + tip: crate::chain::BlockHash, + fetched_at: Instant, + body: BodyBytes, +} + impl Query { #[cfg(not(feature = "liquid"))] pub fn new( @@ -50,6 +61,7 @@ impl Query { config, cached_estimates: RwLock::new((HashMap::new(), None)), cached_relayfee: RwLock::new(None), + cached_block_template: Mutex::new(None), } } @@ -101,6 +113,43 @@ impl Query { Ok(result) } + #[trace] + pub fn getblocktemplate(&self) -> Result { + let tip = self.chain.best_hash(); + + let mut cache = self.cached_block_template.lock().unwrap(); + if let Some(cached) = cache.as_ref() { + if cached.tip == tip + && cached.fetched_at.elapsed() < Duration::from_secs(GETBLOCKTEMPLATE_TTL) + { + return Ok(cached.body.clone()); + } + } + + let value = self + .daemon + .getblocktemplate(block_template_rules(self.config.network_type))?; + let body = BodyBytes::from( + serde_json::to_string(&value).chain_err(|| "failed to serialize getblocktemplate")?, + ); + + match block_template_tip(&value) { + Ok(tip) => { + *cache = Some(CachedBlockTemplate { + tip, + fetched_at: Instant::now(), + body: body.clone(), + }); + } + Err(err) => { + warn!("not caching getblocktemplate response: {}", err); + *cache = None; + } + } + + Ok(body) + } + #[trace] pub fn utxo(&self, scripthash: &[u8]) -> Result> { let mut utxos = self.chain.utxo(scripthash, self.config.utxos_limit)?; @@ -278,6 +327,7 @@ impl Query { asset_db, cached_estimates: RwLock::new((HashMap::new(), None)), cached_relayfee: RwLock::new(None), + cached_block_template: Mutex::new(None), } } @@ -311,3 +361,40 @@ impl Query { Ok((total_num, results)) } } + +#[cfg(not(feature = "liquid"))] +fn block_template_rules(network: Network) -> &'static [&'static str] { + match network { + Network::Signet => &["segwit", "signet"], + _ => &["segwit"], + } +} + +#[cfg(feature = "liquid")] +fn block_template_rules(_network: Network) -> &'static [&'static str] { + &["segwit"] +} + +fn block_template_tip(value: &Value) -> Result { + let previousblockhash = value + .get("previousblockhash") + .and_then(|value| value.as_str()) + .chain_err(|| "getblocktemplate response missing previousblockhash")?; + crate::chain::BlockHash::from_str(previousblockhash) + .chain_err(|| "invalid getblocktemplate previousblockhash") +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + #[test] + fn block_template_tip_parses_previousblockhash() { + let hash = "0000000000000000000000000000000000000000000000000000000000000000"; + let value = json!({ "previousblockhash": hash }); + assert_eq!(super::block_template_tip(&value).unwrap().to_string(), hash); + + assert!(super::block_template_tip(&json!({})).is_err()); + assert!(super::block_template_tip(&json!({ "previousblockhash": "not a hash" })).is_err()); + } +} diff --git a/src/rest.rs b/src/rest.rs index c0ab6954a..850a3a79a 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -1150,6 +1150,15 @@ fn handle_request( json_response(query.estimate_fee_map(), TTL_SHORT) } + (&Method::GET, Some(&"block-template"), None, None, None, None) => { + if !config.enable_mining_rest { + return Err(HttpError::forbidden( + "mining REST endpoints are disabled".to_string(), + )); + } + getblocktemplate_response(query.getblocktemplate()) + } + #[cfg(feature = "liquid")] (&Method::GET, Some(&"assets"), Some(&"registry"), None, None, None) => { let start_index: usize = query_params @@ -1291,14 +1300,86 @@ where } fn json_response(value: T, ttl: u32) -> Result>, HttpError> { + json_response_with_status(value, StatusCode::OK, ttl) +} + +fn json_response_with_status( + value: T, + status: StatusCode, + ttl: u32, +) -> Result>, HttpError> { let value = serde_json::to_string(&value)?; Ok(Response::builder() + .status(status) .header("Content-Type", "application/json") .header("Cache-Control", format!("public, max-age={:}", ttl)) .body(Full::new(Bytes::from(value))) .unwrap()) } +fn json_response_no_store( + value: T, + status: StatusCode, +) -> Result>, HttpError> { + let value = serde_json::to_string(&value)?; + Ok(Response::builder() + .status(status) + .header("Content-Type", "application/json") + .header("Cache-Control", "no-store") + .body(Full::new(Bytes::from(value))) + .unwrap()) +} + +fn json_bytes_response_no_store(body: Bytes) -> Result>, HttpError> { + Ok(Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "application/json") + .header("Cache-Control", "no-store") + .body(Full::new(body)) + .unwrap()) +} + +fn text_response_no_store>( + status: StatusCode, + message: T, +) -> Result>, HttpError> { + Ok(Response::builder() + .status(status) + .header("Content-Type", "text/plain") + .header("Cache-Control", "no-store") + .body(Full::new(message.into())) + .unwrap()) +} + +fn getblocktemplate_response( + result: errors::Result, +) -> Result>, HttpError> { + match result { + Ok(body) => json_bytes_response_no_store(body), + Err(err) => { + if let Some((code, message)) = getblocktemplate_rpc_error(&err) { + return json_response_no_store( + json!({ "error": { "code": code, "message": message } }), + StatusCode::BAD_GATEWAY, + ); + } + if let errors::ErrorKind::Connection(message) = err.kind() { + return text_response_no_store(StatusCode::BAD_GATEWAY, message.clone()); + } + Err(HttpError::from(err)) + } + } +} + +fn getblocktemplate_rpc_error(err: &errors::Error) -> Option<(i64, String)> { + match err.kind() { + errors::ErrorKind::RpcError(code, message, method) if method == "getblocktemplate" => { + Some((*code, message.clone())) + } + _ => None, + } +} + #[trace] fn blocks(query: &Query, start_height: Option) -> Result>, HttpError> { let mut values = Vec::new(); @@ -1381,6 +1462,10 @@ impl HttpError { fn not_found(msg: String) -> Self { HttpError(StatusCode::NOT_FOUND, msg) } + + fn forbidden(msg: String) -> Self { + HttpError(StatusCode::FORBIDDEN, msg) + } } impl From for HttpError { @@ -1455,7 +1540,9 @@ impl From for HttpError { #[cfg(test)] mod tests { - use crate::rest::HttpError; + use crate::{errors, rest::HttpError}; + use http_body_util::BodyExt; + use hyper::StatusCode; use serde_json::Value; use std::collections::HashMap; @@ -1520,4 +1607,63 @@ mod tests { assert!(err.is_err()); } + + #[test] + fn test_getblocktemplate_rpc_error() { + let err: errors::Error = errors::ErrorKind::RpcError( + -8, + "getblocktemplate must be called with the segwit rule set".to_string(), + "getblocktemplate".to_string(), + ) + .into(); + assert_eq!( + super::getblocktemplate_rpc_error(&err), + Some(( + -8, + "getblocktemplate must be called with the segwit rule set".to_string() + )) + ); + + let other_method: errors::Error = + errors::ErrorKind::RpcError(-5, "Block not found".to_string(), "getblock".to_string()) + .into(); + assert_eq!(super::getblocktemplate_rpc_error(&other_method), None); + } + + #[tokio::test] + async fn test_getblocktemplate_response_no_store_errors() { + let warmup: errors::Error = errors::ErrorKind::RpcError( + -28, + "warming up".to_string(), + "getblocktemplate".to_string(), + ) + .into(); + let response = super::getblocktemplate_response(Err(warmup)).unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("no-store") + ); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(body["error"]["code"].as_i64(), Some(-28)); + assert_eq!(body["error"]["message"].as_str(), Some("warming up")); + + let connection: errors::Error = + errors::ErrorKind::Connection("daemon unavailable".to_string()).into(); + let response = super::getblocktemplate_response(Err(connection)).unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("no-store") + ); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&body[..], b"daemon unavailable"); + } } diff --git a/tests/common.rs b/tests/common.rs index 56e05c595..99a7730fd 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -107,6 +107,7 @@ impl TestRunner { light_mode: false, address_search: true, index_unspendables: false, + enable_mining_rest: true, cors: None, precache_scripts: None, utxos_limit: 100, diff --git a/tests/rest.rs b/tests/rest.rs index 37b95855a..6d325f25a 100644 --- a/tests/rest.rs +++ b/tests/rest.rs @@ -450,6 +450,43 @@ fn test_rest_mempool() -> Result<()> { Ok(()) } +#[test] +fn test_rest_getblocktemplate() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); + + let tip = tester.get_best_block_hash()?; + let response = get(rest_addr, "/block-template")?; + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("no-store") + ); + let template: Value = response.into_body().read_json()?; + assert_eq!( + template["previousblockhash"].as_str(), + Some(tip.to_string().as_str()) + ); + assert!(template["transactions"].is_array()); + assert!(template["version"].is_i64() || template["version"].is_u64()); + assert!(template["rules"].is_array()); + assert!(template["bits"].is_string()); + + let cached_template = get_json(rest_addr, "/block-template")?; + assert_eq!(cached_template, template); + + let new_tip = tester.mine()?; + let updated_template = get_json(rest_addr, "/block-template")?; + assert_eq!( + updated_template["previousblockhash"].as_str(), + Some(new_tip.to_string().as_str()) + ); + + rest_handle.stop(); + Ok(()) +} + #[test] fn test_rest_broadcast_tx() -> Result<()> { let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap();