From 5790044da6ea6c3342e9410c59228216512760fa Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 19 May 2026 15:20:35 +0100 Subject: [PATCH 1/6] adds engine api --- Cargo.lock | 378 ++++++++--- Cargo.toml | 1 + crates/common/src/lib.rs | 20 +- crates/common/src/spine.rs | 20 +- crates/common/src/spine/messages.rs | 236 +++++++ crates/engine/Cargo.toml | 30 + crates/engine/src/client.rs | 297 +++++++++ crates/engine/src/error.rs | 17 + crates/engine/src/http.rs | 424 +++++++++++++ crates/engine/src/ipc.rs | 256 ++++++++ crates/engine/src/jwt.rs | 111 ++++ crates/engine/src/lib.rs | 18 + crates/engine/src/req_handlers.rs | 173 ++++++ crates/engine/src/resp_handlers.rs | 391 ++++++++++++ crates/engine/src/tile.rs | 122 ++++ crates/engine/src/types.rs | 929 ++++++++++++++++++++++++++++ memory/MEMORY.md | 3 + memory/feedback_best_method.md | 11 + 18 files changed, 3339 insertions(+), 98 deletions(-) create mode 100644 crates/engine/Cargo.toml create mode 100644 crates/engine/src/client.rs create mode 100644 crates/engine/src/error.rs create mode 100644 crates/engine/src/http.rs create mode 100644 crates/engine/src/ipc.rs create mode 100644 crates/engine/src/jwt.rs create mode 100644 crates/engine/src/lib.rs create mode 100644 crates/engine/src/req_handlers.rs create mode 100644 crates/engine/src/resp_handlers.rs create mode 100644 crates/engine/src/tile.rs create mode 100644 crates/engine/src/types.rs create mode 100644 memory/MEMORY.md create mode 100644 memory/feedback_best_method.md diff --git a/Cargo.lock b/Cargo.lock index e4450c4..00f8546 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,9 +143,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "asn1-rs" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -343,15 +343,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.16.3" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" dependencies = [ "aws-lc-sys", "zeroize", @@ -359,9 +359,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.40.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" dependencies = [ "cc", "cmake", @@ -369,6 +369,58 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -424,6 +476,15 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitcode" version = "0.6.9" @@ -450,15 +511,15 @@ dependencies = [ [[package]] name = "bitcoin-io" -version = "0.1.4" +version = "0.1.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953" +checksum = "11301df0b06f22dea7bb1916403fdd88a371031e495c49b8f96931b28189e175" [[package]] name = "bitcoin_hashes" -version = "0.14.1" +version = "0.14.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26ec84b80c482df901772e931a9a681e26a1b9ee2302edeff23cb30328745c8b" +checksum = "0c9901a56e133a1fc86eeb1113e2591f45f4682451ca893bff494d2f88918e3f" dependencies = [ "bitcoin-io", "hex-conservative", @@ -560,9 +621,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" @@ -625,9 +686,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.61" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -734,9 +795,9 @@ dependencies = [ [[package]] name = "compact_str" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" dependencies = [ "castaway", "cfg-if", @@ -1149,9 +1210,9 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -1210,9 +1271,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "elliptic-curve" @@ -1553,9 +1614,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" @@ -1652,9 +1713,9 @@ checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" [[package]] name = "glam" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f70749695b063ecbf6b62949ccccde2e733ec3ecbbd71d467dca4e5c6c97cca0" +checksum = "8fb167719045debebe9f532320accc7b5c993c5a3b813f5696a11d5ca7bdc57b" [[package]] name = "glob" @@ -1757,9 +1818,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "hashlink" @@ -1906,9 +1967,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ "bytes", "itoa", @@ -1943,6 +2004,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.3.0" @@ -1951,9 +2018,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "eb92f162bf56536459fc83c79b974bb12837acfed43d6bc370a7916d0ae15ecc" dependencies = [ "atomic-waker", "bytes", @@ -1963,6 +2030,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -2190,7 +2258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -2302,6 +2370,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2320,9 +2397,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.97" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" dependencies = [ "cfg-if", "futures-util", @@ -2548,9 +2625,9 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c7892c221730ba55f7196e98b0b8ba5e04b4155651736036628e9f73ed6fc3" +checksum = "9525f3831544f7ae497bde79adf114ef127b0fbbb97edbbf692a80408636421c" dependencies = [ "asn1_der", "bs58", @@ -2558,7 +2635,7 @@ dependencies = [ "hkdf", "k256", "multihash", - "quick-protobuf", + "prost", "rand 0.8.6", "sha2", "thiserror 2.0.18", @@ -2706,7 +2783,7 @@ dependencies = [ "rustls-webpki", "thiserror 2.0.18", "x509-parser 0.17.0", - "yasna", + "yasna 0.5.2", ] [[package]] @@ -2726,9 +2803,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" dependencies = [ "libc", ] @@ -2772,9 +2849,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" dependencies = [ "value-bag", ] @@ -2814,11 +2891,17 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "memmap2" @@ -2847,6 +2930,12 @@ dependencies = [ "libmimalloc-sys", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3073,9 +3162,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-format" @@ -3214,18 +3303,18 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project" -version = "1.1.12" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.12" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -3403,7 +3492,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.11+spec-1.1.0", + "toml_edit 0.25.12+spec-1.1.0", ] [[package]] @@ -3438,6 +3527,29 @@ dependencies = [ "syn", ] +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quanta" version = "0.12.6" @@ -3681,21 +3793,21 @@ dependencies = [ "ring", "rustls-pki-types", "time", - "yasna", + "yasna 0.5.2", ] [[package]] name = "rcgen" -version = "0.14.7" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b99e0098aa4082912d4c649628623db6aba77335e4f4569ff5083a6448b32e" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" dependencies = [ "pem", "ring", "rustls-pki-types", "time", "x509-parser 0.18.1", - "yasna", + "yasna 0.6.0", ] [[package]] @@ -3844,7 +3956,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4008,9 +4120,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -4019,6 +4131,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_spanned" version = "0.6.9" @@ -4028,6 +4151,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yml" version = "0.0.12" @@ -4196,7 +4331,7 @@ dependencies = [ "libc", "quinn-proto", "rand 0.8.6", - "rcgen 0.14.7", + "rcgen 0.14.8", "ring", "rustc-hash", "rustls", @@ -4284,6 +4419,28 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "silver_engine" +version = "0.0.1" +dependencies = [ + "axum", + "base64 0.22.1", + "flux", + "hex", + "hmac", + "httparse", + "mio", + "rustc-hash", + "serde", + "serde_json", + "sha2", + "silver_common", + "thiserror 1.0.69", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "silver_gossip" version = "0.0.1" @@ -4322,7 +4479,7 @@ dependencies = [ "quinn-proto", "rand 0.8.6", "rand_core 0.6.4", - "rcgen 0.14.7", + "rcgen 0.14.8", "ring", "rustls", "secp256k1", @@ -4333,7 +4490,7 @@ dependencies = [ "tracing", "tracing-subscriber", "x509-parser 0.17.0", - "yasna", + "yasna 0.5.2", ] [[package]] @@ -4487,9 +4644,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "str_stack" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" +checksum = "7f446288b699d66d0fd2e30d1cfe7869194312524b3b9252594868ed26ef056a" [[package]] name = "strsim" @@ -4559,6 +4716,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.2" @@ -4736,14 +4899,16 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.2" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2 0.6.3", "tokio-macros", "windows-sys 0.61.2", @@ -4819,14 +4984,14 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.11+spec-1.1.0" +version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow 1.0.2", + "winnow 1.0.3", ] [[package]] @@ -4835,7 +5000,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.2", + "winnow 1.0.3", ] [[package]] @@ -4844,6 +5009,28 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -5113,9 +5300,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.120" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" dependencies = [ "cfg-if", "once_cell", @@ -5126,9 +5313,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.70" +version = "0.4.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af934872acec734c2d80e6617bbb5ff4f12b052dd8e6332b0817bce889516084" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" dependencies = [ "js-sys", "wasm-bindgen", @@ -5136,9 +5323,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.120" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5146,9 +5333,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.120" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" dependencies = [ "bumpalo", "proc-macro2", @@ -5159,9 +5346,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.120" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" dependencies = [ "unicode-ident", ] @@ -5202,9 +5389,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.97" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" dependencies = [ "js-sys", "wasm-bindgen", @@ -5409,6 +5596,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.60.2" @@ -5663,9 +5859,9 @@ dependencies = [ [[package]] name = "winnow" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] @@ -5830,6 +6026,16 @@ dependencies = [ "time", ] +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec", + "time", +] + [[package]] name = "yoke" version = "0.8.2" @@ -5875,9 +6081,9 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] diff --git a/Cargo.toml b/Cargo.toml index 7daf24f..b5224c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/discovery", "crates/e2e", "crates/gossip", + "crates/engine", "crates/network", "crates/peer", "crates/surfer", diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index b4670d2..f151f27 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -18,14 +18,18 @@ pub use crate::{ }, spine::{ ALL_PROTOCOLS, AcquiredRead as TRead, BeaconStateEvent, Consumer as TConsumer, - DataColumnsAvailable, Error as TCacheError, GossipMsgOut, IpBytes, MULTISTREAM_V1, - MultiProducer as TMultiProducer, NewGossipMsg, P2pSend, P2pStreamId, PeerControl, - PeerEvent, PeerStatus, Producer as TProducer, REJECT_RESPONSE, RPC_PROTOCOLS, - RandomAccessConsumer as TRandomAccess, RejectSource, Reservation as TReservation, - RpcInbound, RpcMsg, RpcOutbound, RpcRequest, RpcRequestInbound, RpcRequestOutbound, - RpcResponse, RpcResponseInbound, RpcResponseOutbound, RpcSeverity, SilverSpine, - SilverSpineProducers, StreamProtocol, SyncUpdate, TCache, TCacheProducer, TCacheRead, - TCacheRef, + DataColumnsAvailable, ELSyncStatus, EngineFcuReq, EngineFcuResp, EngineGetBlobsReq, + EngineGetBlobsResp, EngineGetPayloadBodiesByHashReq, EngineGetPayloadBodiesByRangeReq, + EngineGetPayloadBodiesResp, EngineGetPayloadReq, EngineGetPayloadResp, EngineHealthEvent, + EngineNewPayloadReq, EngineNewPayloadResp, EngineRawReq, EngineRawResp, EngineReq, + EngineResp, Error as TCacheError, GossipMsgOut, IpBytes, MAX_BLOBS_PER_BLOCK, + MULTISTREAM_V1, MultiProducer as TMultiProducer, NewGossipMsg, P2pSend, P2pStreamId, + PayloadValidationStatus, PeerControl, PeerEvent, PeerStatus, Producer as TProducer, + REJECT_RESPONSE, RPC_PROTOCOLS, RandomAccessConsumer as TRandomAccess, RejectSource, + Reservation as TReservation, RpcInbound, RpcMsg, RpcOutbound, RpcRequest, + RpcRequestInbound, RpcRequestOutbound, RpcResponse, RpcResponseInbound, + RpcResponseOutbound, RpcSeverity, SilverSpine, SilverSpineProducers, StreamProtocol, + SyncUpdate, TCache, TCacheProducer, TCacheRead, TCacheRef, }, util::{create_self_signed_certificate, decode_varint, encode_varint}, wheel::Wheel, diff --git a/crates/common/src/spine.rs b/crates/common/src/spine.rs index 267e860..56c173d 100644 --- a/crates/common/src/spine.rs +++ b/crates/common/src/spine.rs @@ -2,10 +2,15 @@ use flux::{communication::ShmemData, spine::SpineQueue, spine_derive::from_spine, tile::TileInfo}; pub use messages::{ - BeaconStateEvent, DataColumnsAvailable, GossipMsgOut, IpBytes, NewGossipMsg, P2pSend, - PeerControl, PeerEvent, PeerStatus, RejectSource, RpcInbound, RpcMsg, RpcOutbound, RpcRequest, - RpcRequestInbound, RpcRequestOutbound, RpcResponse, RpcResponseInbound, RpcResponseOutbound, - RpcSeverity, SyncUpdate, + BeaconStateEvent, DataColumnsAvailable, ELSyncStatus, EngineFcuReq, EngineFcuResp, + EngineGetBlobsReq, EngineGetBlobsResp, EngineGetPayloadBodiesByHashReq, + EngineGetPayloadBodiesByRangeReq, EngineGetPayloadBodiesResp, EngineGetPayloadReq, + EngineGetPayloadResp, EngineHealthEvent, EngineNewPayloadReq, EngineNewPayloadResp, + EngineRawReq, EngineRawResp, EngineReq, EngineResp, GossipMsgOut, IpBytes, + MAX_BLOBS_PER_BLOCK, NewGossipMsg, P2pSend, PayloadValidationStatus, PeerControl, PeerEvent, + PeerStatus, RejectSource, RpcInbound, RpcMsg, RpcOutbound, RpcRequest, RpcRequestInbound, + RpcRequestOutbound, RpcResponse, RpcResponseInbound, RpcResponseOutbound, RpcSeverity, + SyncUpdate, }; pub use stream_id::P2pStreamId; pub use stream_protocol::{ @@ -45,4 +50,11 @@ pub struct SilverSpine { pub data_columns: SpineQueue, #[queue(size(2usize.pow(10)))] pub sync_target: SpineQueue, + + #[queue(size(2usize.pow(10)))] + pub engine_reqs: SpineQueue, + #[queue(size(2usize.pow(10)))] + pub engine_resps: SpineQueue, + #[queue(size(2usize.pow(8)))] + pub engine_health: SpineQueue, } diff --git a/crates/common/src/spine/messages.rs b/crates/common/src/spine/messages.rs index 2cc16f9..e1cf059 100644 --- a/crates/common/src/spine/messages.rs +++ b/crates/common/src/spine/messages.rs @@ -596,6 +596,242 @@ pub enum BeaconStateEvent { BlockRejected { block_root: [u8; 32], source: RejectSource }, } +/// Maximum blob commitments per block (Fulu target; increase as the spec +/// evolves). +pub const MAX_BLOBS_PER_BLOCK: usize = 9; + +/// Maximum number of block hashes in a single `getPayloadBodiesByHash` request. +pub const MAX_PAYLOAD_BODIES_PER_REQ: usize = 128; + +/// Execution-payload validation result returned by the EL. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum PayloadValidationStatus { + Valid = 0, + Invalid = 1, + Syncing = 2, + Accepted = 3, +} + +/// A single withdrawal, inlined into `EngineFcuReq` payload attributes. +/// Field order avoids interior padding (all u64s first, then the 20-byte +/// address). +#[derive(Clone, Copy, Debug, Default)] +#[repr(C)] +pub struct WithdrawalInline { + pub index: u64, + pub validator_index: u64, + pub amount: u64, + pub address: [u8; 20], +} + +/// `engine_forkchoiceUpdatedV3` request. Fully inline — no TCache needed. +/// +/// When `has_attrs` is false the `attrs_*` fields are ignored. +/// `attrs_withdrawal_count` gives the number of valid entries in +/// `attrs_withdrawals`; the remainder are zero-filled. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineFcuReq { + pub id: u64, + pub head_block_hash: [u8; 32], + pub safe_block_hash: [u8; 32], + pub finalized_block_hash: [u8; 32], +} + +/// `engine_newPayloadV4` request. +/// +/// `data` is a single TCache entry whose layout is: +/// ```text +/// [u32 LE payload_ssz_len] [payload_ssz_len bytes: ExecutionPayload SSZ] +/// [u8 exec_req_count] [for each: u32 LE len, then bytes] +/// ``` +/// `versioned_hashes[..versioned_hash_count]` are the expected KZG commitment +/// hashes for the blobs carried by this payload. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineNewPayloadReq { + pub id: u64, + pub data: TCacheRead, + pub parent_beacon_block_root: [u8; 32], + pub versioned_hash_count: u8, + pub versioned_hashes: [[u8; 32]; MAX_BLOBS_PER_BLOCK], +} + +/// Response to `engine_forkchoiceUpdatedV3`. Fully inline. +/// +/// `latest_valid_hash` is all-zeros when the EL did not return one. +/// `payload_id` is meaningful only when `has_payload_id` is true. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineFcuResp { + pub id: u64, + pub status: PayloadValidationStatus, + pub latest_valid_hash: [u8; 32], + pub has_payload_id: bool, + pub payload_id: [u8; 8], +} + +/// Response to `engine_newPayloadV4`. Fully inline. +/// +/// `latest_valid_hash` is all-zeros when the EL did not return one. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineNewPayloadResp { + pub id: u64, + pub status: PayloadValidationStatus, + pub latest_valid_hash: [u8; 32], +} + +/// `engine_getPayloadV4` request. The engine tile issues FCU-with-attrs +/// internally, then follows up with getPayloadV4 using the returned payload_id. +/// The caller sees a single request/response pair on the spine. +/// +/// Field layout mirrors `EngineFcuReq` (attrs are always present for payload +/// building). +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineGetPayloadReq { + pub id: u64, + pub head_block_hash: [u8; 32], + pub safe_block_hash: [u8; 32], + pub finalized_block_hash: [u8; 32], + pub attrs_timestamp: u64, + pub attrs_prev_randao: [u8; 32], + pub attrs_fee_recipient: [u8; 20], + pub attrs_parent_beacon_block_root: [u8; 32], + pub attrs_withdrawal_count: u8, + pub attrs_withdrawals: [WithdrawalInline; 16], +} + +/// Response to `EngineGetPayloadReq`. +/// When `ok` is true, `data` is a TCache slot with the JSON-encoded EL +/// response: `{executionPayload, blobsBundle, shouldOverrideBuilder, +/// executionRequests}`. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineGetPayloadResp { + pub id: u64, + pub ok: bool, + pub data: TCacheRead, +} + +/// `engine_getBlobsV2` request. `hashes[..hash_count]` are the versioned hashes +/// derived from the block's KZG commitments. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineGetBlobsReq { + pub id: u64, + pub hash_count: u8, + pub hashes: [[u8; 32]; MAX_BLOBS_PER_BLOCK], +} + +/// Response to `EngineGetBlobsReq`. +/// When `ok` is true, `data` is a TCache slot with the JSON-encoded array of +/// `{blob, proofs}` objects (entries may be `null` for missing blobs). +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineGetBlobsResp { + pub id: u64, + pub ok: bool, + pub data: TCacheRead, +} + +/// `engine_getPayloadBodiesByHashV1` request. +/// `hashes[..hash_count]` are the execution block hashes to fetch bodies for. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineGetPayloadBodiesByHashReq { + pub id: u64, + pub hash_count: u8, + pub hashes: [[u8; 32]; MAX_PAYLOAD_BODIES_PER_REQ], +} + +/// `engine_getPayloadBodiesByRangeV1` request. Fully inline. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineGetPayloadBodiesByRangeReq { + pub id: u64, + pub start: u64, + pub count: u64, +} + +/// Response to either `getPayloadBodiesByHash` or `getPayloadBodiesByRange`. +/// When `ok` is true, `data` is a TCache slot with the JSON-encoded array of +/// payload body objects (entries may be `null` for missing blocks). +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineGetPayloadBodiesResp { + pub id: u64, + pub ok: bool, + pub data: TCacheRead, +} + +/// Passthrough request for any engine/eth JSON-RPC method not handled inline. +/// `body` is a TCache slot containing `{"method":"...","params":[...]}` JSON. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineRawReq { + pub id: u64, + pub body: TCacheRead, +} + +/// Response to an `EngineRawReq`. +/// When `ok` is true, `data` is a TCache slot with the JSON-encoded result +/// value. When `ok` is false the slot is unused. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineRawResp { + pub id: u64, + pub ok: bool, + pub data: TCacheRead, +} + +/// Multiplexed engine request. A single spine queue carries FCU, +/// new-payload, and raw passthrough requests, preserving strict FIFO ordering. +#[derive(Clone, Copy, Debug)] +#[repr(C, u8)] +#[allow(clippy::large_enum_variant)] +pub enum EngineReq { + Fcu(EngineFcuReq), + NewPayload(EngineNewPayloadReq), + GetPayload(EngineGetPayloadReq), + GetBlobs(EngineGetBlobsReq), + GetPayloadBodiesByHash(EngineGetPayloadBodiesByHashReq), + GetPayloadBodiesByRange(EngineGetPayloadBodiesByRangeReq), +} + +/// Multiplexed engine response. +#[derive(Clone, Copy, Debug)] +#[repr(C, u8)] +#[allow(clippy::large_enum_variant)] +pub enum EngineResp { + Fcu(EngineFcuResp), + NewPayload(EngineNewPayloadResp), + GetPayload(EngineGetPayloadResp), + GetBlobs(EngineGetBlobsResp), + GetPayloadBodies(EngineGetPayloadBodiesResp), +} + +/// Sync status of the attached execution layer. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum ELSyncStatus { + Unknown = 0, + Syncing = 1, + Synced = 2, + Offline = 3, +} + +/// Published to the `engine_health` spine queue whenever the EL sync status +/// changes. Other tiles subscribe to suppress block proposals during outages +/// or to gate fork-choice updates on EL liveness. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineHealthEvent { + pub sync_status: ELSyncStatus, +} + impl BeaconStateEvent { pub fn view(&self) -> SszView { match self { diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml new file mode 100644 index 0000000..1fb3f33 --- /dev/null +++ b/crates/engine/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "silver_engine" +edition.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +silver_common.workspace = true +flux.workspace = true +rustc-hash.workspace = true + +base64 = "0.22" +hex = "0.4" +hmac = "0.12" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +sha2 = "0.10" +thiserror = "1.0" +httparse = "1" +mio = { workspace = true } +tracing = "0.1" + +[dev-dependencies] +axum = "0.8.9" +tokio = { version = "1.0", features = ["full"] } +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +[lints] +workspace = true diff --git a/crates/engine/src/client.rs b/crates/engine/src/client.rs new file mode 100644 index 0000000..29c7259 --- /dev/null +++ b/crates/engine/src/client.rs @@ -0,0 +1,297 @@ +use std::time::Duration; + +use mio::{Events, Poll}; +use rustc_hash::FxHashMap; + +use crate::{ + EngineError, JwtSecret, + http::{HttpPool, POOL_SIZE, http_pool_enqueue, poll_http_pool}, + ipc::{IpcPool, ipc_pool_enqueue, poll_ipc_pool}, + types::{B256, ExecutionPayload, ForkchoiceState, PayloadAttributesV3}, +}; + +const EVENTS_CAPACITY: usize = 16; + +const OUR_CAPABILITIES: &[&str] = &[ + "engine_forkchoiceUpdatedV3", + "engine_newPayloadV4", + "engine_getPayloadV4", + "engine_getBlobsV2", + "engine_getPayloadBodiesByHashV1", + "engine_getPayloadBodiesByRangeV1", + "engine_getClientVersionV1", +]; + +#[derive(Clone, Copy)] +pub enum ReqKind { + Capabilities, + ClientVersion, + Syncing, + Fcu(u64), + NewPayload(u64), + GetPayloadFcu(u64), + GetPayloadFetch(u64), + GetBlobs(u64), + GetPayloadBodiesByHash(u64), + GetPayloadBodiesByRange(u64), +} + +enum Transport { + Http(HttpPool), + Ipc(IpcPool), +} + +pub struct EngineClient { + transport: Transport, + poll: Poll, + events: Events, + id: u64, + pending_requests: FxHashMap, +} + +impl EngineClient { + pub fn new(endpoint: impl Into, jwt: JwtSecret) -> Self { + Self { + transport: Transport::Http(HttpPool::new(endpoint.into(), jwt, POOL_SIZE)), + poll: Poll::new().expect("mio Poll::new failed"), + events: Events::with_capacity(EVENTS_CAPACITY), + id: 1, + pending_requests: FxHashMap::default(), + } + } + + pub fn new_ipc(path: impl Into) -> Self { + Self { + transport: Transport::Ipc(IpcPool::new(path.into(), POOL_SIZE)), + poll: Poll::new().expect("mio Poll::new failed"), + events: Events::with_capacity(EVENTS_CAPACITY), + id: 1, + pending_requests: FxHashMap::default(), + } + } +} + +fn next_id(id: &mut u64) -> u64 { + let v = *id; + *id += 1; + v +} + +fn make_rpc_body( + id: &mut u64, + method: &str, + params: serde_json::Value, +) -> (u64, serde_json::Value) { + let rpc_id = next_id(id); + let body = serde_json::json!({ + "jsonrpc": "2.0", + "method": method, + "params": params, + "id": rpc_id, + }); + (rpc_id, body) +} + +fn enqueue(c: &mut EngineClient, rpc_id: u64, body: &serde_json::Value) { + match &mut c.transport { + Transport::Http(p) => http_pool_enqueue(p, rpc_id, body, &mut c.poll), + Transport::Ipc(p) => ipc_pool_enqueue(p, rpc_id, body, &mut c.poll), + } +} + +pub fn send_fcu( + c: &mut EngineClient, + state: ForkchoiceState, + attrs: Option, + req_id: u64, +) { + let (id, body) = + make_rpc_body(&mut c.id, "engine_forkchoiceUpdatedV3", serde_json::json!([state, attrs])); + enqueue(c, id, &body); + if attrs.is_some() { + c.pending_requests.insert(id, ReqKind::Fcu(req_id)); + } else { + c.pending_requests.insert(id, ReqKind::GetPayloadFcu(req_id)); + } +} + +pub fn send_new_payload( + c: &mut EngineClient, + payload: ExecutionPayload, + versioned_hashes: Vec, + parent_beacon_block_root: B256, + execution_requests: Vec>, + req_id: u64, +) { + let hashes: Vec = + versioned_hashes.iter().map(|h| format!("0x{}", hex::encode(h))).collect(); + let parent = format!("0x{}", hex::encode(parent_beacon_block_root)); + let requests: Vec = + execution_requests.iter().map(|r| format!("0x{}", hex::encode(r))).collect(); + let (id, body) = make_rpc_body( + &mut c.id, + "engine_newPayloadV4", + serde_json::json!([payload, hashes, parent, requests]), + ); + enqueue(c, id, &body); + c.pending_requests.insert(id, ReqKind::NewPayload(req_id)); +} + +pub fn get_payload(c: &mut EngineClient, payload_id: [u8; 8], req_id: u64) { + let id_hex = format!("0x{}", hex::encode(payload_id)); + let (id, body) = make_rpc_body(&mut c.id, "engine_getPayloadV4", serde_json::json!([id_hex])); + enqueue(c, id, &body); + c.pending_requests.insert(id, ReqKind::GetPayloadFetch(req_id)); +} + +pub fn get_blobs(c: &mut EngineClient, params: serde_json::Value, req_id: u64) { + let (id, body) = make_rpc_body(&mut c.id, "engine_getBlobsV2", params); + enqueue(c, id, &body); + c.pending_requests.insert(id, ReqKind::GetBlobs(req_id)); +} + +pub fn get_payload_bodies_by_hash(c: &mut EngineClient, params: serde_json::Value, req_id: u64) { + let (id, body) = make_rpc_body(&mut c.id, "engine_getPayloadBodiesByHashV1", params); + enqueue(c, id, &body); + c.pending_requests.insert(id, ReqKind::GetPayloadBodiesByHash(req_id)); +} + +pub fn get_payload_bodies_by_range(c: &mut EngineClient, params: serde_json::Value, req_id: u64) { + let (id, body) = make_rpc_body(&mut c.id, "engine_getPayloadBodiesByRangeV1", params); + enqueue(c, id, &body); + c.pending_requests.insert(id, ReqKind::GetPayloadBodiesByRange(req_id)); +} + +pub fn get_sync_status(c: &mut EngineClient) { + let (id, body) = make_rpc_body(&mut c.id, "eth_syncing", serde_json::json!([])); + enqueue(c, id, &body); + c.pending_requests.insert(id, ReqKind::Syncing); +} + +pub fn exchange_capabilities(c: &mut EngineClient) { + let (id, body) = make_rpc_body( + &mut c.id, + "engine_exchangeCapabilities", + serde_json::json!(OUR_CAPABILITIES), + ); + enqueue(c, id, &body); + c.pending_requests.insert(id, ReqKind::Capabilities); +} + +pub fn get_client_version(c: &mut EngineClient) { + let (id, body) = make_rpc_body(&mut c.id, "engine_getClientVersionV1", serde_json::json!([{}])); + enqueue(c, id, &body); + c.pending_requests.insert(id, ReqKind::ClientVersion); +} + +/// Drive I/O, calling `on_complete(req_kind, response)` for each finished RPC. +pub fn poll(c: &mut EngineClient, mut on_complete: F) +where + F: FnMut(ReqKind, Result), +{ + c.poll.poll(&mut c.events, Some(Duration::ZERO)).ok(); + let EngineClient { transport, events, poll, pending_requests, .. } = c; + match transport { + Transport::Http(p) => poll_http_pool(p, events, poll, &mut |rpc_id, res| { + if let Some(req_kind) = pending_requests.remove(&rpc_id) { + on_complete(req_kind, res.and_then(extract_result)); + } + }), + Transport::Ipc(p) => poll_ipc_pool(p, events, poll, &mut |rpc_id, res| { + if let Some(req_kind) = pending_requests.remove(&rpc_id) { + on_complete(req_kind, res.and_then(extract_result)); + } + }), + } +} + +fn extract_result(resp: serde_json::Value) -> Result { + if let Some(err) = resp.get("error") { + return Err(EngineError::Rpc(err.clone())); + } + resp.get("result").cloned().ok_or(EngineError::MissingResult) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn next_id_returns_current_then_increments() { + let mut id = 1u64; + assert_eq!(next_id(&mut id), 1); + assert_eq!(next_id(&mut id), 2); + assert_eq!(id, 3); + } + + #[test] + fn next_id_starts_from_arbitrary_value() { + let mut id = 100u64; + assert_eq!(next_id(&mut id), 100); + assert_eq!(id, 101); + } + + #[test] + fn make_rpc_body_has_correct_structure() { + let mut id = 1u64; + let (rpc_id, body) = make_rpc_body(&mut id, "eth_test", serde_json::json!(["param"])); + assert_eq!(rpc_id, 1); + assert_eq!(id, 2); + assert_eq!(body["jsonrpc"], "2.0"); + assert_eq!(body["method"], "eth_test"); + assert_eq!(body["params"], serde_json::json!(["param"])); + assert_eq!(body["id"], 1); + } + + #[test] + fn make_rpc_body_ids_increase_across_calls() { + let mut id = 5u64; + let (id1, body1) = make_rpc_body(&mut id, "m1", serde_json::json!([])); + let (id2, body2) = make_rpc_body(&mut id, "m2", serde_json::json!([])); + assert_eq!(id1, 5); + assert_eq!(id2, 6); + assert_eq!(body1["id"], 5); + assert_eq!(body2["id"], 6); + } + + #[test] + fn extract_result_returns_result_field() { + let resp = serde_json::json!({"jsonrpc": "2.0", "id": 1, "result": {"key": "value"}}); + assert_eq!(extract_result(resp).unwrap(), serde_json::json!({"key": "value"})); + } + + #[test] + fn extract_result_returns_rpc_error_when_error_present() { + let resp = serde_json::json!({"jsonrpc": "2.0", "id": 1, "error": {"code": -32600, "message": "bad"}}); + assert!(matches!(extract_result(resp), Err(EngineError::Rpc(_)))); + } + + #[test] + fn extract_result_error_contains_error_payload() { + let error_val = serde_json::json!({"code": -32700, "message": "parse error"}); + let resp = serde_json::json!({"jsonrpc": "2.0", "id": 1, "error": error_val}); + match extract_result(resp) { + Err(EngineError::Rpc(v)) => assert_eq!(v["code"], -32700), + other => panic!("unexpected: {other:?}"), + } + } + + #[test] + fn extract_result_error_takes_precedence_over_result() { + let resp = + serde_json::json!({"jsonrpc": "2.0", "id": 1, "error": {"code": 1}, "result": "ok"}); + assert!(matches!(extract_result(resp), Err(EngineError::Rpc(_)))); + } + + #[test] + fn extract_result_missing_result_when_neither_field_present() { + let resp = serde_json::json!({"jsonrpc": "2.0", "id": 1}); + assert!(matches!(extract_result(resp), Err(EngineError::MissingResult))); + } + + #[test] + fn extract_result_null_result_is_ok() { + let resp = serde_json::json!({"jsonrpc": "2.0", "id": 1, "result": null}); + assert_eq!(extract_result(resp).unwrap(), serde_json::Value::Null); + } +} diff --git a/crates/engine/src/error.rs b/crates/engine/src/error.rs new file mode 100644 index 0000000..b94116d --- /dev/null +++ b/crates/engine/src/error.rs @@ -0,0 +1,17 @@ +#[derive(Debug, thiserror::Error)] +pub enum EngineError { + #[error("http: {0}")] + Http(String), + #[error("json-rpc error: {0}")] + Rpc(serde_json::Value), + #[error("missing result field in response")] + MissingResult, + #[error("serde: {0}")] + Json(#[from] serde_json::Error), + #[error("jwt: {0}")] + Jwt(String), + #[error("ipc: {0}")] + Ipc(String), + #[error("ssz: {0}")] + Ssz(String), +} diff --git a/crates/engine/src/http.rs b/crates/engine/src/http.rs new file mode 100644 index 0000000..214b4b3 --- /dev/null +++ b/crates/engine/src/http.rs @@ -0,0 +1,424 @@ +use std::{ + io::{self, Read, Write}, + net::{SocketAddr, ToSocketAddrs}, +}; + +use mio::{Events, Interest, Poll, Token, net::TcpStream}; + +use crate::{EngineError, JwtSecret}; + +pub(crate) const POOL_SIZE: usize = 4; + +#[derive(PartialEq)] +enum State { + Disconnected, + Connecting, + Connected, +} + +pub(crate) struct HttpConnection { + endpoint: String, + host: String, + jwt: JwtSecret, + token: Token, + stream: Option, + state: State, + addr: Option, + pending: Option<(u64, Vec)>, + write_pos: usize, + in_flight: Option, + read_buf: Vec, + read_offset: usize, +} + +impl HttpConnection { + pub(crate) fn new(endpoint: String, jwt: JwtSecret, token: Token) -> Self { + let host = endpoint + .trim_start_matches("http://") + .split('/') + .next() + .unwrap_or("localhost") + .to_string(); + Self { + endpoint, + host, + jwt, + token, + stream: None, + state: State::Disconnected, + addr: None, + pending: None, + write_pos: 0, + in_flight: None, + read_buf: Vec::new(), + read_offset: 0, + } + } +} + +pub(crate) fn http_is_free(t: &HttpConnection) -> bool { + t.in_flight.is_none() && t.pending.is_none() +} + +pub(crate) fn http_enqueue( + t: &mut HttpConnection, + rpc_id: u64, + body: &serde_json::Value, + poll: &mut Poll, +) { + let json = serde_json::to_vec(body).unwrap_or_default(); + let bearer = t.jwt.bearer_token(); + let bytes = build_request(&t.host, &json, bearer, true); + t.pending = Some((rpc_id, bytes)); + + match t.state { + State::Disconnected => http_connect(t, poll), + State::Connected => http_set_interest(t, poll, Interest::READABLE | Interest::WRITABLE), + State::Connecting => {} + } +} + +pub(crate) fn http_poll( + t: &mut HttpConnection, + events: &Events, + poll: &mut Poll, + on_complete: &mut F, +) where + F: FnMut(u64, Result), +{ + for event in events.iter() { + if event.token() != t.token { + continue; + } + match t.state { + State::Disconnected => {} + State::Connecting => { + if event.is_error() || event.is_read_closed() || event.is_write_closed() { + http_on_error(t, poll, on_complete, "connect failed"); + break; + } + if event.is_writable() { + let connected = t.stream.as_ref().and_then(|s| s.peer_addr().ok()).is_some(); + if connected { + t.state = State::Connected; + let interest = if t.pending.is_none() { + Interest::READABLE + } else { + Interest::READABLE | Interest::WRITABLE + }; + http_set_interest(t, poll, interest); + } else { + http_on_error(t, poll, on_complete, "connect failed"); + break; + } + } + } + State::Connected => { + if event.is_error() || event.is_read_closed() { + http_on_error(t, poll, on_complete, "connection lost"); + break; + } + if event.is_writable() { + if let Err(e) = http_do_write(t) { + let msg = e.to_string(); + http_on_error(t, poll, on_complete, &msg); + break; + } + let interest = if t.pending.is_none() { + Interest::READABLE + } else { + Interest::READABLE | Interest::WRITABLE + }; + http_set_interest(t, poll, interest); + } + if event.is_readable() { + if let Err(e) = http_do_read(t, on_complete) { + let msg = e.to_string(); + http_on_error(t, poll, on_complete, &msg); + break; + } + } + } + } + } +} + +fn http_connect(t: &mut HttpConnection, poll: &mut Poll) { + let addr = if let Some(a) = t.addr { + a + } else { + match parse_addr(&t.endpoint) { + Ok(a) => { + t.addr = Some(a); + a + } + Err(e) => { + tracing::warn!("engine http: resolve failed for {}: {e}", t.endpoint); + return; + } + } + }; + match TcpStream::connect(addr) { + Ok(mut stream) => { + if poll.registry().register(&mut stream, t.token, Interest::WRITABLE).is_ok() { + t.stream = Some(stream); + t.state = State::Connecting; + } + } + Err(e) => tracing::warn!("engine http: connect error: {e}"), + } +} + +fn http_do_write(t: &mut HttpConnection) -> io::Result<()> { + if let Some((_, bytes)) = t.pending.as_ref() { + let stream = t.stream.as_mut().unwrap(); + loop { + match stream.write(&bytes[t.write_pos..]) { + Ok(0) => break, + Ok(n) => { + t.write_pos += n; + if t.write_pos == bytes.len() { + let (rpc_id, _) = t.pending.take().unwrap(); + t.in_flight = Some(rpc_id); + t.write_pos = 0; + break; + } + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, + Err(e) => return Err(e), + } + } + } + Ok(()) +} + +fn http_do_read(t: &mut HttpConnection, on_complete: &mut F) -> io::Result<()> +where + F: FnMut(u64, Result), +{ + let stream = t.stream.as_mut().unwrap(); + let mut buf = [0u8; 8192]; + loop { + match stream.read(&mut buf) { + Ok(0) => return Err(io::Error::new(io::ErrorKind::ConnectionReset, "eof")), + Ok(n) => { + t.read_buf.extend_from_slice(&buf[..n]); + loop { + match try_parse_response(&t.read_buf[t.read_offset..]) { + Some((result, consumed)) => { + t.read_offset += consumed; + if let Some(rpc_id) = t.in_flight.take() { + on_complete(rpc_id, result); + } + } + None => break, + } + } + if t.read_offset == t.read_buf.len() { + t.read_buf.clear(); + t.read_offset = 0; + } + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, + Err(e) => return Err(e), + } + } + Ok(()) +} + +fn http_on_error(t: &mut HttpConnection, poll: &mut Poll, on_complete: &mut F, msg: &str) +where + F: FnMut(u64, Result), +{ + tracing::warn!("engine http: {msg}"); + let err = msg.to_string(); + if let Some(rpc_id) = t.in_flight.take() { + on_complete(rpc_id, Err(EngineError::Http(err.clone()))); + } + if let Some((rpc_id, _)) = t.pending.take() { + on_complete(rpc_id, Err(EngineError::Http(err.clone()))); + } + t.write_pos = 0; + t.read_buf.clear(); + t.read_offset = 0; + if let Some(mut stream) = t.stream.take() { + let _ = poll.registry().deregister(&mut stream); + } + t.state = State::Disconnected; +} + +fn http_set_interest(t: &mut HttpConnection, poll: &mut Poll, interest: Interest) { + if let Some(stream) = t.stream.as_mut() { + let _ = poll.registry().reregister(stream, t.token, interest); + } +} + +pub(crate) struct HttpPool { + connections: Vec, + endpoint: String, + jwt: JwtSecret, +} + +impl HttpPool { + pub(crate) fn new(endpoint: String, jwt: JwtSecret, size: usize) -> Self { + let connections = (0..size) + .map(|i| HttpConnection::new(endpoint.clone(), jwt.clone(), Token(i))) + .collect(); + Self { connections, endpoint, jwt } + } +} + +pub(crate) fn http_pool_enqueue( + pool: &mut HttpPool, + rpc_id: u64, + body: &serde_json::Value, + poll: &mut Poll, +) { + if let Some(conn) = pool.connections.iter_mut().find(|c| http_is_free(c)) { + http_enqueue(conn, rpc_id, body, poll); + } else { + let mut new_conn = HttpConnection::new( + pool.endpoint.clone(), + pool.jwt.clone(), + Token(pool.connections.len()), + ); + http_enqueue(&mut new_conn, rpc_id, body, poll); + pool.connections.push(new_conn); + } +} + +pub(crate) fn poll_http_pool( + pool: &mut HttpPool, + events: &Events, + poll: &mut Poll, + on_complete: &mut F, +) where + F: FnMut(u64, Result), +{ + for conn in &mut pool.connections { + http_poll(conn, events, poll, on_complete); + } +} + +fn build_request(host: &str, json: &[u8], bearer: &str, keep_alive: bool) -> Vec { + let connection = if keep_alive { "keep-alive" } else { "close" }; + let header = format!( + "POST / HTTP/1.1\r\nHost: {host}\r\nContent-Type: application/json\r\n\ + Content-Length: {len}\r\nAuthorization: {bearer}\r\nConnection: {connection}\r\n\r\n", + len = json.len(), + ); + let mut bytes = header.into_bytes(); + bytes.extend_from_slice(json); + bytes +} + +fn parse_addr(endpoint: &str) -> io::Result { + let hostport = endpoint.trim_start_matches("http://").split('/').next().unwrap_or(endpoint); + hostport + .to_socket_addrs()? + .next() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no address resolved")) +} + +pub(crate) fn try_parse_response( + buf: &[u8], +) -> Option<(Result, usize)> { + let mut headers = [httparse::EMPTY_HEADER; 32]; + let mut resp = httparse::Response::new(&mut headers); + + let header_end = match resp.parse(buf) { + Ok(httparse::Status::Complete(n)) => n, + Ok(httparse::Status::Partial) => return None, + Err(e) => { + return Some((Err(EngineError::Http(format!("httparse: {e}"))), buf.len())); + } + }; + + let cl_bytes = headers.iter().find(|h| h.name.eq_ignore_ascii_case("content-length"))?.value; + if !cl_bytes.iter().all(|b| b.is_ascii_digit()) { + return Some((Err(EngineError::Http("invalid Content-Length".into())), buf.len())); + } + let content_length: usize = + cl_bytes.iter().copied().fold(0usize, |acc, b| acc * 10 + (b - b'0') as usize); + + let total = header_end + content_length; + if buf.len() < total { + return None; + } + + let body = &buf[header_end..total]; + Some((serde_json::from_slice(body).map_err(EngineError::Json), total)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn http_response(body: &[u8]) -> Vec { + let header = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n", + body.len() + ); + let mut buf = header.into_bytes(); + buf.extend_from_slice(body); + buf + } + + #[test] + fn parse_complete_response() { + let body = br#"{"jsonrpc":"2.0","id":1,"result":true}"#; + let buf = http_response(body); + let (result, consumed) = try_parse_response(&buf).unwrap(); + assert!(result.is_ok()); + assert_eq!(consumed, buf.len()); + } + + #[test] + fn parse_incomplete_headers_returns_none() { + let partial = b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n"; + assert!(try_parse_response(partial).is_none()); + } + + #[test] + fn parse_complete_headers_but_truncated_body_returns_none() { + let body = br#"{"result":1}"#; + let mut buf = http_response(body); + buf.truncate(buf.len() - 3); + assert!(try_parse_response(&buf).is_none()); + } + + #[test] + fn parse_two_responses_sequentially() { + let body = br#"{"id":1}"#; + let single = http_response(body); + let mut buf = single.clone(); + buf.extend_from_slice(&single); + + let (r1, n1) = try_parse_response(&buf).unwrap(); + assert!(r1.is_ok()); + let (r2, n2) = try_parse_response(&buf[n1..]).unwrap(); + assert!(r2.is_ok()); + assert_eq!(n1, n2, "both responses are the same size"); + assert_eq!(n1 + n2, buf.len()); + } + + #[test] + fn parse_missing_content_length_returns_none() { + let buf = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{}"; + assert!(try_parse_response(buf).is_none(), "no Content-Length header → None"); + } + + #[test] + fn parse_large_body_correct_consumed_count() { + let body: Vec = (0u8..=255).cycle().take(4096).collect(); + // wrap in a JSON string so serde accepts it + let json_body = serde_json::to_vec(&serde_json::Value::String( + String::from_utf8_lossy(&body).into_owned(), + )) + .unwrap(); + let buf = http_response(&json_body); + let (result, consumed) = try_parse_response(&buf).unwrap(); + assert!(result.is_ok()); + assert_eq!(consumed, buf.len()); + } +} diff --git a/crates/engine/src/ipc.rs b/crates/engine/src/ipc.rs new file mode 100644 index 0000000..93f9dfb --- /dev/null +++ b/crates/engine/src/ipc.rs @@ -0,0 +1,256 @@ +use std::{ + collections::VecDeque, + io::{self, Read, Write}, + path::PathBuf, +}; + +use mio::{Events, Interest, Poll, Token, net::UnixStream}; + +use crate::EngineError; + +#[derive(PartialEq)] +enum State { + Disconnected, + Connecting, + Connected, +} + +pub(crate) struct IpcTransport { + path: PathBuf, + token: Token, + stream: Option, + state: State, + send_queue: VecDeque<(u64, Vec)>, + write_pos: usize, + in_flight: Option, + read_buf: Vec, + read_offset: usize, +} + +impl IpcTransport { + pub(crate) fn new(path: String, token: Token) -> Self { + Self { + path: PathBuf::from(path), + token, + stream: None, + state: State::Disconnected, + send_queue: VecDeque::new(), + write_pos: 0, + in_flight: None, + read_buf: Vec::new(), + read_offset: 0, + } + } +} + +pub(crate) fn ipc_is_free(t: &IpcTransport) -> bool { + t.in_flight.is_none() && t.send_queue.is_empty() +} + +pub(crate) fn ipc_enqueue( + t: &mut IpcTransport, + rpc_id: u64, + body: &serde_json::Value, + poll: &mut Poll, +) { + let mut bytes = serde_json::to_vec(body).unwrap_or_default(); + bytes.push(b'\n'); + t.send_queue.push_back((rpc_id, bytes)); + + match t.state { + State::Disconnected => ipc_connect(t, poll), + State::Connected => ipc_set_interest(t, poll, Interest::READABLE | Interest::WRITABLE), + State::Connecting => {} + } +} + +pub(crate) fn ipc_poll( + t: &mut IpcTransport, + events: &Events, + poll: &mut Poll, + on_complete: &mut F, +) where + F: FnMut(u64, Result), +{ + for event in events.iter() { + if event.token() != t.token { + continue; + } + match t.state { + State::Disconnected => {} + State::Connecting => { + if event.is_error() || event.is_read_closed() || event.is_write_closed() { + ipc_on_error(t, poll, on_complete, "connect failed"); + break; + } + if event.is_writable() { + // Unix socket connect completes on first WRITABLE event + t.state = State::Connected; + let interest = if t.send_queue.is_empty() { + Interest::READABLE + } else { + Interest::READABLE | Interest::WRITABLE + }; + ipc_set_interest(t, poll, interest); + } + } + State::Connected => { + if event.is_error() || event.is_read_closed() { + ipc_on_error(t, poll, on_complete, "ipc connection lost"); + break; + } + if event.is_writable() { + if let Err(e) = ipc_do_write(t) { + let msg = e.to_string(); + ipc_on_error(t, poll, on_complete, &msg); + break; + } + let interest = if t.send_queue.is_empty() { + Interest::READABLE + } else { + Interest::READABLE | Interest::WRITABLE + }; + ipc_set_interest(t, poll, interest); + } + if event.is_readable() { + if let Err(e) = ipc_do_read(t, on_complete) { + let msg = e.to_string(); + ipc_on_error(t, poll, on_complete, &msg); + break; + } + } + } + } + } +} + +fn ipc_connect(t: &mut IpcTransport, poll: &mut Poll) { + match UnixStream::connect(&t.path) { + Ok(mut stream) => { + if poll.registry().register(&mut stream, t.token, Interest::WRITABLE).is_ok() { + t.stream = Some(stream); + t.state = State::Connecting; + } + } + Err(e) => tracing::warn!("engine ipc: connect error: {e}"), + } +} + +fn ipc_do_write(t: &mut IpcTransport) -> io::Result<()> { + while let Some((_, bytes)) = t.send_queue.front() { + let stream = t.stream.as_mut().unwrap(); + match stream.write(&bytes[t.write_pos..]) { + Ok(0) => break, + Ok(n) => { + t.write_pos += n; + if t.write_pos == bytes.len() { + let (rpc_id, _) = t.send_queue.pop_front().unwrap(); + t.in_flight = Some(rpc_id); + t.write_pos = 0; + } + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, + Err(e) => return Err(e), + } + } + Ok(()) +} + +fn ipc_do_read(t: &mut IpcTransport, on_complete: &mut F) -> io::Result<()> +where + F: FnMut(u64, Result), +{ + let stream = t.stream.as_mut().unwrap(); + let mut buf = [0u8; 8192]; + loop { + match stream.read(&mut buf) { + Ok(0) => return Err(io::Error::new(io::ErrorKind::ConnectionReset, "eof")), + Ok(n) => { + t.read_buf.extend_from_slice(&buf[..n]); + while let Some(rel) = t.read_buf[t.read_offset..].iter().position(|&b| b == b'\n') { + let end = t.read_offset + rel; + if let Some(rpc_id) = t.in_flight { + let result = serde_json::from_slice(&t.read_buf[t.read_offset..end]) + .map_err(EngineError::Json); + on_complete(rpc_id, result); + } + t.read_offset = end + 1; + } + if t.read_offset == t.read_buf.len() { + t.read_buf.clear(); + t.read_offset = 0; + } + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, + Err(e) => return Err(e), + } + } + Ok(()) +} + +fn ipc_on_error(t: &mut IpcTransport, poll: &mut Poll, on_complete: &mut F, msg: &str) +where + F: FnMut(u64, Result), +{ + tracing::warn!("engine ipc: {msg}"); + let err = msg.to_string(); + if let Some(rpc_id) = t.in_flight.take() { + on_complete(rpc_id, Err(EngineError::Ipc(err.clone()))); + } + for (rpc_id, _) in t.send_queue.drain(..) { + on_complete(rpc_id, Err(EngineError::Ipc(err.clone()))); + } + t.write_pos = 0; + t.read_buf.clear(); + t.read_offset = 0; + if let Some(mut stream) = t.stream.take() { + let _ = poll.registry().deregister(&mut stream); + } + t.state = State::Disconnected; +} + +fn ipc_set_interest(t: &mut IpcTransport, poll: &mut Poll, interest: Interest) { + if let Some(stream) = t.stream.as_mut() { + let _ = poll.registry().reregister(stream, t.token, interest); + } +} + +pub(crate) struct IpcPool { + connections: Vec, + path: String, +} + +impl IpcPool { + pub(crate) fn new(path: String, size: usize) -> Self { + let connections = (0..size).map(|i| IpcTransport::new(path.clone(), Token(i))).collect(); + Self { connections, path } + } +} + +pub(crate) fn ipc_pool_enqueue( + pool: &mut IpcPool, + rpc_id: u64, + body: &serde_json::Value, + poll: &mut Poll, +) { + if let Some(conn) = pool.connections.iter_mut().find(|c| ipc_is_free(c)) { + ipc_enqueue(conn, rpc_id, body, poll); + } else { + let mut new_conn = IpcTransport::new(pool.path.clone(), Token(pool.connections.len())); + ipc_enqueue(&mut new_conn, rpc_id, body, poll); + pool.connections.push(new_conn); + } +} + +pub(crate) fn poll_ipc_pool( + pool: &mut IpcPool, + events: &Events, + poll: &mut Poll, + on_complete: &mut F, +) where + F: FnMut(u64, Result), +{ + for conn in &mut pool.connections { + ipc_poll(conn, events, poll, on_complete); + } +} diff --git a/crates/engine/src/jwt.rs b/crates/engine/src/jwt.rs new file mode 100644 index 0000000..797f565 --- /dev/null +++ b/crates/engine/src/jwt.rs @@ -0,0 +1,111 @@ +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; + +use crate::EngineError; + +type HmacSha256 = Hmac; + +// base64url({"alg":"HS256","typ":"JWT"}) — constant, computed once +const HEADER_B64: &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; + +#[derive(Clone)] +pub struct JwtSecret { + secret: [u8; 32], + cached_iat: u64, + cached_token: String, +} + +impl JwtSecret { + pub fn from_hex(s: &str) -> Result { + let s = s.trim().strip_prefix("0x").unwrap_or(s); + let bytes = hex::decode(s).map_err(|e| EngineError::Jwt(e.to_string()))?; + let arr: [u8; 32] = bytes + .try_into() + .map_err(|_| EngineError::Jwt("secret must be 32 bytes (64 hex chars)".into()))?; + Ok(Self { secret: arr, cached_iat: 0, cached_token: String::new() }) + } + + pub fn from_file(path: &std::path::Path) -> Result { + let s = std::fs::read_to_string(path).map_err(|e| EngineError::Jwt(e.to_string()))?; + Self::from_hex(s.trim()) + } + + /// Returns a cached `Bearer ` valid for the current second. + /// Recomputes (one HMAC-SHA256) only when the Unix timestamp advances. + pub fn bearer_token(&mut self) -> &str { + let iat = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs(); + + if iat == self.cached_iat && !self.cached_token.is_empty() { + return &self.cached_token; + } + + let payload_b64 = URL_SAFE_NO_PAD.encode(format!(r#"{{"iat":{iat}}}"#)); + let signing_input = format!("{HEADER_B64}.{payload_b64}"); + + let mut mac = HmacSha256::new_from_slice(&self.secret).unwrap(); + mac.update(signing_input.as_bytes()); + let sig_b64 = URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()); + + self.cached_iat = iat; + self.cached_token = format!("Bearer {signing_input}.{sig_b64}"); + &self.cached_token + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ZERO_KEY: &str = "0000000000000000000000000000000000000000000000000000000000000000"; + + #[test] + fn from_hex_accepts_zero_key() { + assert!(JwtSecret::from_hex(ZERO_KEY).is_ok()); + } + + #[test] + fn from_hex_strips_0x_prefix() { + let with_prefix = JwtSecret::from_hex(&format!("0x{ZERO_KEY}")); + let without_prefix = JwtSecret::from_hex(ZERO_KEY); + assert!(with_prefix.is_ok()); + assert!(without_prefix.is_ok()); + } + + #[test] + fn from_hex_rejects_wrong_length() { + assert!(JwtSecret::from_hex("deadbeef").is_err()); + assert!(JwtSecret::from_hex("").is_err()); + } + + #[test] + fn bearer_token_is_three_part_jwt() { + let mut jwt = JwtSecret::from_hex(ZERO_KEY).unwrap(); + let token = jwt.bearer_token(); + + assert!(token.starts_with("Bearer "), "should start with 'Bearer '"); + let jwt_str = token.strip_prefix("Bearer ").unwrap(); + let parts: Vec<&str> = jwt_str.split('.').collect(); + assert_eq!(parts.len(), 3, "JWT must have header.payload.signature"); + + // Header is always the same constant + assert_eq!(parts[0], HEADER_B64); + + // Payload must base64-decode to JSON containing "iat" + let payload_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(parts[1]) + .expect("payload must be valid base64url"); + let payload: serde_json::Value = + serde_json::from_slice(&payload_bytes).expect("payload must be valid JSON"); + assert!(payload.get("iat").is_some(), "JWT payload must contain 'iat'"); + } + + #[test] + fn bearer_token_cached_within_same_second() { + let mut jwt = JwtSecret::from_hex(ZERO_KEY).unwrap(); + let t1 = jwt.bearer_token().to_owned(); + let t2 = jwt.bearer_token().to_owned(); + assert_eq!(t1, t2, "consecutive calls in the same second must return the same token"); + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs new file mode 100644 index 0000000..367a38d --- /dev/null +++ b/crates/engine/src/lib.rs @@ -0,0 +1,18 @@ +mod client; +mod error; +mod http; +mod ipc; +mod jwt; +mod req_handlers; +mod resp_handlers; +pub mod tile; +mod types; + +pub use client::EngineClient; +pub use error::EngineError; +pub use jwt::JwtSecret; +pub use tile::EngineTile; +pub use types::{ + B256, ExecutionAddress, ExecutionPayload, ForkchoiceState, PayloadAttributesV3, Withdrawal, + encode_new_payload_data, +}; diff --git a/crates/engine/src/req_handlers.rs b/crates/engine/src/req_handlers.rs new file mode 100644 index 0000000..fdbfca4 --- /dev/null +++ b/crates/engine/src/req_handlers.rs @@ -0,0 +1,173 @@ +use flux::spine::FluxSpine; +use silver_common::{ + EngineFcuReq, EngineGetBlobsReq, EngineGetPayloadBodiesByHashReq, + EngineGetPayloadBodiesByRangeReq, EngineGetPayloadReq, EngineNewPayloadReq, + EngineNewPayloadResp, EngineReq, EngineResp, PayloadValidationStatus, SilverSpine, + TRandomAccess, +}; + +use crate::{ + EngineClient, + client::{ + get_blobs, get_payload_bodies_by_hash, get_payload_bodies_by_range, send_fcu, + send_new_payload, + }, + types::{ForkchoiceState, PayloadAttributesV3, Withdrawal, decode_new_payload_data}, +}; + +#[inline] +pub(crate) fn handle_request( + client: &mut EngineClient, + req_consumer: &mut TRandomAccess, + req: &EngineReq, + producers: &mut ::Producers, +) { + match req { + EngineReq::Fcu(r) => handle_fcu(client, r), + EngineReq::NewPayload(r) => handle_new_payload(client, req_consumer, r, producers), + EngineReq::GetBlobs(r) => handle_get_blobs(client, r), + EngineReq::GetPayloadBodiesByHash(r) => handle_get_payload_bodies_by_hash(client, r), + EngineReq::GetPayloadBodiesByRange(r) => handle_get_payload_bodies_by_range(client, r), + EngineReq::GetPayload(r) => handle_get_payload(client, *r), + } +} + +#[inline] +fn handle_fcu(client: &mut EngineClient, r: &EngineFcuReq) { + let state = ForkchoiceState { + head_block_hash: r.head_block_hash, + safe_block_hash: r.safe_block_hash, + finalized_block_hash: r.finalized_block_hash, + }; + send_fcu(client, state, None, r.id); +} + +#[inline] +fn handle_new_payload( + client: &mut EngineClient, + req_consumer: &mut TRandomAccess, + r: &EngineNewPayloadReq, + producers: &mut ::Producers, +) { + let acquired = req_consumer.acquire(r.data); + let bytes = match acquired.buffer() { + Ok((b, _)) => b.to_owned(), + Err(e) => { + tracing::warn!("engine tile: failed to read payload data: {e}"); + producers + .engine_resps + .produce(&EngineResp::NewPayload(invalid_new_payload_resp(r.id)).into()); + return; + } + }; + + let (payload, exec_requests) = match decode_new_payload_data(&bytes) { + Ok(v) => v, + Err(e) => { + tracing::warn!("engine tile: failed to decode payload: {e}"); + producers + .engine_resps + .produce(&EngineResp::NewPayload(invalid_new_payload_resp(r.id)).into()); + return; + } + }; + + drop(acquired); + req_consumer.free(); + + let n = r.versioned_hash_count as usize; + let versioned_hashes = r.versioned_hashes[..n].to_vec(); + send_new_payload( + client, + payload, + versioned_hashes, + r.parent_beacon_block_root, + exec_requests, + r.id, + ); +} + +#[inline] +fn handle_get_blobs(client: &mut EngineClient, r: &EngineGetBlobsReq) { + let n = r.hash_count as usize; + let hashes: Vec = + r.hashes[..n].iter().map(|h| format!("0x{}", hex::encode(h))).collect(); + get_blobs(client, serde_json::json!([hashes]), r.id); +} + +#[inline] +fn handle_get_payload_bodies_by_hash( + client: &mut EngineClient, + r: &EngineGetPayloadBodiesByHashReq, +) { + let n = r.hash_count as usize; + let hashes: Vec = + r.hashes[..n].iter().map(|h| format!("0x{}", hex::encode(h))).collect(); + get_payload_bodies_by_hash(client, serde_json::json!([hashes]), r.id); +} + +#[inline] +fn handle_get_payload_bodies_by_range( + client: &mut EngineClient, + r: &EngineGetPayloadBodiesByRangeReq, +) { + let start_hex = format!("0x{:x}", r.start); + let count_hex = format!("0x{:x}", r.count); + get_payload_bodies_by_range(client, serde_json::json!([start_hex, count_hex]), r.id); +} + +#[inline] +fn handle_get_payload(client: &mut EngineClient, r: EngineGetPayloadReq) { + let n = r.attrs_withdrawal_count as usize; + let withdrawals = r.attrs_withdrawals[..n] + .iter() + .map(|w| Withdrawal { + index: w.index, + validator_index: w.validator_index, + address: w.address, + amount: w.amount, + }) + .collect(); + let state = ForkchoiceState { + head_block_hash: r.head_block_hash, + safe_block_hash: r.safe_block_hash, + finalized_block_hash: r.finalized_block_hash, + }; + let attrs = Some(PayloadAttributesV3 { + timestamp: r.attrs_timestamp, + prev_randao: r.attrs_prev_randao, + suggested_fee_recipient: r.attrs_fee_recipient, + withdrawals, + parent_beacon_block_root: r.attrs_parent_beacon_block_root, + }); + send_fcu(client, state, attrs, r.id); +} + +#[inline] +fn invalid_new_payload_resp(id: u64) -> EngineNewPayloadResp { + EngineNewPayloadResp { + id, + status: PayloadValidationStatus::Invalid, + latest_valid_hash: [0u8; 32], + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn invalid_new_payload_resp_fields() { + let resp = invalid_new_payload_resp(99); + assert_eq!(resp.id, 99); + assert_eq!(resp.status, PayloadValidationStatus::Invalid); + assert_eq!(resp.latest_valid_hash, [0u8; 32]); + } + + #[test] + fn invalid_new_payload_resp_preserves_id() { + for id in [0, 1, u64::MAX] { + assert_eq!(invalid_new_payload_resp(id).id, id); + } + } +} diff --git a/crates/engine/src/resp_handlers.rs b/crates/engine/src/resp_handlers.rs new file mode 100644 index 0000000..31c334f --- /dev/null +++ b/crates/engine/src/resp_handlers.rs @@ -0,0 +1,391 @@ +use flux::spine::SpineAdapter; +use silver_common::{ + ELSyncStatus, EngineFcuResp, EngineGetBlobsResp, EngineGetPayloadBodiesResp, + EngineGetPayloadResp, EngineHealthEvent, EngineNewPayloadResp, EngineResp, + PayloadValidationStatus, SilverSpine, TCacheProducer, TCacheRead, TProducer, +}; + +use crate::{ + EngineError, + types::{ + BlobAndProofV1, ExecutionPayloadBodyV1, ForkchoiceUpdatedResult, GetPayloadV4Response, + PayloadStatus, encode_get_blobs_data, encode_get_payload_data, encode_payload_bodies_data, + }, +}; + +#[inline] +pub(crate) fn handle_capabilities_response(response: Result) { + match response { + Ok(val) => { + let arr = val.as_array().map(|a| a.as_slice()).unwrap_or_default(); + let has = |m: &str| arr.iter().any(|v| v.as_str() == Some(m)); + if !has("engine_forkchoiceUpdatedV3") { + tracing::warn!("engine tile: EL does not support engine_forkchoiceUpdatedV3"); + } + if !has("engine_newPayloadV4") { + tracing::warn!("engine tile: EL does not support engine_newPayloadV4"); + } + if !has("engine_getPayloadBodiesByHashV1") { + tracing::warn!("engine tile: EL does not support engine_getPayloadBodiesByHashV1"); + } + if !has("engine_getPayloadBodiesByRangeV1") { + tracing::warn!("engine tile: EL does not support engine_getPayloadBodiesByRangeV1"); + } + tracing::info!("engine tile: capabilities negotiated"); + } + Err(e) => { + tracing::warn!("engine tile: engine_exchangeCapabilities failed: {e}"); + } + } +} + +#[inline] +pub(crate) fn handle_client_version_response(response: Result) { + match response { + Ok(val) => { + if let Some(client) = val.as_array().and_then(|a| a.first()) { + let name = client + .get("clientName") + .or_else(|| client.get("name")) + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + let version = client.get("version").and_then(|v| v.as_str()).unwrap_or("?"); + tracing::info!("engine tile: EL client {name} {version}"); + } + } + Err(e) => { + tracing::warn!("engine tile: engine_getClientVersionV1 failed: {e}"); + } + } +} + +#[inline] +pub(crate) fn handle_sync_response( + response: Result, + adapter: &mut SpineAdapter, + sync_status: &mut ELSyncStatus, + healthcheck_pending: &mut bool, +) { + *healthcheck_pending = false; + let new_status = parse_sync_status(response); + publish_health_if_changed(adapter, sync_status, new_status); +} + +#[inline] +pub(crate) fn handle_fcu_response( + spine_id: u64, + response: Result, + adapter: &mut SpineAdapter, +) { + let resp = match response.and_then(|v| { + serde_json::from_value::(v).map_err(EngineError::Json) + }) { + Ok(r) => { + let status = status_from_str(&r.payload_status.status); + let latest_valid_hash = r.payload_status.latest_valid_hash.unwrap_or([0u8; 32]); + let (has_payload_id, payload_id) = match r.payload_id { + Some(id) => (true, id), + None => (false, [0u8; 8]), + }; + tracing::info!("engine tile: forkchoiceUpdated → {:?}", status); + EngineFcuResp { id: spine_id, status, latest_valid_hash, has_payload_id, payload_id } + } + Err(e) => { + tracing::warn!("engine tile: forkchoiceUpdated error: {e}"); + EngineFcuResp { + id: spine_id, + status: PayloadValidationStatus::Invalid, + latest_valid_hash: [0u8; 32], + has_payload_id: false, + payload_id: [0u8; 8], + } + } + }; + adapter.produce(EngineResp::Fcu(resp)); +} + +#[inline] +pub(crate) fn handle_new_payload_response( + spine_id: u64, + response: Result, + adapter: &mut SpineAdapter, +) { + let resp = match response + .and_then(|v| serde_json::from_value::(v).map_err(EngineError::Json)) + { + Ok(ps) => { + let status = status_from_str(&ps.status); + let latest_valid_hash = ps.latest_valid_hash.unwrap_or([0u8; 32]); + tracing::info!("engine tile: newPayload → {:?}", status); + EngineNewPayloadResp { id: spine_id, status, latest_valid_hash } + } + Err(e) => { + tracing::warn!("engine tile: newPayload error: {e}"); + EngineNewPayloadResp { + id: spine_id, + status: PayloadValidationStatus::Invalid, + latest_valid_hash: [0u8; 32], + } + } + }; + adapter.produce(EngineResp::NewPayload(resp)); +} + +#[inline] +pub(crate) fn handle_get_payload_fcu( + spine_id: u64, + response: Result, + adapter: &mut SpineAdapter, + pending_get_payload: &mut Option<([u8; 8], u64)>, +) { + match response.and_then(|v| { + serde_json::from_value::(v).map_err(EngineError::Json) + }) { + Ok(fcu) => match fcu.payload_id { + Some(payload_id) => { + *pending_get_payload = Some((payload_id, spine_id)); + } + None => { + tracing::warn!("engine tile: getPayload FCU returned no payload_id"); + adapter.produce(EngineResp::GetPayload(get_payload_error(spine_id))); + } + }, + Err(e) => { + tracing::warn!("engine tile: getPayload FCU error: {e}"); + adapter.produce(EngineResp::GetPayload(get_payload_error(spine_id))); + } + } +} + +#[inline] +pub(crate) fn handle_get_payload_fetch( + spine_id: u64, + response: Result, + adapter: &mut SpineAdapter, + resp_producer: &mut TProducer, +) { + let resp = match response { + Ok(v) => match serde_json::from_value::(v) { + Ok(p) => { + let payload_ssz = p.execution_payload.to_ssz(); + let data = encode_get_payload_data( + &payload_ssz, + &p.blobs_bundle.commitments, + &p.blobs_bundle.proofs, + &p.blobs_bundle.blobs, + p.should_override_builder, + &p.execution_requests, + ); + match write_tcache(resp_producer, &data) { + Some(data) => EngineGetPayloadResp { id: spine_id, ok: true, data }, + None => { + tracing::warn!("engine tile: getPayload TCache full"); + get_payload_error(spine_id) + } + } + } + Err(e) => { + tracing::warn!("engine tile: getPayload parse error: {e}"); + get_payload_error(spine_id) + } + }, + Err(e) => { + tracing::warn!("engine tile: getPayload error: {e}"); + get_payload_error(spine_id) + } + }; + adapter.produce(EngineResp::GetPayload(resp)); +} + +#[inline] +pub(crate) fn handle_get_blobs_response( + spine_id: u64, + response: Result, + adapter: &mut SpineAdapter, + resp_producer: &mut TProducer, +) { + let items = match response { + Ok(v) => match serde_json::from_value::>>(v) { + Ok(items) => items, + Err(e) => { + tracing::warn!("engine tile: getBlobsV2 parse error: {e}"); + adapter.produce(EngineResp::GetBlobs(get_blobs_error(spine_id))); + return; + } + }, + Err(e) => { + tracing::warn!("engine tile: getBlobsV2 error: {e}"); + adapter.produce(EngineResp::GetBlobs(get_blobs_error(spine_id))); + return; + } + }; + let data = encode_get_blobs_data(&items); + let resp = match write_tcache(resp_producer, &data) { + Some(data) => EngineGetBlobsResp { id: spine_id, ok: true, data }, + None => { + tracing::warn!("engine tile: getBlobsV2 TCache full"); + get_blobs_error(spine_id) + } + }; + adapter.produce(EngineResp::GetBlobs(resp)); +} + +#[inline] +pub(crate) fn handle_get_payload_bodies_response( + spine_id: u64, + response: Result, + adapter: &mut SpineAdapter, + resp_producer: &mut TProducer, +) { + let bodies = match response { + Ok(v) => match serde_json::from_value::>>(v) { + Ok(bodies) => bodies, + Err(e) => { + tracing::warn!("engine tile: getPayloadBodies parse error: {e}"); + adapter.produce(EngineResp::GetPayloadBodies(get_payload_bodies_error(spine_id))); + return; + } + }, + Err(e) => { + tracing::warn!("engine tile: getPayloadBodies error: {e}"); + adapter.produce(EngineResp::GetPayloadBodies(get_payload_bodies_error(spine_id))); + return; + } + }; + let data = encode_payload_bodies_data(&bodies); + let resp = match write_tcache(resp_producer, &data) { + Some(data) => EngineGetPayloadBodiesResp { id: spine_id, ok: true, data }, + None => { + tracing::warn!("engine tile: getPayloadBodies TCache full"); + get_payload_bodies_error(spine_id) + } + }; + adapter.produce(EngineResp::GetPayloadBodies(resp)); +} + +#[inline] +fn publish_health_if_changed( + adapter: &mut SpineAdapter, + sync_status: &mut ELSyncStatus, + new_status: ELSyncStatus, +) { + if new_status != *sync_status { + *sync_status = new_status; + adapter.produce(EngineHealthEvent { sync_status: new_status }); + tracing::info!("engine tile: EL health → {:?}", new_status); + } +} + +#[inline] +fn parse_sync_status(response: Result) -> ELSyncStatus { + match response { + Ok(val) if val.as_bool() == Some(false) => { + tracing::info!("engine tile: EL synced"); + ELSyncStatus::Synced + } + Ok(_) => { + tracing::info!("engine tile: EL syncing"); + ELSyncStatus::Syncing + } + Err(e) => { + tracing::warn!("engine tile: eth_syncing failed: {e}"); + ELSyncStatus::Offline + } + } +} + +#[inline] +fn status_from_str(s: &str) -> PayloadValidationStatus { + match s { + "VALID" => PayloadValidationStatus::Valid, + "SYNCING" => PayloadValidationStatus::Syncing, + "ACCEPTED" => PayloadValidationStatus::Accepted, + _ => PayloadValidationStatus::Invalid, + } +} + +#[inline] +fn write_tcache(producer: &mut TProducer, data: &[u8]) -> Option { + use std::io::Write as _; + let mut res = + producer.reserve(data.len(), false).or_else(|| producer.reserve(data.len(), false))?; + res.write_all(data).ok()?; + res.flush().ok()?; + Some(res.read()) +} + +#[inline] +fn get_payload_error(id: u64) -> EngineGetPayloadResp { + EngineGetPayloadResp { id, ok: false, data: unsafe { std::mem::zeroed() } } +} + +#[inline] +fn get_blobs_error(id: u64) -> EngineGetBlobsResp { + EngineGetBlobsResp { id, ok: false, data: unsafe { std::mem::zeroed() } } +} + +#[inline] +fn get_payload_bodies_error(id: u64) -> EngineGetPayloadBodiesResp { + EngineGetPayloadBodiesResp { id, ok: false, data: unsafe { std::mem::zeroed() } } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_from_str_valid() { + assert_eq!(status_from_str("VALID"), PayloadValidationStatus::Valid); + } + + #[test] + fn status_from_str_syncing() { + assert_eq!(status_from_str("SYNCING"), PayloadValidationStatus::Syncing); + } + + #[test] + fn status_from_str_accepted() { + assert_eq!(status_from_str("ACCEPTED"), PayloadValidationStatus::Accepted); + } + + #[test] + fn status_from_str_unknown_maps_to_invalid() { + assert_eq!(status_from_str("INVALID"), PayloadValidationStatus::Invalid); + assert_eq!(status_from_str(""), PayloadValidationStatus::Invalid); + assert_eq!(status_from_str("valid"), PayloadValidationStatus::Invalid); + assert_eq!(status_from_str("UNKNOWN_STATUS"), PayloadValidationStatus::Invalid); + } + + #[test] + fn parse_sync_status_synced_on_false() { + assert_eq!(parse_sync_status(Ok(serde_json::Value::Bool(false))), ELSyncStatus::Synced); + } + + #[test] + fn parse_sync_status_syncing_on_object() { + let syncing_obj = serde_json::json!({ + "startingBlock": "0x0", + "currentBlock": "0x100", + "highestBlock": "0x200" + }); + assert_eq!(parse_sync_status(Ok(syncing_obj)), ELSyncStatus::Syncing); + } + + #[test] + fn parse_sync_status_syncing_on_true() { + assert_eq!(parse_sync_status(Ok(serde_json::Value::Bool(true))), ELSyncStatus::Syncing); + } + + #[test] + fn parse_sync_status_offline_on_error() { + assert_eq!( + parse_sync_status(Err(EngineError::Http("connection refused".into()))), + ELSyncStatus::Offline + ); + } + + #[test] + fn parse_sync_status_offline_on_rpc_error() { + assert_eq!(parse_sync_status(Err(EngineError::MissingResult)), ELSyncStatus::Offline); + } +} diff --git a/crates/engine/src/tile.rs b/crates/engine/src/tile.rs new file mode 100644 index 0000000..92b58ff --- /dev/null +++ b/crates/engine/src/tile.rs @@ -0,0 +1,122 @@ +use std::time::{Duration, Instant}; + +use flux::{spine::SpineAdapter, tile::Tile}; +use silver_common::{ELSyncStatus, EngineReq, SilverSpine, TProducer, TRandomAccess}; + +use crate::{ + EngineClient, + client::{ + ReqKind, exchange_capabilities, get_client_version, get_payload, get_sync_status, poll, + }, + req_handlers::handle_request, + resp_handlers::*, +}; + +const HEALTHCHECK_INTERVAL: Duration = Duration::from_secs(10); + +pub struct EngineTile { + pub client: EngineClient, + pub req_consumer: TRandomAccess, + pub resp_producer: TProducer, + + first_run: bool, + healthcheck_pending: bool, + healthcheck_deadline: Instant, + sync_status: ELSyncStatus, +} + +impl Tile for EngineTile { + fn loop_body(&mut self, adapter: &mut SpineAdapter) { + adapter.consume(|req: EngineReq, producers| { + handle_request(&mut self.client, &mut self.req_consumer, &req, producers); + }); + self.spin(adapter); + } +} + +impl EngineTile { + pub fn new( + client: EngineClient, + req_consumer: TRandomAccess, + resp_producer: TProducer, + ) -> Self { + Self { + client, + req_consumer, + resp_producer, + + first_run: true, + healthcheck_pending: false, + healthcheck_deadline: Instant::now(), + sync_status: ELSyncStatus::Unknown, + } + } + + fn spin(&mut self, adapter: &mut SpineAdapter) { + let mut pending_get_payload: Option<([u8; 8], u64)> = None; + + { + let Self { + client, + resp_producer, + first_run, + healthcheck_pending, + healthcheck_deadline, + sync_status, + .. + } = self; + + if !*healthcheck_pending && Instant::now() >= *healthcheck_deadline { + run_healthcheck(client, first_run, healthcheck_pending, healthcheck_deadline); + } + + poll(client, |req_kind, response| match req_kind { + ReqKind::Capabilities => handle_capabilities_response(response), + ReqKind::ClientVersion => handle_client_version_response(response), + ReqKind::Syncing => { + handle_sync_response(response, adapter, sync_status, healthcheck_pending) + } + ReqKind::Fcu(spine_id) => handle_fcu_response(spine_id, response, adapter), + ReqKind::NewPayload(spine_id) => { + handle_new_payload_response(spine_id, response, adapter) + } + ReqKind::GetPayloadFcu(spine_id) => { + handle_get_payload_fcu(spine_id, response, adapter, &mut pending_get_payload) + } + ReqKind::GetPayloadFetch(spine_id) => { + handle_get_payload_fetch(spine_id, response, adapter, resp_producer) + } + ReqKind::GetBlobs(spine_id) => { + handle_get_blobs_response(spine_id, response, adapter, resp_producer) + } + ReqKind::GetPayloadBodiesByHash(spine_id) | + ReqKind::GetPayloadBodiesByRange(spine_id) => { + handle_get_payload_bodies_response(spine_id, response, adapter, resp_producer); + } + }); + } + + // GetPayloadFcu enqueues a follow-up RPC; done here so the closure + // above doesn't need to borrow client while poll holds it. + if let Some((payload_id, spine_id)) = pending_get_payload { + get_payload(&mut self.client, payload_id, spine_id); + } + } +} + +fn run_healthcheck( + client: &mut EngineClient, + first_run: &mut bool, + healthcheck_pending: &mut bool, + healthcheck_deadline: &mut Instant, +) { + if *first_run { + exchange_capabilities(client); + get_client_version(client); + *first_run = false; + } + + get_sync_status(client); + *healthcheck_deadline = Instant::now() + HEALTHCHECK_INTERVAL; + *healthcheck_pending = true; +} diff --git a/crates/engine/src/types.rs b/crates/engine/src/types.rs new file mode 100644 index 0000000..470eac2 --- /dev/null +++ b/crates/engine/src/types.rs @@ -0,0 +1,929 @@ +use serde::{Deserialize, Deserializer, Serialize, Serializer, ser::SerializeSeq}; + +pub type B256 = [u8; 32]; +pub type ExecutionAddress = [u8; 20]; + +// Engine API wire-format helpers: binary ↔ "0x" JSON. +// Each submodule is a serde `with` target. +mod wire { + use super::*; + + fn decode_hex_into(s: &str) -> Result<[u8; N], String> { + let s = s.strip_prefix("0x").unwrap_or(s); + let mut out = [0u8; N]; + hex::decode_to_slice(s, &mut out).map_err(|e| e.to_string())?; + Ok(out) + } + + fn decode_hex(s: &str) -> Result, String> { + hex::decode(s.strip_prefix("0x").unwrap_or(s)).map_err(|e| e.to_string()) + } + + // 32-byte hash/root ↔ "0x<64 hex>" + pub mod b256 { + use super::*; + pub fn serialize(v: &B256, s: S) -> Result { + s.serialize_str(&format!("0x{}", hex::encode(v))) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + decode_hex_into::<32>(&String::deserialize(d)?).map_err(serde::de::Error::custom) + } + } + + // Option: null or "0x<64 hex>" + pub mod opt_b256 { + use super::*; + pub fn serialize(v: &Option, s: S) -> Result { + match v { + Some(b) => s.serialize_str(&format!("0x{}", hex::encode(b))), + None => s.serialize_none(), + } + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + match Option::::deserialize(d)? { + None => Ok(None), + Some(s) => decode_hex_into::<32>(&s).map(Some).map_err(serde::de::Error::custom), + } + } + } + + // 20-byte execution address ↔ "0x<40 hex>" + pub mod addr { + use super::*; + pub fn serialize(v: &ExecutionAddress, s: S) -> Result { + s.serialize_str(&format!("0x{}", hex::encode(v))) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + decode_hex_into::<20>(&String::deserialize(d)?).map_err(serde::de::Error::custom) + } + } + + // u64 ↔ Ethereum QUANTITY ("0x0", "0x1a", ...) + pub mod quantity { + use super::*; + pub fn serialize(v: &u64, s: S) -> Result { + s.serialize_str(&format!("0x{:x}", v)) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + let s = String::deserialize(d)?; + let s = s.strip_prefix("0x").unwrap_or(&s); + u64::from_str_radix(s, 16).map_err(serde::de::Error::custom) + } + } + + // [u8; 32] LE ↔ Ethereum QUANTITY (uint256, no leading zeros) + // Used for base_fee_per_gas, which beacon_state stores as 32-byte LE. + pub mod u256_le { + use super::*; + pub fn serialize(v: &[u8; 32], s: S) -> Result { + let mut be = *v; + be.reverse(); + let h = hex::encode(be); + let trimmed = h.trim_start_matches('0'); + s.serialize_str(&format!("0x{}", if trimmed.is_empty() { "0" } else { trimmed })) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<[u8; 32], D::Error> { + let s = String::deserialize(d)?; + let s = s.strip_prefix("0x").unwrap_or(&s); + let padded = format!("{:0>64}", s); + let mut out = [0u8; 32]; + hex::decode_to_slice(&padded, &mut out).map_err(serde::de::Error::custom)?; + out.reverse(); + Ok(out) + } + } + + // [u8; 256] ↔ "0x<512 hex>" (logs_bloom) + pub mod bytes256 { + use super::*; + pub fn serialize(v: &[u8; 256], s: S) -> Result { + s.serialize_str(&format!("0x{}", hex::encode(v))) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<[u8; 256], D::Error> { + decode_hex_into::<256>(&String::deserialize(d)?).map_err(serde::de::Error::custom) + } + } + + // Vec ↔ "0x" (extra_data, blobs, proofs, ...) + pub mod data { + use super::*; + pub fn serialize(v: &Vec, s: S) -> Result { + s.serialize_str(&format!("0x{}", hex::encode(v))) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + decode_hex(&String::deserialize(d)?).map_err(serde::de::Error::custom) + } + } + + // Vec> ↔ array of "0x" (transactions, execution requests) + pub mod data_list { + use super::*; + pub fn serialize(v: &Vec>, s: S) -> Result { + let mut seq = s.serialize_seq(Some(v.len()))?; + for item in v { + seq.serialize_element(&format!("0x{}", hex::encode(item)))?; + } + seq.end() + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result>, D::Error> { + Vec::::deserialize(d)? + .iter() + .map(|s| decode_hex(s).map_err(serde::de::Error::custom)) + .collect() + } + } + + // Option<[u8; 8]> ↔ null or "0x<16 hex>" (payloadId) + pub mod opt_payload_id { + use super::*; + pub fn serialize(v: &Option<[u8; 8]>, s: S) -> Result { + match v { + Some(b) => s.serialize_str(&format!("0x{}", hex::encode(b))), + None => s.serialize_none(), + } + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + match Option::::deserialize(d)? { + None => Ok(None), + Some(s) => decode_hex_into::<8>(&s).map(Some).map_err(serde::de::Error::custom), + } + } + } +} + +// --- forkchoiceUpdated --- + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ForkchoiceState { + #[serde(with = "wire::b256")] + pub head_block_hash: B256, + #[serde(with = "wire::b256")] + pub safe_block_hash: B256, + #[serde(with = "wire::b256")] + pub finalized_block_hash: B256, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PayloadAttributesV3 { + #[serde(with = "wire::quantity")] + pub timestamp: u64, + #[serde(with = "wire::b256")] + pub prev_randao: B256, + #[serde(with = "wire::addr")] + pub suggested_fee_recipient: ExecutionAddress, + pub withdrawals: Vec, + #[serde(with = "wire::b256")] + pub parent_beacon_block_root: B256, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ForkchoiceUpdatedResult { + pub payload_status: PayloadStatus, + #[serde(with = "wire::opt_payload_id")] + pub payload_id: Option<[u8; 8]>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PayloadStatus { + pub status: String, + #[serde(with = "wire::opt_b256")] + pub latest_valid_hash: Option, + pub validation_error: Option, +} + +// --- getPayloadV4 response --- + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct GetPayloadV4Response { + pub execution_payload: ExecutionPayload, + pub blobs_bundle: BlobsBundle, + pub should_override_builder: bool, + #[serde(default, with = "wire::data_list")] + pub execution_requests: Vec>, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct BlobsBundle { + #[serde(with = "wire::data_list")] + pub commitments: Vec>, + #[serde(with = "wire::data_list")] + pub proofs: Vec>, + #[serde(with = "wire::data_list")] + pub blobs: Vec>, +} + +// --- getBlobsV2 response --- + +#[derive(Debug, Deserialize)] +pub(crate) struct BlobAndProofV1 { + #[serde(with = "wire::data")] + pub blob: Vec, + #[serde(with = "wire::data")] + pub proof: Vec, +} + +// --- getPayloadBodiesByHashV1 / ByRangeV1 response --- + +#[derive(Debug, Deserialize)] +pub(crate) struct ExecutionPayloadBodyV1 { + #[serde(with = "wire::data_list")] + pub transactions: Vec>, + pub withdrawals: Option>, +} + +// --- newPayload --- + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExecutionPayload { + #[serde(with = "wire::b256")] + pub parent_hash: B256, + #[serde(with = "wire::addr")] + pub fee_recipient: ExecutionAddress, + #[serde(with = "wire::b256")] + pub state_root: B256, + #[serde(with = "wire::b256")] + pub receipts_root: B256, + #[serde(with = "wire::bytes256")] + pub logs_bloom: [u8; 256], + #[serde(with = "wire::b256")] + pub prev_randao: B256, + #[serde(with = "wire::quantity")] + pub block_number: u64, + #[serde(with = "wire::quantity")] + pub gas_limit: u64, + #[serde(with = "wire::quantity")] + pub gas_used: u64, + #[serde(with = "wire::quantity")] + pub timestamp: u64, + #[serde(with = "wire::data")] + pub extra_data: Vec, + #[serde(with = "wire::u256_le")] + pub base_fee_per_gas: [u8; 32], + #[serde(with = "wire::b256")] + pub block_hash: B256, + #[serde(with = "wire::data_list")] + pub transactions: Vec>, + pub withdrawals: Vec, + #[serde(with = "wire::quantity")] + pub blob_gas_used: u64, + #[serde(with = "wire::quantity")] + pub excess_blob_gas: u64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Withdrawal { + #[serde(with = "wire::quantity")] + pub index: u64, + #[serde(with = "wire::quantity")] + pub validator_index: u64, + #[serde(with = "wire::addr")] + pub address: ExecutionAddress, + #[serde(with = "wire::quantity")] + pub amount: u64, +} + +// --------------------------------------------------------------------------- +// SSZ encode / decode for ExecutionPayload +// +// Deneb/Pectra ExecutionPayload SSZ layout (field order per spec): +// +// Fixed part (528 bytes): +// parent_hash [u8;32] 0..32 +// fee_recipient [u8;20] 32..52 +// state_root [u8;32] 52..84 +// receipts_root [u8;32] 84..116 +// logs_bloom [u8;256] 116..372 +// prev_randao [u8;32] 372..404 +// block_number u64 LE 404..412 +// gas_limit u64 LE 412..420 +// gas_used u64 LE 420..428 +// timestamp u64 LE 428..436 +// extra_data offset u32 LE 436..440 (variable) +// base_fee_per_gas [u8;32] 440..472 (uint256 LE) +// block_hash [u8;32] 472..504 +// transactions off u32 LE 504..508 (variable) +// withdrawals off u32 LE 508..512 (variable) +// blob_gas_used u64 LE 512..520 +// excess_blob_gas u64 LE 520..528 +// +// Variable part (appended in declaration order): +// extra_data bytes +// transactions SSZ List (u32 offsets header + raw tx bytes) +// withdrawals 44-byte fixed chunks (index u64, validator_index u64, +// address [u8;20], amount u64) +// --------------------------------------------------------------------------- + +const PAYLOAD_FIXED_LEN: usize = 528; + +impl ExecutionPayload { + pub fn from_ssz(ssz: &[u8]) -> Result { + if ssz.len() < PAYLOAD_FIXED_LEN { + return Err(crate::EngineError::Ssz(format!( + "payload too short: {} < {PAYLOAD_FIXED_LEN}", + ssz.len() + ))); + } + let u32_at = + |off: usize| u32::from_le_bytes(ssz[off..off + 4].try_into().unwrap()) as usize; + let u64_at = |off: usize| u64::from_le_bytes(ssz[off..off + 8].try_into().unwrap()); + + let extra_data_off = u32_at(436); + let transactions_off = u32_at(504); + let withdrawals_off = u32_at(508); + + if extra_data_off < PAYLOAD_FIXED_LEN || + transactions_off < extra_data_off || + withdrawals_off < transactions_off || + ssz.len() < withdrawals_off + { + return Err(crate::EngineError::Ssz("invalid variable offsets".into())); + } + + let extra_data = ssz[extra_data_off..transactions_off].to_vec(); + let transactions = decode_transactions(&ssz[transactions_off..withdrawals_off])?; + let withdrawals = decode_withdrawals(&ssz[withdrawals_off..])?; + + Ok(ExecutionPayload { + parent_hash: ssz[0..32].try_into().unwrap(), + fee_recipient: ssz[32..52].try_into().unwrap(), + state_root: ssz[52..84].try_into().unwrap(), + receipts_root: ssz[84..116].try_into().unwrap(), + logs_bloom: ssz[116..372].try_into().unwrap(), + prev_randao: ssz[372..404].try_into().unwrap(), + block_number: u64_at(404), + gas_limit: u64_at(412), + gas_used: u64_at(420), + timestamp: u64_at(428), + extra_data, + base_fee_per_gas: ssz[440..472].try_into().unwrap(), + block_hash: ssz[472..504].try_into().unwrap(), + transactions, + withdrawals, + blob_gas_used: u64_at(512), + excess_blob_gas: u64_at(520), + }) + } + + pub fn to_ssz(&self) -> Vec { + let tx_ssz = encode_transactions(&self.transactions); + let w_ssz = encode_withdrawals(&self.withdrawals); + + let extra_data_off = PAYLOAD_FIXED_LEN as u32; + let transactions_off = extra_data_off + self.extra_data.len() as u32; + let withdrawals_off = transactions_off + tx_ssz.len() as u32; + + let mut buf = Vec::with_capacity( + PAYLOAD_FIXED_LEN + self.extra_data.len() + tx_ssz.len() + w_ssz.len(), + ); + + buf.extend_from_slice(&self.parent_hash); + buf.extend_from_slice(&self.fee_recipient); + buf.extend_from_slice(&self.state_root); + buf.extend_from_slice(&self.receipts_root); + buf.extend_from_slice(&self.logs_bloom); + buf.extend_from_slice(&self.prev_randao); + buf.extend_from_slice(&self.block_number.to_le_bytes()); + buf.extend_from_slice(&self.gas_limit.to_le_bytes()); + buf.extend_from_slice(&self.gas_used.to_le_bytes()); + buf.extend_from_slice(&self.timestamp.to_le_bytes()); + buf.extend_from_slice(&extra_data_off.to_le_bytes()); + buf.extend_from_slice(&self.base_fee_per_gas); + buf.extend_from_slice(&self.block_hash); + buf.extend_from_slice(&transactions_off.to_le_bytes()); + buf.extend_from_slice(&withdrawals_off.to_le_bytes()); + buf.extend_from_slice(&self.blob_gas_used.to_le_bytes()); + buf.extend_from_slice(&self.excess_blob_gas.to_le_bytes()); + + buf.extend_from_slice(&self.extra_data); + buf.extend_from_slice(&tx_ssz); + buf.extend_from_slice(&w_ssz); + buf + } +} + +/// Encodes transactions as SSZ `List`. The first `n * 4` bytes are +/// u32 LE offsets, each the absolute byte position of that transaction within +/// this buffer. Transaction bytes follow immediately after the offset header. +/// The count is recoverable on decode as `first_offset / 4`. +fn encode_transactions(txs: &[Vec]) -> Vec { + if txs.is_empty() { + return vec![]; + } + let header_bytes = txs.len() * 4; + let mut offset = header_bytes as u32; + let mut buf = Vec::with_capacity(header_bytes + txs.iter().map(|t| t.len()).sum::()); + for tx in txs { + buf.extend_from_slice(&offset.to_le_bytes()); + offset += tx.len() as u32; + } + for tx in txs { + buf.extend_from_slice(tx); + } + buf +} + +fn decode_transactions(data: &[u8]) -> Result>, crate::EngineError> { + if data.is_empty() { + return Ok(vec![]); + } + if data.len() < 4 { + return Err(crate::EngineError::Ssz("transactions header too short".into())); + } + let first_off = u32::from_le_bytes(data[0..4].try_into().unwrap()) as usize; + if first_off.is_multiple_of(4) || first_off < 4 { + return Err(crate::EngineError::Ssz("invalid transaction list offset".into())); + } + let n = first_off / 4; + if data.len() < n * 4 { + return Err(crate::EngineError::Ssz("transaction offsets exceed data".into())); + } + let offsets: Vec = (0..n) + .map(|i| u32::from_le_bytes(data[i * 4..i * 4 + 4].try_into().unwrap()) as usize) + .collect(); + offsets + .iter() + .enumerate() + .map(|(i, &start)| { + let end = if i + 1 < n { offsets[i + 1] } else { data.len() }; + if start > data.len() || end > data.len() || start > end { + return Err(crate::EngineError::Ssz("transaction slice out of bounds".into())); + } + Ok(data[start..end].to_vec()) + }) + .collect() +} + +/// Each `Withdrawal` is 44 bytes: index u64, validator_index u64, address +/// [u8;20], amount u64. +fn encode_withdrawals(ws: &[Withdrawal]) -> Vec { + let mut buf = Vec::with_capacity(ws.len() * 44); + for w in ws { + buf.extend_from_slice(&w.index.to_le_bytes()); + buf.extend_from_slice(&w.validator_index.to_le_bytes()); + buf.extend_from_slice(&w.address); + buf.extend_from_slice(&w.amount.to_le_bytes()); + } + buf +} + +fn decode_withdrawals(data: &[u8]) -> Result, crate::EngineError> { + if data.len().is_multiple_of(44) { + return Err(crate::EngineError::Ssz(format!( + "withdrawals length {} is not a multiple of 44", + data.len() + ))); + } + Ok(data + .chunks(44) + .map(|c| Withdrawal { + index: u64::from_le_bytes(c[0..8].try_into().unwrap()), + validator_index: u64::from_le_bytes(c[8..16].try_into().unwrap()), + address: c[16..36].try_into().unwrap(), + amount: u64::from_le_bytes(c[36..44].try_into().unwrap()), + }) + .collect()) +} + +// --------------------------------------------------------------------------- +// TCache data framing for engine_newPayloadV4 +// +// Layout of the single TCache entry written by the producer (BridgeTile or +// BeaconStateTile) and read by EngineTile: +// +// [u32 LE payload_ssz_len] +// [payload_ssz_len bytes: ExecutionPayload SSZ] +// [u8 exec_req_count] +// [for each exec request: u32 LE len, then len bytes] +// --------------------------------------------------------------------------- + +pub fn encode_new_payload_data(payload_ssz: &[u8], exec_requests: &[Vec]) -> Vec { + let count = exec_requests.len().min(255) as u8; + let total = + 4 + payload_ssz.len() + 1 + exec_requests.iter().map(|r| 4 + r.len()).sum::(); + let mut buf = Vec::with_capacity(total); + buf.extend_from_slice(&(payload_ssz.len() as u32).to_le_bytes()); + buf.extend_from_slice(payload_ssz); + buf.push(count); + for r in exec_requests.iter().take(count as usize) { + buf.extend_from_slice(&(r.len() as u32).to_le_bytes()); + buf.extend_from_slice(r); + } + buf +} + +pub(crate) fn decode_new_payload_data( + data: &[u8], +) -> Result<(ExecutionPayload, Vec>), crate::EngineError> { + if data.len() < 5 { + return Err(crate::EngineError::Ssz("new-payload data too short".into())); + } + let payload_len = u32::from_le_bytes(data[0..4].try_into().unwrap()) as usize; + if data.len() < 4 + payload_len + 1 { + return Err(crate::EngineError::Ssz("new-payload data truncated".into())); + } + let payload = ExecutionPayload::from_ssz(&data[4..4 + payload_len])?; + let mut pos = 4 + payload_len; + let count = data[pos] as usize; + pos += 1; + let mut exec_requests = Vec::with_capacity(count); + for _ in 0..count { + if data.len() < pos + 4 { + return Err(crate::EngineError::Ssz("exec-request length truncated".into())); + } + let req_len = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()) as usize; + pos += 4; + if data.len() < pos + req_len { + return Err(crate::EngineError::Ssz("exec-request body truncated".into())); + } + exec_requests.push(data[pos..pos + req_len].to_vec()); + pos += req_len; + } + Ok((payload, exec_requests)) +} + +// --------------------------------------------------------------------------- +// TCache data framing for engine_getPayloadV4 +// +// Layout: +// [u32 LE payload_ssz_len] +// [payload_ssz_len bytes: ExecutionPayload SSZ] +// [u8 blob_count] +// for each blob: +// [48 bytes commitment] +// [48 bytes proof] +// [u32 LE blob_len][blob_len bytes blob] +// [u8 should_override_builder] +// [u8 exec_req_count] +// for each exec request: [u32 LE len][len bytes] +// --------------------------------------------------------------------------- + +pub fn encode_get_payload_data( + payload_ssz: &[u8], + commitments: &[Vec], + proofs: &[Vec], + blobs: &[Vec], + should_override_builder: bool, + exec_requests: &[Vec], +) -> Vec { + let blob_count = commitments.len().min(proofs.len()).min(blobs.len()).min(255) as u8; + let exec_count = exec_requests.len().min(255) as u8; + let blob_data_size: usize = blobs.iter().take(blob_count as usize).map(|b| 4 + b.len()).sum(); + let total = 4 + + payload_ssz.len() + + 1 + + blob_count as usize * (48 + 48) + + blob_data_size + + 2 + + exec_requests.iter().take(exec_count as usize).map(|r| 4 + r.len()).sum::(); + let mut buf = Vec::with_capacity(total); + buf.extend_from_slice(&(payload_ssz.len() as u32).to_le_bytes()); + buf.extend_from_slice(payload_ssz); + buf.push(blob_count); + for i in 0..blob_count as usize { + let mut c48 = [0u8; 48]; + let c = &commitments[i]; + c48[..c.len().min(48)].copy_from_slice(&c[..c.len().min(48)]); + buf.extend_from_slice(&c48); + let mut p48 = [0u8; 48]; + let p = &proofs[i]; + p48[..p.len().min(48)].copy_from_slice(&p[..p.len().min(48)]); + buf.extend_from_slice(&p48); + buf.extend_from_slice(&(blobs[i].len() as u32).to_le_bytes()); + buf.extend_from_slice(&blobs[i]); + } + buf.push(should_override_builder as u8); + buf.push(exec_count); + for r in exec_requests.iter().take(exec_count as usize) { + buf.extend_from_slice(&(r.len() as u32).to_le_bytes()); + buf.extend_from_slice(r); + } + buf +} + +// --------------------------------------------------------------------------- +// TCache data framing for engine_getBlobsV2 +// +// Layout: +// [u32 LE count] +// for each item: +// [u8 present] (0 = null) +// if present: +// [48 bytes proof] +// [u32 LE blob_len][blob_len bytes blob] +// --------------------------------------------------------------------------- + +pub fn encode_get_blobs_data(items: &[Option]) -> Vec { + let mut buf = Vec::new(); + buf.extend_from_slice(&(items.len() as u32).to_le_bytes()); + for item in items { + match item { + None => buf.push(0), + Some(bap) => { + buf.push(1); + let mut p48 = [0u8; 48]; + let p = &bap.proof; + p48[..p.len().min(48)].copy_from_slice(&p[..p.len().min(48)]); + buf.extend_from_slice(&p48); + buf.extend_from_slice(&(bap.blob.len() as u32).to_le_bytes()); + buf.extend_from_slice(&bap.blob); + } + } + } + buf +} + +// --------------------------------------------------------------------------- +// TCache data framing for engine_getPayloadBodiesByHashV1 / ByRangeV1 +// +// Layout: +// [u32 LE count] +// for each body: +// [u8 present] (0 = null) +// if present: +// [u32 LE tx_count] +// for each tx: [u32 LE len][bytes] +// [u32 LE withdrawal_count] +// for each withdrawal: 44 bytes (index u64 + validator_index u64 + +// address [u8;20] + amount u64) +// --------------------------------------------------------------------------- + +pub fn encode_payload_bodies_data(bodies: &[Option]) -> Vec { + let mut buf = Vec::new(); + buf.extend_from_slice(&(bodies.len() as u32).to_le_bytes()); + for body in bodies { + match body { + None => buf.push(0), + Some(b) => { + buf.push(1); + buf.extend_from_slice(&(b.transactions.len() as u32).to_le_bytes()); + for tx in &b.transactions { + buf.extend_from_slice(&(tx.len() as u32).to_le_bytes()); + buf.extend_from_slice(tx); + } + let withdrawals = b.withdrawals.as_deref().unwrap_or(&[]); + buf.extend_from_slice(&(withdrawals.len() as u32).to_le_bytes()); + for w in withdrawals { + buf.extend_from_slice(&w.index.to_le_bytes()); + buf.extend_from_slice(&w.validator_index.to_le_bytes()); + buf.extend_from_slice(&w.address); + buf.extend_from_slice(&w.amount.to_le_bytes()); + } + } + } + } + buf +} + +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_payload() -> ExecutionPayload { + ExecutionPayload { + parent_hash: [0x11; 32], + fee_recipient: [0x22; 20], + state_root: [0x33; 32], + receipts_root: [0x44; 32], + logs_bloom: [0x55; 256], + prev_randao: [0x66; 32], + block_number: 12_345, + gas_limit: 30_000_000, + gas_used: 21_000, + timestamp: 1_700_000_000, + extra_data: b"extra".to_vec(), + base_fee_per_gas: [0x77; 32], + block_hash: [0x88; 32], + transactions: vec![vec![1, 2, 3], vec![4, 5, 6, 7, 8]], + withdrawals: vec![ + Withdrawal { index: 1, validator_index: 42, address: [0x99; 20], amount: 1000 }, + Withdrawal { index: 2, validator_index: 43, address: [0xaa; 20], amount: 2000 }, + ], + blob_gas_used: 131_072, + excess_blob_gas: 262_144, + } + } + + #[test] + fn ssz_round_trip_with_all_fields() { + let original = sample_payload(); + let decoded = ExecutionPayload::from_ssz(&original.to_ssz()).unwrap(); + assert_eq!(original, decoded); + } + + #[test] + fn ssz_round_trip_empty_variable_fields() { + let mut p = sample_payload(); + p.extra_data = vec![]; + p.transactions = vec![]; + p.withdrawals = vec![]; + assert_eq!(p, ExecutionPayload::from_ssz(&p.to_ssz()).unwrap()); + } + + #[test] + fn ssz_round_trip_large_extra_data() { + let mut p = sample_payload(); + p.extra_data = vec![0xffu8; 32]; + assert_eq!(p, ExecutionPayload::from_ssz(&p.to_ssz()).unwrap()); + } + + #[test] + fn ssz_round_trip_many_transactions() { + let mut p = sample_payload(); + p.transactions = (0u8..=20).map(|i| vec![i; i as usize + 1]).collect(); + assert_eq!(p, ExecutionPayload::from_ssz(&p.to_ssz()).unwrap()); + } + + #[test] + fn ssz_too_short_returns_error() { + assert!(ExecutionPayload::from_ssz(&[0u8; 100]).is_err()); + } + + #[test] + fn ssz_exact_fixed_length_no_variable_data() { + // A 528-byte all-zero buffer has zero offsets, which fail the + // `extra_data_off >= PAYLOAD_FIXED_LEN` guard. + assert!(ExecutionPayload::from_ssz(&[0u8; 528]).is_err()); + } + + #[test] + fn ssz_variable_offsets_at_correct_positions() { + let p = sample_payload(); + let ssz = p.to_ssz(); + + let extra_off = u32::from_le_bytes(ssz[436..440].try_into().unwrap()) as usize; + let tx_off = u32::from_le_bytes(ssz[504..508].try_into().unwrap()) as usize; + let wd_off = u32::from_le_bytes(ssz[508..512].try_into().unwrap()) as usize; + + assert_eq!(extra_off, 528, "extra_data offset must point past fixed section"); + assert_eq!(tx_off, 528 + p.extra_data.len()); + // withdrawals follow the SSZ transaction list + assert!(wd_off > tx_off); + assert_eq!(wd_off, ssz.len() - p.withdrawals.len() * 44); + } + + // ----------------------------------------------------------------------- + // Transactions SSZ helpers + // ----------------------------------------------------------------------- + + #[test] + fn transactions_empty_round_trip() { + let encoded = encode_transactions(&[]); + assert!(encoded.is_empty()); + let decoded = decode_transactions(&encoded).unwrap(); + assert!(decoded.is_empty()); + } + + #[test] + fn transactions_single_round_trip() { + let txs = vec![vec![0xde, 0xad, 0xbe, 0xef]]; + assert_eq!(txs, decode_transactions(&encode_transactions(&txs)).unwrap()); + } + + #[test] + fn transactions_multi_varying_lengths_round_trip() { + let txs: Vec> = (1u8..=5).map(|i| vec![i; i as usize * 3]).collect(); + assert_eq!(txs, decode_transactions(&encode_transactions(&txs)).unwrap()); + } + + // ----------------------------------------------------------------------- + // Withdrawals SSZ helpers + // ----------------------------------------------------------------------- + + #[test] + fn withdrawals_round_trip() { + let ws = vec![ + Withdrawal { index: 10, validator_index: 20, address: [0x01; 20], amount: 500 }, + Withdrawal { index: 11, validator_index: 21, address: [0x02; 20], amount: 600 }, + ]; + let decoded = decode_withdrawals(&encode_withdrawals(&ws)).unwrap(); + assert_eq!(ws, decoded); + } + + #[test] + fn withdrawals_non_multiple_of_44_returns_error() { + assert!(decode_withdrawals(&[0u8; 45]).is_err()); + } + + // ----------------------------------------------------------------------- + // TCache framing: encode_new_payload_data / decode_new_payload_data + // ----------------------------------------------------------------------- + + #[test] + fn new_payload_framing_round_trip_no_exec_requests() { + let p = sample_payload(); + let ssz = p.to_ssz(); + let enc = encode_new_payload_data(&ssz, &[]); + let (decoded_p, decoded_reqs) = decode_new_payload_data(&enc).unwrap(); + assert_eq!(p, decoded_p); + assert!(decoded_reqs.is_empty()); + } + + #[test] + fn new_payload_framing_round_trip_with_exec_requests() { + let p = sample_payload(); + let ssz = p.to_ssz(); + let reqs = vec![vec![1u8, 2, 3], vec![0xde, 0xad]]; + let enc = encode_new_payload_data(&ssz, &reqs); + let (decoded_p, decoded_reqs) = decode_new_payload_data(&enc).unwrap(); + assert_eq!(p, decoded_p); + assert_eq!(reqs, decoded_reqs); + } + + #[test] + fn new_payload_framing_truncated_returns_error() { + let enc = encode_new_payload_data(&sample_payload().to_ssz(), &[]); + assert!(decode_new_payload_data(&enc[..4]).is_err()); + } + + // ----------------------------------------------------------------------- + // Wire format: serialize ↔ deserialize round-trips + // ----------------------------------------------------------------------- + + #[test] + fn wire_b256_round_trip() { + let state = ForkchoiceState { + head_block_hash: [0xab; 32], + safe_block_hash: [0x00; 32], + finalized_block_hash: [0xff; 32], + }; + let json: ForkchoiceState = + serde_json::from_str(&serde_json::to_string(&state).unwrap()).unwrap(); + assert_eq!(state.head_block_hash, json.head_block_hash); + assert_eq!(state.safe_block_hash, json.safe_block_hash); + assert_eq!(state.finalized_block_hash, json.finalized_block_hash); + } + + #[test] + fn wire_b256_correct_hex_encoding() { + let state = ForkchoiceState { + head_block_hash: [0xab; 32], + safe_block_hash: [0; 32], + finalized_block_hash: [0; 32], + }; + let json = serde_json::to_value(&state).unwrap(); + let hex = json["headBlockHash"].as_str().unwrap(); + assert!(hex.starts_with("0x")); + assert_eq!(hex.len(), 2 + 64); + assert_eq!(&hex[2..4], "ab"); + } + + #[test] + fn wire_quantity_round_trip() { + // Withdrawal uses quantity for all numeric fields + let w = Withdrawal { index: 0, validator_index: 0xdeadbeef, address: [0; 20], amount: 0 }; + let json: Withdrawal = serde_json::from_str(&serde_json::to_string(&w).unwrap()).unwrap(); + assert_eq!(w.validator_index, json.validator_index); + } + + #[test] + fn wire_u256_le_zero_round_trip() { + let mut p = sample_payload(); + p.base_fee_per_gas = [0u8; 32]; + let json = serde_json::to_value(&p).unwrap(); + assert_eq!(json["baseFeePerGas"].as_str().unwrap(), "0x0"); + let decoded: ExecutionPayload = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.base_fee_per_gas, [0u8; 32]); + } + + #[test] + fn wire_u256_le_nonzero_round_trip() { + let mut p = sample_payload(); + p.base_fee_per_gas = [0u8; 32]; + p.base_fee_per_gas[0] = 1; // LE: value = 1 + let json = serde_json::to_value(&p).unwrap(); + assert_eq!(json["baseFeePerGas"].as_str().unwrap(), "0x1"); + let decoded: ExecutionPayload = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.base_fee_per_gas, p.base_fee_per_gas); + } + + #[test] + fn wire_opt_b256_none_round_trip() { + let v: PayloadStatus = serde_json::from_str( + r#"{"status":"VALID","latestValidHash":null,"validationError":null}"#, + ) + .unwrap(); + assert_eq!(v.latest_valid_hash, None); + let json = serde_json::to_string(&v).unwrap(); + let v2: PayloadStatus = serde_json::from_str(&json).unwrap(); + assert_eq!(v2.latest_valid_hash, None); + } + + #[test] + fn wire_opt_payload_id_round_trip() { + let fcu_json = r#"{"payloadStatus":{"status":"VALID","latestValidHash":null,"validationError":null},"payloadId":"0x0102030405060708"}"#; + let fcu: ForkchoiceUpdatedResult = serde_json::from_str(fcu_json).unwrap(); + assert_eq!(fcu.payload_id, Some([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08])); + let roundtrip: ForkchoiceUpdatedResult = + serde_json::from_str(&serde_json::to_string(&fcu).unwrap()).unwrap(); + assert_eq!(fcu.payload_id, roundtrip.payload_id); + } +} diff --git a/memory/MEMORY.md b/memory/MEMORY.md new file mode 100644 index 0000000..dcbb949 --- /dev/null +++ b/memory/MEMORY.md @@ -0,0 +1,3 @@ +# Memory Index + +- [Always seek the best method](feedback_best_method.md) — Never settle for "fine" or "good enough" — always find the state of the art approach diff --git a/memory/feedback_best_method.md b/memory/feedback_best_method.md new file mode 100644 index 0000000..f8a80d6 --- /dev/null +++ b/memory/feedback_best_method.md @@ -0,0 +1,11 @@ +--- +name: Always seek the best method +description: Never settle for "fine" or "good enough" — always find the state of the art approach +type: feedback +--- + +Always seek out the best method for anything. The state of the art. Never settle for "good enough" or "fine." + +**Why:** Explicitly corrected when I said a linear scan was "fine" for in-flight lookup given small pool size — the right call was to use a HashMap for O(1) lookup regardless. + +**How to apply:** When evaluating implementation choices, don't stop at "acceptable given constraints." Find the best approach unconditionally, then apply it. From bd5c34ca7e083447977386817763c725149a41f9 Mon Sep 17 00:00:00 2001 From: owen Date: Mon, 1 Jun 2026 15:45:26 +0100 Subject: [PATCH 2/6] switch to simd json and fix perf issues --- Cargo.lock | 310 +++++----- Cargo.toml | 4 + crates/bin/Cargo.toml | 1 + crates/common/src/lib.rs | 25 +- crates/common/src/spine.rs | 14 +- crates/common/src/spine/messages.rs | 54 +- crates/common/src/spine/tcache.rs | 317 ++++++++-- crates/common/src/spine/tcache/consumer.rs | 4 +- crates/common/src/spine/tcache/producer.rs | 2 +- crates/engine/Cargo.toml | 27 +- crates/engine/examples/engine_tile_relay.rs | 86 +++ crates/engine/src/client.rs | 130 +++-- crates/engine/src/error.rs | 4 +- crates/engine/src/http.rs | 365 ++++++------ crates/engine/src/ipc.rs | 76 +-- crates/engine/src/jwt.rs | 8 +- crates/engine/src/req_handlers.rs | 37 +- crates/engine/src/resp_handlers.rs | 248 ++++---- crates/engine/src/tile.rs | 32 +- crates/engine/src/types.rs | 606 ++++++++++++++------ memory/MEMORY.md | 3 - memory/feedback_best_method.md | 11 - 22 files changed, 1510 insertions(+), 854 deletions(-) create mode 100644 crates/engine/examples/engine_tile_relay.rs delete mode 100644 memory/MEMORY.md delete mode 100644 memory/feedback_best_method.md diff --git a/Cargo.lock b/Cargo.lock index 00f8546..3ef60f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,12 +117,56 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -369,58 +413,6 @@ dependencies = [ "fs_extra", ] -[[package]] -name = "axum" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" -dependencies = [ - "axum-core", - "bytes", - "form_urlencoded", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde_core", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "axum-core" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "sync_wrapper", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "backtrace" version = "0.3.76" @@ -766,6 +758,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -774,8 +767,22 @@ version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -793,6 +800,12 @@ dependencies = [ "cc", ] +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "compact_str" version = "0.8.2" @@ -1411,6 +1424,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + [[package]] name = "flux" version = "0.0.38" @@ -1492,6 +1514,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1796,6 +1824,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "halfbrown" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7ed2f2edad8a14c8186b847909a41fbb9c3eafa44f88bd891114ed5019da09" +dependencies = [ + "hashbrown 0.16.1", + "serde", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -1813,7 +1851,18 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -2004,12 +2053,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "humantime" version = "2.3.0" @@ -2030,7 +2073,6 @@ dependencies = [ "http", "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -2352,6 +2394,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.10.5" @@ -2891,12 +2939,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - [[package]] name = "memchr" version = "2.8.1" @@ -2930,12 +2972,6 @@ dependencies = [ "libmimalloc-sys", ] -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3232,6 +3268,12 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "oorandom" version = "11.1.5" @@ -3830,6 +3872,26 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "regex" version = "1.12.3" @@ -4131,17 +4193,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "serde_path_to_error" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" -dependencies = [ - "itoa", - "serde", - "serde_core", -] - [[package]] name = "serde_spanned" version = "0.6.9" @@ -4151,18 +4202,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "serde_yml" version = "0.0.12" @@ -4278,6 +4317,7 @@ dependencies = [ name = "silver" version = "0.0.1" dependencies = [ + "clap", "flux", "hex", "mimalloc", @@ -4423,7 +4463,6 @@ dependencies = [ name = "silver_engine" version = "0.0.1" dependencies = [ - "axum", "base64 0.22.1", "flux", "hex", @@ -4432,11 +4471,10 @@ dependencies = [ "mio", "rustc-hash", "serde", - "serde_json", "sha2", "silver_common", + "simd-json", "thiserror 1.0.69", - "tokio", "tracing", "tracing-subscriber", ] @@ -4538,6 +4576,26 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd-json" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4255126f310d2ba20048db6321c81ab376f6a6735608bf11f0785c41f01f64e3" +dependencies = [ + "halfbrown", + "ref-cast", + "serde", + "serde_json", + "simdutf8", + "value-trait", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "1.0.3" @@ -4716,12 +4774,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" - [[package]] name = "synstructure" version = "0.13.2" @@ -4906,9 +4958,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", - "signal-hook-registry", "socket2 0.6.3", "tokio-macros", "windows-sys 0.61.2", @@ -5009,28 +5059,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" -[[package]] -name = "tower" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - [[package]] name = "tower-service" version = "0.3.3" @@ -5226,6 +5254,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -5249,6 +5283,18 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ba6f5989077681266825251a52748b8c1d8a4ad098cc37e440103d0ea717fc0" +[[package]] +name = "value-trait" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e80f0c733af0720a501b3905d22e2f97662d8eacfe082a75ed7ffb5ab08cb59" +dependencies = [ + "float-cmp", + "halfbrown", + "itoa", + "ryu", +] + [[package]] name = "version_check" version = "0.9.5" @@ -6061,18 +6107,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "bce33a6288fa3f072a8c2c7d0f2fdbb90e28298f0135c1f99b96c3db2efcc60b" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "8fd425244944f4ab65ccff928e7323354c5a018c75838362fdce749dfad2ee1e" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index b5224c9..32cafc0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,6 +92,8 @@ hashtree-rs = "0.2" hdrhistogram = "7.5.4" hex = { version = "0.4", features = ["serde"] } hkdf = "0.12" +hmac = "0.12" +httparse = "1" libc = "0.2" mimalloc = "0.1.52" mio = { version = "1.0.4", features = ["net", "os-poll"] } @@ -108,7 +110,9 @@ rustc-hash = "2" secp256k1 = { version = "0.30", features = ["global-context", "rand", "hashes"] } rustls = "0.23.37" serde = { version = "1.0.228", features = ["derive"] } +clap = { version = "4", features = ["derive"] } serde_json = "1.0.135" +simd-json = "0.17.0" toml = "0.8" sha2 = "0.10" sha2-const-stable = "0.1" diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index 2070a69..3a138e4 100644 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -14,6 +14,7 @@ silver_gossip.workspace = true silver_network.workspace = true silver_peer.workspace = true +clap.workspace = true flux.workspace = true hex.workspace = true mimalloc.workspace = true diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index f151f27..6bb0480 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -18,18 +18,19 @@ pub use crate::{ }, spine::{ ALL_PROTOCOLS, AcquiredRead as TRead, BeaconStateEvent, Consumer as TConsumer, - DataColumnsAvailable, ELSyncStatus, EngineFcuReq, EngineFcuResp, EngineGetBlobsReq, - EngineGetBlobsResp, EngineGetPayloadBodiesByHashReq, EngineGetPayloadBodiesByRangeReq, - EngineGetPayloadBodiesResp, EngineGetPayloadReq, EngineGetPayloadResp, EngineHealthEvent, - EngineNewPayloadReq, EngineNewPayloadResp, EngineRawReq, EngineRawResp, EngineReq, - EngineResp, Error as TCacheError, GossipMsgOut, IpBytes, MAX_BLOBS_PER_BLOCK, - MULTISTREAM_V1, MultiProducer as TMultiProducer, NewGossipMsg, P2pSend, P2pStreamId, - PayloadValidationStatus, PeerControl, PeerEvent, PeerStatus, Producer as TProducer, - REJECT_RESPONSE, RPC_PROTOCOLS, RandomAccessConsumer as TRandomAccess, RejectSource, - Reservation as TReservation, RpcInbound, RpcMsg, RpcOutbound, RpcRequest, - RpcRequestInbound, RpcRequestOutbound, RpcResponse, RpcResponseInbound, - RpcResponseOutbound, RpcSeverity, SilverSpine, SilverSpineProducers, StreamProtocol, - SyncUpdate, TCache, TCacheProducer, TCacheRead, TCacheRef, + DataColumnsAvailable, ELSyncStatus, EngineFcuReq, EngineFcuResp, EngineGetPayloadReq, + EngineGetBlobsReq, EngineGetBlobsResp, EngineGetPayloadBodiesByHashReq, + EngineGetPayloadBodiesByRangeReq, EngineGetPayloadBodiesResp, EngineGetPayloadResp, + EngineHealthEvent, EngineNewPayloadReq, EngineNewPayloadResp, EnginePreparePayloadReq, + EngineReq, EngineResp, Error as TCacheError, GossipMsgOut, IpBytes, MAX_BLOBS_PER_BLOCK, + MAX_PAYLOAD_BODIES_PER_REQ, MULTISTREAM_V1, MultiProducer as TMultiProducer, NewGossipMsg, + P2pSend, P2pStreamId, PayloadValidationStatus, PeerControl, PeerEvent, PeerStatus, + Producer as TProducer, REJECT_RESPONSE, RPC_PROTOCOLS, + RandomAccessConsumer as TRandomAccess, RejectSource, Reservation as TReservation, + RpcInbound, RpcMsg, RpcOutbound, RpcRequest, RpcRequestInbound, RpcRequestOutbound, + RpcResponse, RpcResponseInbound, RpcResponseOutbound, RpcSeverity, SilverSpine, + SilverSpineProducers, StreamProtocol, SyncUpdate, TCache, TCacheProducer, TCacheRead, + TCacheRef, WithdrawalInline, }, util::{create_self_signed_certificate, decode_varint, encode_varint}, wheel::Wheel, diff --git a/crates/common/src/spine.rs b/crates/common/src/spine.rs index 56c173d..14518a6 100644 --- a/crates/common/src/spine.rs +++ b/crates/common/src/spine.rs @@ -3,14 +3,14 @@ use flux::{communication::ShmemData, spine::SpineQueue, spine_derive::from_spine, tile::TileInfo}; pub use messages::{ BeaconStateEvent, DataColumnsAvailable, ELSyncStatus, EngineFcuReq, EngineFcuResp, - EngineGetBlobsReq, EngineGetBlobsResp, EngineGetPayloadBodiesByHashReq, - EngineGetPayloadBodiesByRangeReq, EngineGetPayloadBodiesResp, EngineGetPayloadReq, - EngineGetPayloadResp, EngineHealthEvent, EngineNewPayloadReq, EngineNewPayloadResp, - EngineRawReq, EngineRawResp, EngineReq, EngineResp, GossipMsgOut, IpBytes, - MAX_BLOBS_PER_BLOCK, NewGossipMsg, P2pSend, PayloadValidationStatus, PeerControl, PeerEvent, - PeerStatus, RejectSource, RpcInbound, RpcMsg, RpcOutbound, RpcRequest, RpcRequestInbound, + EngineGetPayloadReq, EngineGetBlobsReq, EngineGetBlobsResp, EngineGetPayloadBodiesByHashReq, + EngineGetPayloadBodiesByRangeReq, EngineGetPayloadBodiesResp, EngineGetPayloadResp, + EngineHealthEvent, EngineNewPayloadReq, EngineNewPayloadResp, EnginePreparePayloadReq, + EngineReq, EngineResp, GossipMsgOut, IpBytes, MAX_BLOBS_PER_BLOCK, MAX_PAYLOAD_BODIES_PER_REQ, + NewGossipMsg, P2pSend, PayloadValidationStatus, PeerControl, PeerEvent, PeerStatus, + RejectSource, RpcInbound, RpcMsg, RpcOutbound, RpcRequest, RpcRequestInbound, RpcRequestOutbound, RpcResponse, RpcResponseInbound, RpcResponseOutbound, RpcSeverity, - SyncUpdate, + SyncUpdate, WithdrawalInline, }; pub use stream_id::P2pStreamId; pub use stream_protocol::{ diff --git a/crates/common/src/spine/messages.rs b/crates/common/src/spine/messages.rs index e1cf059..b515c84 100644 --- a/crates/common/src/spine/messages.rs +++ b/crates/common/src/spine/messages.rs @@ -598,7 +598,7 @@ pub enum BeaconStateEvent { /// Maximum blob commitments per block (Fulu target; increase as the spec /// evolves). -pub const MAX_BLOBS_PER_BLOCK: usize = 9; +pub const MAX_BLOBS_PER_BLOCK: usize = 21; /// Maximum number of block hashes in a single `getPayloadBodiesByHash` request. pub const MAX_PAYLOAD_BODIES_PER_REQ: usize = 128; @@ -683,15 +683,16 @@ pub struct EngineNewPayloadResp { pub latest_valid_hash: [u8; 32], } -/// `engine_getPayloadV4` request. The engine tile issues FCU-with-attrs -/// internally, then follows up with getPayloadV4 using the returned payload_id. -/// The caller sees a single request/response pair on the spine. + +/// The engine tile sends `engine_forkchoiceUpdatedV3` with payload attributes +/// and returns the `payload_id` assigned by the EL. The caller should use. +/// this to fetch the built payload. /// /// Field layout mirrors `EngineFcuReq` (attrs are always present for payload /// building). #[derive(Clone, Copy, Debug)] #[repr(C)] -pub struct EngineGetPayloadReq { +pub struct EnginePreparePayloadReq { pub id: u64, pub head_block_hash: [u8; 32], pub safe_block_hash: [u8; 32], @@ -704,10 +705,16 @@ pub struct EngineGetPayloadReq { pub attrs_withdrawals: [WithdrawalInline; 16], } +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct EngineGetPayloadReq { + pub id: u64, + pub payload_id: [u8; 8], +} + /// Response to `EngineGetPayloadReq`. -/// When `ok` is true, `data` is a TCache slot with the JSON-encoded EL -/// response: `{executionPayload, blobsBundle, shouldOverrideBuilder, -/// executionRequests}`. +/// When `ok` is true, `data` is a TCache slot with the encoded EL payload: +/// `{executionPayload, blobsBundle, shouldOverrideBuilder, executionRequests}`. #[derive(Clone, Copy, Debug)] #[repr(C)] pub struct EngineGetPayloadResp { @@ -727,8 +734,9 @@ pub struct EngineGetBlobsReq { } /// Response to `EngineGetBlobsReq`. -/// When `ok` is true, `data` is a TCache slot with the JSON-encoded array of -/// `{blob, proofs}` objects (entries may be `null` for missing blobs). +/// When `ok` is true, `data` is a TCache slot with binary-encoded blobs: +/// `[u32 count] ([u8 present] [u8 proof_count] [48B proof]* [u32 blob_len] [blob bytes])*` +/// `present == 0` means the entry is null (blob missing). #[derive(Clone, Copy, Debug)] #[repr(C)] pub struct EngineGetBlobsResp { @@ -757,8 +765,9 @@ pub struct EngineGetPayloadBodiesByRangeReq { } /// Response to either `getPayloadBodiesByHash` or `getPayloadBodiesByRange`. -/// When `ok` is true, `data` is a TCache slot with the JSON-encoded array of -/// payload body objects (entries may be `null` for missing blocks). +/// When `ok` is true, `data` is a TCache slot with binary-encoded bodies: +/// `[u32 count] ([u8 present] [u32 tx_count] ([u32 tx_len][tx bytes])* [u32 withdrawal_count] ([u32 index][u32 validator_index][20B address][u64 amount])*)*` +/// `present == 0` means the entry is null (block missing). #[derive(Clone, Copy, Debug)] #[repr(C)] pub struct EngineGetPayloadBodiesResp { @@ -767,26 +776,6 @@ pub struct EngineGetPayloadBodiesResp { pub data: TCacheRead, } -/// Passthrough request for any engine/eth JSON-RPC method not handled inline. -/// `body` is a TCache slot containing `{"method":"...","params":[...]}` JSON. -#[derive(Clone, Copy, Debug)] -#[repr(C)] -pub struct EngineRawReq { - pub id: u64, - pub body: TCacheRead, -} - -/// Response to an `EngineRawReq`. -/// When `ok` is true, `data` is a TCache slot with the JSON-encoded result -/// value. When `ok` is false the slot is unused. -#[derive(Clone, Copy, Debug)] -#[repr(C)] -pub struct EngineRawResp { - pub id: u64, - pub ok: bool, - pub data: TCacheRead, -} - /// Multiplexed engine request. A single spine queue carries FCU, /// new-payload, and raw passthrough requests, preserving strict FIFO ordering. #[derive(Clone, Copy, Debug)] @@ -795,6 +784,7 @@ pub struct EngineRawResp { pub enum EngineReq { Fcu(EngineFcuReq), NewPayload(EngineNewPayloadReq), + PreparePayload(EnginePreparePayloadReq), GetPayload(EngineGetPayloadReq), GetBlobs(EngineGetBlobsReq), GetPayloadBodiesByHash(EngineGetPayloadBodiesByHashReq), diff --git a/crates/common/src/spine/tcache.rs b/crates/common/src/spine/tcache.rs index 0b146f1..bb54466 100644 --- a/crates/common/src/spine/tcache.rs +++ b/crates/common/src/spine/tcache.rs @@ -19,6 +19,13 @@ const MAGIC: [u8; 3] = [0xEA, 0x51, 0xEE]; const MAX_CONSUMERS: usize = 64; const ALIGN: usize = size_of::(); +// TCacheHead (528 B) rounded up to Slot alignment (32 B). +const DATA_OFFSET: usize = const { + let h = size_of::(); + let a = size_of::(); + (h + a - 1) & !(a - 1) +}; + mod consumer; mod metrics; mod producer; @@ -27,26 +34,58 @@ use metrics::TCacheMetrics; /// Single or multi producer, multi consumer cache buffer with a Tail /// -/// _ /)---(\ -/// \\ (/ . . \) -/// \\__)-\(*)/ -/// \_ (_ -/// (___/-(____) _ -/// +/// _ /)---(\ +/// \\ (/ . . \) +/// \\__)-\(*)/ +/// \_ (_ +/// (___/-(____) _ pub struct TCache { - /// Descriptive label for this TCache instance — used as the - /// `{name}` in `counters-tcache-{name}` for the metrics layer, - /// and useful for tracing. Stable for the lifetime of the - /// allocation. name: &'static str, - head: TCacheHead, + /// Flat buffer: [TCacheHead | padding | data…] + /// Aligned to ALIGN. DATA_OFFSET bytes precede the data region. + ptr: *mut u8, + /// Data capacity (excludes the header). len: u32, - data: Box<[u8]>, + backing: Backing, /// `None` when name is empty, or if mmap'ing the metrics file /// fails (tile continues to function without surfer visibility). metrics: Option, } +unsafe impl Send for TCache {} +unsafe impl Sync for TCache {} + +enum Backing { + Heap { + layout: Layout, + }, + #[cfg(unix)] + Shmem { + fd: libc::c_int, + total_len: usize, + owner: bool, + name: std::ffi::CString, + }, +} + +impl Drop for TCache { + fn drop(&mut self) { + match &self.backing { + Backing::Heap { layout } => unsafe { + alloc::dealloc(self.ptr, *layout); + }, + #[cfg(unix)] + Backing::Shmem { fd, total_len, owner, name } => unsafe { + libc::munmap(self.ptr as *mut libc::c_void, *total_len); + libc::close(*fd); + if *owner { + libc::shm_unlink(name.as_ptr()); + } + }, + } + } +} + #[derive(Copy, Clone, Debug)] #[repr(C)] pub struct TCacheRef { @@ -84,14 +123,14 @@ impl TCache { /// (e.g. `"gossip_in"`); the metrics layer uses it to produce /// `counters-tcache-{name}`. pub fn producer(name: &'static str, n: usize) -> Producer { - let tcache = Self::alloc_tache(name, n); + let tcache = Self::alloc_heap(name, n); let space = tcache.len; Producer { cache: Box::into_raw(tcache), seq: 0, space } } /// Create a multi-producer t-cache. pub fn multi_producer(name: &'static str, n: usize) -> MultiProducer { - let tcache = Self::alloc_tache(name, n); + let tcache = Self::alloc_heap(name, n); let len = tcache.len; MultiProducer::new(Box::into_raw(tcache), len) } @@ -100,12 +139,32 @@ impl TCache { self.name } + /// Attach to a named shmem segment as a producer, creating it if needed. + /// Either side (producer or consumer) may start first. `n` must be + /// identical on both sides. + #[cfg(unix)] + pub fn shm_producer(name: &str, n: usize) -> Producer { + let tcache = Self::attach_shmem(name, n); + let space = tcache.len; + Producer { cache: Box::into_raw(tcache), seq: 0, space } + } + + /// Attach to a named shmem segment as a random-access consumer, creating + /// it if needed. Either side (producer or consumer) may start first. `n` + /// must be identical on both sides. + #[cfg(unix)] + pub fn shm_consumer(name: &str, n: usize) -> Result { + let tcache = Box::into_raw(Self::attach_shmem(name, n)); + // SAFETY: tcache is a valid Box-allocated TCache; we intentionally + // leak it here (same pattern as producer/multi_producer). + unsafe { (*tcache).random_access("", true) } + } + pub fn consumer(&self, name: &'static str) -> Result { - // find start seq - let seq = self.head.seq.load(Ordering::Acquire); + let seq = self.head().seq.load(Ordering::Acquire); let index = self - .head + .head() .tails .iter() .position(|t| { @@ -132,11 +191,10 @@ impl TCache { name: &'static str, auto_free: bool, ) -> Result { - // find start seq - let seq = self.head.seq.load(Ordering::Acquire); + let seq = self.head().seq.load(Ordering::Acquire); let index = self - .head + .head() .tails .iter() .position(|t| { @@ -195,7 +253,7 @@ impl TCache { fn min_tail(&self, seq: u64) -> u64 { let mut min = seq; - for tail in &self.head.tails { + for tail in &self.head().tails { min = min.min(tail.load(Ordering::Acquire)); } min @@ -203,7 +261,7 @@ impl TCache { fn read(&self, seq: u64) -> Result<(&[u8], u64, Nanos), Error> { let idx = self.index(seq); - let slot: &Slot = (&self.data[idx..]).into(); + let slot: &Slot = self.slot_at(idx); if slot.magic != MAGIC { return Err(Error::NoMagic); } @@ -217,7 +275,12 @@ impl TCache { } Ok(( - &self.data[slot.data_start as usize..slot.data_end as usize], + unsafe { + slice::from_raw_parts( + self.data_ptr().add(slot.data_start as usize), + slot.data_end as usize - slot.data_start as usize, + ) + }, slot.reservation_len as u64, slot.reserve_ns, )) @@ -225,7 +288,7 @@ impl TCache { fn slot_ts(&self, seq: u64) -> Result { let idx = self.index(seq); - let slot: &Slot = (&self.data[idx..]).into(); + let slot: &Slot = self.slot_at(idx); if slot.magic != MAGIC { return Err(Error::NoMagic); } @@ -237,21 +300,16 @@ impl TCache { Ok(slot.reserve_ns) } - /// For a reservation at the requested seq and length, what reservation - /// length is required? Handles slot header and buffer wrapping. fn reserve_len(&self, seq: u64, requested_len: usize) -> usize { let mut data_len = requested_len + size_of::(); - let reserve_seq = seq; - let idx = self.index(reserve_seq); + let idx = self.index(seq); if idx + data_len > self.len as usize { - // would wrap, allocate at start of buffer to return contiguous slice let offset = self.len as usize - idx; data_len += offset; } - // Aligned so that contiguous `size_of::()` always available align::(data_len) } @@ -262,7 +320,6 @@ impl TCache { let idx = self.index(reserve_seq); let data_offset = if idx + data_len > self.len as usize { - // would wrap, allocate at start of buffer to return contiguous slice let offset = self.len as usize - idx; data_len += offset; offset @@ -270,7 +327,6 @@ impl TCache { size_of::() }; - // Aligned so that contiguous `size_of::()` always available let reserve_len = align::(data_len); (space >= reserve_len as u32).then(|| { @@ -278,7 +334,7 @@ impl TCache { let end = start + len as usize; let slot: &mut Slot = unsafe { - let mut_ptr = self.data[idx..idx + size_of::()].as_ptr() as *mut u8; + let mut_ptr = self.data_ptr().add(idx); slice::from_raw_parts_mut(mut_ptr, size_of::()).into() }; @@ -294,12 +350,12 @@ impl TCache { }) } - /// SAFETY: This must only calleded by a single `Reservation` owner for a + /// SAFETY: This must only be called by a single `Reservation` owner for a /// given `seq`. #[allow(clippy::mut_from_ref)] fn write(&self, seq: u64) -> Result<&mut [u8], Error> { let idx = self.index(seq); - let slot: &Slot = (&self.data[idx..]).into(); + let slot: &Slot = self.slot_at(idx); if slot.magic != MAGIC { return Err(Error::NoMagic); } @@ -310,12 +366,8 @@ impl TCache { } unsafe { - let mut_ptr = - self.data[slot.data_start as usize..slot.data_end as usize].as_ptr() as *mut u8; - Ok(slice::from_raw_parts_mut( - mut_ptr, - slot.data_end as usize - slot.data_start as usize, - )) + let ptr = self.data_ptr().add(slot.data_start as usize); + Ok(slice::from_raw_parts_mut(ptr, slot.data_end as usize - slot.data_start as usize)) } } @@ -323,7 +375,7 @@ impl TCache { let idx = self.index(seq); let slot: &mut Slot = unsafe { - let mut_ptr = self.data[idx..idx + size_of::()].as_ptr() as *mut u8; + let mut_ptr = self.data_ptr().add(idx); slice::from_raw_parts_mut(mut_ptr, size_of::()).into() }; if success { @@ -338,40 +390,182 @@ impl TCache { self.record_head(new_head); } - fn alloc_tache(name: &'static str, size: usize) -> Box { + // --- backing accessors --- + + #[inline] + pub(super) fn head(&self) -> &TCacheHead { + // SAFETY: ptr is always valid and aligned; TCacheHead is at offset 0. + unsafe { &*(self.ptr as *const TCacheHead) } + } + + #[inline] + fn data_ptr(&self) -> *mut u8 { + // SAFETY: ptr + DATA_OFFSET is within the allocated buffer. + unsafe { self.ptr.add(DATA_OFFSET) } + } + + #[inline] + fn slot_at(&self, idx: usize) -> &Slot { + unsafe { + let ptr = self.data_ptr().add(idx); + &*(slice::from_raw_parts(ptr, size_of::()).as_ptr() as *const Slot) + } + } + + // --- allocators --- + + fn alloc_heap(name: &'static str, size: usize) -> Box { assert!( size.is_power_of_two() && size.is_multiple_of(ALIGN), - "n is not mutiple of {ALIGN}" + "n must be a power-of-two multiple of {ALIGN}" ); - let layout = Layout::from_size_align(size, ALIGN).unwrap(); + let total = DATA_OFFSET + size; + let layout = Layout::from_size_align(total, ALIGN).unwrap(); let metrics = if name.is_empty() { None } else { - // MAX_CONSUMERS — overcount is fine for the file size; surfer - // ignores never-set tails (they stay at u64::MAX sentinel). TCacheMetrics::new(name, MAX_CONSUMERS, size as u64) .map_err(|e| tracing::warn!(?name, ?e, "TCacheMetrics::new failed")) .ok() }; - unsafe { - let ptr = alloc::alloc_zeroed(layout); - if ptr.is_null() { + let ptr = unsafe { + let p = alloc::alloc_zeroed(layout); + if p.is_null() { alloc::handle_alloc_error(layout); } - let data = Box::from_raw(std::ptr::slice_from_raw_parts_mut(ptr, size)); + p + }; + Self::init_head(ptr); + Box::new(Self { name, ptr, len: size as u32, backing: Backing::Heap { layout }, metrics }) + } + + /// Attach to a named shmem segment, creating it if it doesn't exist yet. + /// Safe to call from multiple processes concurrently: `O_CREAT | O_EXCL` + /// is the atomic gate — exactly one caller becomes the creator (sizes + + /// inits the segment), all others wait until it is ready. + /// + /// Readiness is signalled by `ready == u64::MAX`, set as the last + /// store in `init_head`. + #[cfg(unix)] + fn attach_shmem(name: &str, size: usize) -> Box { + use std::ffi::CString; + assert!( + size.is_power_of_two() && size.is_multiple_of(ALIGN), + "n must be a power-of-two multiple of {ALIGN}" + ); + let total = DATA_OFFSET + size; + let cname = CString::new(name).expect("shmem name must not contain null bytes"); + + // Race to create. Exactly one caller wins O_EXCL; losers open existing. + let fd = unsafe { + libc::shm_open(cname.as_ptr(), libc::O_CREAT | libc::O_EXCL | libc::O_RDWR, 0o600) + }; + + if fd >= 0 { + // Creator: size the segment, map it, init. + // On any failure, unlink so joiners don't spin forever on a zombie segment. + struct Cleanup(i32, *const libc::c_char); + impl Drop for Cleanup { + fn drop(&mut self) { + unsafe { libc::shm_unlink(self.1) }; + unsafe { libc::close(self.0) }; + } + } + let cleanup = Cleanup(fd, cname.as_ptr()); + + let rc = unsafe { libc::ftruncate(fd, total as libc::off_t) }; + assert_eq!(rc, 0, "ftruncate({name}): {}", std::io::Error::last_os_error()); + + let ptr = unsafe { + libc::mmap( + std::ptr::null_mut(), + total, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + fd, + 0, + ) + }; + assert_ne!(ptr, libc::MAP_FAILED, "mmap({name}): {}", std::io::Error::last_os_error()); + + let ptr = ptr as *mut u8; + // Sets ready = u64::MAX as the last store (release) in init_head. + Self::init_head(ptr); + std::mem::forget(cleanup); // init complete, segment is valid + Box::new(Self { + name: "", + ptr, + len: size as u32, + backing: Backing::Shmem { fd, total_len: total, owner: true, name: cname }, + metrics: None, + }) + } else { + // Joiner: open existing, wait for creator to finish sizing + init. + assert_eq!( + std::io::Error::last_os_error().raw_os_error(), + Some(libc::EEXIST), + "shm_open({name}): {}", + std::io::Error::last_os_error(), + ); + + let fd = loop { + let fd = unsafe { libc::shm_open(cname.as_ptr(), libc::O_RDWR, 0) }; + if fd >= 0 { + break fd; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + }; + + // Wait for ftruncate. + loop { + let mut st: libc::stat = unsafe { std::mem::zeroed() }; + unsafe { libc::fstat(fd, &mut st) }; + if st.st_size as usize >= total { + break; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + + let ptr = unsafe { + libc::mmap( + std::ptr::null_mut(), + total, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + fd, + 0, + ) + }; + assert_ne!(ptr, libc::MAP_FAILED, "mmap({name}): {}", std::io::Error::last_os_error()); + + // Wait for init_head (acquire load pairs with creator's release store on + // ready). + let head = unsafe { &*(ptr as *const TCacheHead) }; + while head.ready.load(Ordering::Acquire) != u64::MAX { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + Box::new(Self { - name, - head: TCacheHead { - seq: AtomicU64::new(0), - tails: array::from_fn(|_| AtomicU64::new(u64::MAX)), - }, - len: data.len() as u32, - data, - metrics, + name: "", + ptr: ptr as *mut u8, + len: size as u32, + backing: Backing::Shmem { fd, total_len: total, owner: false, name: cname }, + metrics: None, }) } } + fn init_head(ptr: *mut u8) { + // SAFETY: ptr is freshly allocated/mapped with at least DATA_OFFSET bytes. + unsafe { + let head = &mut *(ptr as *mut TCacheHead); + head.seq = AtomicU64::new(0); + head.tails = array::from_fn(|_| AtomicU64::new(u64::MAX)); + // Release store — pairs with the joiner's acquire load on ready. + head.ready.store(u64::MAX, Ordering::Release); + } + } + #[inline] pub(super) fn record_head(&self, seq: u64) { if let Some(m) = &self.metrics { @@ -395,9 +589,12 @@ impl TCache { } #[repr(C)] -struct TCacheHead { +pub(super) struct TCacheHead { seq: AtomicU64, tails: [AtomicU64; MAX_CONSUMERS], + /// Written last in init_head as the readiness signal; never modified after + /// init. + ready: AtomicU64, } #[repr(C)] diff --git a/crates/common/src/spine/tcache/consumer.rs b/crates/common/src/spine/tcache/consumer.rs index d2d7c94..c836508 100644 --- a/crates/common/src/spine/tcache/consumer.rs +++ b/crates/common/src/spine/tcache/consumer.rs @@ -85,7 +85,7 @@ impl Consumer { pub fn free(&mut self) { //tracing::warn!("consumer free: {}", self.seq); self.seq = self.next_seq; - self.cache.head.tails[self.index].store(self.seq, Ordering::Release); + self.cache.head().tails[self.index].store(self.seq, Ordering::Release); self.cache.record_tail(self.index, self.seq); } } @@ -123,7 +123,7 @@ impl RandomAccessConsumer { if tail != u64::MAX { //tracing::warn!("Random consumer free {tail}"); - self.cache.head.tails[self.index].store(tail, Ordering::Release); + self.cache.head().tails[self.index].store(tail, Ordering::Release); self.cache.record_tail(self.index, tail); } } diff --git a/crates/common/src/spine/tcache/producer.rs b/crates/common/src/spine/tcache/producer.rs index bd5d939..6373893 100644 --- a/crates/common/src/spine/tcache/producer.rs +++ b/crates/common/src/spine/tcache/producer.rs @@ -12,7 +12,7 @@ pub trait TCacheProducer: SealedProducer { fn publish_head(&self) { let tcache = unsafe { &*self.tcache() }; let seq = self.seq(); - tcache.head.seq.store(seq, Ordering::Release); + tcache.head().seq.store(seq, Ordering::Release); tcache.record_head(seq); } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 1fb3f33..22a1d37 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -6,25 +6,22 @@ rust-version.workspace = true version.workspace = true [dependencies] -silver_common.workspace = true +base64.workspace = true flux.workspace = true +hex.workspace = true +hmac.workspace = true +httparse.workspace = true +mio.workspace = true rustc-hash.workspace = true - -base64 = "0.22" -hex = "0.4" -hmac = "0.12" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -sha2 = "0.10" -thiserror = "1.0" -httparse = "1" -mio = { workspace = true } -tracing = "0.1" +serde.workspace = true +simd-json.workspace = true +sha2.workspace = true +silver_common.workspace = true +thiserror.workspace = true +tracing.workspace = true [dev-dependencies] -axum = "0.8.9" -tokio = { version = "1.0", features = ["full"] } -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber.workspace = true [lints] workspace = true diff --git a/crates/engine/examples/engine_tile_relay.rs b/crates/engine/examples/engine_tile_relay.rs new file mode 100644 index 0000000..83f7a3f --- /dev/null +++ b/crates/engine/examples/engine_tile_relay.rs @@ -0,0 +1,86 @@ +// Run the engine tile standalone against a local EL node. +// +// A separate feeder process can write EngineReq messages over the shared +// memory spine and large SSZ payloads into the shmem TCache segments. +// +// Usage: +// cargo run -p silver_engine --example engine_tile_relay -- \ +// --endpoint http://127.0.0.1:8551 \ +// --jwt-secret /path/to/jwtsecret +// +// Press Ctrl-C to stop. + +use std::sync::atomic::Ordering; + +use flux::tile::{TileConfig, attach_tile}; +use silver_common::{SilverSpine, TCache}; +use silver_engine::{EngineClient, EngineTile, JwtSecret}; +use tracing_subscriber::EnvFilter; + +const ENGINE_TCACHE_SIZE: usize = 1 << 24; // 16 MiB + +const REQ_SHMEM_NAME: &str = "/silver-engine-req"; +const RESP_SHMEM_NAME: &str = "/silver-engine-resp"; + +fn main() { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env().add_directive(tracing::Level::INFO.into())) + .init(); + + let (endpoint, jwt) = parse_args(); + tracing::info!(%endpoint, "starting engine tile"); + + // Shmem-backed TCache pair. The req segment is written by the feeder + // process; the resp segment is written here and read by the feeder. + let req_consumer = + TCache::shm_consumer(REQ_SHMEM_NAME, ENGINE_TCACHE_SIZE).expect("req shmem consumer"); + let resp_producer = TCache::shm_producer(RESP_SHMEM_NAME, ENGINE_TCACHE_SIZE); + + let tile = EngineTile::new(EngineClient::new(endpoint, jwt), req_consumer, resp_producer); + + let spine = SilverSpine::new(None); + spine.start(None, None, |scoped| { + let stop = std::sync::Arc::clone(&scoped.stop_flag); + attach_tile(tile, scoped, TileConfig::background(None, None)); + + while stop.load(Ordering::Relaxed) == 0 { + std::thread::sleep(std::time::Duration::from_millis(100)); + } + }); +} + +fn parse_args() -> (String, JwtSecret) { + let args: Vec = std::env::args().collect(); + let mut endpoint = "http://127.0.0.1:8551".to_string(); + let mut jwt_arg: Option = None; + + let mut i = 1; + while i < args.len() { + match args[i].as_str() { + "--endpoint" => { + i += 1; + endpoint = args[i].clone(); + } + "--jwt-secret" => { + i += 1; + jwt_arg = Some(args[i].clone()); + } + _ => {} + } + i += 1; + } + + let jwt = match jwt_arg { + Some(s) if std::path::Path::new(&s).exists() => { + JwtSecret::from_file(std::path::Path::new(&s)) + .unwrap_or_else(|e| panic!("invalid jwt file {s}: {e}")) + } + Some(s) => JwtSecret::from_hex(&s).unwrap_or_else(|e| panic!("invalid jwt hex: {e}")), + _ => { + eprintln!("--jwt-secret required"); + std::process::exit(1); + } + }; + + (endpoint, jwt) +} diff --git a/crates/engine/src/client.rs b/crates/engine/src/client.rs index 29c7259..903030c 100644 --- a/crates/engine/src/client.rs +++ b/crates/engine/src/client.rs @@ -2,19 +2,25 @@ use std::time::Duration; use mio::{Events, Poll}; use rustc_hash::FxHashMap; +use simd_json::prelude::ValueObjectAccess; use crate::{ EngineError, JwtSecret, - http::{HttpPool, POOL_SIZE, http_pool_enqueue, poll_http_pool}, + http::{HttpPool, http_pool_enqueue, poll_http_pool}, ipc::{IpcPool, ipc_pool_enqueue, poll_ipc_pool}, types::{B256, ExecutionPayload, ForkchoiceState, PayloadAttributesV3}, }; const EVENTS_CAPACITY: usize = 16; +// Sized for the largest expected outgoing request: newPayload with a full block +// (~30M gas of transactions, hex-encoded in JSON). +const SCRATCH_CAPACITY: usize = 10 * 1024 * 1024; + const OUR_CAPABILITIES: &[&str] = &[ "engine_forkchoiceUpdatedV3", "engine_newPayloadV4", + "engine_getPayloadV3", "engine_getPayloadV4", "engine_getBlobsV2", "engine_getPayloadBodiesByHashV1", @@ -29,7 +35,6 @@ pub enum ReqKind { Syncing, Fcu(u64), NewPayload(u64), - GetPayloadFcu(u64), GetPayloadFetch(u64), GetBlobs(u64), GetPayloadBodiesByHash(u64), @@ -47,26 +52,32 @@ pub struct EngineClient { events: Events, id: u64, pending_requests: FxHashMap, + pub get_payload_method: &'static str, + scratch: Vec, } impl EngineClient { pub fn new(endpoint: impl Into, jwt: JwtSecret) -> Self { Self { - transport: Transport::Http(HttpPool::new(endpoint.into(), jwt, POOL_SIZE)), + transport: Transport::Http(HttpPool::new(endpoint.into(), jwt)), poll: Poll::new().expect("mio Poll::new failed"), events: Events::with_capacity(EVENTS_CAPACITY), id: 1, pending_requests: FxHashMap::default(), + get_payload_method: "engine_getPayloadV3", + scratch: Vec::with_capacity(SCRATCH_CAPACITY), } } pub fn new_ipc(path: impl Into) -> Self { Self { - transport: Transport::Ipc(IpcPool::new(path.into(), POOL_SIZE)), + transport: Transport::Ipc(IpcPool::new(path.into())), poll: Poll::new().expect("mio Poll::new failed"), events: Events::with_capacity(EVENTS_CAPACITY), id: 1, pending_requests: FxHashMap::default(), + get_payload_method: "engine_getPayloadV3", + scratch: Vec::with_capacity(SCRATCH_CAPACITY), } } } @@ -80,10 +91,10 @@ fn next_id(id: &mut u64) -> u64 { fn make_rpc_body( id: &mut u64, method: &str, - params: serde_json::Value, -) -> (u64, serde_json::Value) { + params: simd_json::OwnedValue, +) -> (u64, simd_json::OwnedValue) { let rpc_id = next_id(id); - let body = serde_json::json!({ + let body = simd_json::json!({ "jsonrpc": "2.0", "method": method, "params": params, @@ -92,10 +103,15 @@ fn make_rpc_body( (rpc_id, body) } -fn enqueue(c: &mut EngineClient, rpc_id: u64, body: &serde_json::Value) { +fn enqueue(c: &mut EngineClient, rpc_id: u64, body: &simd_json::OwnedValue) { + c.scratch.clear(); + if let Err(e) = simd_json::to_writer(&mut c.scratch, body) { + tracing::warn!("failed to serialize RPC body: {e}"); + return; + } match &mut c.transport { - Transport::Http(p) => http_pool_enqueue(p, rpc_id, body, &mut c.poll), - Transport::Ipc(p) => ipc_pool_enqueue(p, rpc_id, body, &mut c.poll), + Transport::Http(p) => http_pool_enqueue(p, rpc_id, &c.scratch, &mut c.poll), + Transport::Ipc(p) => ipc_pool_enqueue(p, rpc_id, &c.scratch, &mut c.poll), } } @@ -106,13 +122,9 @@ pub fn send_fcu( req_id: u64, ) { let (id, body) = - make_rpc_body(&mut c.id, "engine_forkchoiceUpdatedV3", serde_json::json!([state, attrs])); + make_rpc_body(&mut c.id, "engine_forkchoiceUpdatedV3", simd_json::json!([state, attrs])); enqueue(c, id, &body); - if attrs.is_some() { - c.pending_requests.insert(id, ReqKind::Fcu(req_id)); - } else { - c.pending_requests.insert(id, ReqKind::GetPayloadFcu(req_id)); - } + c.pending_requests.insert(id, ReqKind::Fcu(req_id)); } pub fn send_new_payload( @@ -131,7 +143,7 @@ pub fn send_new_payload( let (id, body) = make_rpc_body( &mut c.id, "engine_newPayloadV4", - serde_json::json!([payload, hashes, parent, requests]), + simd_json::json!([payload, hashes, parent, requests]), ); enqueue(c, id, &body); c.pending_requests.insert(id, ReqKind::NewPayload(req_id)); @@ -139,73 +151,93 @@ pub fn send_new_payload( pub fn get_payload(c: &mut EngineClient, payload_id: [u8; 8], req_id: u64) { let id_hex = format!("0x{}", hex::encode(payload_id)); - let (id, body) = make_rpc_body(&mut c.id, "engine_getPayloadV4", serde_json::json!([id_hex])); + let (id, body) = make_rpc_body(&mut c.id, c.get_payload_method, simd_json::json!([id_hex])); enqueue(c, id, &body); c.pending_requests.insert(id, ReqKind::GetPayloadFetch(req_id)); } -pub fn get_blobs(c: &mut EngineClient, params: serde_json::Value, req_id: u64) { +pub fn get_blobs(c: &mut EngineClient, params: simd_json::OwnedValue, req_id: u64) { let (id, body) = make_rpc_body(&mut c.id, "engine_getBlobsV2", params); enqueue(c, id, &body); c.pending_requests.insert(id, ReqKind::GetBlobs(req_id)); } -pub fn get_payload_bodies_by_hash(c: &mut EngineClient, params: serde_json::Value, req_id: u64) { +pub fn get_payload_bodies_by_hash( + c: &mut EngineClient, + params: simd_json::OwnedValue, + req_id: u64, +) { let (id, body) = make_rpc_body(&mut c.id, "engine_getPayloadBodiesByHashV1", params); enqueue(c, id, &body); c.pending_requests.insert(id, ReqKind::GetPayloadBodiesByHash(req_id)); } -pub fn get_payload_bodies_by_range(c: &mut EngineClient, params: serde_json::Value, req_id: u64) { +pub fn get_payload_bodies_by_range( + c: &mut EngineClient, + params: simd_json::OwnedValue, + req_id: u64, +) { let (id, body) = make_rpc_body(&mut c.id, "engine_getPayloadBodiesByRangeV1", params); enqueue(c, id, &body); c.pending_requests.insert(id, ReqKind::GetPayloadBodiesByRange(req_id)); } pub fn get_sync_status(c: &mut EngineClient) { - let (id, body) = make_rpc_body(&mut c.id, "eth_syncing", serde_json::json!([])); + let (id, body) = make_rpc_body(&mut c.id, "eth_syncing", simd_json::json!([])); enqueue(c, id, &body); c.pending_requests.insert(id, ReqKind::Syncing); } pub fn exchange_capabilities(c: &mut EngineClient) { + // Spec: params = [capabilitiesArray] — the array is wrapped in an outer params + // array. let (id, body) = make_rpc_body( &mut c.id, "engine_exchangeCapabilities", - serde_json::json!(OUR_CAPABILITIES), + simd_json::json!([OUR_CAPABILITIES]), ); enqueue(c, id, &body); c.pending_requests.insert(id, ReqKind::Capabilities); } pub fn get_client_version(c: &mut EngineClient) { - let (id, body) = make_rpc_body(&mut c.id, "engine_getClientVersionV1", serde_json::json!([{}])); + // Spec: params = [ClientVersionV1] with required fields + // code/name/version/commit. + let (id, body) = make_rpc_body( + &mut c.id, + "engine_getClientVersionV1", + simd_json::json!([{"code": "GE", "name": "silver", "version": "0.1.0", "commit": "00000000"}]), + ); enqueue(c, id, &body); c.pending_requests.insert(id, ReqKind::ClientVersion); } -/// Drive I/O, calling `on_complete(req_kind, response)` for each finished RPC. +/// Drive I/O, calling `on_complete(req_kind, raw_body)` for each finished RPC. +/// Raw bytes are the full HTTP/IPC response body; handlers parse them as +/// needed. pub fn poll(c: &mut EngineClient, mut on_complete: F) where - F: FnMut(ReqKind, Result), + F: FnMut(ReqKind, Result, EngineError>), { c.poll.poll(&mut c.events, Some(Duration::ZERO)).ok(); let EngineClient { transport, events, poll, pending_requests, .. } = c; match transport { Transport::Http(p) => poll_http_pool(p, events, poll, &mut |rpc_id, res| { if let Some(req_kind) = pending_requests.remove(&rpc_id) { - on_complete(req_kind, res.and_then(extract_result)); + on_complete(req_kind, res); } }), Transport::Ipc(p) => poll_ipc_pool(p, events, poll, &mut |rpc_id, res| { if let Some(req_kind) = pending_requests.remove(&rpc_id) { - on_complete(req_kind, res.and_then(extract_result)); + on_complete(req_kind, res); } }), } } -fn extract_result(resp: serde_json::Value) -> Result { +pub(crate) fn extract_result( + resp: simd_json::OwnedValue, +) -> Result { if let Some(err) = resp.get("error") { return Err(EngineError::Rpc(err.clone())); } @@ -214,6 +246,8 @@ fn extract_result(resp: serde_json::Value) -> Result assert_eq!(v["code"], -32700), + Err(EngineError::Rpc(v)) => assert_eq!(v["code"].as_i64(), Some(-32700)), other => panic!("unexpected: {other:?}"), } } @@ -279,19 +313,19 @@ mod tests { #[test] fn extract_result_error_takes_precedence_over_result() { let resp = - serde_json::json!({"jsonrpc": "2.0", "id": 1, "error": {"code": 1}, "result": "ok"}); + simd_json::json!({"jsonrpc": "2.0", "id": 1, "error": {"code": 1}, "result": "ok"}); assert!(matches!(extract_result(resp), Err(EngineError::Rpc(_)))); } #[test] fn extract_result_missing_result_when_neither_field_present() { - let resp = serde_json::json!({"jsonrpc": "2.0", "id": 1}); + let resp = simd_json::json!({"jsonrpc": "2.0", "id": 1}); assert!(matches!(extract_result(resp), Err(EngineError::MissingResult))); } #[test] fn extract_result_null_result_is_ok() { - let resp = serde_json::json!({"jsonrpc": "2.0", "id": 1, "result": null}); - assert_eq!(extract_result(resp).unwrap(), serde_json::Value::Null); + let resp = simd_json::json!({"jsonrpc": "2.0", "id": 1, "result": null}); + assert!(extract_result(resp).unwrap().is_null()); } } diff --git a/crates/engine/src/error.rs b/crates/engine/src/error.rs index b94116d..fd3e956 100644 --- a/crates/engine/src/error.rs +++ b/crates/engine/src/error.rs @@ -3,11 +3,11 @@ pub enum EngineError { #[error("http: {0}")] Http(String), #[error("json-rpc error: {0}")] - Rpc(serde_json::Value), + Rpc(simd_json::OwnedValue), #[error("missing result field in response")] MissingResult, #[error("serde: {0}")] - Json(#[from] serde_json::Error), + Json(#[from] simd_json::Error), #[error("jwt: {0}")] Jwt(String), #[error("ipc: {0}")] diff --git a/crates/engine/src/http.rs b/crates/engine/src/http.rs index 214b4b3..fab4005 100644 --- a/crates/engine/src/http.rs +++ b/crates/engine/src/http.rs @@ -7,32 +7,32 @@ use mio::{Events, Interest, Poll, Token, net::TcpStream}; use crate::{EngineError, JwtSecret}; -pub(crate) const POOL_SIZE: usize = 4; +// Sized for the largest expected EL response: getPayload with a full blobsBundle +// (~21 blobs × 256 KB hex-encoded + execution payload transactions). +const READ_BUF_CAPACITY: usize = 10 * 1024 * 1024; -#[derive(PartialEq)] -enum State { +enum Conn { Disconnected, - Connecting, - Connected, + Connecting(TcpStream), + Connected(TcpStream), } -pub(crate) struct HttpConnection { +struct HttpConnection { endpoint: String, host: String, jwt: JwtSecret, token: Token, - stream: Option, - state: State, + conn: Conn, addr: Option, + in_flight: Option, pending: Option<(u64, Vec)>, write_pos: usize, - in_flight: Option, read_buf: Vec, read_offset: usize, } impl HttpConnection { - pub(crate) fn new(endpoint: String, jwt: JwtSecret, token: Token) -> Self { + fn new(endpoint: String, jwt: JwtSecret, token: Token) -> Self { let host = endpoint .trim_start_matches("http://") .split('/') @@ -44,101 +44,118 @@ impl HttpConnection { host, jwt, token, - stream: None, - state: State::Disconnected, + conn: Conn::Disconnected, addr: None, pending: None, write_pos: 0, in_flight: None, - read_buf: Vec::new(), + read_buf: Vec::with_capacity(READ_BUF_CAPACITY), read_offset: 0, } } } -pub(crate) fn http_is_free(t: &HttpConnection) -> bool { +fn http_is_free(t: &HttpConnection) -> bool { t.in_flight.is_none() && t.pending.is_none() } -pub(crate) fn http_enqueue( - t: &mut HttpConnection, - rpc_id: u64, - body: &serde_json::Value, - poll: &mut Poll, -) { - let json = serde_json::to_vec(body).unwrap_or_default(); +fn http_enqueue(t: &mut HttpConnection, rpc_id: u64, body: &[u8], poll: &mut Poll) { + debug_assert!(t.in_flight.is_none() && t.pending.is_none(), "enqueue on busy connection"); let bearer = t.jwt.bearer_token(); - let bytes = build_request(&t.host, &json, bearer, true); + let bytes = build_request(&t.host, body, bearer, true); t.pending = Some((rpc_id, bytes)); - match t.state { - State::Disconnected => http_connect(t, poll), - State::Connected => http_set_interest(t, poll, Interest::READABLE | Interest::WRITABLE), - State::Connecting => {} + // matches! borrows t.conn transiently, freeing it before the function call + // below. + if matches!(t.conn, Conn::Disconnected) { + http_connect(t, poll); + } else if matches!(t.conn, Conn::Connected(_)) { + http_set_interest(&mut t.conn, t.token, poll, Interest::READABLE | Interest::WRITABLE); } } -pub(crate) fn http_poll( - t: &mut HttpConnection, - events: &Events, - poll: &mut Poll, - on_complete: &mut F, -) where - F: FnMut(u64, Result), +fn http_poll(t: &mut HttpConnection, events: &Events, poll: &mut Poll, on_complete: &mut F) +where + F: FnMut(u64, Result, EngineError>), { for event in events.iter() { if event.token() != t.token { continue; } - match t.state { - State::Disconnected => {} - State::Connecting => { - if event.is_error() || event.is_read_closed() || event.is_write_closed() { - http_on_error(t, poll, on_complete, "connect failed"); - break; - } - if event.is_writable() { - let connected = t.stream.as_ref().and_then(|s| s.peer_addr().ok()).is_some(); - if connected { - t.state = State::Connected; - let interest = if t.pending.is_none() { - Interest::READABLE - } else { - Interest::READABLE | Interest::WRITABLE - }; - http_set_interest(t, poll, interest); - } else { - http_on_error(t, poll, on_complete, "connect failed"); - break; - } - } + if matches!(t.conn, Conn::Connecting(_)) { + if event.is_error() || event.is_read_closed() || event.is_write_closed() { + http_on_error(t, poll, on_complete, "connect failed"); + break; } - State::Connected => { - if event.is_error() || event.is_read_closed() { - http_on_error(t, poll, on_complete, "connection lost"); - break; - } - if event.is_writable() { - if let Err(e) = http_do_write(t) { - let msg = e.to_string(); - http_on_error(t, poll, on_complete, &msg); - break; - } + if event.is_writable() { + // Take ownership to inspect peer_addr and transition state atomically. + let Conn::Connecting(stream) = std::mem::replace(&mut t.conn, Conn::Disconnected) + else { + unreachable!() + }; + if stream.peer_addr().is_ok() { + t.conn = Conn::Connected(stream); let interest = if t.pending.is_none() { Interest::READABLE } else { Interest::READABLE | Interest::WRITABLE }; - http_set_interest(t, poll, interest); + http_set_interest(&mut t.conn, t.token, poll, interest); + } else { + t.conn = Conn::Connecting(stream); + http_on_error(t, poll, on_complete, "connect failed"); + break; } - if event.is_readable() { - if let Err(e) = http_do_read(t, on_complete) { - let msg = e.to_string(); - http_on_error(t, poll, on_complete, &msg); - break; - } + } + } else if matches!(t.conn, Conn::Connected(_)) { + if event.is_error() { + http_on_error(t, poll, on_complete, "connection error"); + break; + } + if event.is_writable() { + let result = { + let Conn::Connected(stream) = &mut t.conn else { unreachable!() }; + http_do_write(stream, &mut t.pending, &mut t.write_pos, &mut t.in_flight) + }; + if let Err(e) = result { + let msg = e.to_string(); + http_on_error(t, poll, on_complete, &msg); + break; + } + let interest = if t.pending.is_none() { + Interest::READABLE + } else { + Interest::READABLE | Interest::WRITABLE + }; + http_set_interest(&mut t.conn, t.token, poll, interest); + } + if event.is_readable() { + // Drain data before checking is_read_closed: when the remote + // sends a response + FIN in one exchange (EPOLLIN|EPOLLRDHUP), + // we must read the response first. http_do_read returns Err on + // EOF, so the break below covers that close path too. + let result = { + let Conn::Connected(stream) = &mut t.conn else { unreachable!() }; + http_do_read( + stream, + &mut t.in_flight, + &mut t.read_buf, + &mut t.read_offset, + on_complete, + ) + }; + if let Err(e) = result { + let msg = e.to_string(); + http_on_error(t, poll, on_complete, &msg); + break; } } + if event.is_read_closed() { + // Remote closed with no (more) data — in_flight will never get + // a response. + http_on_error(t, poll, on_complete, "connection closed"); + break; + } } } } @@ -153,7 +170,7 @@ fn http_connect(t: &mut HttpConnection, poll: &mut Poll) { a } Err(e) => { - tracing::warn!("engine http: resolve failed for {}: {e}", t.endpoint); + tracing::warn!("resolve failed for {}: {e}", t.endpoint); return; } } @@ -161,26 +178,30 @@ fn http_connect(t: &mut HttpConnection, poll: &mut Poll) { match TcpStream::connect(addr) { Ok(mut stream) => { if poll.registry().register(&mut stream, t.token, Interest::WRITABLE).is_ok() { - t.stream = Some(stream); - t.state = State::Connecting; + t.conn = Conn::Connecting(stream); } } - Err(e) => tracing::warn!("engine http: connect error: {e}"), + Err(e) => tracing::warn!("connect error: {e}"), } } -fn http_do_write(t: &mut HttpConnection) -> io::Result<()> { - if let Some((_, bytes)) = t.pending.as_ref() { - let stream = t.stream.as_mut().unwrap(); +fn http_do_write( + stream: &mut TcpStream, + pending: &mut Option<(u64, Vec)>, + write_pos: &mut usize, + in_flight: &mut Option, +) -> io::Result<()> { + if let Some((_, bytes)) = pending.as_ref() { loop { - match stream.write(&bytes[t.write_pos..]) { + match stream.write(&bytes[*write_pos..]) { Ok(0) => break, Ok(n) => { - t.write_pos += n; - if t.write_pos == bytes.len() { - let (rpc_id, _) = t.pending.take().unwrap(); - t.in_flight = Some(rpc_id); - t.write_pos = 0; + *write_pos += n; + if *write_pos == bytes.len() { + //SAFETY: can unwrap here because we're in pending is Some branch + let (rpc_id, _) = pending.take().unwrap(); + *in_flight = Some(rpc_id); + *write_pos = 0; break; } } @@ -192,35 +213,45 @@ fn http_do_write(t: &mut HttpConnection) -> io::Result<()> { Ok(()) } -fn http_do_read(t: &mut HttpConnection, on_complete: &mut F) -> io::Result<()> +fn http_do_read( + stream: &mut TcpStream, + in_flight: &mut Option, + read_buf: &mut Vec, + read_offset: &mut usize, + on_complete: &mut F, +) -> io::Result<()> where - F: FnMut(u64, Result), + F: FnMut(u64, Result, EngineError>), { - let stream = t.stream.as_mut().unwrap(); - let mut buf = [0u8; 8192]; loop { - match stream.read(&mut buf) { - Ok(0) => return Err(io::Error::new(io::ErrorKind::ConnectionReset, "eof")), + let base = read_buf.len(); + read_buf.resize(base + READ_BUF_CAPACITY, 0); + match stream.read(&mut read_buf[base..]) { + Ok(0) => { + read_buf.truncate(base); + return Err(io::Error::new(io::ErrorKind::ConnectionReset, "eof")); + } Ok(n) => { - t.read_buf.extend_from_slice(&buf[..n]); - loop { - match try_parse_response(&t.read_buf[t.read_offset..]) { - Some((result, consumed)) => { - t.read_offset += consumed; - if let Some(rpc_id) = t.in_flight.take() { - on_complete(rpc_id, result); - } - } - None => break, + read_buf.truncate(base + n); + while let Some((result, consumed)) = try_parse_response(&read_buf[*read_offset..]) { + *read_offset += consumed; + if let Some(rpc_id) = in_flight.take() { + on_complete(rpc_id, result); } } - if t.read_offset == t.read_buf.len() { - t.read_buf.clear(); - t.read_offset = 0; + if *read_offset == read_buf.len() { + read_buf.clear(); + *read_offset = 0; } } - Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, - Err(e) => return Err(e), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + read_buf.truncate(base); + break; + } + Err(e) => { + read_buf.truncate(base); + return Err(e); + } } } Ok(()) @@ -228,9 +259,9 @@ where fn http_on_error(t: &mut HttpConnection, poll: &mut Poll, on_complete: &mut F, msg: &str) where - F: FnMut(u64, Result), + F: FnMut(u64, Result, EngineError>), { - tracing::warn!("engine http: {msg}"); + tracing::warn!("{msg}"); let err = msg.to_string(); if let Some(rpc_id) = t.in_flight.take() { on_complete(rpc_id, Err(EngineError::Http(err.clone()))); @@ -241,65 +272,21 @@ where t.write_pos = 0; t.read_buf.clear(); t.read_offset = 0; - if let Some(mut stream) = t.stream.take() { + let old = std::mem::replace(&mut t.conn, Conn::Disconnected); + if let Conn::Connecting(mut stream) | Conn::Connected(mut stream) = old { let _ = poll.registry().deregister(&mut stream); } - t.state = State::Disconnected; -} - -fn http_set_interest(t: &mut HttpConnection, poll: &mut Poll, interest: Interest) { - if let Some(stream) = t.stream.as_mut() { - let _ = poll.registry().reregister(stream, t.token, interest); - } -} - -pub(crate) struct HttpPool { - connections: Vec, - endpoint: String, - jwt: JwtSecret, -} - -impl HttpPool { - pub(crate) fn new(endpoint: String, jwt: JwtSecret, size: usize) -> Self { - let connections = (0..size) - .map(|i| HttpConnection::new(endpoint.clone(), jwt.clone(), Token(i))) - .collect(); - Self { connections, endpoint, jwt } - } } -pub(crate) fn http_pool_enqueue( - pool: &mut HttpPool, - rpc_id: u64, - body: &serde_json::Value, - poll: &mut Poll, -) { - if let Some(conn) = pool.connections.iter_mut().find(|c| http_is_free(c)) { - http_enqueue(conn, rpc_id, body, poll); - } else { - let mut new_conn = HttpConnection::new( - pool.endpoint.clone(), - pool.jwt.clone(), - Token(pool.connections.len()), - ); - http_enqueue(&mut new_conn, rpc_id, body, poll); - pool.connections.push(new_conn); - } -} - -pub(crate) fn poll_http_pool( - pool: &mut HttpPool, - events: &Events, - poll: &mut Poll, - on_complete: &mut F, -) where - F: FnMut(u64, Result), -{ - for conn in &mut pool.connections { - http_poll(conn, events, poll, on_complete); - } +fn http_set_interest(conn: &mut Conn, token: Token, poll: &mut Poll, interest: Interest) { + let stream = match conn { + Conn::Connecting(s) | Conn::Connected(s) => s, + Conn::Disconnected => return, + }; + let _ = poll.registry().reregister(stream, token, interest); } +// Connection helper functions fn build_request(host: &str, json: &[u8], bearer: &str, keep_alive: bool) -> Vec { let connection = if keep_alive { "keep-alive" } else { "close" }; let header = format!( @@ -320,9 +307,7 @@ fn parse_addr(endpoint: &str) -> io::Result { .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no address resolved")) } -pub(crate) fn try_parse_response( - buf: &[u8], -) -> Option<(Result, usize)> { +fn try_parse_response(buf: &[u8]) -> Option<(Result, EngineError>, usize)> { let mut headers = [httparse::EMPTY_HEADER; 32]; let mut resp = httparse::Response::new(&mut headers); @@ -346,8 +331,47 @@ pub(crate) fn try_parse_response( return None; } - let body = &buf[header_end..total]; - Some((serde_json::from_slice(body).map_err(EngineError::Json), total)) + Some((Ok(buf[header_end..total].to_vec()), total)) +} + +pub(crate) struct HttpPool { + connections: Vec, + endpoint: String, + jwt: JwtSecret, +} + +impl HttpPool { + pub(crate) fn new(endpoint: String, jwt: JwtSecret) -> Self { + let connections = vec![HttpConnection::new(endpoint.clone(), jwt.clone(), Token(0))]; + Self { connections, endpoint, jwt } + } +} + +pub(crate) fn http_pool_enqueue(pool: &mut HttpPool, rpc_id: u64, body: &[u8], poll: &mut Poll) { + if let Some(conn) = pool.connections.iter_mut().find(|c| http_is_free(c)) { + http_enqueue(conn, rpc_id, body, poll); + } else { + let mut new_conn = HttpConnection::new( + pool.endpoint.clone(), + pool.jwt.clone(), + Token(pool.connections.len()), + ); + http_enqueue(&mut new_conn, rpc_id, body, poll); + pool.connections.push(new_conn); + } +} + +pub(crate) fn poll_http_pool( + pool: &mut HttpPool, + events: &Events, + poll: &mut Poll, + on_complete: &mut F, +) where + F: FnMut(u64, Result, EngineError>), +{ + for conn in &mut pool.connections { + http_poll(conn, events, poll, on_complete); + } } #[cfg(test)] @@ -410,12 +434,7 @@ mod tests { #[test] fn parse_large_body_correct_consumed_count() { - let body: Vec = (0u8..=255).cycle().take(4096).collect(); - // wrap in a JSON string so serde accepts it - let json_body = serde_json::to_vec(&serde_json::Value::String( - String::from_utf8_lossy(&body).into_owned(), - )) - .unwrap(); + let json_body: Vec = (0u8..=255).cycle().take(4096).collect(); let buf = http_response(&json_body); let (result, consumed) = try_parse_response(&buf).unwrap(); assert!(result.is_ok()); diff --git a/crates/engine/src/ipc.rs b/crates/engine/src/ipc.rs index 93f9dfb..faea69d 100644 --- a/crates/engine/src/ipc.rs +++ b/crates/engine/src/ipc.rs @@ -8,6 +8,10 @@ use mio::{Events, Interest, Poll, Token, net::UnixStream}; use crate::EngineError; +// Sized for the largest expected EL response: getPayload with a full blobsBundle +// (~21 blobs × 256 KB hex-encoded + execution payload transactions). +const READ_BUF_CAPACITY: usize = 10 * 1024 * 1024; + #[derive(PartialEq)] enum State { Disconnected, @@ -37,7 +41,7 @@ impl IpcTransport { send_queue: VecDeque::new(), write_pos: 0, in_flight: None, - read_buf: Vec::new(), + read_buf: Vec::with_capacity(READ_BUF_CAPACITY), read_offset: 0, } } @@ -47,13 +51,8 @@ pub(crate) fn ipc_is_free(t: &IpcTransport) -> bool { t.in_flight.is_none() && t.send_queue.is_empty() } -pub(crate) fn ipc_enqueue( - t: &mut IpcTransport, - rpc_id: u64, - body: &serde_json::Value, - poll: &mut Poll, -) { - let mut bytes = serde_json::to_vec(body).unwrap_or_default(); +pub(crate) fn ipc_enqueue(t: &mut IpcTransport, rpc_id: u64, body: &[u8], poll: &mut Poll) { + let mut bytes = body.to_vec(); bytes.push(b'\n'); t.send_queue.push_back((rpc_id, bytes)); @@ -70,7 +69,7 @@ pub(crate) fn ipc_poll( poll: &mut Poll, on_complete: &mut F, ) where - F: FnMut(u64, Result), + F: FnMut(u64, Result, EngineError>), { for event in events.iter() { if event.token() != t.token { @@ -79,12 +78,14 @@ pub(crate) fn ipc_poll( match t.state { State::Disconnected => {} State::Connecting => { - if event.is_error() || event.is_read_closed() || event.is_write_closed() { - ipc_on_error(t, poll, on_complete, "connect failed"); - break; - } if event.is_writable() { - // Unix socket connect completes on first WRITABLE event + // is_error/is_write_closed flags are not reliable; use + // take_error() (getsockopt SO_ERROR) as the authoritative check. + let err = t.stream.as_ref().and_then(|s| s.take_error().ok()).flatten(); + if let Some(e) = err { + ipc_on_error(t, poll, on_complete, &e.to_string()); + break; + } t.state = State::Connected; let interest = if t.send_queue.is_empty() { Interest::READABLE @@ -132,7 +133,7 @@ fn ipc_connect(t: &mut IpcTransport, poll: &mut Poll) { t.state = State::Connecting; } } - Err(e) => tracing::warn!("engine ipc: connect error: {e}"), + Err(e) => tracing::warn!("connect error: {e}"), } } @@ -140,7 +141,7 @@ fn ipc_do_write(t: &mut IpcTransport) -> io::Result<()> { while let Some((_, bytes)) = t.send_queue.front() { let stream = t.stream.as_mut().unwrap(); match stream.write(&bytes[t.write_pos..]) { - Ok(0) => break, + Ok(0) => return Err(io::Error::new(io::ErrorKind::WriteZero, "write returned 0")), Ok(n) => { t.write_pos += n; if t.write_pos == bytes.len() { @@ -158,21 +159,23 @@ fn ipc_do_write(t: &mut IpcTransport) -> io::Result<()> { fn ipc_do_read(t: &mut IpcTransport, on_complete: &mut F) -> io::Result<()> where - F: FnMut(u64, Result), + F: FnMut(u64, Result, EngineError>), { let stream = t.stream.as_mut().unwrap(); - let mut buf = [0u8; 8192]; loop { - match stream.read(&mut buf) { - Ok(0) => return Err(io::Error::new(io::ErrorKind::ConnectionReset, "eof")), + let base = t.read_buf.len(); + t.read_buf.resize(base + READ_BUF_CAPACITY, 0); + match stream.read(&mut t.read_buf[base..]) { + Ok(0) => { + t.read_buf.truncate(base); + return Err(io::Error::new(io::ErrorKind::ConnectionReset, "eof")); + } Ok(n) => { - t.read_buf.extend_from_slice(&buf[..n]); + t.read_buf.truncate(base + n); while let Some(rel) = t.read_buf[t.read_offset..].iter().position(|&b| b == b'\n') { let end = t.read_offset + rel; if let Some(rpc_id) = t.in_flight { - let result = serde_json::from_slice(&t.read_buf[t.read_offset..end]) - .map_err(EngineError::Json); - on_complete(rpc_id, result); + on_complete(rpc_id, Ok(t.read_buf[t.read_offset..end].to_vec())); } t.read_offset = end + 1; } @@ -181,8 +184,14 @@ where t.read_offset = 0; } } - Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, - Err(e) => return Err(e), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + t.read_buf.truncate(base); + break; + } + Err(e) => { + t.read_buf.truncate(base); + return Err(e); + } } } Ok(()) @@ -190,7 +199,7 @@ where fn ipc_on_error(t: &mut IpcTransport, poll: &mut Poll, on_complete: &mut F, msg: &str) where - F: FnMut(u64, Result), + F: FnMut(u64, Result, EngineError>), { tracing::warn!("engine ipc: {msg}"); let err = msg.to_string(); @@ -221,18 +230,13 @@ pub(crate) struct IpcPool { } impl IpcPool { - pub(crate) fn new(path: String, size: usize) -> Self { - let connections = (0..size).map(|i| IpcTransport::new(path.clone(), Token(i))).collect(); + pub(crate) fn new(path: String) -> Self { + let connections = vec![IpcTransport::new(path.clone(), Token(0))]; Self { connections, path } } } -pub(crate) fn ipc_pool_enqueue( - pool: &mut IpcPool, - rpc_id: u64, - body: &serde_json::Value, - poll: &mut Poll, -) { +pub(crate) fn ipc_pool_enqueue(pool: &mut IpcPool, rpc_id: u64, body: &[u8], poll: &mut Poll) { if let Some(conn) = pool.connections.iter_mut().find(|c| ipc_is_free(c)) { ipc_enqueue(conn, rpc_id, body, poll); } else { @@ -248,7 +252,7 @@ pub(crate) fn poll_ipc_pool( poll: &mut Poll, on_complete: &mut F, ) where - F: FnMut(u64, Result), + F: FnMut(u64, Result, EngineError>), { for conn in &mut pool.connections { ipc_poll(conn, events, poll, on_complete); diff --git a/crates/engine/src/jwt.rs b/crates/engine/src/jwt.rs index 797f565..f2a5785 100644 --- a/crates/engine/src/jwt.rs +++ b/crates/engine/src/jwt.rs @@ -56,6 +56,8 @@ impl JwtSecret { #[cfg(test)] mod tests { + use simd_json::prelude::ValueObjectAccess; + use super::*; const ZERO_KEY: &str = "0000000000000000000000000000000000000000000000000000000000000000"; @@ -93,11 +95,11 @@ mod tests { assert_eq!(parts[0], HEADER_B64); // Payload must base64-decode to JSON containing "iat" - let payload_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + let mut payload_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD .decode(parts[1]) .expect("payload must be valid base64url"); - let payload: serde_json::Value = - serde_json::from_slice(&payload_bytes).expect("payload must be valid JSON"); + let payload: simd_json::OwnedValue = + simd_json::from_slice(&mut payload_bytes).expect("payload must be valid JSON"); assert!(payload.get("iat").is_some(), "JWT payload must contain 'iat'"); } diff --git a/crates/engine/src/req_handlers.rs b/crates/engine/src/req_handlers.rs index fdbfca4..4e8c338 100644 --- a/crates/engine/src/req_handlers.rs +++ b/crates/engine/src/req_handlers.rs @@ -1,15 +1,15 @@ use flux::spine::FluxSpine; use silver_common::{ - EngineFcuReq, EngineGetBlobsReq, EngineGetPayloadBodiesByHashReq, - EngineGetPayloadBodiesByRangeReq, EngineGetPayloadReq, EngineNewPayloadReq, - EngineNewPayloadResp, EngineReq, EngineResp, PayloadValidationStatus, SilverSpine, + EngineFcuReq, EngineGetPayloadReq, EngineGetBlobsReq, EngineGetPayloadBodiesByHashReq, + EngineGetPayloadBodiesByRangeReq, EngineNewPayloadReq, EngineNewPayloadResp, + EnginePreparePayloadReq, EngineReq, EngineResp, PayloadValidationStatus, SilverSpine, TRandomAccess, }; use crate::{ EngineClient, client::{ - get_blobs, get_payload_bodies_by_hash, get_payload_bodies_by_range, send_fcu, + get_blobs, get_payload, get_payload_bodies_by_hash, get_payload_bodies_by_range, send_fcu, send_new_payload, }, types::{ForkchoiceState, PayloadAttributesV3, Withdrawal, decode_new_payload_data}, @@ -25,15 +25,17 @@ pub(crate) fn handle_request( match req { EngineReq::Fcu(r) => handle_fcu(client, r), EngineReq::NewPayload(r) => handle_new_payload(client, req_consumer, r, producers), + EngineReq::PreparePayload(r) => handle_prepare_payload(client, *r), + EngineReq::GetPayload(r) => handle_get_payload(client, *r), EngineReq::GetBlobs(r) => handle_get_blobs(client, r), EngineReq::GetPayloadBodiesByHash(r) => handle_get_payload_bodies_by_hash(client, r), EngineReq::GetPayloadBodiesByRange(r) => handle_get_payload_bodies_by_range(client, r), - EngineReq::GetPayload(r) => handle_get_payload(client, *r), } } #[inline] fn handle_fcu(client: &mut EngineClient, r: &EngineFcuReq) { + tracing::info!(head = %hex::encode(&r.head_block_hash[..4]), id = r.id, "FCU ← spine"); let state = ForkchoiceState { head_block_hash: r.head_block_hash, safe_block_hash: r.safe_block_hash, @@ -53,7 +55,7 @@ fn handle_new_payload( let bytes = match acquired.buffer() { Ok((b, _)) => b.to_owned(), Err(e) => { - tracing::warn!("engine tile: failed to read payload data: {e}"); + tracing::warn!("failed to read payload data: {e}"); producers .engine_resps .produce(&EngineResp::NewPayload(invalid_new_payload_resp(r.id)).into()); @@ -64,7 +66,7 @@ fn handle_new_payload( let (payload, exec_requests) = match decode_new_payload_data(&bytes) { Ok(v) => v, Err(e) => { - tracing::warn!("engine tile: failed to decode payload: {e}"); + tracing::warn!("failed to decode payload: {e}"); producers .engine_resps .produce(&EngineResp::NewPayload(invalid_new_payload_resp(r.id)).into()); @@ -92,7 +94,7 @@ fn handle_get_blobs(client: &mut EngineClient, r: &EngineGetBlobsReq) { let n = r.hash_count as usize; let hashes: Vec = r.hashes[..n].iter().map(|h| format!("0x{}", hex::encode(h))).collect(); - get_blobs(client, serde_json::json!([hashes]), r.id); + get_blobs(client, simd_json::json!([hashes]), r.id); } #[inline] @@ -103,7 +105,7 @@ fn handle_get_payload_bodies_by_hash( let n = r.hash_count as usize; let hashes: Vec = r.hashes[..n].iter().map(|h| format!("0x{}", hex::encode(h))).collect(); - get_payload_bodies_by_hash(client, serde_json::json!([hashes]), r.id); + get_payload_bodies_by_hash(client, simd_json::json!([hashes]), r.id); } #[inline] @@ -113,11 +115,12 @@ fn handle_get_payload_bodies_by_range( ) { let start_hex = format!("0x{:x}", r.start); let count_hex = format!("0x{:x}", r.count); - get_payload_bodies_by_range(client, serde_json::json!([start_hex, count_hex]), r.id); + get_payload_bodies_by_range(client, simd_json::json!([start_hex, count_hex]), r.id); } #[inline] -fn handle_get_payload(client: &mut EngineClient, r: EngineGetPayloadReq) { +fn handle_prepare_payload(client: &mut EngineClient, r: EnginePreparePayloadReq) { + tracing::info!(head = %hex::encode(&r.head_block_hash[..4]), id = r.id, "preparePayload ← spine"); let n = r.attrs_withdrawal_count as usize; let withdrawals = r.attrs_withdrawals[..n] .iter() @@ -143,11 +146,19 @@ fn handle_get_payload(client: &mut EngineClient, r: EngineGetPayloadReq) { send_fcu(client, state, attrs, r.id); } +#[inline] +fn handle_get_payload(client: &mut EngineClient, r: EngineGetPayloadReq) { + tracing::info!(payload_id = %hex::encode(r.payload_id), id = r.id, "fetchPayload ← spine"); + get_payload(client, r.payload_id, r.id); +} + #[inline] fn invalid_new_payload_resp(id: u64) -> EngineNewPayloadResp { + // Internal error (TCache read/decode failure) — use SYNCING, not INVALID. + // INVALID tells the CL the block is definitively bad; we don't know that here. EngineNewPayloadResp { id, - status: PayloadValidationStatus::Invalid, + status: PayloadValidationStatus::Syncing, latest_valid_hash: [0u8; 32], } } @@ -160,7 +171,7 @@ mod tests { fn invalid_new_payload_resp_fields() { let resp = invalid_new_payload_resp(99); assert_eq!(resp.id, 99); - assert_eq!(resp.status, PayloadValidationStatus::Invalid); + assert_eq!(resp.status, PayloadValidationStatus::Syncing); assert_eq!(resp.latest_valid_hash, [0u8; 32]); } diff --git a/crates/engine/src/resp_handlers.rs b/crates/engine/src/resp_handlers.rs index 31c334f..b053653 100644 --- a/crates/engine/src/resp_handlers.rs +++ b/crates/engine/src/resp_handlers.rs @@ -4,44 +4,63 @@ use silver_common::{ EngineGetPayloadResp, EngineHealthEvent, EngineNewPayloadResp, EngineResp, PayloadValidationStatus, SilverSpine, TCacheProducer, TCacheRead, TProducer, }; +use simd_json::prelude::{ValueAsArray, ValueAsScalar, ValueObjectAccess}; use crate::{ EngineError, + client::extract_result, types::{ - BlobAndProofV1, ExecutionPayloadBodyV1, ForkchoiceUpdatedResult, GetPayloadV4Response, - PayloadStatus, encode_get_blobs_data, encode_get_payload_data, encode_payload_bodies_data, + ForkchoiceUpdatedResult, PayloadStatus, + json_get_blobs_to_tcache, json_get_payload_bodies_to_tcache, json_get_payload_to_tcache, }, }; +// Parse raw response bytes into an extracted OwnedValue result field. +// Used by all handlers except getPayload, which uses the zero-alloc path. #[inline] -pub(crate) fn handle_capabilities_response(response: Result) { - match response { +fn parse_rpc_response( + raw: Result, EngineError>, +) -> Result { + raw.and_then(|mut b| simd_json::from_slice(&mut b).map_err(EngineError::Json)) + .and_then(extract_result) +} + +#[inline] +pub(crate) fn handle_capabilities_response(response: Result, EngineError>) -> &'static str { + match parse_rpc_response(response) { Ok(val) => { let arr = val.as_array().map(|a| a.as_slice()).unwrap_or_default(); let has = |m: &str| arr.iter().any(|v| v.as_str() == Some(m)); if !has("engine_forkchoiceUpdatedV3") { - tracing::warn!("engine tile: EL does not support engine_forkchoiceUpdatedV3"); + tracing::warn!("EL does not support engine_forkchoiceUpdatedV3"); } if !has("engine_newPayloadV4") { - tracing::warn!("engine tile: EL does not support engine_newPayloadV4"); + tracing::warn!("EL does not support engine_newPayloadV4"); } if !has("engine_getPayloadBodiesByHashV1") { - tracing::warn!("engine tile: EL does not support engine_getPayloadBodiesByHashV1"); + tracing::warn!("EL does not support engine_getPayloadBodiesByHashV1"); } if !has("engine_getPayloadBodiesByRangeV1") { - tracing::warn!("engine tile: EL does not support engine_getPayloadBodiesByRangeV1"); + tracing::warn!("EL does not support engine_getPayloadBodiesByRangeV1"); } - tracing::info!("engine tile: capabilities negotiated"); + let method = if has("engine_getPayloadV4") { + "engine_getPayloadV4" + } else { + "engine_getPayloadV3" + }; + tracing::info!("capabilities negotiated, using {method}"); + method } Err(e) => { - tracing::warn!("engine tile: engine_exchangeCapabilities failed: {e}"); + tracing::warn!("engine_exchangeCapabilities failed: {e}"); + "engine_getPayloadV3" } } } #[inline] -pub(crate) fn handle_client_version_response(response: Result) { - match response { +pub(crate) fn handle_client_version_response(response: Result, EngineError>) { + match parse_rpc_response(response) { Ok(val) => { if let Some(client) = val.as_array().and_then(|a| a.first()) { let name = client @@ -50,35 +69,35 @@ pub(crate) fn handle_client_version_response(response: Result { - tracing::warn!("engine tile: engine_getClientVersionV1 failed: {e}"); + tracing::warn!("engine_getClientVersionV1 failed: {e}"); } } } #[inline] pub(crate) fn handle_sync_response( - response: Result, + response: Result, EngineError>, adapter: &mut SpineAdapter, sync_status: &mut ELSyncStatus, healthcheck_pending: &mut bool, ) { *healthcheck_pending = false; - let new_status = parse_sync_status(response); + let new_status = parse_sync_status(parse_rpc_response(response)); publish_health_if_changed(adapter, sync_status, new_status); } #[inline] pub(crate) fn handle_fcu_response( spine_id: u64, - response: Result, + response: Result, EngineError>, adapter: &mut SpineAdapter, ) { - let resp = match response.and_then(|v| { - serde_json::from_value::(v).map_err(EngineError::Json) + let resp = match parse_rpc_response(response).and_then(|v| { + simd_json::serde::from_owned_value::(v).map_err(EngineError::Json) }) { Ok(r) => { let status = status_from_str(&r.payload_status.status); @@ -87,14 +106,23 @@ pub(crate) fn handle_fcu_response( Some(id) => (true, id), None => (false, [0u8; 8]), }; - tracing::info!("engine tile: forkchoiceUpdated → {:?}", status); + tracing::info!( + status = %r.payload_status.status, + latest_valid_hash = %r.payload_status.latest_valid_hash + .map(|h| hex::encode(&h[..4])) + .unwrap_or_else(|| "null".into()), + id = spine_id, + "FCU → Reth" + ); EngineFcuResp { id: spine_id, status, latest_valid_hash, has_payload_id, payload_id } } Err(e) => { - tracing::warn!("engine tile: forkchoiceUpdated error: {e}"); + tracing::warn!("forkchoiceUpdated error: {e}"); + // Transport/parse errors are not proof of invalidity — use SYNCING + // so the CL doesn't treat an unreachable EL as a bad block. EngineFcuResp { id: spine_id, - status: PayloadValidationStatus::Invalid, + status: PayloadValidationStatus::Syncing, latest_valid_hash: [0u8; 32], has_payload_id: false, payload_id: [0u8; 8], @@ -107,23 +135,23 @@ pub(crate) fn handle_fcu_response( #[inline] pub(crate) fn handle_new_payload_response( spine_id: u64, - response: Result, + response: Result, EngineError>, adapter: &mut SpineAdapter, ) { - let resp = match response - .and_then(|v| serde_json::from_value::(v).map_err(EngineError::Json)) - { + let resp = match parse_rpc_response(response).and_then(|v| { + simd_json::serde::from_owned_value::(v).map_err(EngineError::Json) + }) { Ok(ps) => { let status = status_from_str(&ps.status); let latest_valid_hash = ps.latest_valid_hash.unwrap_or([0u8; 32]); - tracing::info!("engine tile: newPayload → {:?}", status); + tracing::info!("newPayload → {:?}", status); EngineNewPayloadResp { id: spine_id, status, latest_valid_hash } } Err(e) => { - tracing::warn!("engine tile: newPayload error: {e}"); + tracing::warn!("newPayload error: {e}"); EngineNewPayloadResp { id: spine_id, - status: PayloadValidationStatus::Invalid, + status: PayloadValidationStatus::Syncing, latest_valid_hash: [0u8; 32], } } @@ -131,66 +159,36 @@ pub(crate) fn handle_new_payload_response( adapter.produce(EngineResp::NewPayload(resp)); } -#[inline] -pub(crate) fn handle_get_payload_fcu( - spine_id: u64, - response: Result, - adapter: &mut SpineAdapter, - pending_get_payload: &mut Option<([u8; 8], u64)>, -) { - match response.and_then(|v| { - serde_json::from_value::(v).map_err(EngineError::Json) - }) { - Ok(fcu) => match fcu.payload_id { - Some(payload_id) => { - *pending_get_payload = Some((payload_id, spine_id)); - } - None => { - tracing::warn!("engine tile: getPayload FCU returned no payload_id"); - adapter.produce(EngineResp::GetPayload(get_payload_error(spine_id))); - } - }, - Err(e) => { - tracing::warn!("engine tile: getPayload FCU error: {e}"); - adapter.produce(EngineResp::GetPayload(get_payload_error(spine_id))); - } - } -} - #[inline] pub(crate) fn handle_get_payload_fetch( spine_id: u64, - response: Result, + response: Result, EngineError>, adapter: &mut SpineAdapter, resp_producer: &mut TProducer, + scratch: &mut Vec, ) { let resp = match response { - Ok(v) => match serde_json::from_value::(v) { - Ok(p) => { - let payload_ssz = p.execution_payload.to_ssz(); - let data = encode_get_payload_data( - &payload_ssz, - &p.blobs_bundle.commitments, - &p.blobs_bundle.proofs, - &p.blobs_bundle.blobs, - p.should_override_builder, - &p.execution_requests, - ); - match write_tcache(resp_producer, &data) { - Some(data) => EngineGetPayloadResp { id: spine_id, ok: true, data }, + Ok(mut raw) => { + scratch.clear(); + match json_get_payload_to_tcache(&mut raw, scratch) { + Ok(()) => match write_tcache(resp_producer, scratch) { + Some(data) => { + tracing::info!(id = spine_id, "getPayload ok"); + EngineGetPayloadResp { id: spine_id, ok: true, data } + } None => { - tracing::warn!("engine tile: getPayload TCache full"); + tracing::warn!("getPayload TCache full"); get_payload_error(spine_id) } + }, + Err(e) => { + tracing::warn!("getPayload parse error: {e}"); + get_payload_error(spine_id) } } - Err(e) => { - tracing::warn!("engine tile: getPayload parse error: {e}"); - get_payload_error(spine_id) - } - }, + } Err(e) => { - tracing::warn!("engine tile: getPayload error: {e}"); + tracing::warn!("getPayload error: {e}"); get_payload_error(spine_id) } }; @@ -200,30 +198,33 @@ pub(crate) fn handle_get_payload_fetch( #[inline] pub(crate) fn handle_get_blobs_response( spine_id: u64, - response: Result, + response: Result, EngineError>, adapter: &mut SpineAdapter, resp_producer: &mut TProducer, + scratch: &mut Vec, ) { - let items = match response { - Ok(v) => match serde_json::from_value::>>(v) { - Ok(items) => items, - Err(e) => { - tracing::warn!("engine tile: getBlobsV2 parse error: {e}"); - adapter.produce(EngineResp::GetBlobs(get_blobs_error(spine_id))); - return; + let resp = match response { + Ok(mut raw) => { + scratch.clear(); + match json_get_blobs_to_tcache(&mut raw, scratch) { + Ok(()) => match write_tcache(resp_producer, scratch) { + Some(data) => { + tracing::info!(id = spine_id, "getBlobsV2 ok"); + EngineGetBlobsResp { id: spine_id, ok: true, data } + } + None => { + tracing::warn!("getBlobsV2 TCache full"); + get_blobs_error(spine_id) + } + }, + Err(e) => { + tracing::warn!("getBlobsV2 parse error: {e}"); + get_blobs_error(spine_id) + } } - }, - Err(e) => { - tracing::warn!("engine tile: getBlobsV2 error: {e}"); - adapter.produce(EngineResp::GetBlobs(get_blobs_error(spine_id))); - return; } - }; - let data = encode_get_blobs_data(&items); - let resp = match write_tcache(resp_producer, &data) { - Some(data) => EngineGetBlobsResp { id: spine_id, ok: true, data }, - None => { - tracing::warn!("engine tile: getBlobsV2 TCache full"); + Err(e) => { + tracing::warn!("getBlobsV2 error: {e}"); get_blobs_error(spine_id) } }; @@ -233,30 +234,33 @@ pub(crate) fn handle_get_blobs_response( #[inline] pub(crate) fn handle_get_payload_bodies_response( spine_id: u64, - response: Result, + response: Result, EngineError>, adapter: &mut SpineAdapter, resp_producer: &mut TProducer, + scratch: &mut Vec, ) { - let bodies = match response { - Ok(v) => match serde_json::from_value::>>(v) { - Ok(bodies) => bodies, - Err(e) => { - tracing::warn!("engine tile: getPayloadBodies parse error: {e}"); - adapter.produce(EngineResp::GetPayloadBodies(get_payload_bodies_error(spine_id))); - return; + let resp = match response { + Ok(mut raw) => { + scratch.clear(); + match json_get_payload_bodies_to_tcache(&mut raw, scratch) { + Ok(()) => match write_tcache(resp_producer, scratch) { + Some(data) => { + tracing::info!(id = spine_id, "getPayloadBodies ok"); + EngineGetPayloadBodiesResp { id: spine_id, ok: true, data } + } + None => { + tracing::warn!("getPayloadBodies TCache full"); + get_payload_bodies_error(spine_id) + } + }, + Err(e) => { + tracing::warn!("getPayloadBodies parse error: {e}"); + get_payload_bodies_error(spine_id) + } } - }, - Err(e) => { - tracing::warn!("engine tile: getPayloadBodies error: {e}"); - adapter.produce(EngineResp::GetPayloadBodies(get_payload_bodies_error(spine_id))); - return; } - }; - let data = encode_payload_bodies_data(&bodies); - let resp = match write_tcache(resp_producer, &data) { - Some(data) => EngineGetPayloadBodiesResp { id: spine_id, ok: true, data }, - None => { - tracing::warn!("engine tile: getPayloadBodies TCache full"); + Err(e) => { + tracing::warn!("getPayloadBodies error: {e}"); get_payload_bodies_error(spine_id) } }; @@ -272,23 +276,23 @@ fn publish_health_if_changed( if new_status != *sync_status { *sync_status = new_status; adapter.produce(EngineHealthEvent { sync_status: new_status }); - tracing::info!("engine tile: EL health → {:?}", new_status); + tracing::info!("EL health → {:?}", new_status); } } #[inline] -fn parse_sync_status(response: Result) -> ELSyncStatus { +fn parse_sync_status(response: Result) -> ELSyncStatus { match response { Ok(val) if val.as_bool() == Some(false) => { - tracing::info!("engine tile: EL synced"); + tracing::info!("EL synced"); ELSyncStatus::Synced } Ok(_) => { - tracing::info!("engine tile: EL syncing"); + tracing::info!("EL syncing"); ELSyncStatus::Syncing } Err(e) => { - tracing::warn!("engine tile: eth_syncing failed: {e}"); + tracing::warn!("eth_syncing failed: {e}"); ELSyncStatus::Offline } } @@ -358,12 +362,12 @@ mod tests { #[test] fn parse_sync_status_synced_on_false() { - assert_eq!(parse_sync_status(Ok(serde_json::Value::Bool(false))), ELSyncStatus::Synced); + assert_eq!(parse_sync_status(Ok(simd_json::json!(false))), ELSyncStatus::Synced); } #[test] fn parse_sync_status_syncing_on_object() { - let syncing_obj = serde_json::json!({ + let syncing_obj = simd_json::json!({ "startingBlock": "0x0", "currentBlock": "0x100", "highestBlock": "0x200" @@ -373,7 +377,7 @@ mod tests { #[test] fn parse_sync_status_syncing_on_true() { - assert_eq!(parse_sync_status(Ok(serde_json::Value::Bool(true))), ELSyncStatus::Syncing); + assert_eq!(parse_sync_status(Ok(simd_json::json!(true))), ELSyncStatus::Syncing); } #[test] diff --git a/crates/engine/src/tile.rs b/crates/engine/src/tile.rs index 92b58ff..9ff97f7 100644 --- a/crates/engine/src/tile.rs +++ b/crates/engine/src/tile.rs @@ -5,9 +5,7 @@ use silver_common::{ELSyncStatus, EngineReq, SilverSpine, TProducer, TRandomAcce use crate::{ EngineClient, - client::{ - ReqKind, exchange_capabilities, get_client_version, get_payload, get_sync_status, poll, - }, + client::{ReqKind, exchange_capabilities, get_client_version, get_sync_status, poll}, req_handlers::handle_request, resp_handlers::*, }; @@ -23,6 +21,9 @@ pub struct EngineTile { healthcheck_pending: bool, healthcheck_deadline: Instant, sync_status: ELSyncStatus, + // Reusable scratch buffer for the JSON→SSZ getPayload conversion. + // Cleared on each use; capacity is retained across calls. + scratch: Vec, } impl Tile for EngineTile { @@ -49,16 +50,18 @@ impl EngineTile { healthcheck_pending: false, healthcheck_deadline: Instant::now(), sync_status: ELSyncStatus::Unknown, + scratch: Vec::new(), } } fn spin(&mut self, adapter: &mut SpineAdapter) { - let mut pending_get_payload: Option<([u8; 8], u64)> = None; + let mut negotiated_get_payload_method: Option<&'static str> = None; { let Self { client, resp_producer, + scratch, first_run, healthcheck_pending, healthcheck_deadline, @@ -71,7 +74,9 @@ impl EngineTile { } poll(client, |req_kind, response| match req_kind { - ReqKind::Capabilities => handle_capabilities_response(response), + ReqKind::Capabilities => { + negotiated_get_payload_method = Some(handle_capabilities_response(response)); + } ReqKind::ClientVersion => handle_client_version_response(response), ReqKind::Syncing => { handle_sync_response(response, adapter, sync_status, healthcheck_pending) @@ -80,26 +85,23 @@ impl EngineTile { ReqKind::NewPayload(spine_id) => { handle_new_payload_response(spine_id, response, adapter) } - ReqKind::GetPayloadFcu(spine_id) => { - handle_get_payload_fcu(spine_id, response, adapter, &mut pending_get_payload) - } ReqKind::GetPayloadFetch(spine_id) => { - handle_get_payload_fetch(spine_id, response, adapter, resp_producer) + handle_get_payload_fetch(spine_id, response, adapter, resp_producer, scratch) } ReqKind::GetBlobs(spine_id) => { - handle_get_blobs_response(spine_id, response, adapter, resp_producer) + handle_get_blobs_response(spine_id, response, adapter, resp_producer, scratch) } ReqKind::GetPayloadBodiesByHash(spine_id) | ReqKind::GetPayloadBodiesByRange(spine_id) => { - handle_get_payload_bodies_response(spine_id, response, adapter, resp_producer); + handle_get_payload_bodies_response( + spine_id, response, adapter, resp_producer, scratch, + ); } }); } - // GetPayloadFcu enqueues a follow-up RPC; done here so the closure - // above doesn't need to borrow client while poll holds it. - if let Some((payload_id, spine_id)) = pending_get_payload { - get_payload(&mut self.client, payload_id, spine_id); + if let Some(method) = negotiated_get_payload_method { + self.client.get_payload_method = method; } } } diff --git a/crates/engine/src/types.rs b/crates/engine/src/types.rs index 470eac2..5fef83e 100644 --- a/crates/engine/src/types.rs +++ b/crates/engine/src/types.rs @@ -195,47 +195,6 @@ pub struct PayloadStatus { pub validation_error: Option, } -// --- getPayloadV4 response --- - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct GetPayloadV4Response { - pub execution_payload: ExecutionPayload, - pub blobs_bundle: BlobsBundle, - pub should_override_builder: bool, - #[serde(default, with = "wire::data_list")] - pub execution_requests: Vec>, -} - -#[derive(Debug, Deserialize)] -pub(crate) struct BlobsBundle { - #[serde(with = "wire::data_list")] - pub commitments: Vec>, - #[serde(with = "wire::data_list")] - pub proofs: Vec>, - #[serde(with = "wire::data_list")] - pub blobs: Vec>, -} - -// --- getBlobsV2 response --- - -#[derive(Debug, Deserialize)] -pub(crate) struct BlobAndProofV1 { - #[serde(with = "wire::data")] - pub blob: Vec, - #[serde(with = "wire::data")] - pub proof: Vec, -} - -// --- getPayloadBodiesByHashV1 / ByRangeV1 response --- - -#[derive(Debug, Deserialize)] -pub(crate) struct ExecutionPayloadBodyV1 { - #[serde(with = "wire::data_list")] - pub transactions: Vec>, - pub withdrawals: Option>, -} - // --- newPayload --- #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -437,7 +396,7 @@ fn decode_transactions(data: &[u8]) -> Result>, crate::EngineError> return Err(crate::EngineError::Ssz("transactions header too short".into())); } let first_off = u32::from_le_bytes(data[0..4].try_into().unwrap()) as usize; - if first_off.is_multiple_of(4) || first_off < 4 { + if !first_off.is_multiple_of(4) || first_off < 4 { return Err(crate::EngineError::Ssz("invalid transaction list offset".into())); } let n = first_off / 4; @@ -474,7 +433,7 @@ fn encode_withdrawals(ws: &[Withdrawal]) -> Vec { } fn decode_withdrawals(data: &[u8]) -> Result, crate::EngineError> { - if data.len().is_multiple_of(44) { + if !data.len().is_multiple_of(44) { return Err(crate::EngineError::Ssz(format!( "withdrawals length {} is not a multiple of 44", data.len() @@ -549,142 +508,364 @@ pub(crate) fn decode_new_payload_data( } // --------------------------------------------------------------------------- -// TCache data framing for engine_getPayloadV4 +// Zero-alloc JSON → TCache frame converter for engine_getPayloadV4 // -// Layout: -// [u32 LE payload_ssz_len] -// [payload_ssz_len bytes: ExecutionPayload SSZ] -// [u8 blob_count] -// for each blob: -// [48 bytes commitment] -// [48 bytes proof] -// [u32 LE blob_len][blob_len bytes blob] -// [u8 should_override_builder] -// [u8 exec_req_count] -// for each exec request: [u32 LE len][len bytes] +// Parses the raw HTTP response body (full JSON-RPC envelope) using +// simd_json BorrowedValue so all string values borrow from the input +// buffer with no intermediate allocations. Hex fields are decoded +// directly into `out` with hex::decode_to_slice. +// +// `out` is cleared by the caller. On success it contains exactly the +// TCache frame (same layout as encode_get_payload_data). // --------------------------------------------------------------------------- -pub fn encode_get_payload_data( - payload_ssz: &[u8], - commitments: &[Vec], - proofs: &[Vec], - blobs: &[Vec], - should_override_builder: bool, - exec_requests: &[Vec], -) -> Vec { +fn hex_to_fixed(s: &str) -> Result<[u8; N], crate::EngineError> { + let s = s.strip_prefix("0x").unwrap_or(s); + let mut out = [0u8; N]; + hex::decode_to_slice(s, &mut out).map_err(|e| crate::EngineError::Ssz(e.to_string()))?; + Ok(out) +} + +// Append decoded bytes of hex string `s` to `out`. +fn hex_extend(s: &str, out: &mut Vec) -> Result<(), crate::EngineError> { + let s = s.strip_prefix("0x").unwrap_or(s); + let base = out.len(); + out.resize(base + s.len() / 2, 0); + hex::decode_to_slice(s, &mut out[base..]).map_err(|e| crate::EngineError::Ssz(e.to_string())) +} + +// Append exactly N bytes: decode up to N bytes from hex, zero-pad remainder. +fn hex_extend_clamped( + s: &str, + out: &mut Vec, +) -> Result<(), crate::EngineError> { + let s = s.strip_prefix("0x").unwrap_or(s); + let n = (s.len() / 2).min(N); + let mut buf = [0u8; N]; + if n > 0 { + hex::decode_to_slice(&s[..n * 2], &mut buf[..n]) + .map_err(|e| crate::EngineError::Ssz(e.to_string()))?; + } + out.extend_from_slice(&buf); + Ok(()) +} + +fn parse_quantity(s: &str) -> Result { + let s = s.strip_prefix("0x").unwrap_or(s); + u64::from_str_radix(s, 16).map_err(|e| crate::EngineError::Ssz(e.to_string())) +} + +// Parse Ethereum QUANTITY "0x..." to [u8; 32] little-endian (uint256). +fn parse_u256_le(s: &str) -> Result<[u8; 32], crate::EngineError> { + let s = s.strip_prefix("0x").unwrap_or(s); + if s.len() > 64 { + return Err(crate::EngineError::Ssz("u256 too large".into())); + } + let mut padded = [b'0'; 64]; + padded[64 - s.len()..].copy_from_slice(s.as_bytes()); + let mut arr = [0u8; 32]; + hex::decode_to_slice(padded, &mut arr).map_err(|e| crate::EngineError::Ssz(e.to_string()))?; + arr.reverse(); + Ok(arr) +} + +fn fstr<'a>( + v: &'a simd_json::BorrowedValue<'_>, + field: &str, +) -> Result<&'a str, crate::EngineError> { + use simd_json::prelude::{ValueAsScalar, ValueObjectAccess}; + v.get(field) + .and_then(|f| f.as_str()) + .ok_or_else(|| crate::EngineError::Ssz(format!("missing field: {field}"))) +} + +// Write SSZ transaction list (offset header + tx bytes) from JSON hex-string +// array. +fn encode_txs_json( + arr: &[simd_json::BorrowedValue<'_>], + out: &mut Vec, +) -> Result<(), crate::EngineError> { + use simd_json::prelude::ValueAsScalar; + if arr.is_empty() { + return Ok(()); + } + let n = arr.len(); + let header_base = out.len(); + out.resize(header_base + n * 4, 0); + let mut offset = (n * 4) as u32; + for (i, tx) in arr.iter().enumerate() { + let s = tx.as_str().ok_or_else(|| crate::EngineError::Ssz("tx not a string".into()))?; + out[header_base + i * 4..header_base + i * 4 + 4].copy_from_slice(&offset.to_le_bytes()); + offset += (s.strip_prefix("0x").unwrap_or(s).len() / 2) as u32; + } + for tx in arr { + let s = tx.as_str().ok_or_else(|| crate::EngineError::Ssz("tx not a string".into()))?; + hex_extend(s, out)?; + } + Ok(()) +} + +fn encode_withdrawals_json( + arr: &[simd_json::BorrowedValue<'_>], + out: &mut Vec, +) -> Result<(), crate::EngineError> { + for w in arr { + out.extend_from_slice(&parse_quantity(fstr(w, "index")?)?.to_le_bytes()); + out.extend_from_slice(&parse_quantity(fstr(w, "validatorIndex")?)?.to_le_bytes()); + out.extend_from_slice(&hex_to_fixed::<20>(fstr(w, "address")?)?); + out.extend_from_slice(&parse_quantity(fstr(w, "amount")?)?.to_le_bytes()); + } + Ok(()) +} + +/// Parse a raw `engine_getPayloadV4` JSON-RPC response body and write the +/// TCache frame directly into `out` (same layout as `encode_get_payload_data`). +/// +/// `raw` is mutated in-place by simd_json's SIMD parser. `out` must be empty +/// on entry and is filled with exactly the frame bytes on success. +/// +/// Eliminates all intermediate allocations vs the serde path: +/// one `Vec` output, hex decoded directly from the borrowed JSON strings. +pub fn json_get_payload_to_tcache( + raw: &mut [u8], + out: &mut Vec, +) -> Result<(), crate::EngineError> { + use simd_json::prelude::{ValueAsArray, ValueAsScalar, ValueObjectAccess}; + + let root = simd_json::to_borrowed_value(raw).map_err(crate::EngineError::Json)?; + + if root.get("error").is_some() { + return Err(crate::EngineError::Ssz("rpc error in getPayload response".into())); + } + let result = root.get("result").ok_or(crate::EngineError::MissingResult)?; + + let ep = result + .get("executionPayload") + .ok_or_else(|| crate::EngineError::Ssz("missing executionPayload".into()))?; + let bb = result + .get("blobsBundle") + .ok_or_else(|| crate::EngineError::Ssz("missing blobsBundle".into()))?; + let should_override = + result.get("shouldOverrideBuilder").and_then(|v| v.as_bool()).unwrap_or(false); + let exec_requests = result + .get("executionRequests") + .and_then(|v| v.as_array()) + .map(|a| a.as_slice()) + .unwrap_or(&[]); + + // TCache frame header: placeholder for payload SSZ length. + let tcache_hdr = out.len(); + out.extend_from_slice(&[0u8; 4]); + + // ExecutionPayload SSZ — fixed section (528 bytes). + let ssz_start = out.len(); + let f = ssz_start; + out.resize(f + PAYLOAD_FIXED_LEN, 0); + + out[f..f + 32].copy_from_slice(&hex_to_fixed::<32>(fstr(ep, "parentHash")?)?); + out[f + 32..f + 52].copy_from_slice(&hex_to_fixed::<20>(fstr(ep, "feeRecipient")?)?); + out[f + 52..f + 84].copy_from_slice(&hex_to_fixed::<32>(fstr(ep, "stateRoot")?)?); + out[f + 84..f + 116].copy_from_slice(&hex_to_fixed::<32>(fstr(ep, "receiptsRoot")?)?); + out[f + 116..f + 372].copy_from_slice(&hex_to_fixed::<256>(fstr(ep, "logsBloom")?)?); + out[f + 372..f + 404].copy_from_slice(&hex_to_fixed::<32>(fstr(ep, "prevRandao")?)?); + out[f + 404..f + 412].copy_from_slice(&parse_quantity(fstr(ep, "blockNumber")?)?.to_le_bytes()); + out[f + 412..f + 420].copy_from_slice(&parse_quantity(fstr(ep, "gasLimit")?)?.to_le_bytes()); + out[f + 420..f + 428].copy_from_slice(&parse_quantity(fstr(ep, "gasUsed")?)?.to_le_bytes()); + out[f + 428..f + 436].copy_from_slice(&parse_quantity(fstr(ep, "timestamp")?)?.to_le_bytes()); + // [f+436..f+440] extra_data offset — patched below + out[f + 440..f + 472].copy_from_slice(&parse_u256_le(fstr(ep, "baseFeePerGas")?)?); + out[f + 472..f + 504].copy_from_slice(&hex_to_fixed::<32>(fstr(ep, "blockHash")?)?); + // [f+504..f+508] transactions offset — patched below + // [f+508..f+512] withdrawals offset — patched below + out[f + 512..f + 520].copy_from_slice(&parse_quantity(fstr(ep, "blobGasUsed")?)?.to_le_bytes()); + out[f + 520..f + 528] + .copy_from_slice(&parse_quantity(fstr(ep, "excessBlobGas")?)?.to_le_bytes()); + + // Variable fields — patch offsets into fixed section as we go. + let extra_off = (out.len() - ssz_start) as u32; + out[f + 436..f + 440].copy_from_slice(&extra_off.to_le_bytes()); + hex_extend(fstr(ep, "extraData")?, out)?; + + let txs_off = (out.len() - ssz_start) as u32; + out[f + 504..f + 508].copy_from_slice(&txs_off.to_le_bytes()); + let txs = ep + .get("transactions") + .and_then(|v| v.as_array()) + .map(|a| a.as_slice()) + .ok_or_else(|| crate::EngineError::Ssz("missing transactions".into()))?; + encode_txs_json(txs, out)?; + + let ws_off = (out.len() - ssz_start) as u32; + out[f + 508..f + 512].copy_from_slice(&ws_off.to_le_bytes()); + let ws = ep + .get("withdrawals") + .and_then(|v| v.as_array()) + .map(|a| a.as_slice()) + .ok_or_else(|| crate::EngineError::Ssz("missing withdrawals".into()))?; + encode_withdrawals_json(ws, out)?; + + // Patch payload SSZ length into TCache header. + let payload_ssz_len = (out.len() - ssz_start) as u32; + out[tcache_hdr..tcache_hdr + 4].copy_from_slice(&payload_ssz_len.to_le_bytes()); + + // Blobs bundle. + let commitments = + bb.get("commitments").and_then(|v| v.as_array()).map(|a| a.as_slice()).unwrap_or(&[]); + let proofs = bb.get("proofs").and_then(|v| v.as_array()).map(|a| a.as_slice()).unwrap_or(&[]); + let blobs = bb.get("blobs").and_then(|v| v.as_array()).map(|a| a.as_slice()).unwrap_or(&[]); let blob_count = commitments.len().min(proofs.len()).min(blobs.len()).min(255) as u8; - let exec_count = exec_requests.len().min(255) as u8; - let blob_data_size: usize = blobs.iter().take(blob_count as usize).map(|b| 4 + b.len()).sum(); - let total = 4 + - payload_ssz.len() + - 1 + - blob_count as usize * (48 + 48) + - blob_data_size + - 2 + - exec_requests.iter().take(exec_count as usize).map(|r| 4 + r.len()).sum::(); - let mut buf = Vec::with_capacity(total); - buf.extend_from_slice(&(payload_ssz.len() as u32).to_le_bytes()); - buf.extend_from_slice(payload_ssz); - buf.push(blob_count); + out.push(blob_count); + for i in 0..blob_count as usize { - let mut c48 = [0u8; 48]; - let c = &commitments[i]; - c48[..c.len().min(48)].copy_from_slice(&c[..c.len().min(48)]); - buf.extend_from_slice(&c48); - let mut p48 = [0u8; 48]; - let p = &proofs[i]; - p48[..p.len().min(48)].copy_from_slice(&p[..p.len().min(48)]); - buf.extend_from_slice(&p48); - buf.extend_from_slice(&(blobs[i].len() as u32).to_le_bytes()); - buf.extend_from_slice(&blobs[i]); - } - buf.push(should_override_builder as u8); - buf.push(exec_count); - for r in exec_requests.iter().take(exec_count as usize) { - buf.extend_from_slice(&(r.len() as u32).to_le_bytes()); - buf.extend_from_slice(r); + hex_extend_clamped::<48>(commitments[i].as_str().unwrap_or("0x"), out)?; + hex_extend_clamped::<48>(proofs[i].as_str().unwrap_or("0x"), out)?; + let b_s = blobs[i].as_str().unwrap_or("0x"); + let b_hex = b_s.strip_prefix("0x").unwrap_or(b_s); + out.extend_from_slice(&((b_hex.len() / 2) as u32).to_le_bytes()); + hex_extend(b_s, out)?; } - buf + + out.push(should_override as u8); + + let exec_count = exec_requests.len().min(255) as u8; + out.push(exec_count); + for req in exec_requests.iter().take(exec_count as usize) { + let s = req.as_str().unwrap_or("0x"); + let hex_s = s.strip_prefix("0x").unwrap_or(s); + out.extend_from_slice(&((hex_s.len() / 2) as u32).to_le_bytes()); + hex_extend(s, out)?; + } + + Ok(()) } // --------------------------------------------------------------------------- -// TCache data framing for engine_getBlobsV2 +// Zero-alloc JSON → TCache frame converter for engine_getBlobsV2 // -// Layout: -// [u32 LE count] -// for each item: -// [u8 present] (0 = null) -// if present: -// [48 bytes proof] -// [u32 LE blob_len][blob_len bytes blob] +// Same approach as json_get_payload_to_tcache: BorrowedValue borrows from the +// input buffer; hex decoded directly into `out` with hex::decode_to_slice. +// Wire layout: [u32 count] [u8 present] [u8 proof_count] [proof_count*48B] [u32 blob_len] [blob bytes] // --------------------------------------------------------------------------- -pub fn encode_get_blobs_data(items: &[Option]) -> Vec { - let mut buf = Vec::new(); - buf.extend_from_slice(&(items.len() as u32).to_le_bytes()); +pub fn json_get_blobs_to_tcache(raw: &mut [u8], out: &mut Vec) -> Result<(), crate::EngineError> { + use simd_json::prelude::{TypedScalarValue, ValueAsArray, ValueAsScalar, ValueObjectAccess}; + + let root = simd_json::to_borrowed_value(raw).map_err(crate::EngineError::Json)?; + let result = root.get("result").ok_or(crate::EngineError::MissingResult)?; + + if result.is_null() { + out.extend_from_slice(&0u32.to_le_bytes()); + return Ok(()); + } + + let items = result.as_array().ok_or_else(|| crate::EngineError::Ssz("getBlobsV2 result not array".into()))?; + out.extend_from_slice(&(items.len() as u32).to_le_bytes()); + for item in items { - match item { - None => buf.push(0), - Some(bap) => { - buf.push(1); - let mut p48 = [0u8; 48]; - let p = &bap.proof; - p48[..p.len().min(48)].copy_from_slice(&p[..p.len().min(48)]); - buf.extend_from_slice(&p48); - buf.extend_from_slice(&(bap.blob.len() as u32).to_le_bytes()); - buf.extend_from_slice(&bap.blob); - } + if item.is_null() { + out.push(0); + continue; + } + out.push(1); + + let proofs = item + .get("proofs") + .and_then(|v| v.as_array()) + .ok_or_else(|| crate::EngineError::Ssz("missing proofs".into()))?; + let proof_count = proofs.len().min(255) as u8; + out.push(proof_count); + for p in &proofs[..proof_count as usize] { + let s = p.as_str().ok_or_else(|| crate::EngineError::Ssz("proof not a string".into()))?; + hex_extend_clamped::<48>(s, out)?; } + + let blob_s = item + .get("blob") + .and_then(|v| v.as_str()) + .ok_or_else(|| crate::EngineError::Ssz("missing blob".into()))?; + let blob_s = blob_s.strip_prefix("0x").unwrap_or(blob_s); + let blob_len = blob_s.len() / 2; + out.extend_from_slice(&(blob_len as u32).to_le_bytes()); + let base = out.len(); + out.resize(base + blob_len, 0); + hex::decode_to_slice(blob_s, &mut out[base..]) + .map_err(|e| crate::EngineError::Ssz(e.to_string()))?; } - buf + + Ok(()) } // --------------------------------------------------------------------------- -// TCache data framing for engine_getPayloadBodiesByHashV1 / ByRangeV1 +// Zero-alloc JSON → TCache frame converter for +// engine_getPayloadBodiesByHashV1 / ByRangeV1 // -// Layout: -// [u32 LE count] -// for each body: -// [u8 present] (0 = null) -// if present: -// [u32 LE tx_count] -// for each tx: [u32 LE len][bytes] -// [u32 LE withdrawal_count] -// for each withdrawal: 44 bytes (index u64 + validator_index u64 + -// address [u8;20] + amount u64) +// Wire layout: [u32 count] [u8 present] [u32 tx_count] [u32 len][tx bytes]... [u32 withdrawal_count] [44B each] // --------------------------------------------------------------------------- -pub fn encode_payload_bodies_data(bodies: &[Option]) -> Vec { - let mut buf = Vec::new(); - buf.extend_from_slice(&(bodies.len() as u32).to_le_bytes()); - for body in bodies { - match body { - None => buf.push(0), - Some(b) => { - buf.push(1); - buf.extend_from_slice(&(b.transactions.len() as u32).to_le_bytes()); - for tx in &b.transactions { - buf.extend_from_slice(&(tx.len() as u32).to_le_bytes()); - buf.extend_from_slice(tx); - } - let withdrawals = b.withdrawals.as_deref().unwrap_or(&[]); - buf.extend_from_slice(&(withdrawals.len() as u32).to_le_bytes()); - for w in withdrawals { - buf.extend_from_slice(&w.index.to_le_bytes()); - buf.extend_from_slice(&w.validator_index.to_le_bytes()); - buf.extend_from_slice(&w.address); - buf.extend_from_slice(&w.amount.to_le_bytes()); - } - } +pub fn json_get_payload_bodies_to_tcache( + raw: &mut [u8], + out: &mut Vec, +) -> Result<(), crate::EngineError> { + use simd_json::prelude::{TypedScalarValue, ValueAsArray, ValueAsScalar, ValueObjectAccess}; + + let root = simd_json::to_borrowed_value(raw).map_err(crate::EngineError::Json)?; + let items = root + .get("result") + .and_then(|v| v.as_array()) + .ok_or(crate::EngineError::MissingResult)?; + + out.extend_from_slice(&(items.len() as u32).to_le_bytes()); + + for item in items { + if item.is_null() { + out.push(0); + continue; + } + out.push(1); + + let txs = item + .get("transactions") + .and_then(|v| v.as_array()) + .ok_or_else(|| crate::EngineError::Ssz("missing transactions".into()))?; + out.extend_from_slice(&(txs.len() as u32).to_le_bytes()); + for tx in txs { + let s = tx.as_str().ok_or_else(|| crate::EngineError::Ssz("tx not a string".into()))?; + let s = s.strip_prefix("0x").unwrap_or(s); + let len = s.len() / 2; + out.extend_from_slice(&(len as u32).to_le_bytes()); + let base = out.len(); + out.resize(base + len, 0); + hex::decode_to_slice(s, &mut out[base..]) + .map_err(|e| crate::EngineError::Ssz(e.to_string()))?; + } + + let withdrawals: &[simd_json::BorrowedValue<'_>] = item + .get("withdrawals") + .and_then(|v| v.as_array()) + .map(|a| a.as_slice()) + .unwrap_or(&[]); + out.extend_from_slice(&(withdrawals.len() as u32).to_le_bytes()); + for w in withdrawals { + let index = parse_quantity(fstr(w, "index")?)?; + let validator_index = parse_quantity(fstr(w, "validatorIndex")?)?; + let address = hex_to_fixed::<20>(fstr(w, "address")?)?; + let amount = parse_quantity(fstr(w, "amount")?)?; + out.extend_from_slice(&index.to_le_bytes()); + out.extend_from_slice(&validator_index.to_le_bytes()); + out.extend_from_slice(&address); + out.extend_from_slice(&amount.to_le_bytes()); } } - buf + + Ok(()) } // --------------------------------------------------------------------------- #[cfg(test)] mod tests { + use simd_json::{owned::to_value, prelude::ValueAsScalar, serde::from_owned_value}; + use super::*; fn sample_payload() -> ExecutionPayload { @@ -855,8 +1036,10 @@ mod tests { safe_block_hash: [0x00; 32], finalized_block_hash: [0xff; 32], }; - let json: ForkchoiceState = - serde_json::from_str(&serde_json::to_string(&state).unwrap()).unwrap(); + let json: ForkchoiceState = { + let mut bytes = simd_json::to_vec(&state).unwrap(); + simd_json::from_slice(&mut bytes).unwrap() + }; assert_eq!(state.head_block_hash, json.head_block_hash); assert_eq!(state.safe_block_hash, json.safe_block_hash); assert_eq!(state.finalized_block_hash, json.finalized_block_hash); @@ -869,7 +1052,7 @@ mod tests { safe_block_hash: [0; 32], finalized_block_hash: [0; 32], }; - let json = serde_json::to_value(&state).unwrap(); + let json = to_value(&mut simd_json::to_vec(&state).unwrap()).unwrap(); let hex = json["headBlockHash"].as_str().unwrap(); assert!(hex.starts_with("0x")); assert_eq!(hex.len(), 2 + 64); @@ -880,7 +1063,10 @@ mod tests { fn wire_quantity_round_trip() { // Withdrawal uses quantity for all numeric fields let w = Withdrawal { index: 0, validator_index: 0xdeadbeef, address: [0; 20], amount: 0 }; - let json: Withdrawal = serde_json::from_str(&serde_json::to_string(&w).unwrap()).unwrap(); + let json: Withdrawal = { + let mut bytes = simd_json::to_vec(&w).unwrap(); + simd_json::from_slice(&mut bytes).unwrap() + }; assert_eq!(w.validator_index, json.validator_index); } @@ -888,9 +1074,9 @@ mod tests { fn wire_u256_le_zero_round_trip() { let mut p = sample_payload(); p.base_fee_per_gas = [0u8; 32]; - let json = serde_json::to_value(&p).unwrap(); + let json = to_value(&mut simd_json::to_vec(&p).unwrap()).unwrap(); assert_eq!(json["baseFeePerGas"].as_str().unwrap(), "0x0"); - let decoded: ExecutionPayload = serde_json::from_value(json).unwrap(); + let decoded: ExecutionPayload = from_owned_value(json).unwrap(); assert_eq!(decoded.base_fee_per_gas, [0u8; 32]); } @@ -899,31 +1085,117 @@ mod tests { let mut p = sample_payload(); p.base_fee_per_gas = [0u8; 32]; p.base_fee_per_gas[0] = 1; // LE: value = 1 - let json = serde_json::to_value(&p).unwrap(); + let json = to_value(&mut simd_json::to_vec(&p).unwrap()).unwrap(); assert_eq!(json["baseFeePerGas"].as_str().unwrap(), "0x1"); - let decoded: ExecutionPayload = serde_json::from_value(json).unwrap(); + let decoded: ExecutionPayload = from_owned_value(json).unwrap(); assert_eq!(decoded.base_fee_per_gas, p.base_fee_per_gas); } #[test] fn wire_opt_b256_none_round_trip() { - let v: PayloadStatus = serde_json::from_str( - r#"{"status":"VALID","latestValidHash":null,"validationError":null}"#, - ) - .unwrap(); + let v: PayloadStatus = { + let mut bytes = + br#"{"status":"VALID","latestValidHash":null,"validationError":null}"#.to_vec(); + simd_json::from_slice(&mut bytes).unwrap() + }; assert_eq!(v.latest_valid_hash, None); - let json = serde_json::to_string(&v).unwrap(); - let v2: PayloadStatus = serde_json::from_str(&json).unwrap(); + let v2: PayloadStatus = { + let mut bytes = simd_json::to_vec(&v).unwrap(); + simd_json::from_slice(&mut bytes).unwrap() + }; assert_eq!(v2.latest_valid_hash, None); } #[test] fn wire_opt_payload_id_round_trip() { let fcu_json = r#"{"payloadStatus":{"status":"VALID","latestValidHash":null,"validationError":null},"payloadId":"0x0102030405060708"}"#; - let fcu: ForkchoiceUpdatedResult = serde_json::from_str(fcu_json).unwrap(); + let fcu: ForkchoiceUpdatedResult = { + let mut bytes = fcu_json.as_bytes().to_vec(); + simd_json::from_slice(&mut bytes).unwrap() + }; assert_eq!(fcu.payload_id, Some([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08])); - let roundtrip: ForkchoiceUpdatedResult = - serde_json::from_str(&serde_json::to_string(&fcu).unwrap()).unwrap(); + let roundtrip: ForkchoiceUpdatedResult = { + let mut bytes = simd_json::to_vec(&fcu).unwrap(); + simd_json::from_slice(&mut bytes).unwrap() + }; assert_eq!(fcu.payload_id, roundtrip.payload_id); } + + // Verify json_get_payload_to_tcache produces identical bytes to the old + // serde path (ExecutionPayload → to_ssz → encode_get_payload_data). + #[test] + fn json_get_payload_to_tcache_matches_serde_path() { + let payload = sample_payload(); + let commitments: Vec> = vec![vec![0xc0; 48], vec![0xc1; 48]]; + let proofs: Vec> = vec![vec![0xd0; 48], vec![0xd1; 48]]; + let blobs: Vec> = vec![vec![0xb0; 128], vec![0xb1; 64]]; + let exec_requests: Vec> = vec![vec![0x01, 0x02], vec![0x03]]; + let should_override = false; + + // Reference output via the old serde path. + let payload_ssz = payload.to_ssz(); + let blob_count = 2u8; + let exec_count = exec_requests.len().min(255) as u8; + let blob_data_size: usize = blobs.iter().map(|b| 4 + b.len()).sum(); + let total = 4 + + payload_ssz.len() + + 1 + + blob_count as usize * (48 + 48) + + blob_data_size + + 2 + + exec_requests.iter().map(|r| 4 + r.len()).sum::(); + let mut reference = Vec::with_capacity(total); + reference.extend_from_slice(&(payload_ssz.len() as u32).to_le_bytes()); + reference.extend_from_slice(&payload_ssz); + reference.push(blob_count); + for i in 0..blob_count as usize { + let mut c48 = [0u8; 48]; + let c = &commitments[i]; + c48[..c.len().min(48)].copy_from_slice(&c[..c.len().min(48)]); + reference.extend_from_slice(&c48); + let mut p48 = [0u8; 48]; + let p = &proofs[i]; + p48[..p.len().min(48)].copy_from_slice(&p[..p.len().min(48)]); + reference.extend_from_slice(&p48); + reference.extend_from_slice(&(blobs[i].len() as u32).to_le_bytes()); + reference.extend_from_slice(&blobs[i]); + } + reference.push(should_override as u8); + reference.push(exec_count); + for r in &exec_requests { + reference.extend_from_slice(&(r.len() as u32).to_le_bytes()); + reference.extend_from_slice(r); + } + + // Build the JSON-RPC response envelope that json_get_payload_to_tcache expects. + let commitments_hex: Vec = + commitments.iter().map(|c| format!("0x{}", hex::encode(c))).collect(); + let proofs_hex: Vec = + proofs.iter().map(|p| format!("0x{}", hex::encode(p))).collect(); + let blobs_hex: Vec = + blobs.iter().map(|b| format!("0x{}", hex::encode(b))).collect(); + let exec_hex: Vec = + exec_requests.iter().map(|r| format!("0x{}", hex::encode(r))).collect(); + + let envelope = simd_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "result": { + "executionPayload": payload, + "blobsBundle": { + "commitments": commitments_hex, + "proofs": proofs_hex, + "blobs": blobs_hex, + }, + "shouldOverrideBuilder": should_override, + "executionRequests": exec_hex, + } + }); + let mut raw = simd_json::to_vec(&envelope).unwrap(); + + let mut out = Vec::new(); + json_get_payload_to_tcache(&mut raw, &mut out).expect("json_get_payload_to_tcache failed"); + + assert_eq!(out, reference, "json path output differs from serde path"); + } } diff --git a/memory/MEMORY.md b/memory/MEMORY.md deleted file mode 100644 index dcbb949..0000000 --- a/memory/MEMORY.md +++ /dev/null @@ -1,3 +0,0 @@ -# Memory Index - -- [Always seek the best method](feedback_best_method.md) — Never settle for "fine" or "good enough" — always find the state of the art approach diff --git a/memory/feedback_best_method.md b/memory/feedback_best_method.md deleted file mode 100644 index f8a80d6..0000000 --- a/memory/feedback_best_method.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -name: Always seek the best method -description: Never settle for "fine" or "good enough" — always find the state of the art approach -type: feedback ---- - -Always seek out the best method for anything. The state of the art. Never settle for "good enough" or "fine." - -**Why:** Explicitly corrected when I said a linear scan was "fine" for in-flight lookup given small pool size — the right call was to use a HashMap for O(1) lookup regardless. - -**How to apply:** When evaluating implementation choices, don't stop at "acceptable given constraints." Find the best approach unconditionally, then apply it. From 50461c29b53d578aa39da9851371f52476704274 Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 2 Jun 2026 10:04:00 +0100 Subject: [PATCH 3/6] fix lint --- crates/common/src/lib.rs | 10 +++++----- crates/common/src/spine.rs | 16 +++++++-------- crates/common/src/spine/messages.rs | 10 +++++----- crates/engine/src/http.rs | 5 +++-- crates/engine/src/ipc.rs | 5 +++-- crates/engine/src/req_handlers.rs | 8 ++++---- crates/engine/src/resp_handlers.rs | 4 ++-- crates/engine/src/tile.rs | 6 +++++- crates/engine/src/types.rs | 31 ++++++++++++++++------------- 9 files changed, 52 insertions(+), 43 deletions(-) diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 6bb0480..1a49a04 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -18,11 +18,11 @@ pub use crate::{ }, spine::{ ALL_PROTOCOLS, AcquiredRead as TRead, BeaconStateEvent, Consumer as TConsumer, - DataColumnsAvailable, ELSyncStatus, EngineFcuReq, EngineFcuResp, EngineGetPayloadReq, - EngineGetBlobsReq, EngineGetBlobsResp, EngineGetPayloadBodiesByHashReq, - EngineGetPayloadBodiesByRangeReq, EngineGetPayloadBodiesResp, EngineGetPayloadResp, - EngineHealthEvent, EngineNewPayloadReq, EngineNewPayloadResp, EnginePreparePayloadReq, - EngineReq, EngineResp, Error as TCacheError, GossipMsgOut, IpBytes, MAX_BLOBS_PER_BLOCK, + DataColumnsAvailable, ELSyncStatus, EngineFcuReq, EngineFcuResp, EngineGetBlobsReq, + EngineGetBlobsResp, EngineGetPayloadBodiesByHashReq, EngineGetPayloadBodiesByRangeReq, + EngineGetPayloadBodiesResp, EngineGetPayloadReq, EngineGetPayloadResp, EngineHealthEvent, + EngineNewPayloadReq, EngineNewPayloadResp, EnginePreparePayloadReq, EngineReq, EngineResp, + Error as TCacheError, GossipMsgOut, IpBytes, MAX_BLOBS_PER_BLOCK, MAX_PAYLOAD_BODIES_PER_REQ, MULTISTREAM_V1, MultiProducer as TMultiProducer, NewGossipMsg, P2pSend, P2pStreamId, PayloadValidationStatus, PeerControl, PeerEvent, PeerStatus, Producer as TProducer, REJECT_RESPONSE, RPC_PROTOCOLS, diff --git a/crates/common/src/spine.rs b/crates/common/src/spine.rs index 14518a6..cba1342 100644 --- a/crates/common/src/spine.rs +++ b/crates/common/src/spine.rs @@ -3,14 +3,14 @@ use flux::{communication::ShmemData, spine::SpineQueue, spine_derive::from_spine, tile::TileInfo}; pub use messages::{ BeaconStateEvent, DataColumnsAvailable, ELSyncStatus, EngineFcuReq, EngineFcuResp, - EngineGetPayloadReq, EngineGetBlobsReq, EngineGetBlobsResp, EngineGetPayloadBodiesByHashReq, - EngineGetPayloadBodiesByRangeReq, EngineGetPayloadBodiesResp, EngineGetPayloadResp, - EngineHealthEvent, EngineNewPayloadReq, EngineNewPayloadResp, EnginePreparePayloadReq, - EngineReq, EngineResp, GossipMsgOut, IpBytes, MAX_BLOBS_PER_BLOCK, MAX_PAYLOAD_BODIES_PER_REQ, - NewGossipMsg, P2pSend, PayloadValidationStatus, PeerControl, PeerEvent, PeerStatus, - RejectSource, RpcInbound, RpcMsg, RpcOutbound, RpcRequest, RpcRequestInbound, - RpcRequestOutbound, RpcResponse, RpcResponseInbound, RpcResponseOutbound, RpcSeverity, - SyncUpdate, WithdrawalInline, + EngineGetBlobsReq, EngineGetBlobsResp, EngineGetPayloadBodiesByHashReq, + EngineGetPayloadBodiesByRangeReq, EngineGetPayloadBodiesResp, EngineGetPayloadReq, + EngineGetPayloadResp, EngineHealthEvent, EngineNewPayloadReq, EngineNewPayloadResp, + EnginePreparePayloadReq, EngineReq, EngineResp, GossipMsgOut, IpBytes, MAX_BLOBS_PER_BLOCK, + MAX_PAYLOAD_BODIES_PER_REQ, NewGossipMsg, P2pSend, PayloadValidationStatus, PeerControl, + PeerEvent, PeerStatus, RejectSource, RpcInbound, RpcMsg, RpcOutbound, RpcRequest, + RpcRequestInbound, RpcRequestOutbound, RpcResponse, RpcResponseInbound, RpcResponseOutbound, + RpcSeverity, SyncUpdate, WithdrawalInline, }; pub use stream_id::P2pStreamId; pub use stream_protocol::{ diff --git a/crates/common/src/spine/messages.rs b/crates/common/src/spine/messages.rs index b515c84..c5ae337 100644 --- a/crates/common/src/spine/messages.rs +++ b/crates/common/src/spine/messages.rs @@ -683,7 +683,6 @@ pub struct EngineNewPayloadResp { pub latest_valid_hash: [u8; 32], } - /// The engine tile sends `engine_forkchoiceUpdatedV3` with payload attributes /// and returns the `payload_id` assigned by the EL. The caller should use. /// this to fetch the built payload. @@ -735,8 +734,8 @@ pub struct EngineGetBlobsReq { /// Response to `EngineGetBlobsReq`. /// When `ok` is true, `data` is a TCache slot with binary-encoded blobs: -/// `[u32 count] ([u8 present] [u8 proof_count] [48B proof]* [u32 blob_len] [blob bytes])*` -/// `present == 0` means the entry is null (blob missing). +/// `[u32 count] ([u8 present] [u8 proof_count] [48B proof]* [u32 blob_len] +/// [blob bytes])*` `present == 0` means the entry is null (blob missing). #[derive(Clone, Copy, Debug)] #[repr(C)] pub struct EngineGetBlobsResp { @@ -766,8 +765,9 @@ pub struct EngineGetPayloadBodiesByRangeReq { /// Response to either `getPayloadBodiesByHash` or `getPayloadBodiesByRange`. /// When `ok` is true, `data` is a TCache slot with binary-encoded bodies: -/// `[u32 count] ([u8 present] [u32 tx_count] ([u32 tx_len][tx bytes])* [u32 withdrawal_count] ([u32 index][u32 validator_index][20B address][u64 amount])*)*` -/// `present == 0` means the entry is null (block missing). +/// `[u32 count] ([u8 present] [u32 tx_count] ([u32 tx_len][tx bytes])* [u32 +/// withdrawal_count] ([u32 index][u32 validator_index][20B address][u64 +/// amount])*)*` `present == 0` means the entry is null (block missing). #[derive(Clone, Copy, Debug)] #[repr(C)] pub struct EngineGetPayloadBodiesResp { diff --git a/crates/engine/src/http.rs b/crates/engine/src/http.rs index fab4005..86a05ae 100644 --- a/crates/engine/src/http.rs +++ b/crates/engine/src/http.rs @@ -7,8 +7,9 @@ use mio::{Events, Interest, Poll, Token, net::TcpStream}; use crate::{EngineError, JwtSecret}; -// Sized for the largest expected EL response: getPayload with a full blobsBundle -// (~21 blobs × 256 KB hex-encoded + execution payload transactions). +// Sized for the largest expected EL response: getPayload with a full +// blobsBundle (~21 blobs × 256 KB hex-encoded + execution payload +// transactions). const READ_BUF_CAPACITY: usize = 10 * 1024 * 1024; enum Conn { diff --git a/crates/engine/src/ipc.rs b/crates/engine/src/ipc.rs index faea69d..8bd9316 100644 --- a/crates/engine/src/ipc.rs +++ b/crates/engine/src/ipc.rs @@ -8,8 +8,9 @@ use mio::{Events, Interest, Poll, Token, net::UnixStream}; use crate::EngineError; -// Sized for the largest expected EL response: getPayload with a full blobsBundle -// (~21 blobs × 256 KB hex-encoded + execution payload transactions). +// Sized for the largest expected EL response: getPayload with a full +// blobsBundle (~21 blobs × 256 KB hex-encoded + execution payload +// transactions). const READ_BUF_CAPACITY: usize = 10 * 1024 * 1024; #[derive(PartialEq)] diff --git a/crates/engine/src/req_handlers.rs b/crates/engine/src/req_handlers.rs index 4e8c338..f3df84d 100644 --- a/crates/engine/src/req_handlers.rs +++ b/crates/engine/src/req_handlers.rs @@ -1,9 +1,9 @@ use flux::spine::FluxSpine; use silver_common::{ - EngineFcuReq, EngineGetPayloadReq, EngineGetBlobsReq, EngineGetPayloadBodiesByHashReq, - EngineGetPayloadBodiesByRangeReq, EngineNewPayloadReq, EngineNewPayloadResp, - EnginePreparePayloadReq, EngineReq, EngineResp, PayloadValidationStatus, SilverSpine, - TRandomAccess, + EngineFcuReq, EngineGetBlobsReq, EngineGetPayloadBodiesByHashReq, + EngineGetPayloadBodiesByRangeReq, EngineGetPayloadReq, EngineNewPayloadReq, + EngineNewPayloadResp, EnginePreparePayloadReq, EngineReq, EngineResp, PayloadValidationStatus, + SilverSpine, TRandomAccess, }; use crate::{ diff --git a/crates/engine/src/resp_handlers.rs b/crates/engine/src/resp_handlers.rs index b053653..dea59e7 100644 --- a/crates/engine/src/resp_handlers.rs +++ b/crates/engine/src/resp_handlers.rs @@ -10,8 +10,8 @@ use crate::{ EngineError, client::extract_result, types::{ - ForkchoiceUpdatedResult, PayloadStatus, - json_get_blobs_to_tcache, json_get_payload_bodies_to_tcache, json_get_payload_to_tcache, + ForkchoiceUpdatedResult, PayloadStatus, json_get_blobs_to_tcache, + json_get_payload_bodies_to_tcache, json_get_payload_to_tcache, }, }; diff --git a/crates/engine/src/tile.rs b/crates/engine/src/tile.rs index 9ff97f7..4f407a0 100644 --- a/crates/engine/src/tile.rs +++ b/crates/engine/src/tile.rs @@ -94,7 +94,11 @@ impl EngineTile { ReqKind::GetPayloadBodiesByHash(spine_id) | ReqKind::GetPayloadBodiesByRange(spine_id) => { handle_get_payload_bodies_response( - spine_id, response, adapter, resp_producer, scratch, + spine_id, + response, + adapter, + resp_producer, + scratch, ); } }); diff --git a/crates/engine/src/types.rs b/crates/engine/src/types.rs index 5fef83e..0119f85 100644 --- a/crates/engine/src/types.rs +++ b/crates/engine/src/types.rs @@ -744,10 +744,14 @@ pub fn json_get_payload_to_tcache( // // Same approach as json_get_payload_to_tcache: BorrowedValue borrows from the // input buffer; hex decoded directly into `out` with hex::decode_to_slice. -// Wire layout: [u32 count] [u8 present] [u8 proof_count] [proof_count*48B] [u32 blob_len] [blob bytes] +// Wire layout: [u32 count] [u8 present] [u8 proof_count] [proof_count*48B] [u32 +// blob_len] [blob bytes] // --------------------------------------------------------------------------- -pub fn json_get_blobs_to_tcache(raw: &mut [u8], out: &mut Vec) -> Result<(), crate::EngineError> { +pub fn json_get_blobs_to_tcache( + raw: &mut [u8], + out: &mut Vec, +) -> Result<(), crate::EngineError> { use simd_json::prelude::{TypedScalarValue, ValueAsArray, ValueAsScalar, ValueObjectAccess}; let root = simd_json::to_borrowed_value(raw).map_err(crate::EngineError::Json)?; @@ -758,7 +762,9 @@ pub fn json_get_blobs_to_tcache(raw: &mut [u8], out: &mut Vec) -> Result<(), return Ok(()); } - let items = result.as_array().ok_or_else(|| crate::EngineError::Ssz("getBlobsV2 result not array".into()))?; + let items = result + .as_array() + .ok_or_else(|| crate::EngineError::Ssz("getBlobsV2 result not array".into()))?; out.extend_from_slice(&(items.len() as u32).to_le_bytes()); for item in items { @@ -775,7 +781,8 @@ pub fn json_get_blobs_to_tcache(raw: &mut [u8], out: &mut Vec) -> Result<(), let proof_count = proofs.len().min(255) as u8; out.push(proof_count); for p in &proofs[..proof_count as usize] { - let s = p.as_str().ok_or_else(|| crate::EngineError::Ssz("proof not a string".into()))?; + let s = + p.as_str().ok_or_else(|| crate::EngineError::Ssz("proof not a string".into()))?; hex_extend_clamped::<48>(s, out)?; } @@ -799,7 +806,8 @@ pub fn json_get_blobs_to_tcache(raw: &mut [u8], out: &mut Vec) -> Result<(), // Zero-alloc JSON → TCache frame converter for // engine_getPayloadBodiesByHashV1 / ByRangeV1 // -// Wire layout: [u32 count] [u8 present] [u32 tx_count] [u32 len][tx bytes]... [u32 withdrawal_count] [44B each] +// Wire layout: [u32 count] [u8 present] [u32 tx_count] [u32 len][tx bytes]... +// [u32 withdrawal_count] [44B each] // --------------------------------------------------------------------------- pub fn json_get_payload_bodies_to_tcache( @@ -809,10 +817,8 @@ pub fn json_get_payload_bodies_to_tcache( use simd_json::prelude::{TypedScalarValue, ValueAsArray, ValueAsScalar, ValueObjectAccess}; let root = simd_json::to_borrowed_value(raw).map_err(crate::EngineError::Json)?; - let items = root - .get("result") - .and_then(|v| v.as_array()) - .ok_or(crate::EngineError::MissingResult)?; + let items = + root.get("result").and_then(|v| v.as_array()).ok_or(crate::EngineError::MissingResult)?; out.extend_from_slice(&(items.len() as u32).to_le_bytes()); @@ -839,11 +845,8 @@ pub fn json_get_payload_bodies_to_tcache( .map_err(|e| crate::EngineError::Ssz(e.to_string()))?; } - let withdrawals: &[simd_json::BorrowedValue<'_>] = item - .get("withdrawals") - .and_then(|v| v.as_array()) - .map(|a| a.as_slice()) - .unwrap_or(&[]); + let withdrawals: &[simd_json::BorrowedValue<'_>] = + item.get("withdrawals").and_then(|v| v.as_array()).map(|a| a.as_slice()).unwrap_or(&[]); out.extend_from_slice(&(withdrawals.len() as u32).to_le_bytes()); for w in withdrawals { let index = parse_quantity(fstr(w, "index")?)?; From c5da5f2c5bc452c24b6e9e6c69961f1f3ea8471f Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 2 Jun 2026 12:14:11 +0100 Subject: [PATCH 4/6] rm needless text from logs --- crates/engine/src/ipc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/src/ipc.rs b/crates/engine/src/ipc.rs index 8bd9316..50fb14b 100644 --- a/crates/engine/src/ipc.rs +++ b/crates/engine/src/ipc.rs @@ -202,7 +202,7 @@ fn ipc_on_error(t: &mut IpcTransport, poll: &mut Poll, on_complete: &mut F, m where F: FnMut(u64, Result, EngineError>), { - tracing::warn!("engine ipc: {msg}"); + tracing::warn!("{msg}"); let err = msg.to_string(); if let Some(rpc_id) = t.in_flight.take() { on_complete(rpc_id, Err(EngineError::Ipc(err.clone()))); From 0ca80b0b1473d899574d920e39d8f4b1c5d1f9d5 Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 2 Jun 2026 12:33:29 +0100 Subject: [PATCH 5/6] improve tests --- crates/engine/src/types.rs | 127 +++++++----------- crates/engine/testdata/get_payload_tcache.bin | Bin 0 -> 1047 bytes 2 files changed, 46 insertions(+), 81 deletions(-) create mode 100644 crates/engine/testdata/get_payload_tcache.bin diff --git a/crates/engine/src/types.rs b/crates/engine/src/types.rs index 0119f85..f94743b 100644 --- a/crates/engine/src/types.rs +++ b/crates/engine/src/types.rs @@ -871,6 +871,12 @@ mod tests { use super::*; + const SAMPLE_PAYLOAD_SSZ: &[u8] = include_bytes!("../testdata/sample_payload.ssz"); + const EMPTY_VAR_PAYLOAD_SSZ: &[u8] = include_bytes!("../testdata/empty_var_payload.ssz"); + const LARGE_EXTRA_PAYLOAD_SSZ: &[u8] = include_bytes!("../testdata/large_extra_payload.ssz"); + const MANY_TX_PAYLOAD_SSZ: &[u8] = include_bytes!("../testdata/many_tx_payload.ssz"); + const GET_PAYLOAD_TCACHE: &[u8] = include_bytes!("../testdata/get_payload_tcache.bin"); + fn sample_payload() -> ExecutionPayload { ExecutionPayload { parent_hash: [0x11; 32], @@ -897,33 +903,36 @@ mod tests { } #[test] - fn ssz_round_trip_with_all_fields() { - let original = sample_payload(); - let decoded = ExecutionPayload::from_ssz(&original.to_ssz()).unwrap(); - assert_eq!(original, decoded); + fn ssz_encode_sample_payload() { + assert_eq!(sample_payload().to_ssz(), SAMPLE_PAYLOAD_SSZ); } #[test] - fn ssz_round_trip_empty_variable_fields() { + fn ssz_decode_sample_payload() { + assert_eq!(ExecutionPayload::from_ssz(SAMPLE_PAYLOAD_SSZ).unwrap(), sample_payload()); + } + + #[test] + fn ssz_decode_empty_variable_fields() { let mut p = sample_payload(); p.extra_data = vec![]; p.transactions = vec![]; p.withdrawals = vec![]; - assert_eq!(p, ExecutionPayload::from_ssz(&p.to_ssz()).unwrap()); + assert_eq!(ExecutionPayload::from_ssz(EMPTY_VAR_PAYLOAD_SSZ).unwrap(), p); } #[test] - fn ssz_round_trip_large_extra_data() { + fn ssz_decode_large_extra_data() { let mut p = sample_payload(); p.extra_data = vec![0xffu8; 32]; - assert_eq!(p, ExecutionPayload::from_ssz(&p.to_ssz()).unwrap()); + assert_eq!(ExecutionPayload::from_ssz(LARGE_EXTRA_PAYLOAD_SSZ).unwrap(), p); } #[test] - fn ssz_round_trip_many_transactions() { + fn ssz_decode_many_transactions() { let mut p = sample_payload(); p.transactions = (0u8..=20).map(|i| vec![i; i as usize + 1]).collect(); - assert_eq!(p, ExecutionPayload::from_ssz(&p.to_ssz()).unwrap()); + assert_eq!(ExecutionPayload::from_ssz(MANY_TX_PAYLOAD_SSZ).unwrap(), p); } #[test] @@ -940,18 +949,17 @@ mod tests { #[test] fn ssz_variable_offsets_at_correct_positions() { - let p = sample_payload(); - let ssz = p.to_ssz(); + let ssz = SAMPLE_PAYLOAD_SSZ; let extra_off = u32::from_le_bytes(ssz[436..440].try_into().unwrap()) as usize; let tx_off = u32::from_le_bytes(ssz[504..508].try_into().unwrap()) as usize; let wd_off = u32::from_le_bytes(ssz[508..512].try_into().unwrap()) as usize; assert_eq!(extra_off, 528, "extra_data offset must point past fixed section"); - assert_eq!(tx_off, 528 + p.extra_data.len()); + assert_eq!(tx_off, 528 + 5); // extra_data = b"extra" (5 bytes) // withdrawals follow the SSZ transaction list assert!(wd_off > tx_off); - assert_eq!(wd_off, ssz.len() - p.withdrawals.len() * 44); + assert_eq!(wd_off, ssz.len() - 2 * 44); // 2 withdrawals × 44 bytes each } // ----------------------------------------------------------------------- @@ -1003,28 +1011,24 @@ mod tests { #[test] fn new_payload_framing_round_trip_no_exec_requests() { - let p = sample_payload(); - let ssz = p.to_ssz(); - let enc = encode_new_payload_data(&ssz, &[]); + let enc = encode_new_payload_data(SAMPLE_PAYLOAD_SSZ, &[]); let (decoded_p, decoded_reqs) = decode_new_payload_data(&enc).unwrap(); - assert_eq!(p, decoded_p); + assert_eq!(sample_payload(), decoded_p); assert!(decoded_reqs.is_empty()); } #[test] fn new_payload_framing_round_trip_with_exec_requests() { - let p = sample_payload(); - let ssz = p.to_ssz(); let reqs = vec![vec![1u8, 2, 3], vec![0xde, 0xad]]; - let enc = encode_new_payload_data(&ssz, &reqs); + let enc = encode_new_payload_data(SAMPLE_PAYLOAD_SSZ, &reqs); let (decoded_p, decoded_reqs) = decode_new_payload_data(&enc).unwrap(); - assert_eq!(p, decoded_p); + assert_eq!(sample_payload(), decoded_p); assert_eq!(reqs, decoded_reqs); } #[test] fn new_payload_framing_truncated_returns_error() { - let enc = encode_new_payload_data(&sample_payload().to_ssz(), &[]); + let enc = encode_new_payload_data(SAMPLE_PAYLOAD_SSZ, &[]); assert!(decode_new_payload_data(&enc[..4]).is_err()); } @@ -1124,62 +1128,25 @@ mod tests { assert_eq!(fcu.payload_id, roundtrip.payload_id); } - // Verify json_get_payload_to_tcache produces identical bytes to the old - // serde path (ExecutionPayload → to_ssz → encode_get_payload_data). #[test] - fn json_get_payload_to_tcache_matches_serde_path() { + fn json_get_payload_to_tcache_output() { let payload = sample_payload(); - let commitments: Vec> = vec![vec![0xc0; 48], vec![0xc1; 48]]; - let proofs: Vec> = vec![vec![0xd0; 48], vec![0xd1; 48]]; - let blobs: Vec> = vec![vec![0xb0; 128], vec![0xb1; 64]]; - let exec_requests: Vec> = vec![vec![0x01, 0x02], vec![0x03]]; - let should_override = false; - - // Reference output via the old serde path. - let payload_ssz = payload.to_ssz(); - let blob_count = 2u8; - let exec_count = exec_requests.len().min(255) as u8; - let blob_data_size: usize = blobs.iter().map(|b| 4 + b.len()).sum(); - let total = 4 + - payload_ssz.len() + - 1 + - blob_count as usize * (48 + 48) + - blob_data_size + - 2 + - exec_requests.iter().map(|r| 4 + r.len()).sum::(); - let mut reference = Vec::with_capacity(total); - reference.extend_from_slice(&(payload_ssz.len() as u32).to_le_bytes()); - reference.extend_from_slice(&payload_ssz); - reference.push(blob_count); - for i in 0..blob_count as usize { - let mut c48 = [0u8; 48]; - let c = &commitments[i]; - c48[..c.len().min(48)].copy_from_slice(&c[..c.len().min(48)]); - reference.extend_from_slice(&c48); - let mut p48 = [0u8; 48]; - let p = &proofs[i]; - p48[..p.len().min(48)].copy_from_slice(&p[..p.len().min(48)]); - reference.extend_from_slice(&p48); - reference.extend_from_slice(&(blobs[i].len() as u32).to_le_bytes()); - reference.extend_from_slice(&blobs[i]); - } - reference.push(should_override as u8); - reference.push(exec_count); - for r in &exec_requests { - reference.extend_from_slice(&(r.len() as u32).to_le_bytes()); - reference.extend_from_slice(r); - } - - // Build the JSON-RPC response envelope that json_get_payload_to_tcache expects. - let commitments_hex: Vec = - commitments.iter().map(|c| format!("0x{}", hex::encode(c))).collect(); - let proofs_hex: Vec = - proofs.iter().map(|p| format!("0x{}", hex::encode(p))).collect(); - let blobs_hex: Vec = - blobs.iter().map(|b| format!("0x{}", hex::encode(b))).collect(); - let exec_hex: Vec = - exec_requests.iter().map(|r| format!("0x{}", hex::encode(r))).collect(); - + let commitments_hex: Vec = vec![[0xc0u8; 48], [0xc1u8; 48]] + .iter() + .map(|c| format!("0x{}", hex::encode(c))) + .collect(); + let proofs_hex: Vec = vec![[0xd0u8; 48], [0xd1u8; 48]] + .iter() + .map(|p| format!("0x{}", hex::encode(p))) + .collect(); + let blobs_hex: Vec = vec![vec![0xb0u8; 128], vec![0xb1u8; 64]] + .iter() + .map(|b| format!("0x{}", hex::encode(b))) + .collect(); + let exec_hex: Vec = vec![vec![0x01u8, 0x02], vec![0x03u8]] + .iter() + .map(|r| format!("0x{}", hex::encode(r))) + .collect(); let envelope = simd_json::json!({ "jsonrpc": "2.0", "id": 1, @@ -1190,15 +1157,13 @@ mod tests { "proofs": proofs_hex, "blobs": blobs_hex, }, - "shouldOverrideBuilder": should_override, + "shouldOverrideBuilder": false, "executionRequests": exec_hex, } }); let mut raw = simd_json::to_vec(&envelope).unwrap(); - let mut out = Vec::new(); json_get_payload_to_tcache(&mut raw, &mut out).expect("json_get_payload_to_tcache failed"); - - assert_eq!(out, reference, "json path output differs from serde path"); + assert_eq!(out, GET_PAYLOAD_TCACHE); } } diff --git a/crates/engine/testdata/get_payload_tcache.bin b/crates/engine/testdata/get_payload_tcache.bin new file mode 100644 index 0000000000000000000000000000000000000000..67ccb83ab803340feb9562d7f21504ddfca2c702 GIT binary patch literal 1047 zcmb Date: Tue, 2 Jun 2026 12:46:54 +0100 Subject: [PATCH 6/6] rm dead code --- crates/engine/src/ipc.rs | 14 ++-- crates/engine/src/types.rs | 99 +++---------------------- crates/engine/testdata/tx_multi.bin | Bin 0 -> 65 bytes crates/engine/testdata/tx_single.bin | Bin 0 -> 8 bytes crates/engine/testdata/withdrawals.bin | Bin 0 -> 88 bytes 5 files changed, 17 insertions(+), 96 deletions(-) create mode 100644 crates/engine/testdata/tx_multi.bin create mode 100644 crates/engine/testdata/tx_single.bin create mode 100644 crates/engine/testdata/withdrawals.bin diff --git a/crates/engine/src/ipc.rs b/crates/engine/src/ipc.rs index 50fb14b..2b0178a 100644 --- a/crates/engine/src/ipc.rs +++ b/crates/engine/src/ipc.rs @@ -20,7 +20,7 @@ enum State { Connected, } -pub(crate) struct IpcTransport { +struct IpcTransport { path: PathBuf, token: Token, stream: Option, @@ -48,11 +48,11 @@ impl IpcTransport { } } -pub(crate) fn ipc_is_free(t: &IpcTransport) -> bool { +fn ipc_is_free(t: &IpcTransport) -> bool { t.in_flight.is_none() && t.send_queue.is_empty() } -pub(crate) fn ipc_enqueue(t: &mut IpcTransport, rpc_id: u64, body: &[u8], poll: &mut Poll) { +fn ipc_enqueue(t: &mut IpcTransport, rpc_id: u64, body: &[u8], poll: &mut Poll) { let mut bytes = body.to_vec(); bytes.push(b'\n'); t.send_queue.push_back((rpc_id, bytes)); @@ -64,12 +64,8 @@ pub(crate) fn ipc_enqueue(t: &mut IpcTransport, rpc_id: u64, body: &[u8], poll: } } -pub(crate) fn ipc_poll( - t: &mut IpcTransport, - events: &Events, - poll: &mut Poll, - on_complete: &mut F, -) where +fn ipc_poll(t: &mut IpcTransport, events: &Events, poll: &mut Poll, on_complete: &mut F) +where F: FnMut(u64, Result, EngineError>), { for event in events.iter() { diff --git a/crates/engine/src/types.rs b/crates/engine/src/types.rs index f94743b..65f92c1 100644 --- a/crates/engine/src/types.rs +++ b/crates/engine/src/types.rs @@ -329,63 +329,6 @@ impl ExecutionPayload { excess_blob_gas: u64_at(520), }) } - - pub fn to_ssz(&self) -> Vec { - let tx_ssz = encode_transactions(&self.transactions); - let w_ssz = encode_withdrawals(&self.withdrawals); - - let extra_data_off = PAYLOAD_FIXED_LEN as u32; - let transactions_off = extra_data_off + self.extra_data.len() as u32; - let withdrawals_off = transactions_off + tx_ssz.len() as u32; - - let mut buf = Vec::with_capacity( - PAYLOAD_FIXED_LEN + self.extra_data.len() + tx_ssz.len() + w_ssz.len(), - ); - - buf.extend_from_slice(&self.parent_hash); - buf.extend_from_slice(&self.fee_recipient); - buf.extend_from_slice(&self.state_root); - buf.extend_from_slice(&self.receipts_root); - buf.extend_from_slice(&self.logs_bloom); - buf.extend_from_slice(&self.prev_randao); - buf.extend_from_slice(&self.block_number.to_le_bytes()); - buf.extend_from_slice(&self.gas_limit.to_le_bytes()); - buf.extend_from_slice(&self.gas_used.to_le_bytes()); - buf.extend_from_slice(&self.timestamp.to_le_bytes()); - buf.extend_from_slice(&extra_data_off.to_le_bytes()); - buf.extend_from_slice(&self.base_fee_per_gas); - buf.extend_from_slice(&self.block_hash); - buf.extend_from_slice(&transactions_off.to_le_bytes()); - buf.extend_from_slice(&withdrawals_off.to_le_bytes()); - buf.extend_from_slice(&self.blob_gas_used.to_le_bytes()); - buf.extend_from_slice(&self.excess_blob_gas.to_le_bytes()); - - buf.extend_from_slice(&self.extra_data); - buf.extend_from_slice(&tx_ssz); - buf.extend_from_slice(&w_ssz); - buf - } -} - -/// Encodes transactions as SSZ `List`. The first `n * 4` bytes are -/// u32 LE offsets, each the absolute byte position of that transaction within -/// this buffer. Transaction bytes follow immediately after the offset header. -/// The count is recoverable on decode as `first_offset / 4`. -fn encode_transactions(txs: &[Vec]) -> Vec { - if txs.is_empty() { - return vec![]; - } - let header_bytes = txs.len() * 4; - let mut offset = header_bytes as u32; - let mut buf = Vec::with_capacity(header_bytes + txs.iter().map(|t| t.len()).sum::()); - for tx in txs { - buf.extend_from_slice(&offset.to_le_bytes()); - offset += tx.len() as u32; - } - for tx in txs { - buf.extend_from_slice(tx); - } - buf } fn decode_transactions(data: &[u8]) -> Result>, crate::EngineError> { @@ -419,19 +362,6 @@ fn decode_transactions(data: &[u8]) -> Result>, crate::EngineError> .collect() } -/// Each `Withdrawal` is 44 bytes: index u64, validator_index u64, address -/// [u8;20], amount u64. -fn encode_withdrawals(ws: &[Withdrawal]) -> Vec { - let mut buf = Vec::with_capacity(ws.len() * 44); - for w in ws { - buf.extend_from_slice(&w.index.to_le_bytes()); - buf.extend_from_slice(&w.validator_index.to_le_bytes()); - buf.extend_from_slice(&w.address); - buf.extend_from_slice(&w.amount.to_le_bytes()); - } - buf -} - fn decode_withdrawals(data: &[u8]) -> Result, crate::EngineError> { if !data.len().is_multiple_of(44) { return Err(crate::EngineError::Ssz(format!( @@ -902,11 +832,6 @@ mod tests { } } - #[test] - fn ssz_encode_sample_payload() { - assert_eq!(sample_payload().to_ssz(), SAMPLE_PAYLOAD_SSZ); - } - #[test] fn ssz_decode_sample_payload() { assert_eq!(ExecutionPayload::from_ssz(SAMPLE_PAYLOAD_SSZ).unwrap(), sample_payload()); @@ -966,24 +891,25 @@ mod tests { // Transactions SSZ helpers // ----------------------------------------------------------------------- + const TX_SINGLE: &[u8] = include_bytes!("../testdata/tx_single.bin"); + const TX_MULTI: &[u8] = include_bytes!("../testdata/tx_multi.bin"); + const WITHDRAWALS: &[u8] = include_bytes!("../testdata/withdrawals.bin"); + #[test] - fn transactions_empty_round_trip() { - let encoded = encode_transactions(&[]); - assert!(encoded.is_empty()); - let decoded = decode_transactions(&encoded).unwrap(); - assert!(decoded.is_empty()); + fn transactions_empty_decodes_to_empty() { + assert!(decode_transactions(&[]).unwrap().is_empty()); } #[test] - fn transactions_single_round_trip() { + fn transactions_single_decode() { let txs = vec![vec![0xde, 0xad, 0xbe, 0xef]]; - assert_eq!(txs, decode_transactions(&encode_transactions(&txs)).unwrap()); + assert_eq!(txs, decode_transactions(TX_SINGLE).unwrap()); } #[test] - fn transactions_multi_varying_lengths_round_trip() { + fn transactions_multi_varying_lengths_decode() { let txs: Vec> = (1u8..=5).map(|i| vec![i; i as usize * 3]).collect(); - assert_eq!(txs, decode_transactions(&encode_transactions(&txs)).unwrap()); + assert_eq!(txs, decode_transactions(TX_MULTI).unwrap()); } // ----------------------------------------------------------------------- @@ -991,13 +917,12 @@ mod tests { // ----------------------------------------------------------------------- #[test] - fn withdrawals_round_trip() { + fn withdrawals_decode() { let ws = vec![ Withdrawal { index: 10, validator_index: 20, address: [0x01; 20], amount: 500 }, Withdrawal { index: 11, validator_index: 21, address: [0x02; 20], amount: 600 }, ]; - let decoded = decode_withdrawals(&encode_withdrawals(&ws)).unwrap(); - assert_eq!(ws, decoded); + assert_eq!(ws, decode_withdrawals(WITHDRAWALS).unwrap()); } #[test] diff --git a/crates/engine/testdata/tx_multi.bin b/crates/engine/testdata/tx_multi.bin new file mode 100644 index 0000000000000000000000000000000000000000..3d489302a5cf5d38c24d60753121a2e94fd392a2 GIT binary patch literal 65 icmWe(U|kjVT4F>L)C~vX(lWnf(arC082Lk$^ZZW literal 0 HcmV?d00001