diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 00000000..7da5e644 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,38 @@ +# Silver + +A from-scratch Ethereum beacon node, organised as tiles — independent +pinned-thread components communicating over a typed message spine. + +## Language + +**Tile**: +A component with its own OS thread pinned to a dedicated CPU core, +implementing `loop_body` and attached to the spine. +_Avoid_: service, actor, worker. + +**Spine**: +The process-wide typed message fabric connecting tiles. +_Avoid_: bus, broker. + +**Spine queue**: +A fixed-size lock-free ring on the spine carrying `Copy` messages, broadcast +to consumers. +_Avoid_: channel. + +**TCache**: +The shared-memory bulk store; spine messages carry handles into it instead of +payloads. + +**Hosted crate**: +A transport-free library living inside a tile that owns the loop. Hosted +crates are hardcoded into their tile, not plugins. +_Avoid_: plugin, sub-tile, service. + +**Beacon API**: +The standard Ethereum REST API a beacon node serves; validator clients are +the primary consumers. Served by the `beacon_api` hosted crate. + +**Engine API**: +The standard JSON-RPC protocol between a beacon node and its execution +client. Called by the `engine_api` hosted crate. +_Avoid_: bare "engine" (ambiguous with the execution client itself). diff --git a/Cargo.lock b/Cargo.lock index 2c477c73..2f2f6074 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4427,21 +4427,38 @@ dependencies = [ "mimalloc", "quinn-proto", "rand 0.8.6", + "silver_beacon_api", "silver_beacon_state", "silver_beacon_state_data", + "silver_client_server", "silver_columns", "silver_common", "silver_config", "silver_control", "silver_discovery", - "silver_engine", + "silver_engine_api", "silver_gossip", + "silver_httpcore", "silver_network", "silver_peer", "silver_storage", "tracing", ] +[[package]] +name = "silver_beacon_api" +version = "0.0.1" +dependencies = [ + "hex", + "mio", + "serde", + "serde_json", + "silver_beacon_state_data", + "silver_common", + "silver_httpcore", + "tracing", +] + [[package]] name = "silver_beacon_state" version = "0.0.1" @@ -4493,6 +4510,22 @@ dependencies = [ "serde", ] +[[package]] +name = "silver_client_server" +version = "0.0.1" +dependencies = [ + "flux", + "hex", + "serde_json", + "silver_beacon_api", + "silver_beacon_state_data", + "silver_common", + "silver_config", + "silver_engine_api", + "silver_httpcore", + "tempfile", +] + [[package]] name = "silver_columns" version = "0.0.1" @@ -4631,7 +4664,7 @@ dependencies = [ ] [[package]] -name = "silver_engine" +name = "silver_engine_api" version = "0.0.1" dependencies = [ "base64 0.22.1", @@ -4645,7 +4678,10 @@ dependencies = [ "sha2", "silver_common", "silver_config", + "silver_engine_api", + "silver_httpcore", "simd-json", + "tempfile", "thiserror 1.0.69", "tracing", "tracing-subscriber", @@ -4671,6 +4707,16 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "silver_httpcore" +version = "0.0.1" +dependencies = [ + "httparse", + "mio", + "tempfile", + "tracing", +] + [[package]] name = "silver_metrics" version = "0.0.1" diff --git a/Cargo.toml b/Cargo.toml index 28c0bdb7..1e1bbbe4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,10 @@ [workspace] members = [ + "crates/beacon_api", "crates/beacon_state/data", "crates/beacon_state/tile", "crates/bin", + "crates/client_server", "crates/common", "crates/config", "crates/config/chain_spec", @@ -11,7 +13,8 @@ members = [ "crates/discovery", "crates/e2e", "crates/gossip", - "crates/engine", + "crates/httpcore", + "crates/engine_api", "crates/metrics", "crates/network", "crates/peer", @@ -60,9 +63,11 @@ inherits = "dev" opt-level = 3 [workspace.dependencies] +silver_beacon_api = { path = "crates/beacon_api" } silver_beacon_state = { path = "crates/beacon_state/tile" } silver_beacon_state_data = { path = "crates/beacon_state/data" } silver_chain_spec = { path = "crates/config/chain_spec" } +silver_client_server = { path = "crates/client_server" } silver_columns = { path = "crates/columns" } silver_common = { path = "crates/common" } silver_config = { path = "crates/config" } @@ -71,10 +76,11 @@ silver_ssz = { path = "crates/ssz" } silver_control = { path = "crates/control" } silver_discovery = {path = "crates/discovery" } silver_gossip = {path = "crates/gossip" } +silver_httpcore = { path = "crates/httpcore" } silver_network = {path = "crates/network" } silver_peer = {path = "crates/peer" } silver_storage = { path = "crates/storage" } -silver_engine = { path = "crates/engine"} +silver_engine_api = { path = "crates/engine_api" } flux = { git = "https://github.com/gattaca-com/flux", rev = "d6785f1af35336002476c3d97721fcc67fe76dfd"} flux-utils = { git = "https://github.com/gattaca-com/flux", rev = "d6785f1af35336002476c3d97721fcc67fe76dfd", features = ["bytes"]} flux-profiler = { git = "https://github.com/gattaca-com/flux", rev = "d6785f1af35336002476c3d97721fcc67fe76dfd"} diff --git a/crates/beacon_api/Cargo.toml b/crates/beacon_api/Cargo.toml new file mode 100644 index 00000000..e62d76bf --- /dev/null +++ b/crates/beacon_api/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "silver_beacon_api" +edition.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +hex.workspace = true +mio.workspace = true +silver_beacon_state_data.workspace = true +silver_common.workspace = true +silver_httpcore.workspace = true +serde.workspace = true +tracing.workspace = true +serde_json = "1.0.149" + +[lints] +workspace = true diff --git a/crates/beacon_api/examples/srv.rs b/crates/beacon_api/examples/srv.rs new file mode 100644 index 00000000..e83466be --- /dev/null +++ b/crates/beacon_api/examples/srv.rs @@ -0,0 +1,21 @@ +use std::time::Duration; + +use silver_beacon_api::BeaconApi; +use silver_beacon_state_data::BeaconStateOwner; +use silver_common::{Enr, Identify, Keypair}; +use silver_httpcore::Bind; + +fn main() { + let bind = Bind::parse(&std::env::args().nth(1).unwrap_or_else(|| "0.0.0.0:5051".into())); + let keypair = Keypair::from_secret(&[1u8; 32]).unwrap(); + let local_enr = Enr::empty(keypair.secret_key()).unwrap(); + // Never-published reader: state endpoints answer 503, as pre-bootstrap. + let state = BeaconStateOwner::empty_test(0).reader(); + + let mut api = BeaconApi::new(&bind, 64, &keypair, local_enr, &Identify::default(), state); + println!("serving on {:?}", api.local_addr()); + loop { + api.pump(); + std::thread::sleep(Duration::from_millis(1)); + } +} diff --git a/crates/beacon_api/src/identity.rs b/crates/beacon_api/src/identity.rs new file mode 100644 index 00000000..10f4da35 --- /dev/null +++ b/crates/beacon_api/src/identity.rs @@ -0,0 +1,115 @@ +use serde::{Deserialize, Serialize}; +use silver_common::{Enr, Eth2Addr, Identify, Keypair}; + +#[derive(Debug, Serialize)] +struct IdentityResponse<'a> { + data: &'a Identity, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Identity { + peer_id: String, + enr: String, + p2p_addresses: Vec, + discovery_addresses: Vec, + metadata: Metadata, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Metadata { + seq_number: String, + attnets: String, + syncnets: String, + custody_group_count: String, +} + +pub(crate) fn build_identity_json( + keypair: &Keypair, + local_enr: &Enr, + identify: &Identify, +) -> Vec { + let pid_multiaddr = Eth2Addr::PeerId(keypair.peer_id()).to_string(); + let peer_id_str = pid_multiaddr.strip_prefix("/p2p/").unwrap_or(&pid_multiaddr); + + let mut p2p_addresses = Vec::new(); + if let Some(addr) = identify.tcp_ipv4 { + p2p_addresses.push(format!("/ip4/{}/tcp/{}/p2p/{}", addr.ip(), addr.port(), peer_id_str)); + } + if let Some(addr) = identify.tcp_ipv6 { + p2p_addresses.push(format!("/ip6/{}/tcp/{}/p2p/{}", addr.ip(), addr.port(), peer_id_str)); + } + if let Some(addr) = identify.udp_ipv4 { + p2p_addresses.push(format!( + "/ip4/{}/udp/{}/quic-v1/p2p/{}", + addr.ip(), + addr.port(), + peer_id_str + )); + } + if let Some(addr) = identify.udp_ipv6 { + p2p_addresses.push(format!( + "/ip6/{}/udp/{}/quic-v1/p2p/{}", + addr.ip(), + addr.port(), + peer_id_str + )); + } + + let mut discovery_addresses = Vec::new(); + if let (Some(ip), Some(udp)) = (local_enr.ip4(), local_enr.udp4()) { + discovery_addresses.push(format!("/ip4/{}/udp/{}/p2p/{}", ip, udp, peer_id_str)); + } + if let (Some(ip), Some(udp)) = (local_enr.ip6(), local_enr.udp6()) { + discovery_addresses.push(format!("/ip6/{}/udp/{}/p2p/{}", ip, udp, peer_id_str)); + } + + let identity = Identity { + peer_id: peer_id_str.to_string(), + enr: local_enr.to_base64(), + p2p_addresses, + discovery_addresses, + metadata: Metadata { + seq_number: local_enr.seq().to_string(), + attnets: format!("0x{}", hex::encode(local_enr.attnets().unwrap_or([0u8; 8]))), + syncnets: format!("0x{:02x}", local_enr.syncnets().unwrap_or(0)), + custody_group_count: local_enr.cgc().unwrap_or(4).to_string(), + }, + }; + + serde_json::to_vec(&IdentityResponse { data: &identity }).unwrap() +} + +#[cfg(test)] +mod tests { + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + + use super::*; + + #[test] + fn identity_json_fields_present() { + let kp = Keypair::from_secret(&[1u8; 32]).unwrap(); + let enr = Enr::builder().build(kp.secret_key()).unwrap(); + let body = build_identity_json(&kp, &enr, &Identify::default()); + let v: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let data = &v["data"]; + assert!(data["peer_id"].as_str().is_some_and(|s| !s.is_empty())); + assert!(data["enr"].as_str().is_some_and(|s| s.starts_with("enr:"))); + assert!(data["metadata"]["seq_number"].as_str().is_some()); + assert!(data["metadata"]["attnets"].as_str().is_some_and(|s| s.starts_with("0x"))); + assert!(data["metadata"]["syncnets"].as_str().is_some_and(|s| s.starts_with("0x"))); + } + + #[test] + fn identity_p2p_address_format() { + let kp = Keypair::from_secret(&[1u8; 32]).unwrap(); + let enr = Enr::builder().build(kp.secret_key()).unwrap(); + let mut identify = Identify::default(); + identify.tcp_ipv4 = Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4)), 9000)); + let body = build_identity_json(&kp, &enr, &identify); + let v: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let addrs = v["data"]["p2p_addresses"].as_array().unwrap(); + assert_eq!(addrs.len(), 1); + let addr = addrs[0].as_str().unwrap(); + assert!(addr.starts_with("/ip4/1.2.3.4/tcp/9000/p2p/"), "bad format: {addr}"); + } +} diff --git a/crates/beacon_api/src/lib.rs b/crates/beacon_api/src/lib.rs new file mode 100644 index 00000000..56e01769 --- /dev/null +++ b/crates/beacon_api/src/lib.rs @@ -0,0 +1,7 @@ +mod identity; +mod response; +mod router; +mod routes; +mod server; + +pub use server::BeaconApi; diff --git a/crates/beacon_api/src/response.rs b/crates/beacon_api/src/response.rs new file mode 100644 index 00000000..732aa23c --- /dev/null +++ b/crates/beacon_api/src/response.rs @@ -0,0 +1,65 @@ +use silver_httpcore::frame_response; + +pub(crate) struct Response<'a> { + out: &'a mut Vec, +} + +impl<'a> Response<'a> { + pub(crate) fn new(out: &'a mut Vec) -> Self { + Self { out } + } + + pub(crate) fn json(&mut self, body: &[u8]) { + frame_response(self.out, "200 OK", Some("application/json"), body); + } + + pub(crate) fn empty(&mut self, content_type: &str) { + frame_response(self.out, "200 OK", Some(content_type), b""); + } + + /// Beacon-API error shape: `{"code":,"message":"..."}`. + pub(crate) fn error(&mut self, code: u16, message: &str) { + debug_assert!(!message.contains(['"', '\\']), "message goes into JSON unescaped"); + let status = match code { + 400 => "400 Bad Request", + 405 => "405 Method Not Allowed", + 503 => "503 Service Unavailable", + _ => unreachable!("unmapped error code {code}"), + }; + let body = format!("{{\"code\":{code},\"message\":\"{message}\"}}"); + frame_response(self.out, status, Some("application/json"), body.as_bytes()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn error_writes_status_line_and_json_body() { + let mut out = Vec::new(); + Response::new(&mut out).error(400, "invalid state_id"); + let expected: &[u8] = b"HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\nContent-Length: 41\r\n\r\n{\"code\":400,\"message\":\"invalid state_id\"}"; + assert_eq!(out, expected); + } + + #[test] + fn json_frames_ok_with_content_type() { + let mut out = Vec::new(); + Response::new(&mut out).json(b"{\"data\":1}"); + assert_eq!( + out, + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 10\r\n\r\n{\"data\":1}" + ); + } + + #[test] + fn empty_frames_ok_with_zero_length_body() { + let mut out = Vec::new(); + Response::new(&mut out).empty("text/plain"); + assert_eq!( + out, + b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n" + ); + } +} diff --git a/crates/beacon_api/src/router.rs b/crates/beacon_api/src/router.rs new file mode 100644 index 00000000..f0d258df --- /dev/null +++ b/crates/beacon_api/src/router.rs @@ -0,0 +1,328 @@ +use silver_httpcore::{ParsedRequest, frame_response}; + +use crate::{response::Response, routes::ApiCtx}; + +const MAX_PARAMS: usize = 4; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum Method { + Get, + Post, +} + +impl Method { + fn parse(name: &str) -> Option { + match name { + "GET" => Some(Self::Get), + "POST" => Some(Self::Post), + _ => None, + } + } +} + +pub(crate) type Handler = fn(&Request<'_>, &ApiCtx, &mut Response<'_>); + +// Fields become live with the first parameterised endpoints; until then only +// tests read them. +#[allow(dead_code)] +pub(crate) struct Request<'a> { + pub(crate) method: Method, + pub(crate) path: &'a str, + pub(crate) params: Params<'a>, + pub(crate) query: &'a str, + pub(crate) body: &'a [u8], +} + +pub(crate) struct Params<'a> { + entries: [(&'static str, &'a str); MAX_PARAMS], + len: usize, +} + +impl<'a> Params<'a> { + #[allow(dead_code)] + pub(crate) fn get(&self, name: &str) -> Option<&'a str> { + self.entries[..self.len].iter().find(|(n, _)| *n == name).map(|&(_, value)| value) + } + + fn push(&mut self, name: &'static str, value: &'a str) { + self.entries[self.len] = (name, value); + self.len += 1; + } +} + +impl Default for Params<'_> { + fn default() -> Self { + Self { entries: [("", ""); MAX_PARAMS], len: 0 } + } +} + +enum Seg { + Lit(&'static str), + Param(&'static str), +} + +struct Route { + method: Method, + segs: Vec, + handler: Handler, +} + +impl Route { + fn capture<'p>(&self, path: &'p str) -> Option> { + let mut parts = path.strip_prefix('/')?.split('/'); + let mut params = Params::default(); + for seg in &self.segs { + let part = parts.next()?; + match seg { + Seg::Lit(lit) if *lit == part => {} + Seg::Param(name) => params.push(name, part), + Seg::Lit(_) => return None, + } + } + parts.next().is_none().then_some(params) + } +} + +pub(crate) struct Router { + routes: Vec, +} + +impl Router { + pub(crate) fn new(table: &[(Method, &'static str, Handler)]) -> Self { + let mut routes: Vec = Vec::with_capacity(table.len()); + for &(method, pattern, handler) in table { + let segs = compile(pattern); + assert!( + !routes.iter().any(|r| r.method == method && same_match_set(&r.segs, &segs)), + "duplicate route pattern: {pattern}" + ); + routes.push(Route { method, segs, handler }); + } + Self { routes } + } + + pub(crate) fn dispatch(&self, req: &ParsedRequest<'_>, ctx: &ApiCtx, out: &mut Vec) { + let method = Method::parse(req.method); + let mut path_known = false; + for route in &self.routes { + let Some(params) = route.capture(req.path) else { continue }; + if method != Some(route.method) { + path_known = true; + continue; + } + let request = Request { + method: route.method, + path: req.path, + params, + query: req.query, + body: req.body, + }; + (route.handler)(&request, ctx, &mut Response::new(out)); + return; + } + if path_known { + Response::new(out).error(405, "method not allowed"); + } else { + tracing::warn!("unknown path: {}", req.path); + frame_response(out, "404 Not Found", None, b""); + } + } +} + +fn compile(pattern: &'static str) -> Vec { + let stripped = pattern + .strip_prefix('/') + .unwrap_or_else(|| panic!("route pattern must start with '/': {pattern}")); + let segs: Vec<_> = stripped + .split('/') + .map(|seg| match seg.strip_prefix('{') { + Some(name) => Seg::Param( + name.strip_suffix('}') + .unwrap_or_else(|| panic!("unterminated param in route pattern: {pattern}")), + ), + None => Seg::Lit(seg), + }) + .collect(); + let params = segs.iter().filter(|s| matches!(s, Seg::Param(_))).count(); + assert!(params <= MAX_PARAMS, "route pattern exceeds {MAX_PARAMS} params: {pattern}"); + segs +} + +/// Whether two compiled patterns match exactly the same set of paths — +/// param names don't affect matching, so they are ignored. +fn same_match_set(a: &[Seg], b: &[Seg]) -> bool { + a.len() == b.len() && + a.iter().zip(b).all(|pair| match pair { + (Seg::Lit(x), Seg::Lit(y)) => x == y, + (Seg::Param(_), Seg::Param(_)) => true, + _ => false, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::routes::preboot_ctx; + + fn request<'a>(method: &'a str, path: &'a str) -> ParsedRequest<'a> { + ParsedRequest { method, path, query: "", body: b"", version: 1, keep_alive: true } + } + + fn dispatch(router: &Router, method: &str, path: &str) -> Vec { + let mut out = Vec::new(); + router.dispatch(&request(method, path), &preboot_ctx(), &mut out); + out + } + + fn body(response: &[u8]) -> &[u8] { + let s = std::str::from_utf8(response).unwrap(); + &response[s.find("\r\n\r\n").unwrap() + 4..] + } + + fn first(_req: &Request<'_>, _ctx: &ApiCtx, resp: &mut Response<'_>) { + resp.json(b"first"); + } + + fn second(_req: &Request<'_>, _ctx: &ApiCtx, resp: &mut Response<'_>) { + resp.json(b"second"); + } + + fn echo_params(req: &Request<'_>, _ctx: &ApiCtx, resp: &mut Response<'_>) { + let mut joined = String::new(); + for name in ["state_id", "epoch", "a", "b", "c", "d"] { + if let Some(value) = req.params.get(name) { + joined.push_str(name); + joined.push('='); + joined.push_str(value); + joined.push(';'); + } + } + resp.json(joined.as_bytes()); + } + + fn echo_query_body(req: &Request<'_>, _ctx: &ApiCtx, resp: &mut Response<'_>) { + let mut joined = req.query.as_bytes().to_vec(); + joined.push(b'|'); + joined.extend_from_slice(req.body); + resp.json(&joined); + } + + #[test] + fn literal_route_dispatches_matching_handler() { + let router = Router::new(&[ + (Method::Get, "/eth/v1/node/identity", first), + (Method::Get, "/metrics", second), + ]); + assert_eq!(body(&dispatch(&router, "GET", "/eth/v1/node/identity")), b"first"); + assert_eq!(body(&dispatch(&router, "GET", "/metrics")), b"second"); + } + + #[test] + fn single_param_extracted_by_name() { + let router = Router::new(&[( + Method::Get, + "/eth/v1/beacon/states/{state_id}/finality_checkpoints", + echo_params, + )]); + let resp = dispatch(&router, "GET", "/eth/v1/beacon/states/head/finality_checkpoints"); + assert_eq!(body(&resp), b"state_id=head;"); + } + + #[test] + fn two_params_extracted_by_name() { + let router = + Router::new(&[(Method::Get, "/eth/v1/states/{state_id}/epochs/{epoch}", echo_params)]); + let resp = dispatch(&router, "GET", "/eth/v1/states/0xdead/epochs/42"); + assert_eq!(body(&resp), b"state_id=0xdead;epoch=42;"); + } + + #[test] + fn url_encoded_param_value_passed_through_verbatim() { + let router = Router::new(&[(Method::Get, "/states/{state_id}", echo_params)]); + let resp = dispatch(&router, "GET", "/states/0x1234%2Fabc%20d"); + assert_eq!(body(&resp), b"state_id=0x1234%2Fabc%20d;"); + } + + #[test] + fn query_and_body_reach_handler() { + let router = Router::new(&[(Method::Post, "/submit", echo_query_body)]); + let mut out = Vec::new(); + let req = ParsedRequest { + method: "POST", + path: "/submit", + query: "k=v", + body: b"payload", + version: 1, + keep_alive: true, + }; + router.dispatch(&req, &preboot_ctx(), &mut out); + assert_eq!(body(&out), b"k=v|payload"); + } + + #[test] + fn unmatched_path_gets_bare_404() { + let router = Router::new(&[( + Method::Get, + "/eth/v1/beacon/states/{state_id}/finality_checkpoints", + echo_params, + )]); + let expected: &[u8] = b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n"; + assert_eq!(dispatch(&router, "GET", "/not/real"), expected); + assert_eq!(dispatch(&router, "GET", "/eth/v1/beacon/states/head"), expected, "prefix"); + assert_eq!( + dispatch(&router, "GET", "/eth/v1/beacon/states/head/finality_checkpoints/x"), + expected, + "longer than pattern" + ); + } + + #[test] + fn matched_path_wrong_method_gets_405() { + let router = Router::new(&[(Method::Get, "/metrics", first)]); + let resp = dispatch(&router, "POST", "/metrics"); + assert!(resp.starts_with(b"HTTP/1.1 405 Method Not Allowed\r\n")); + assert_eq!(body(&resp), br#"{"code":405,"message":"method not allowed"}"#); + } + + #[test] + fn unknown_method_gets_405_on_known_path_else_404() { + let router = Router::new(&[(Method::Get, "/metrics", first)]); + assert!(dispatch(&router, "PUT", "/metrics").starts_with(b"HTTP/1.1 405")); + assert!(dispatch(&router, "PUT", "/nope").starts_with(b"HTTP/1.1 404")); + } + + #[test] + fn same_pattern_distinct_methods_dispatch_by_method() { + let router = Router::new(&[ + (Method::Get, "/eth/v1/thing", first), + (Method::Post, "/eth/v1/thing", second), + ]); + assert_eq!(body(&dispatch(&router, "GET", "/eth/v1/thing")), b"first"); + assert_eq!(body(&dispatch(&router, "POST", "/eth/v1/thing")), b"second"); + } + + #[test] + #[should_panic(expected = "duplicate route pattern")] + fn duplicate_pattern_panics_at_init() { + Router::new(&[(Method::Get, "/a/b", first), (Method::Get, "/a/b", second)]); + } + + #[test] + #[should_panic(expected = "duplicate route pattern")] + fn duplicate_modulo_param_names_panics_at_init() { + Router::new(&[(Method::Get, "/a/{x}/c", first), (Method::Get, "/a/{y}/c", second)]); + } + + #[test] + fn four_param_pattern_matches() { + let router = Router::new(&[(Method::Get, "/{a}/{b}/{c}/{d}", echo_params)]); + let resp = dispatch(&router, "GET", "/1/2/3/4"); + assert_eq!(body(&resp), b"a=1;b=2;c=3;d=4;"); + } + + #[test] + #[should_panic(expected = "exceeds 4 params")] + fn fifth_param_panics_at_init() { + Router::new(&[(Method::Get, "/{a}/{b}/{c}/{d}/{e}", echo_params)]); + } +} diff --git a/crates/beacon_api/src/routes.rs b/crates/beacon_api/src/routes.rs new file mode 100644 index 00000000..66aad9ee --- /dev/null +++ b/crates/beacon_api/src/routes.rs @@ -0,0 +1,180 @@ +#[cfg(test)] +use silver_beacon_state_data::BeaconStateOwner; +use silver_beacon_state_data::{BeaconStateReader, StateReadView}; +use silver_common::{Enr, Identify, Keypair}; + +use crate::{ + identity::build_identity_json, + response::Response, + router::{Handler, Method, Request}, +}; + +const METRICS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; + +pub(crate) const ROUTES: &[(Method, &str, Handler)] = + &[(Method::Get, "/eth/v1/node/identity", identity), (Method::Get, "/metrics", metrics)]; + +pub(crate) struct ApiCtx { + pub(crate) identity_json: Vec, + pub(crate) state: BeaconStateReader, +} + +impl ApiCtx { + pub(crate) fn new( + keypair: &Keypair, + local_enr: &Enr, + identify: &Identify, + state: BeaconStateReader, + ) -> Self { + Self { identity_json: build_identity_json(keypair, local_enr, identify), state } + } + + #[allow(dead_code)] + pub(crate) fn read_state_or_503( + &self, + resp: &mut Response<'_>, + read: impl Fn(StateReadView<'_>) -> R, + ) -> Option { + let result = self.state.read(&read); + if result.is_none() { + resp.error(503, "beacon node not initialized"); + } + result + } +} + +fn identity(_req: &Request<'_>, ctx: &ApiCtx, resp: &mut Response<'_>) { + resp.json(&ctx.identity_json); +} + +fn metrics(_req: &Request<'_>, _ctx: &ApiCtx, resp: &mut Response<'_>) { + resp.empty(METRICS_CONTENT_TYPE); +} + +/// Never-published reader: `read` yields `None`, as on a node before +/// bootstrap. +#[cfg(test)] +pub(crate) fn preboot_ctx() -> ApiCtx { + ApiCtx { identity_json: Vec::new(), state: BeaconStateOwner::empty_test(0).reader() } +} + +#[cfg(test)] +mod tests { + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + + use silver_beacon_state_data::BeaconState; + use silver_httpcore::ParsedRequest; + + use super::*; + use crate::router::Router; + + /// Wire bytes the pre-table implementation produced for these exact + /// inputs (captured before the table dispatch landed). + const GOLDEN_IDENTITY: &str = "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 478\r\n\r\n{\"data\":{\"peer_id\":\"16Uiu2HAmEWQnHq2jLKJypwVnVoQeFCULuyop6atvq2eWjYSUjzNi\",\"enr\":\"enr:-HW4QFVim6voTojjE-JbeUF0GPFRcqmWxgqgJ8-tXE5hh9PFTQSCwUJPHY_61U3Wvzi6OGrvJfb6KNjNpw4Q18sNL_sBgmlkgnY0iXNlY3AyNTZrMaEDG4TFVnsSZECZXT7VqroFZdceGDRgSBn_nBf16dXdB48\",\"p2p_addresses\":[\"/ip4/1.2.3.4/tcp/9000/p2p/16Uiu2HAmEWQnHq2jLKJypwVnVoQeFCULuyop6atvq2eWjYSUjzNi\"],\"discovery_addresses\":[],\"metadata\":{\"seq_number\":\"1\",\"attnets\":\"0x0000000000000000\",\"syncnets\":\"0x00\",\"custody_group_count\":\"4\"}}}"; + + fn fixture_ctx() -> ApiCtx { + let kp = Keypair::from_secret(&[1u8; 32]).unwrap(); + let enr = Enr::builder().build(kp.secret_key()).unwrap(); + let mut identify = Identify::default(); + identify.tcp_ipv4 = Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4)), 9000)); + ApiCtx::new(&kp, &enr, &identify, BeaconStateOwner::empty_test(0).reader()) + } + + fn get(router: &Router, ctx: &ApiCtx, path: &str) -> Vec { + let mut out = Vec::new(); + let req = ParsedRequest { + method: "GET", + path, + query: "", + body: b"", + version: 1, + keep_alive: true, + }; + router.dispatch(&req, ctx, &mut out); + out + } + + fn body(response: &[u8]) -> &[u8] { + let s = std::str::from_utf8(response).unwrap(); + &response[s.find("\r\n\r\n").unwrap() + 4..] + } + + #[test] + fn identity_wire_bytes_match_pre_table_implementation() { + let router = Router::new(ROUTES); + let resp = get(&router, &fixture_ctx(), "/eth/v1/node/identity"); + assert_eq!(std::str::from_utf8(&resp).unwrap(), GOLDEN_IDENTITY); + } + + #[test] + fn identity_content_length_matches_body() { + let router = Router::new(ROUTES); + let resp = get(&router, &fixture_ctx(), "/eth/v1/node/identity"); + let s = std::str::from_utf8(&resp).unwrap(); + let header_end = s.find("\r\n\r\n").unwrap(); + let cl: usize = s[..header_end] + .lines() + .find(|l| l.to_ascii_lowercase().starts_with("content-length:")) + .unwrap() + .split(':') + .nth(1) + .unwrap() + .trim() + .parse() + .unwrap(); + assert_eq!(cl, s[header_end + 4..].len()); + } + + #[test] + fn metrics_response_valid_prometheus_format() { + let router = Router::new(ROUTES); + let resp = get(&router, &preboot_ctx(), "/metrics"); + let s = std::str::from_utf8(&resp).unwrap(); + assert!(s.starts_with("HTTP/1.1 200 OK\r\n")); + assert!(s.contains("text/plain; version=0.0.4; charset=utf-8")); + assert_eq!(body(&resp), b""); + } + + #[test] + fn unknown_path_returns_404() { + let router = Router::new(ROUTES); + let resp = get(&router, &preboot_ctx(), "/not/real"); + assert_eq!(resp, b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n"); + } + + #[test] + fn events_returns_404_v1_defers_sse_clients_poll() { + let router = Router::new(ROUTES); + let resp = get(&router, &preboot_ctx(), "/eth/v1/events"); + assert_eq!(resp, b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n"); + } + + fn genesis_root(_req: &Request<'_>, ctx: &ApiCtx, resp: &mut Response<'_>) { + let Some(root) = ctx.read_state_or_503(resp, |view| view.imm.genesis_validators_root) + else { + return; + }; + resp.json(hex::encode(root).as_bytes()); + } + + #[test] + fn state_route_503_before_bootstrap() { + let router = Router::new(&[(Method::Get, "/test/genesis_root", genesis_root)]); + let resp = get(&router, &preboot_ctx(), "/test/genesis_root"); + assert!(resp.starts_with(b"HTTP/1.1 503 Service Unavailable\r\n")); + assert_eq!(body(&resp), br#"{"code":503,"message":"beacon node not initialized"}"#); + } + + #[test] + fn state_route_reads_published_state() { + let mut owner = BeaconStateOwner::new(BeaconState::empty_test(0)); + let anchor = owner.roll_fresh(); + owner.publish_state_id(anchor); + let ctx = ApiCtx { identity_json: Vec::new(), state: owner.reader() }; + + let router = Router::new(&[(Method::Get, "/test/genesis_root", genesis_root)]); + let resp = get(&router, &ctx, "/test/genesis_root"); + assert!(resp.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert_eq!(body(&resp), hex::encode([0u8; 32]).as_bytes()); + } +} diff --git a/crates/beacon_api/src/server.rs b/crates/beacon_api/src/server.rs new file mode 100644 index 00000000..b8fab32b --- /dev/null +++ b/crates/beacon_api/src/server.rs @@ -0,0 +1,303 @@ +use std::{ + collections::HashMap, + io::{self, Read, Write}, + time::Duration, +}; + +use mio::{Events, Interest, Poll, Token}; +use silver_beacon_state_data::BeaconStateReader; +use silver_common::{Enr, Identify, Keypair}; +use silver_httpcore::{AfterResponse, Bind, Listener, ParsedRequest, ServerConnection, Stream}; + +use crate::{ + router::Router, + routes::{ApiCtx, ROUTES}, +}; + +const LISTENER: Token = Token(0); + +struct Connection { + stream: Stream, + http: ServerConnection, +} + +pub struct BeaconApi { + poll: Poll, + events: Events, + listener: Listener, + max_connections: usize, + current_token: Token, + connections: HashMap, + router: Router, + ctx: ApiCtx, +} + +impl BeaconApi { + pub fn new( + bind: &Bind, + max_connections: usize, + keypair: &Keypair, + local_enr: Enr, + identify: &Identify, + state: BeaconStateReader, + ) -> Self { + let poll = Poll::new().unwrap(); + let mut listener = + Listener::bind(bind).unwrap_or_else(|e| panic!("beacon api bind {bind:?}: {e}")); + poll.registry().register(&mut listener, LISTENER, Interest::READABLE).unwrap(); + + Self { + poll, + events: Events::with_capacity(1024), + listener, + max_connections, + current_token: Token(LISTENER.0 + 1), + connections: HashMap::new(), + router: Router::new(ROUTES), + ctx: ApiCtx::new(keypair, &local_enr, identify, state), + } + } + + pub fn local_addr(&self) -> Bind { + self.listener.local_addr() + } + + pub fn pump(&mut self) -> bool { + self.poll.poll(&mut self.events, Some(Duration::ZERO)).unwrap(); + + let mut did_work = false; + for event in &self.events { + match event.token() { + LISTENER => loop { + let mut stream = match self.listener.accept() { + Ok(stream) => stream, + Err(e) if would_block(&e) => break, + Err(e) => { + tracing::warn!("accept failed: {e}"); + break; + } + }; + + did_work = true; + // Accept-and-close at the cap: with edge-triggered + // registration, leaving the stream in the backlog would go + // silent until the next SYN retriggers the listener. + if self.connections.len() >= self.max_connections { + tracing::warn!( + "beacon api connection cap {} reached, dropping new connection", + self.max_connections + ); + continue; + } + let token = next(&mut self.current_token); + self.poll.registry().register(&mut stream, token, Interest::READABLE).unwrap(); + self.connections + .insert(token, Connection { stream, http: ServerConnection::new() }); + }, + token => { + if let Some(conn) = self.connections.get_mut(&token) { + did_work = true; + match handle_event(self.poll.registry(), conn, event, &|req, out| { + self.router.dispatch(req, &self.ctx, out) + }) { + Ok(true) => { + let _ = self.poll.registry().deregister(&mut conn.stream); + self.connections.remove(&token); + } + Ok(false) => {} + Err(e) => { + tracing::warn!("connection error: {e}"); + let _ = self.poll.registry().deregister(&mut conn.stream); + self.connections.remove(&token); + } + }; + } + } + } + } + + did_work + } +} + +fn handle_event, &mut Vec)>( + registry: &mio::Registry, + conn: &mut Connection, + event: &mio::event::Event, + request_handler: &F, +) -> io::Result { + if event.is_readable() { + loop { + let space = conn.http.read_space()?; + match conn.stream.read(space) { + Ok(0) => return Err(io::Error::from(io::ErrorKind::UnexpectedEof)), + Ok(n) => conn.http.commit_read(n), + Err(e) if would_block(&e) => break, + Err(e) if interrupted(&e) => continue, + Err(e) => return Err(e), + } + } + + if conn.http.dispatch(request_handler) { + registry.reregister(&mut conn.stream, event.token(), Interest::WRITABLE)?; + } + return Ok(false); + } + + if event.is_writable() { + if !conn.http.pending_write().is_empty() { + loop { + match conn.stream.write(conn.http.pending_write()) { + Ok(0) => { + return Err(io::Error::new(io::ErrorKind::WriteZero, "write returned 0")) + } + Ok(n) => { + conn.http.commit_write(n); + if conn.http.pending_write().is_empty() { + break; + } + } + Err(e) if would_block(&e) => return Ok(false), + Err(e) if interrupted(&e) => continue, + Err(e) => return Err(e), + } + } + match conn.http.after_response(request_handler) { + AfterResponse::Close => return Ok(true), + AfterResponse::ResponsePending => { + registry.reregister(&mut conn.stream, event.token(), Interest::WRITABLE)? + } + AfterResponse::AwaitRequest => { + registry.reregister(&mut conn.stream, event.token(), Interest::READABLE)? + } + } + } + return Ok(false); + } + + Ok(false) +} + +fn next(current: &mut Token) -> Token { + let tok = Token(current.0); + let n = current.0.wrapping_add(1); + // Skip Token(0) == LISTENER on wrap to avoid aliasing the accept socket. + current.0 = if n == LISTENER.0 { LISTENER.0 + 1 } else { n }; + tok +} + +fn would_block(err: &io::Error) -> bool { + err.kind() == io::ErrorKind::WouldBlock +} + +fn interrupted(err: &io::Error) -> bool { + err.kind() == io::ErrorKind::Interrupted +} + +#[cfg(test)] +mod tests { + use std::{ + net::{SocketAddr, TcpStream}, + thread::JoinHandle, + time::Instant, + }; + + use silver_beacon_state_data::BeaconStateOwner; + + use super::*; + + #[test] + fn token_wrap_skips_listener() { + let mut cur = Token(usize::MAX); + let assigned = next(&mut cur); + assert_ne!(assigned, LISTENER, "returned token must not alias LISTENER"); + assert_ne!(cur, LISTENER, "next token must not alias LISTENER after wrap"); + assert_eq!(cur.0, LISTENER.0 + 1); + } + + fn pump_until(api: &mut BeaconApi, client: JoinHandle, msg: &str) -> T { + let deadline = Instant::now() + Duration::from_secs(10); + while !client.is_finished() { + assert!(Instant::now() < deadline, "timeout: {msg}"); + api.pump(); + std::thread::sleep(Duration::from_millis(1)); + } + client.join().unwrap() + } + + fn connect(addr: SocketAddr) -> TcpStream { + let stream = TcpStream::connect(addr).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + stream + } + + #[test] + fn connection_cap_drops_excess_then_recovers() { + let keypair = Keypair::from_secret(&[1u8; 32]).unwrap(); + let local_enr = Enr::empty(keypair.secret_key()).unwrap(); + let mut api = BeaconApi::new( + &Bind::parse("127.0.0.1:0"), + 1, + &keypair, + local_enr, + &Identify::default(), + BeaconStateOwner::empty_test(0).reader(), + ); + let Bind::Tcp(addr) = api.local_addr() else { panic!("expected tcp bind") }; + + let held_open = pump_until( + &mut api, + std::thread::spawn(move || { + let mut stream = connect(addr); + write!(stream, "GET /metrics HTTP/1.1\r\nHost: x\r\n\r\n").unwrap(); + let mut response = Vec::new(); + let mut chunk = [0u8; 1024]; + while !response.windows(4).any(|w| w == b"\r\n\r\n") { + let n = stream.read(&mut chunk).unwrap(); + assert!(n > 0, "server closed the first connection"); + response.extend_from_slice(&chunk[..n]); + } + assert!(response.starts_with(b"HTTP/1.1 200 OK\r\n")); + stream + }), + "first client served", + ); + + let denied = pump_until( + &mut api, + std::thread::spawn(move || { + let mut stream = connect(addr); + let _ = write!(stream, "GET /metrics HTTP/1.1\r\nHost: x\r\n\r\n"); + let mut chunk = [0u8; 1024]; + stream.read(&mut chunk) + }), + "second client dropped at cap", + ); + assert!( + !matches!(denied, Ok(n) if n > 0), + "connection over the cap must not be served: {denied:?}" + ); + + drop(held_open); + let deadline = Instant::now() + Duration::from_secs(10); + while !api.connections.is_empty() { + assert!(Instant::now() < deadline, "timeout: closed connection reaped"); + api.pump(); + std::thread::sleep(Duration::from_millis(1)); + } + + let response = pump_until( + &mut api, + std::thread::spawn(move || { + let mut stream = connect(addr); + write!(stream, "GET /metrics HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n") + .unwrap(); + let mut response = Vec::new(); + stream.read_to_end(&mut response).unwrap(); + response + }), + "third client served after the slot freed", + ); + assert!(response.starts_with(b"HTTP/1.1 200 OK\r\n")); + } +} diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index 79f9f6cb..11a264d0 100644 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -6,8 +6,10 @@ rust-version.workspace = true version.workspace = true [dependencies] +silver_beacon_api.workspace = true silver_beacon_state.workspace = true silver_beacon_state_data.workspace = true +silver_client_server.workspace = true silver_columns.workspace = true silver_common.workspace = true silver_config.workspace = true @@ -17,7 +19,8 @@ silver_gossip.workspace = true silver_network.workspace = true silver_peer.workspace = true silver_storage.workspace = true -silver_engine.workspace = true +silver_engine_api.workspace = true +silver_httpcore.workspace = true clap.workspace = true flux.workspace = true diff --git a/crates/bin/src/main.rs b/crates/bin/src/main.rs index 1d50a3df..bfa5479b 100644 --- a/crates/bin/src/main.rs +++ b/crates/bin/src/main.rs @@ -4,8 +4,10 @@ use flux::tile::{TileConfig, attach_tile}; use mimalloc::MiMalloc; use quinn_proto::{Endpoint, EndpointConfig}; use rand::RngCore; +use silver_beacon_api::BeaconApi; use silver_beacon_state::{BeaconStateTile, SlotTicker}; use silver_beacon_state_data::{BeaconState, SLOTS_PER_EPOCH}; +use silver_client_server::ClientServerTile; use silver_columns::tile::DataColumnsTile; #[cfg(feature = "alloc-profile")] use silver_common::metrics::CountingAllocator; @@ -15,8 +17,9 @@ use silver_common::{ use silver_config::Config; use silver_control::Controller; use silver_discovery::{DiscV5, Discovery}; -use silver_engine::EngineTile; +use silver_engine_api::EngineApi; use silver_gossip::GossipHandler; +use silver_httpcore::Bind; use silver_network::{Context, NetworkTile, P2p}; use silver_peer::PeerManager; use silver_storage::{latest_local_checkpoint, tile::StorageTile}; @@ -134,6 +137,7 @@ fn main() -> Result<(), Box> { None, ), ); + let identify = config.identify()?; let p2p_context = Context { gossip_producer: incoming_gossip_producer, gossip_consumer: outgoing_gossip_producer @@ -141,7 +145,7 @@ fn main() -> Result<(), Box> { .random_access("p2p_outgoing_gossip", true)?, rpc_producer: incoming_rpc_producer, rpc_consumer: outgoing_rpc_producer.cache_ref().random_access("p2p_outgoing_rpc", true)?, - identify: Some(ProtoIdentify::from((&config.identify()?, &keypair))), + identify: Some(ProtoIdentify::from((&identify, &keypair))), }; let now = Instant::now(); @@ -228,6 +232,15 @@ fn main() -> Result<(), Box> { !config.disable_weak_subjectivity_check(), state, ); + let beacon_api = BeaconApi::new( + &Bind::parse(config.beacon_api_bind()), + config.beacon_api_max_connections(), + &keypair, + local_enr, + &identify, + beacon_state_tile.reader(), + ); + let state_reader = beacon_state_tile.reader(); let storage_tile = StorageTile::new( @@ -257,12 +270,13 @@ fn main() -> Result<(), Box> { el_producer, ); - let engine_tile = EngineTile::new( + let engine_api = EngineApi::new( config.engine_config(), ssz_gossip_consumer_eng, incoming_rpc_consumer_eng, incoming_engine_resp_producer, ); + let client_server_tile = ClientServerTile { beacon: beacon_api, engine: engine_api }; // Spine let spine = SilverSpine::new(None); @@ -273,7 +287,7 @@ fn main() -> Result<(), Box> { attach_tile(network_tile, scoped_spine, TileConfig::new(2, None)); attach_tile(beacon_state_tile, scoped_spine, TileConfig::new(3, None)); attach_tile(storage_tile, scoped_spine, TileConfig::new(4, None)); - attach_tile(engine_tile, scoped_spine, TileConfig::new(5, None)); + attach_tile(client_server_tile, scoped_spine, TileConfig::new(5, None)); attach_tile(data_columns_tile, scoped_spine, TileConfig::new(6, None)); }); @@ -316,6 +330,11 @@ fn load_config() -> Result { if args.iter().any(|a| a == "--unsafe-no-el") { config = config.with_unsafe_no_el(true); } + if let Some(bind) = + args.iter().position(|a| a == "--beacon-api-bind").and_then(|i| args.get(i + 1)) + { + config = config.with_beacon_api_bind(bind.clone()); + } tracing::info!("loaded config: {config:#?}"); diff --git a/crates/client_server/Cargo.toml b/crates/client_server/Cargo.toml new file mode 100644 index 00000000..29660382 --- /dev/null +++ b/crates/client_server/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "silver_client_server" +edition.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +flux.workspace = true +silver_beacon_api.workspace = true +silver_common.workspace = true +silver_engine_api.workspace = true + +[dev-dependencies] +hex.workspace = true +serde_json.workspace = true +silver_beacon_state_data.workspace = true +silver_config.workspace = true +silver_engine_api = { workspace = true, features = ["test-el"] } +silver_httpcore.workspace = true +tempfile = "3" + +[lints] +workspace = true diff --git a/crates/client_server/src/lib.rs b/crates/client_server/src/lib.rs new file mode 100644 index 00000000..c07638e9 --- /dev/null +++ b/crates/client_server/src/lib.rs @@ -0,0 +1,19 @@ +use flux::{spine::SpineAdapter, tile::Tile}; +use silver_beacon_api::BeaconApi; +use silver_common::SilverSpine; +use silver_engine_api::EngineApi; + +pub struct ClientServerTile { + pub beacon: BeaconApi, + pub engine: EngineApi, +} + +impl Tile for ClientServerTile { + fn loop_body(&mut self, adapter: &mut SpineAdapter) { + self.engine.intake(adapter); + self.engine.spin(adapter); + if self.beacon.pump() { + adapter.mark_work(); + } + } +} diff --git a/crates/client_server/tests/tile.rs b/crates/client_server/tests/tile.rs new file mode 100644 index 00000000..7f07bc33 --- /dev/null +++ b/crates/client_server/tests/tile.rs @@ -0,0 +1,300 @@ +use std::{ + io::{Read, Write}, + net::TcpStream, + os::unix::net::UnixStream, + time::{Duration, Instant}, +}; + +use flux::{spine::SpineAdapter, tile::Tile}; +use silver_beacon_api::BeaconApi; +use silver_beacon_state_data::BeaconStateOwner; +use silver_client_server::ClientServerTile; +use silver_common::{ + EngineFcuReq, EngineReq, EngineResp, Enr, Identify, Keypair, SilverSpine, TCache, + TCacheProducer, +}; +use silver_config::EngineConfig; +use silver_engine_api::{ + EngineApi, + test_el::{FCU_VALID_RESULT, FakeEl, write_jwt}, +}; +use silver_httpcore::Bind; +use tempfile::TempDir; + +struct Injector; +impl Tile for Injector { + fn loop_body(&mut self, _: &mut SpineAdapter) {} +} + +fn beacon(bind: &Bind) -> BeaconApi { + let keypair = Keypair::from_secret(&[1u8; 32]).unwrap(); + let local_enr = Enr::empty(keypair.secret_key()).unwrap(); + BeaconApi::new( + bind, + 64, + &keypair, + local_enr, + &Identify::default(), + BeaconStateOwner::empty_test(0).reader(), + ) +} + +fn engine(config: EngineConfig, tcache_names: [&'static str; 3]) -> EngineApi { + let gossip_p = TCache::producer(tcache_names[0], 1 << 12); + let rpc_p = TCache::producer(tcache_names[1], 1 << 12); + let resp_p = TCache::producer(tcache_names[2], 1 << 12); + EngineApi::new( + config, + gossip_p.cache_ref().random_access("t", true).unwrap(), + rpc_p.cache_ref().random_access("t", true).unwrap(), + resp_p, + ) +} + +fn no_el() -> EngineConfig { + EngineConfig { unsafe_no_el: true, ..EngineConfig::default() } +} + +fn http_get(mut stream: impl Read + Write, path: &str) -> String { + write!(stream, "GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n").unwrap(); + stream.flush().unwrap(); + let mut response = Vec::new(); + stream.read_to_end(&mut response).unwrap(); + String::from_utf8(response).unwrap() +} + +fn assert_identity_ok(response: &str) { + assert!(response.starts_with("HTTP/1.1 200 OK\r\n"), "unexpected response: {response}"); + let body = &response[response.find("\r\n\r\n").unwrap() + 4..]; + let json: serde_json::Value = serde_json::from_str(body).unwrap(); + assert!(json["data"]["peer_id"].as_str().is_some_and(|id| !id.is_empty())); + assert!(json["data"]["enr"].as_str().unwrap().starts_with("enr:")); + assert!(json["data"]["metadata"]["seq_number"].is_string()); +} + +fn fcu_req(byte: u8) -> EngineReq { + EngineReq::Fcu(EngineFcuReq { + block_root: [byte; 32], + head_block_hash: [byte; 32], + safe_block_hash: [0u8; 32], + finalized_block_hash: [0u8; 32], + }) +} + +fn head_block_hash_json(byte: u8) -> String { + format!("\"headBlockHash\":\"0x{}\"", hex::encode([byte; 32])) +} + +#[test] +fn serves_identity_over_tcp() { + let base = TempDir::new().unwrap(); + let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None)); + let mut tile = ClientServerTile { + beacon: beacon(&Bind::parse("127.0.0.1:0")), + engine: engine(no_el(), ["cs_tcp_gossip", "cs_tcp_rpc", "cs_tcp_resp"]), + }; + let mut adapter = SpineAdapter::connect_tile(&tile, &mut *spine); + + let Bind::Tcp(addr) = tile.beacon.local_addr() else { panic!("expected tcp bind") }; + assert_ne!(addr.port(), 0, "port-0 bind must resolve to an ephemeral port"); + + let client = std::thread::spawn(move || { + let stream = TcpStream::connect(addr).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + http_get(stream, "/eth/v1/node/identity") + }); + + let deadline = Instant::now() + Duration::from_secs(10); + while !client.is_finished() { + assert!(Instant::now() < deadline, "timeout: identity over tcp"); + tile.loop_body(&mut adapter); + std::thread::sleep(Duration::from_millis(1)); + } + assert_identity_ok(&client.join().unwrap()); +} + +#[test] +fn serves_identity_over_uds() { + let base = TempDir::new().unwrap(); + let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None)); + let socket = base.path().join("beacon_api.sock"); + let mut tile = ClientServerTile { + beacon: beacon(&Bind::Unix(socket.clone())), + engine: engine(no_el(), ["cs_uds_gossip", "cs_uds_rpc", "cs_uds_resp"]), + }; + let mut adapter = SpineAdapter::connect_tile(&tile, &mut *spine); + + assert_eq!(tile.beacon.local_addr(), Bind::Unix(socket.clone())); + + let client = std::thread::spawn(move || { + let stream = UnixStream::connect(&socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + http_get(stream, "/eth/v1/node/identity") + }); + + let deadline = Instant::now() + Duration::from_secs(10); + while !client.is_finished() { + assert!(Instant::now() < deadline, "timeout: identity over uds"); + tile.loop_body(&mut adapter); + std::thread::sleep(Duration::from_millis(1)); + } + assert_identity_ok(&client.join().unwrap()); +} + +/// ADR 0004's core claim: all pumps are non-blocking, so an unanswered EL +/// call never stalls beacon-api serving, and the EL completion still lands +/// once the response arrives. +#[test] +fn serves_beacon_api_while_engine_call_in_flight() { + let base = TempDir::new().unwrap(); + let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None)); + let (mut el, endpoint) = FakeEl::tcp(); + let jwt_path = write_jwt(base.path()); + + let config = EngineConfig { + execution_endpoint: endpoint, + jwt_secret: jwt_path.to_str().unwrap().to_string(), + ..EngineConfig::default() + }; + let mut tile = ClientServerTile { + beacon: beacon(&Bind::parse("127.0.0.1:0")), + engine: engine(config, ["cs_flight_gossip", "cs_flight_rpc", "cs_flight_resp"]), + }; + let mut adapter = SpineAdapter::connect_tile(&tile, &mut *spine); + let mut inj = SpineAdapter::connect_tile(&Injector, &mut *spine); + inj.consume(|_: EngineResp, _| {}); + + let deadline = Instant::now() + Duration::from_secs(10); + let mut crank = |tile: &mut ClientServerTile, el: &mut FakeEl, msg: &str| { + assert!(Instant::now() < deadline, "timeout: {msg}"); + tile.loop_body(&mut adapter); + el.pump(); + std::thread::sleep(Duration::from_millis(1)); + }; + + // Crank until the startup healthcheck trio is on the wire: the tile's + // EngineReq cursor initializes on its first consume, so injecting before + // the first loop_body would be skipped. The trio stays unanswered — three + // more in-flight EL calls. + while el.requests.len() < 3 { + crank(&mut tile, &mut el, "startup healthcheck trio"); + } + + inj.produce(fcu_req(42)); + let fcu_on_wire = + |el: &FakeEl| el.requests.iter().position(|r| r.method == "engine_forkchoiceUpdatedV3"); + while fcu_on_wire(&el).is_none() { + crank(&mut tile, &mut el, "fcu on the wire"); + } + + // The FCU (and the startup healthcheck trio) sit unanswered on the EL; + // the API request must be served anyway. + let Bind::Tcp(addr) = tile.beacon.local_addr() else { panic!("expected tcp bind") }; + let client = std::thread::spawn(move || { + let stream = TcpStream::connect(addr).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + http_get(stream, "/eth/v1/node/identity") + }); + while !client.is_finished() { + crank(&mut tile, &mut el, "identity served while fcu in flight"); + } + assert_identity_ok(&client.join().unwrap()); + + let mut completed = Vec::new(); + inj.consume(|resp: EngineResp, _| { + if let EngineResp::Fcu(r) = resp { + completed.push(r.block_root); + } + }); + assert!(completed.is_empty(), "engine call must still be in flight after the API response"); + + el.respond(fcu_on_wire(&el).unwrap(), FCU_VALID_RESULT); + while completed.is_empty() { + crank(&mut tile, &mut el, "fcu completion on the spine"); + inj.consume(|resp: EngineResp, _| { + if let EngineResp::Fcu(r) = resp { + completed.push(r.block_root); + } + }); + } + assert_eq!(completed, vec![[42u8; 32]]); +} + +/// (cap+1) concurrent spine requests with `max_connections = cap`: the +/// last one must stay queued on the spine until a completion frees a +/// connection, and completions must correlate out of order. +#[test] +fn pool_cap_gates_spine_intake() { + let base = TempDir::new().unwrap(); + let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None)); + let (mut el, endpoint) = FakeEl::tcp(); + let jwt_path = write_jwt(base.path()); + + let config = EngineConfig { + execution_endpoint: endpoint, + jwt_secret: jwt_path.to_str().unwrap().to_string(), + max_connections: 3, + ..EngineConfig::default() + }; + let mut tile = ClientServerTile { + beacon: beacon(&Bind::parse("127.0.0.1:0")), + engine: engine(config, ["cs_cap_gossip", "cs_cap_rpc", "cs_cap_resp"]), + }; + let mut adapter = SpineAdapter::connect_tile(&tile, &mut *spine); + let mut inj = SpineAdapter::connect_tile(&Injector, &mut *spine); + inj.consume(|_: EngineResp, _| {}); + + let deadline = Instant::now() + Duration::from_secs(10); + let mut crank = |tile: &mut ClientServerTile, el: &mut FakeEl, msg: &str| { + assert!(Instant::now() < deadline, "timeout: {msg}"); + tile.loop_body(&mut adapter); + el.pump(); + std::thread::sleep(Duration::from_millis(1)); + }; + + // First loop_body fires the startup healthcheck trio; answer it so all + // three pooled connections are free before the capped scenario. + while el.requests.len() < 3 { + crank(&mut tile, &mut el, "startup healthcheck trio"); + } + for i in 0..3 { + el.respond(i, "false"); + } + + for byte in [11u8, 12, 13, 14] { + inj.produce(fcu_req(byte)); + } + + let fcu_count = |el: &FakeEl| { + el.requests.iter().filter(|r| r.method == "engine_forkchoiceUpdatedV3").count() + }; + while fcu_count(&el) < 3 { + crank(&mut tile, &mut el, "first three FCUs sent"); + } + for _ in 0..50 { + crank(&mut tile, &mut el, "cap holds"); + assert_eq!(fcu_count(&el), 3, "4th request must wait while pool is at cap"); + } + + // Free one connection by answering the SECOND fcu; the gated request + // must then be sent, and the completion must carry the responded + // request's block root. + let second = el + .requests + .iter() + .position(|r| r.body.contains(&head_block_hash_json(12))) + .expect("fcu for root 12 on the wire"); + el.respond(second, FCU_VALID_RESULT); + + while fcu_count(&el) < 4 { + crank(&mut tile, &mut el, "gated FCU sent after a connection freed"); + } + + let mut completed = Vec::new(); + inj.consume(|resp: EngineResp, _| { + if let EngineResp::Fcu(r) = resp { + completed.push(r.block_root); + } + }); + assert_eq!(completed, vec![[12u8; 32]], "out-of-order completion correlated"); +} diff --git a/crates/config/src/engine_config.rs b/crates/config/src/engine_config.rs index 718cf84a..f48d9405 100644 --- a/crates/config/src/engine_config.rs +++ b/crates/config/src/engine_config.rs @@ -4,6 +4,10 @@ fn default_tcache_size() -> usize { 2 << 24 } +fn default_max_connections() -> usize { + 32 +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct EngineConfig { pub execution_endpoint: String, @@ -11,6 +15,8 @@ pub struct EngineConfig { pub jwt_secret: String, #[serde(default = "default_tcache_size")] pub incoming_engine_resp_tcache_size: usize, + #[serde(default = "default_max_connections")] + pub max_connections: usize, /// Unsafe testing mode: do not connect to the EL. The engine tile answers /// every spine request with a synthetic VALID response. Lets the CL run /// without an execution client. Never enable in production. @@ -24,6 +30,7 @@ impl Default for EngineConfig { execution_endpoint: "http://localhost:8551".into(), jwt_secret: "0".into(), incoming_engine_resp_tcache_size: 2 << 24, + max_connections: 32, unsafe_no_el: false, } } diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 4e965e5a..55674e07 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -33,6 +33,10 @@ const fn default_u64() -> u64 { V } +fn default_beacon_api_bind() -> String { + "0.0.0.0:5051".into() +} + fn default_data_dir() -> String { std::env::home_dir() .and_then(|mut b| { @@ -122,6 +126,11 @@ pub struct Config { data_storage_dir: String, #[serde(default)] engine_config: EngineConfig, + /// TCP `addr:port` or a unix socket path. + #[serde(default = "default_beacon_api_bind")] + beacon_api_bind: String, + #[serde(default = "default_usize::<64>")] + beacon_api_max_connections: usize, #[serde(default)] disable_weak_subjectivity_check: bool, } @@ -156,6 +165,8 @@ impl Config { outgoing_rpc_tcache_size: 2 << 24, // ssz data_storage_dir: default_data_dir(), engine_config: Default::default(), + beacon_api_bind: default_beacon_api_bind(), + beacon_api_max_connections: 64, disable_weak_subjectivity_check: false, } } @@ -208,6 +219,16 @@ impl Config { self } + pub fn with_beacon_api_bind(mut self, bind: String) -> Self { + self.beacon_api_bind = bind; + self + } + + pub fn with_beacon_api_max_connections(mut self, max: usize) -> Self { + self.beacon_api_max_connections = max; + self + } + pub fn keypair(&self) -> Result { Keypair::from_secret(&self.secret_key) } @@ -336,6 +357,14 @@ impl Config { self.engine_config.clone() } + pub fn beacon_api_bind(&self) -> &str { + &self.beacon_api_bind + } + + pub fn beacon_api_max_connections(&self) -> usize { + self.beacon_api_max_connections + } + pub fn disable_weak_subjectivity_check(&self) -> bool { self.disable_weak_subjectivity_check } @@ -366,6 +395,24 @@ mod tests { assert_eq!(cfg.next_fork_epoch, u64::MAX); assert_eq!(cfg.supported_protocols().unwrap().len(), 11); assert_eq!(cfg.gossip_topics().unwrap().len(), 8); + assert_eq!(cfg.beacon_api_bind(), "0.0.0.0:5051"); + assert_eq!(cfg.beacon_api_max_connections(), 64); + } + + #[test] + fn builder_sets_beacon_api_bind() { + let cfg = Config::new([1u8; 32], [0u8; 4], [0u8; 4], 0); + assert_eq!(cfg.beacon_api_bind(), "0.0.0.0:5051"); + let cfg = cfg.with_beacon_api_bind("/run/beacon.sock".into()); + assert_eq!(cfg.beacon_api_bind(), "/run/beacon.sock"); + } + + #[test] + fn builder_sets_beacon_api_max_connections() { + let cfg = Config::new([1u8; 32], [0u8; 4], [0u8; 4], 0); + assert_eq!(cfg.beacon_api_max_connections(), 64); + let cfg = cfg.with_beacon_api_max_connections(2); + assert_eq!(cfg.beacon_api_max_connections(), 2); } #[test] diff --git a/crates/engine/src/http.rs b/crates/engine/src/http.rs deleted file mode 100644 index fee7deef..00000000 --- a/crates/engine/src/http.rs +++ /dev/null @@ -1,474 +0,0 @@ -use std::{ - io::{self, Read, Write}, - net::{SocketAddr, ToSocketAddrs}, -}; - -use mio::{Events, Interest, Poll, Token, net::TcpStream}; - -use crate::{EngineError, JwtSecret}; - -// Sized for the largest expected EL response: getPayload with a full -// blobsBundle (~21 blobs × 256 KB hex-encoded + execution payload -// transactions). -const READ_BUF_CAPACITY: usize = 10 * 1024 * 1024; - -// Sized for the largest expected outgoing request: newPayload with a full -// block (~30M gas of transactions, hex-encoded in JSON) plus HTTP headers. -const WRITE_BUF_CAPACITY: usize = 10 * 1024 * 1024; - -enum Conn { - Disconnected, - Connecting(TcpStream), - Connected(TcpStream), -} - -struct HttpConnection { - endpoint: String, - host: String, - jwt: JwtSecret, - token: Token, - conn: Conn, - addr: Option, - in_flight: Option, - pending_id: Option, - write_buf: Vec, - write_pos: usize, - read_buf: Vec, - read_offset: usize, - // Cached from the first read of the current response; zero = not yet parsed. - response_header_end: usize, - response_total: usize, // header_end + content_length -} - -impl HttpConnection { - fn new(endpoint: String, jwt: JwtSecret, token: Token) -> Self { - let host = endpoint - .trim_start_matches("http://") - .split('/') - .next() - .unwrap_or("localhost") - .to_string(); - Self { - endpoint, - host, - jwt, - token, - conn: Conn::Disconnected, - addr: None, - pending_id: None, - write_buf: Vec::with_capacity(WRITE_BUF_CAPACITY), - write_pos: 0, - in_flight: None, - read_buf: Vec::with_capacity(READ_BUF_CAPACITY), - read_offset: 0, - response_header_end: 0, - response_total: 0, - } - } -} - -fn http_is_free(t: &HttpConnection) -> bool { - t.in_flight.is_none() && t.pending_id.is_none() -} - -fn http_enqueue(t: &mut HttpConnection, rpc_id: u64, body: &[u8], poll: &mut Poll) { - debug_assert!(t.in_flight.is_none() && t.pending_id.is_none(), "enqueue on busy connection"); - let bearer = t.jwt.bearer_token(); - build_request_into(&mut t.write_buf, &t.host, body, bearer, true); - t.pending_id = Some(rpc_id); - t.write_pos = 0; - - // matches! borrows t.conn transiently, freeing it before the function call - // below. - if matches!(t.conn, Conn::Disconnected) { - http_connect(t, poll); - } else if matches!(t.conn, Conn::Connected(_)) { - http_set_interest(&mut t.conn, t.token, poll, Interest::READABLE | Interest::WRITABLE); - } -} - -fn http_poll(t: &mut HttpConnection, events: &Events, poll: &mut Poll, on_complete: &mut F) -where - F: FnMut(u64, Result<&mut [u8], EngineError>), -{ - for event in events.iter() { - if event.token() != t.token { - continue; - } - if matches!(t.conn, Conn::Connecting(_)) { - if event.is_error() || event.is_read_closed() || event.is_write_closed() { - http_on_error(t, poll, on_complete, "connect failed"); - break; - } - if event.is_writable() { - // Take ownership to inspect peer_addr and transition state atomically. - let Conn::Connecting(stream) = std::mem::replace(&mut t.conn, Conn::Disconnected) - else { - unreachable!() - }; - if stream.peer_addr().is_ok() { - t.conn = Conn::Connected(stream); - let interest = if t.pending_id.is_none() { - Interest::READABLE - } else { - Interest::READABLE | Interest::WRITABLE - }; - http_set_interest(&mut t.conn, t.token, poll, interest); - } else { - t.conn = Conn::Connecting(stream); - http_on_error(t, poll, on_complete, "connect failed"); - break; - } - } - } else if matches!(t.conn, Conn::Connected(_)) { - if event.is_error() { - http_on_error(t, poll, on_complete, "connection error"); - break; - } - if event.is_writable() { - let result = { - let Conn::Connected(stream) = &mut t.conn else { unreachable!() }; - http_do_write( - stream, - &mut t.pending_id, - &t.write_buf, - &mut t.write_pos, - &mut t.in_flight, - ) - }; - if let Err(e) = result { - let msg = e.to_string(); - http_on_error(t, poll, on_complete, &msg); - break; - } - let interest = if t.pending_id.is_none() { - Interest::READABLE - } else { - Interest::READABLE | Interest::WRITABLE - }; - http_set_interest(&mut t.conn, t.token, poll, interest); - } - if event.is_readable() { - // Drain data before checking is_read_closed: when the remote - // sends a response + FIN in one exchange (EPOLLIN|EPOLLRDHUP), - // we must read the response first. http_do_read returns Err on - // EOF, so the break below covers that close path too. - let result = { - let Conn::Connected(stream) = &mut t.conn else { unreachable!() }; - http_do_read( - stream, - &mut t.in_flight, - &mut t.read_buf, - &mut t.read_offset, - &mut t.response_header_end, - &mut t.response_total, - on_complete, - ) - }; - if let Err(e) = result { - let msg = e.to_string(); - http_on_error(t, poll, on_complete, &msg); - break; - } - } - if event.is_read_closed() { - // Remote closed with no (more) data — in_flight will never get - // a response. - http_on_error(t, poll, on_complete, "connection closed"); - break; - } - } - } -} - -fn http_connect(t: &mut HttpConnection, poll: &mut Poll) { - let addr = if let Some(a) = t.addr { - a - } else { - match parse_addr(&t.endpoint) { - Ok(a) => { - t.addr = Some(a); - a - } - Err(e) => { - tracing::warn!("resolve failed for {}: {e}", t.endpoint); - return; - } - } - }; - match TcpStream::connect(addr) { - Ok(mut stream) => { - if poll.registry().register(&mut stream, t.token, Interest::WRITABLE).is_ok() { - t.conn = Conn::Connecting(stream); - } - } - Err(e) => tracing::warn!("connect error: {e}"), - } -} - -fn http_do_write( - stream: &mut TcpStream, - pending_id: &mut Option, - write_buf: &[u8], - write_pos: &mut usize, - in_flight: &mut Option, -) -> io::Result<()> { - if pending_id.is_some() { - loop { - match stream.write(&write_buf[*write_pos..]) { - Ok(0) => break, - Ok(n) => { - *write_pos += n; - if *write_pos == write_buf.len() { - *in_flight = pending_id.take(); - *write_pos = 0; - break; - } - } - Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, - Err(e) => return Err(e), - } - } - } - Ok(()) -} - -fn http_do_read( - stream: &mut TcpStream, - in_flight: &mut Option, - read_buf: &mut Vec, - read_offset: &mut usize, - response_header_end: &mut usize, - response_total: &mut usize, - on_complete: &mut F, -) -> io::Result<()> -where - F: FnMut(u64, Result<&mut [u8], EngineError>), -{ - loop { - // Deliver if a complete response is already buffered. - if *response_total > 0 && read_buf.len() - *read_offset >= *response_total { - if let Some(rpc_id) = in_flight.take() { - let start = *read_offset + *response_header_end; - let end = *read_offset + *response_total; - on_complete(rpc_id, Ok(&mut read_buf[start..end])); - } - *read_offset += *response_total; - *response_header_end = 0; - *response_total = 0; - if *read_offset == read_buf.len() { - read_buf.clear(); - *read_offset = 0; - } - continue; - } - - let want = if *response_total > 0 { - // Know total size; read exactly the remaining bytes. - *response_total - (read_buf.len() - *read_offset) - } else { - // Headers not yet parsed; 4096 covers any realistic HTTP response header. - 4096 - }; - - let base = read_buf.len(); - read_buf.resize(base + want, 0); - match stream.read(&mut read_buf[base..]) { - Ok(0) => { - return Err(io::Error::new(io::ErrorKind::ConnectionReset, "eof")); - } - Ok(n) => { - read_buf.truncate(base + n); - if *response_total == 0 { - match try_parse_headers(&read_buf[*read_offset..]) { - Ok(Some((hend, cl))) => { - *response_header_end = hend; - *response_total = hend + cl; - } - Ok(None) => {} // headers still incomplete - Err(e) => { - return Err(io::Error::new(io::ErrorKind::InvalidData, e.to_string())); - } - } - } - } - Err(e) if e.kind() == io::ErrorKind::WouldBlock => { - read_buf.truncate(base); - break; - } - Err(e) => { - return Err(e); - } - } - } - Ok(()) -} - -fn http_on_error(t: &mut HttpConnection, poll: &mut Poll, on_complete: &mut F, msg: &str) -where - F: FnMut(u64, Result<&mut [u8], EngineError>), -{ - tracing::warn!("{msg}"); - let err = msg.to_string(); - if let Some(rpc_id) = t.in_flight.take() { - on_complete(rpc_id, Err(EngineError::Http(err.clone()))); - } - if let Some(rpc_id) = t.pending_id.take() { - on_complete(rpc_id, Err(EngineError::Http(err.clone()))); - } - t.write_pos = 0; - t.read_buf.clear(); - t.read_offset = 0; - t.response_header_end = 0; - t.response_total = 0; - let old = std::mem::replace(&mut t.conn, Conn::Disconnected); - if let Conn::Connecting(mut stream) | Conn::Connected(mut stream) = old { - let _ = poll.registry().deregister(&mut stream); - } -} - -fn http_set_interest(conn: &mut Conn, token: Token, poll: &mut Poll, interest: Interest) { - let stream = match conn { - Conn::Connecting(s) | Conn::Connected(s) => s, - Conn::Disconnected => return, - }; - let _ = poll.registry().reregister(stream, token, interest); -} - -// Connection helper functions -fn build_request_into(buf: &mut Vec, host: &str, json: &[u8], bearer: &str, keep_alive: bool) { - use std::io::Write as _; - let connection = if keep_alive { "keep-alive" } else { "close" }; - buf.clear(); - // SAFETY: Vec's io::Write impl is infallible. - write!( - buf, - "POST / HTTP/1.1\r\nHost: {host}\r\nContent-Type: application/json\r\n\ - Content-Length: {len}\r\nAuthorization: {bearer}\r\nConnection: {connection}\r\n\r\n", - len = json.len(), - ) - .unwrap(); - buf.extend_from_slice(json); -} - -fn parse_addr(endpoint: &str) -> io::Result { - let hostport = endpoint.trim_start_matches("http://").split('/').next().unwrap_or(endpoint); - hostport - .to_socket_addrs()? - .next() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no address resolved")) -} - -// Returns (header_end, content_length) when headers are complete, None if -// partial. -fn try_parse_headers(buf: &[u8]) -> Result, EngineError> { - let mut headers = [httparse::EMPTY_HEADER; 32]; - let mut resp = httparse::Response::new(&mut headers); - let header_end = match resp.parse(buf) { - Ok(httparse::Status::Complete(n)) => n, - Ok(httparse::Status::Partial) => return Ok(None), - Err(e) => return Err(EngineError::Http(format!("httparse: {e}"))), - }; - match headers.iter().find(|h| h.name.eq_ignore_ascii_case("content-length")) { - Some(h) if h.value.iter().all(|b| b.is_ascii_digit()) => { - let cl = h.value.iter().copied().fold(0usize, |acc, b| acc * 10 + (b - b'0') as usize); - Ok(Some((header_end, cl))) - } - Some(_) => Err(EngineError::Http("invalid Content-Length".into())), - None => Err(EngineError::Http("missing Content-Length".into())), - } -} - -pub(crate) struct HttpPool { - connections: Vec, - endpoint: String, - jwt: JwtSecret, -} - -impl HttpPool { - pub(crate) fn new(endpoint: String, jwt: JwtSecret) -> Self { - let connections = vec![HttpConnection::new(endpoint.clone(), jwt.clone(), Token(0))]; - Self { connections, endpoint, jwt } - } -} - -pub(crate) fn http_pool_enqueue(pool: &mut HttpPool, rpc_id: u64, body: &[u8], poll: &mut Poll) { - if let Some(conn) = pool.connections.iter_mut().find(|c| http_is_free(c)) { - http_enqueue(conn, rpc_id, body, poll); - } else { - let mut new_conn = HttpConnection::new( - pool.endpoint.clone(), - pool.jwt.clone(), - Token(pool.connections.len()), - ); - http_enqueue(&mut new_conn, rpc_id, body, poll); - pool.connections.push(new_conn); - } -} - -pub(crate) fn poll_http_pool( - pool: &mut HttpPool, - events: &Events, - poll: &mut Poll, - on_complete: &mut F, -) where - F: FnMut(u64, Result<&mut [u8], EngineError>), -{ - for conn in &mut pool.connections { - http_poll(conn, events, poll, on_complete); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn make_response(body: &[u8]) -> Vec { - let header = format!( - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n", - body.len() - ); - let mut buf = header.into_bytes(); - buf.extend_from_slice(body); - buf - } - - #[test] - fn headers_complete_returns_offsets() { - let body = br#"{"jsonrpc":"2.0","id":1,"result":true}"#; - let buf = make_response(body); - let (hend, cl) = try_parse_headers(&buf).unwrap().unwrap(); - assert_eq!(cl, body.len()); - assert_eq!(hend + cl, buf.len()); - } - - #[test] - fn headers_partial_returns_none() { - let partial = b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n"; - assert!(try_parse_headers(partial).unwrap().is_none()); - } - - #[test] - fn headers_complete_body_incomplete_still_returns_offsets() { - // try_parse_headers only cares about headers; body completeness is the caller's - // job. - let body = br#"{"result":1}"#; - let mut buf = make_response(body); - buf.truncate(buf.len() - 3); - let (hend, cl) = try_parse_headers(&buf).unwrap().unwrap(); - assert_eq!(cl, body.len()); - assert!(buf.len() < hend + cl); - } - - #[test] - fn missing_content_length_is_error() { - let buf = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{}"; - assert!(try_parse_headers(buf).is_err()); - } - - #[test] - fn invalid_content_length_is_error() { - let buf = b"HTTP/1.1 200 OK\r\nContent-Length: abc\r\n\r\n{}"; - assert!(try_parse_headers(buf).is_err()); - } -} diff --git a/crates/engine/src/ipc.rs b/crates/engine/src/ipc.rs deleted file mode 100644 index 2945fc58..00000000 --- a/crates/engine/src/ipc.rs +++ /dev/null @@ -1,267 +0,0 @@ -use std::{ - io::{self, Read, Write}, - path::PathBuf, -}; - -use mio::{Events, Interest, Poll, Token, net::UnixStream}; - -use crate::EngineError; - -// Sized for the largest expected EL response: getPayload with a full -// blobsBundle (~21 blobs × 256 KB hex-encoded + execution payload -// transactions). -const READ_BUF_CAPACITY: usize = 10 * 1024 * 1024; - -// Sized for the largest expected outgoing request: newPayload with a full -// block (~30M gas of transactions, hex-encoded in JSON). -const WRITE_BUF_CAPACITY: usize = 10 * 1024 * 1024; - -#[derive(PartialEq)] -enum State { - Disconnected, - Connecting, - Connected, -} - -struct IpcTransport { - path: PathBuf, - token: Token, - stream: Option, - state: State, - pending_id: Option, - write_buf: Vec, - write_pos: usize, - in_flight: Option, - read_buf: Vec, - read_offset: usize, -} - -impl IpcTransport { - pub(crate) fn new(path: String, token: Token) -> Self { - Self { - path: PathBuf::from(path), - token, - stream: None, - state: State::Disconnected, - pending_id: None, - write_buf: Vec::with_capacity(WRITE_BUF_CAPACITY), - write_pos: 0, - in_flight: None, - read_buf: Vec::with_capacity(READ_BUF_CAPACITY), - read_offset: 0, - } - } -} - -fn ipc_is_free(t: &IpcTransport) -> bool { - t.in_flight.is_none() && t.pending_id.is_none() -} - -fn ipc_enqueue(t: &mut IpcTransport, rpc_id: u64, body: &[u8], poll: &mut Poll) { - t.write_buf.clear(); - t.write_buf.extend_from_slice(body); - t.write_buf.push(b'\n'); - t.pending_id = Some(rpc_id); - t.write_pos = 0; - - match t.state { - State::Disconnected => ipc_connect(t, poll), - State::Connected => ipc_set_interest(t, poll, Interest::READABLE | Interest::WRITABLE), - State::Connecting => {} - } -} - -fn ipc_poll(t: &mut IpcTransport, events: &Events, poll: &mut Poll, on_complete: &mut F) -where - F: FnMut(u64, Result<&mut [u8], EngineError>), -{ - for event in events.iter() { - if event.token() != t.token { - continue; - } - match t.state { - State::Disconnected => {} - State::Connecting => { - if event.is_writable() { - // is_error/is_write_closed flags are not reliable; use - // take_error() (getsockopt SO_ERROR) as the authoritative check. - let err = t.stream.as_ref().and_then(|s| s.take_error().ok()).flatten(); - if let Some(e) = err { - ipc_on_error(t, poll, on_complete, &e.to_string()); - break; - } - t.state = State::Connected; - let interest = if t.pending_id.is_none() { - Interest::READABLE - } else { - Interest::READABLE | Interest::WRITABLE - }; - ipc_set_interest(t, poll, interest); - } - } - State::Connected => { - if event.is_error() || event.is_read_closed() { - ipc_on_error(t, poll, on_complete, "ipc connection lost"); - break; - } - if event.is_writable() { - if let Err(e) = ipc_do_write(t) { - let msg = e.to_string(); - ipc_on_error(t, poll, on_complete, &msg); - break; - } - let interest = if t.pending_id.is_none() { - Interest::READABLE - } else { - Interest::READABLE | Interest::WRITABLE - }; - ipc_set_interest(t, poll, interest); - } - if event.is_readable() { - if let Err(e) = ipc_do_read(t, on_complete) { - let msg = e.to_string(); - ipc_on_error(t, poll, on_complete, &msg); - break; - } - } - } - } - } -} - -fn ipc_connect(t: &mut IpcTransport, poll: &mut Poll) { - match UnixStream::connect(&t.path) { - Ok(mut stream) => { - if poll.registry().register(&mut stream, t.token, Interest::WRITABLE).is_ok() { - t.stream = Some(stream); - t.state = State::Connecting; - } - } - Err(e) => tracing::warn!("connect error: {e}"), - } -} - -fn ipc_do_write(t: &mut IpcTransport) -> io::Result<()> { - if t.pending_id.is_some() { - let stream = t.stream.as_mut().unwrap(); - loop { - match stream.write(&t.write_buf[t.write_pos..]) { - Ok(0) => return Err(io::Error::new(io::ErrorKind::WriteZero, "write returned 0")), - Ok(n) => { - t.write_pos += n; - if t.write_pos == t.write_buf.len() { - t.in_flight = t.pending_id.take(); - t.write_pos = 0; - break; - } - } - Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, - Err(e) => return Err(e), - } - } - } - Ok(()) -} - -fn ipc_do_read(t: &mut IpcTransport, on_complete: &mut F) -> io::Result<()> -where - F: FnMut(u64, Result<&mut [u8], EngineError>), -{ - let stream = t.stream.as_mut().unwrap(); - loop { - let base = t.read_buf.len(); - t.read_buf.resize(base + READ_BUF_CAPACITY, 0); - match stream.read(&mut t.read_buf[base..]) { - Ok(0) => { - t.read_buf.truncate(base); - return Err(io::Error::new(io::ErrorKind::ConnectionReset, "eof")); - } - Ok(n) => { - t.read_buf.truncate(base + n); - while let Some(rel) = t.read_buf[t.read_offset..].iter().position(|&b| b == b'\n') { - let offset = t.read_offset; - let end = offset + rel; - if let Some(rpc_id) = t.in_flight { - on_complete(rpc_id, Ok(&mut t.read_buf[offset..end])); - } - t.read_offset = end + 1; - } - if t.read_offset == t.read_buf.len() { - t.read_buf.clear(); - t.read_offset = 0; - } - } - Err(e) if e.kind() == io::ErrorKind::WouldBlock => { - t.read_buf.truncate(base); - break; - } - Err(e) => { - t.read_buf.truncate(base); - return Err(e); - } - } - } - Ok(()) -} - -fn ipc_on_error(t: &mut IpcTransport, poll: &mut Poll, on_complete: &mut F, msg: &str) -where - F: FnMut(u64, Result<&mut [u8], EngineError>), -{ - tracing::warn!("{msg}"); - let err = msg.to_string(); - if let Some(rpc_id) = t.in_flight.take() { - on_complete(rpc_id, Err(EngineError::Ipc(err.clone()))); - } - if let Some(rpc_id) = t.pending_id.take() { - on_complete(rpc_id, Err(EngineError::Ipc(err.clone()))); - } - t.write_pos = 0; - t.read_buf.clear(); - t.read_offset = 0; - if let Some(mut stream) = t.stream.take() { - let _ = poll.registry().deregister(&mut stream); - } - t.state = State::Disconnected; -} - -fn ipc_set_interest(t: &mut IpcTransport, poll: &mut Poll, interest: Interest) { - if let Some(stream) = t.stream.as_mut() { - let _ = poll.registry().reregister(stream, t.token, interest); - } -} - -pub(crate) struct IpcPool { - connections: Vec, - path: String, -} - -impl IpcPool { - pub(crate) fn new(path: String) -> Self { - let connections = vec![IpcTransport::new(path.clone(), Token(0))]; - Self { connections, path } - } -} - -pub(crate) fn ipc_pool_enqueue(pool: &mut IpcPool, rpc_id: u64, body: &[u8], poll: &mut Poll) { - if let Some(conn) = pool.connections.iter_mut().find(|c| ipc_is_free(c)) { - ipc_enqueue(conn, rpc_id, body, poll); - } else { - let mut new_conn = IpcTransport::new(pool.path.clone(), Token(pool.connections.len())); - ipc_enqueue(&mut new_conn, rpc_id, body, poll); - pool.connections.push(new_conn); - } -} - -pub(crate) fn poll_ipc_pool( - pool: &mut IpcPool, - events: &Events, - poll: &mut Poll, - on_complete: &mut F, -) where - F: FnMut(u64, Result<&mut [u8], EngineError>), -{ - for conn in &mut pool.connections { - ipc_poll(conn, events, poll, on_complete); - } -} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs deleted file mode 100644 index b0e36fea..00000000 --- a/crates/engine/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -mod client; -mod error; -mod http; -mod ipc; -mod jwt; -mod req_handlers; -mod resp_handlers; -pub mod tile; -mod types; - -pub use client::EngineClient; -pub use error::EngineError; -pub use jwt::JwtSecret; -pub use tile::EngineTile; diff --git a/crates/engine/Cargo.toml b/crates/engine_api/Cargo.toml similarity index 65% rename from crates/engine/Cargo.toml rename to crates/engine_api/Cargo.toml index c733a4e1..b0f521f8 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine_api/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "silver_engine" +name = "silver_engine_api" edition.workspace = true repository.workspace = true rust-version.workspace = true @@ -11,17 +11,25 @@ base64.workspace = true flux.workspace = true hex.workspace = true hmac.workspace = true -httparse.workspace = true +httparse = { workspace = true, optional = true } mio.workspace = true rustc-hash.workspace = true serde.workspace = true simd-json.workspace = true sha2.workspace = true silver_common.workspace = true +silver_httpcore.workspace = true thiserror.workspace = true tracing.workspace = true +[features] +# Exposes the `test_el` fake execution client to dependents' tests. +test-el = ["dep:httparse"] + [dev-dependencies] +httparse.workspace = true +silver_engine_api = { workspace = true, features = ["test-el"] } +tempfile = "3" tracing-subscriber.workspace = true [lints] diff --git a/crates/engine/src/tile.rs b/crates/engine_api/src/api.rs similarity index 77% rename from crates/engine/src/tile.rs rename to crates/engine_api/src/api.rs index 4d4e6470..42bed05f 100644 --- a/crates/engine/src/tile.rs +++ b/crates/engine_api/src/api.rs @@ -1,6 +1,6 @@ use std::time::{Duration, Instant}; -use flux::{spine::SpineAdapter, tile::Tile}; +use flux::spine::SpineAdapter; use silver_common::{ ELSyncStatus, EngineHealthEvent, EngineReq, SilverSpine, TProducer, TRandomAccess, }; @@ -15,7 +15,7 @@ use crate::{ const HEALTHCHECK_INTERVAL: Duration = Duration::from_secs(10); -pub struct EngineTile { +pub struct EngineApi { /// `None` in unsafe no-EL testing mode — see /// [`EngineConfig::unsafe_no_el`]. pub client: Option, @@ -32,38 +32,7 @@ pub struct EngineTile { scratch: Vec, } -impl Tile for EngineTile { - fn loop_body(&mut self, adapter: &mut SpineAdapter) { - self.rpc_consumer.free(); - self.gossip_consumer.free(); - - if self.client.is_none() { - // Unsafe no-EL testing mode: report healthy once so peers don't - // gate on EL liveness, then answer every request with VALID. - if self.first_run { - adapter.produce(EngineHealthEvent { sync_status: ELSyncStatus::Synced }); - self.first_run = false; - } - let resp_producer = &mut self.resp_producer; - adapter.consume(|req: EngineReq, producers| { - handle_request_no_el(resp_producer, &req, producers) - }); - return; - } - adapter.consume(|req: EngineReq, producers| { - handle_request( - self.client.as_mut().unwrap(), - &mut self.gossip_consumer, - &mut self.rpc_consumer, - &req, - producers, - ); - }); - self.spin(adapter); - } -} - -impl EngineTile { +impl EngineApi { pub fn new( config: EngineConfig, gossip_consumer: TRandomAccess, @@ -71,12 +40,14 @@ impl EngineTile { resp_producer: TProducer, ) -> Self { let client = if config.unsafe_no_el { - tracing::warn!( - "engine tile in UNSAFE no-EL testing mode: answering all requests VALID" - ); + tracing::warn!("engine api in UNSAFE no-EL testing mode: answering all requests VALID"); None } else { - Some(EngineClient::new(&config.execution_endpoint, &config.jwt_secret)) + Some(EngineClient::new( + &config.execution_endpoint, + &config.jwt_secret, + config.max_connections, + )) }; Self { client, @@ -92,7 +63,43 @@ impl EngineTile { } } - fn spin(&mut self, adapter: &mut SpineAdapter) { + pub fn intake(&mut self, adapter: &mut SpineAdapter) { + self.rpc_consumer.free(); + self.gossip_consumer.free(); + + if self.client.is_none() { + // Unsafe no-EL testing mode: report healthy once so peers don't + // gate on EL liveness, then answer every request with VALID. + if self.first_run { + adapter.produce(EngineHealthEvent { sync_status: ELSyncStatus::Synced }); + self.first_run = false; + } + let resp_producer = &mut self.resp_producer; + adapter.consume(|req: EngineReq, producers| { + handle_request_no_el(resp_producer, &req, producers) + }); + return; + } + // Requests stay queued on the spine while every connection is busy and + // the pool is at max_connections; intake resumes as completions free + // connections. + while self.client.as_ref().unwrap().has_capacity() { + let consumed = adapter.consume_one(|req: EngineReq, producers| { + handle_request( + self.client.as_mut().unwrap(), + &mut self.gossip_consumer, + &mut self.rpc_consumer, + &req, + producers, + ); + }); + if !consumed { + break; + } + } + } + + pub fn spin(&mut self, adapter: &mut SpineAdapter) { let mut negotiated_get_payload_method: Option<&'static str> = None; { @@ -106,10 +113,12 @@ impl EngineTile { sync_status, .. } = self; - // Only reached in EL mode; loop_body returns early otherwise. - let client = client.as_mut().expect("spin without EL client"); + let Some(client) = client.as_mut() else { return }; - if !*healthcheck_pending && Instant::now() >= *healthcheck_deadline { + if !*healthcheck_pending && + Instant::now() >= *healthcheck_deadline && + client.has_capacity() + { run_healthcheck(client, first_run, healthcheck_pending, healthcheck_deadline); } diff --git a/crates/engine/src/client.rs b/crates/engine_api/src/client.rs similarity index 79% rename from crates/engine/src/client.rs rename to crates/engine_api/src/client.rs index 737e2849..48b6ae34 100644 --- a/crates/engine/src/client.rs +++ b/crates/engine_api/src/client.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::{path::PathBuf, time::Duration}; use mio::{Events, Poll}; use rustc_hash::FxHashMap; @@ -6,8 +6,7 @@ use silver_common::merkle::B256; use crate::{ EngineError, JwtSecret, - http::{HttpPool, http_pool_enqueue, poll_http_pool}, - ipc::{IpcPool, ipc_pool_enqueue, poll_ipc_pool}, + pool::{Endpoint, HttpPool}, types::{ ForkchoiceState, PayloadAttributesV3, write_new_payload_params_fulu, write_new_payload_params_gloas, @@ -45,13 +44,8 @@ pub enum ReqKind { GetPayloadBodiesByRange(u64), } -enum Transport { - Http(HttpPool), - Ipc(IpcPool), -} - pub struct EngineClient { - transport: Transport, + pool: HttpPool, poll: Poll, events: Events, id: u64, @@ -61,10 +55,18 @@ pub struct EngineClient { } impl EngineClient { - pub fn new(endpoint: impl Into, jwt: &str) -> Self { + pub fn new(endpoint: &str, jwt: &str, max_connections: usize) -> Self { + Self::with_endpoint(parse_endpoint(endpoint), jwt, max_connections) + } + + pub fn new_uds(path: impl Into, jwt: &str, max_connections: usize) -> Self { + Self::with_endpoint(Endpoint::Uds(path.into()), jwt, max_connections) + } + + fn with_endpoint(endpoint: Endpoint, jwt: &str, max_connections: usize) -> Self { let jwt = JwtSecret::from_file(jwt).unwrap_or_else(|e| panic!("invalid JWT secret: {e}")); Self { - transport: Transport::Http(HttpPool::new(endpoint.into(), jwt)), + pool: HttpPool::new(endpoint, jwt, max_connections), poll: Poll::new().expect("mio Poll::new failed"), events: Events::with_capacity(EVENTS_CAPACITY), id: 1, @@ -74,16 +76,21 @@ impl EngineClient { } } - pub fn new_ipc(path: impl Into) -> Self { - Self { - transport: Transport::Ipc(IpcPool::new(path.into())), - poll: Poll::new().expect("mio Poll::new failed"), - events: Events::with_capacity(EVENTS_CAPACITY), - id: 1, - pending_requests: FxHashMap::default(), - get_payload_method: "engine_getPayloadV3", - scratch: Vec::with_capacity(SCRATCH_CAPACITY), - } + pub fn has_capacity(&self) -> bool { + self.pool.has_capacity() + } +} + +fn parse_endpoint(endpoint: &str) -> Endpoint { + if endpoint.starts_with("http://") { + Endpoint::Http(endpoint.to_string()) + } else if endpoint.contains("://") { + panic!( + "unsupported execution_endpoint scheme (only http:// or a unix socket path): \ + {endpoint}" + ) + } else { + Endpoint::Uds(PathBuf::from(endpoint)) } } @@ -114,10 +121,7 @@ fn enqueue(c: &mut EngineClient, rpc_id: u64, body: &simd_json::OwnedValue) { tracing::warn!("failed to serialize RPC body: {e}"); return; } - match &mut c.transport { - Transport::Http(p) => http_pool_enqueue(p, rpc_id, &c.scratch, &mut c.poll), - Transport::Ipc(p) => ipc_pool_enqueue(p, rpc_id, &c.scratch, &mut c.poll), - } + c.pool.enqueue(rpc_id, &c.scratch, &mut c.poll); } pub fn send_fcu( @@ -168,10 +172,7 @@ fn send_new_payload_request_impl( c.scratch.extend_from_slice(b",\"id\":"); append_decimal_u64(rpc_id, &mut c.scratch); c.scratch.push(b'}'); - match &mut c.transport { - Transport::Http(p) => http_pool_enqueue(p, rpc_id, &c.scratch, &mut c.poll), - Transport::Ipc(p) => ipc_pool_enqueue(p, rpc_id, &c.scratch, &mut c.poll), - } + c.pool.enqueue(rpc_id, &c.scratch, &mut c.poll); c.pending_requests.insert(rpc_id, ReqKind::NewPayload(block_root)); Ok(()) } @@ -256,26 +257,18 @@ pub fn get_client_version(c: &mut EngineClient) { } /// Drive I/O, calling `on_complete(req_kind, raw_body)` for each finished RPC. -/// Raw bytes are the full HTTP/IPC response body; handlers parse them as -/// needed. +/// Raw bytes are the full HTTP response body; handlers parse them as needed. pub fn poll(c: &mut EngineClient, mut on_complete: F) where F: FnMut(ReqKind, Result<&mut [u8], EngineError>), { c.poll.poll(&mut c.events, Some(Duration::ZERO)).ok(); - let EngineClient { transport, events, poll, pending_requests, .. } = c; - match transport { - Transport::Http(p) => poll_http_pool(p, events, poll, &mut |rpc_id, res| { - if let Some(req_kind) = pending_requests.remove(&rpc_id) { - on_complete(req_kind, res); - } - }), - Transport::Ipc(p) => poll_ipc_pool(p, events, poll, &mut |rpc_id, res| { - if let Some(req_kind) = pending_requests.remove(&rpc_id) { - on_complete(req_kind, res); - } - }), - } + let EngineClient { pool, events, poll, pending_requests, .. } = c; + pool.poll_events(events, poll, &mut |rpc_id, res| { + if let Some(req_kind) = pending_requests.remove(&rpc_id) { + on_complete(req_kind, res); + } + }); } #[cfg(test)] @@ -284,6 +277,28 @@ mod tests { use super::*; + #[test] + fn endpoint_http_scheme_parses_to_http() { + assert!(matches!( + parse_endpoint("http://localhost:8551"), + Endpoint::Http(e) if e == "http://localhost:8551" + )); + } + + #[test] + fn endpoint_bare_path_parses_to_uds() { + assert!(matches!( + parse_endpoint("/run/reth/engine.sock"), + Endpoint::Uds(p) if p == std::path::Path::new("/run/reth/engine.sock") + )); + } + + #[test] + #[should_panic(expected = "unsupported execution_endpoint scheme")] + fn endpoint_unknown_scheme_panics() { + parse_endpoint("https://localhost:8551"); + } + #[test] fn next_id_returns_current_then_increments() { let mut id = 1u64; diff --git a/crates/engine/src/error.rs b/crates/engine_api/src/error.rs similarity index 89% rename from crates/engine/src/error.rs rename to crates/engine_api/src/error.rs index bc6fae82..237ab410 100644 --- a/crates/engine/src/error.rs +++ b/crates/engine_api/src/error.rs @@ -10,8 +10,6 @@ pub enum EngineError { Json(#[from] simd_json::Error), #[error("jwt: {0}")] Jwt(String), - #[error("ipc: {0}")] - Ipc(String), #[error("ssz: {0}")] Ssz(String), } diff --git a/crates/engine/src/jwt.rs b/crates/engine_api/src/jwt.rs similarity index 100% rename from crates/engine/src/jwt.rs rename to crates/engine_api/src/jwt.rs diff --git a/crates/engine_api/src/lib.rs b/crates/engine_api/src/lib.rs new file mode 100644 index 00000000..c72f0708 --- /dev/null +++ b/crates/engine_api/src/lib.rs @@ -0,0 +1,17 @@ +mod api; +mod client; +mod error; +mod jwt; +mod pool; +mod req_handlers; +mod resp_handlers; +#[cfg(any(test, feature = "test-el"))] +pub mod test_el; +mod types; + +pub use api::EngineApi; +pub use client::EngineClient; +#[cfg(feature = "test-el")] +pub use client::{ReqKind, poll, send_new_payload}; +pub use error::EngineError; +pub use jwt::JwtSecret; diff --git a/crates/engine_api/src/pool.rs b/crates/engine_api/src/pool.rs new file mode 100644 index 00000000..2d06ba03 --- /dev/null +++ b/crates/engine_api/src/pool.rs @@ -0,0 +1,451 @@ +use std::{ + io::{self, Read, Write}, + net::{SocketAddr, ToSocketAddrs}, + path::PathBuf, +}; + +use mio::{Events, Interest, Poll, Token}; +use silver_httpcore::{ClientConnection, Stream, frame_request}; + +use crate::{EngineError, JwtSecret}; + +// Sized for the largest expected EL response: getPayload with a full +// blobsBundle (~21 blobs × 256 KB hex-encoded + execution payload +// transactions). +const READ_BUF_CAPACITY: usize = 10 * 1024 * 1024; + +// Sized for the largest expected outgoing request: newPayload with a full +// block (~30M gas of transactions, hex-encoded in JSON) plus HTTP headers. +const WRITE_BUF_CAPACITY: usize = 10 * 1024 * 1024; + +#[derive(Clone)] +pub(crate) enum Endpoint { + Http(String), + Uds(PathBuf), +} + +impl Endpoint { + fn host(&self) -> String { + match self { + Self::Http(endpoint) => endpoint + .trim_start_matches("http://") + .split('/') + .next() + .unwrap_or("localhost") + .to_string(), + Self::Uds(_) => "localhost".to_string(), + } + } +} + +enum Conn { + Disconnected, + Connecting(Stream), + Connected(Stream), +} + +struct PooledConnection { + endpoint: Endpoint, + host: String, + jwt: JwtSecret, + token: Token, + conn: Conn, + addr: Option, + machine: ClientConnection, + in_flight: Option, + pending_id: Option, +} + +impl PooledConnection { + fn new(endpoint: Endpoint, jwt: JwtSecret, token: Token) -> Self { + let host = endpoint.host(); + Self { + endpoint, + host, + jwt, + token, + conn: Conn::Disconnected, + addr: None, + machine: ClientConnection::with_capacity(READ_BUF_CAPACITY, WRITE_BUF_CAPACITY), + in_flight: None, + pending_id: None, + } + } + + fn is_free(&self) -> bool { + self.in_flight.is_none() && self.pending_id.is_none() + } + + fn enqueue(&mut self, rpc_id: u64, body: &[u8], poll: &mut Poll) { + debug_assert!(self.is_free(), "enqueue on busy connection"); + let out = self.machine.begin_request(); + frame_request(out, &self.host, body, Some(self.jwt.bearer_token()), true); + self.pending_id = Some(rpc_id); + + match self.conn { + Conn::Disconnected => self.connect(poll), + Conn::Connected(_) => self.update_interest(poll), + Conn::Connecting(_) => {} + } + } + + fn handle_events(&mut self, events: &Events, poll: &mut Poll, on_complete: &mut F) + where + F: FnMut(u64, Result<&mut [u8], EngineError>), + { + for event in events.iter() { + if event.token() != self.token { + continue; + } + match &self.conn { + Conn::Disconnected => {} + Conn::Connecting(stream) => { + if event.is_error() || event.is_read_closed() || event.is_write_closed() { + self.fail(poll, on_complete, "connect failed"); + break; + } + if event.is_writable() { + if stream.connect_complete().is_ok() { + let Conn::Connecting(stream) = + std::mem::replace(&mut self.conn, Conn::Disconnected) + else { + unreachable!() + }; + self.conn = Conn::Connected(stream); + self.update_interest(poll); + } else { + self.fail(poll, on_complete, "connect failed"); + break; + } + } + } + Conn::Connected(_) => { + if event.is_error() { + self.fail(poll, on_complete, "connection error"); + break; + } + if event.is_writable() { + if let Err(e) = self.do_write() { + let msg = e.to_string(); + self.fail(poll, on_complete, &msg); + break; + } + self.update_interest(poll); + } + if event.is_readable() { + // Drain data before checking is_read_closed: when the + // remote sends a response + FIN in one exchange + // (EPOLLIN|EPOLLRDHUP), we must read the response + // first. do_read returns Err on EOF, so the break + // below covers that close path too. + if let Err(e) = self.do_read(on_complete) { + let msg = e.to_string(); + self.fail(poll, on_complete, &msg); + break; + } + } + if event.is_read_closed() { + // Remote closed with no (more) data — in_flight will + // never get a response. + self.fail(poll, on_complete, "connection closed"); + break; + } + } + } + } + } + + fn connect(&mut self, poll: &mut Poll) { + let stream = match &self.endpoint { + Endpoint::Http(endpoint) => { + let addr = if let Some(a) = self.addr { + a + } else { + match parse_addr(endpoint) { + Ok(a) => { + self.addr = Some(a); + a + } + Err(e) => { + tracing::warn!("resolve failed for {endpoint}: {e}"); + return; + } + } + }; + Stream::connect_tcp(addr) + } + Endpoint::Uds(path) => Stream::connect_uds(path), + }; + match stream { + Ok(mut stream) => { + if poll.registry().register(&mut stream, self.token, Interest::WRITABLE).is_ok() { + self.conn = Conn::Connecting(stream); + } + } + Err(e) => tracing::warn!("connect error: {e}"), + } + } + + fn do_write(&mut self) -> io::Result<()> { + if self.pending_id.is_none() { + return Ok(()); + } + let Self { conn, machine, pending_id, in_flight, .. } = self; + let Conn::Connected(stream) = conn else { return Ok(()) }; + loop { + match stream.write(machine.pending_write()) { + Ok(0) => break, + Ok(n) => { + machine.commit_write(n); + if machine.pending_write().is_empty() { + *in_flight = pending_id.take(); + break; + } + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, + Err(e) => return Err(e), + } + } + Ok(()) + } + + fn do_read(&mut self, on_complete: &mut F) -> io::Result<()> + where + F: FnMut(u64, Result<&mut [u8], EngineError>), + { + let Self { conn, machine, in_flight, .. } = self; + let Conn::Connected(stream) = conn else { return Ok(()) }; + loop { + while let Some(body) = machine.take_response() { + if let Some(rpc_id) = in_flight.take() { + on_complete(rpc_id, Ok(body)); + } + } + match stream.read(machine.read_space()) { + Ok(0) => return Err(io::Error::new(io::ErrorKind::ConnectionReset, "eof")), + Ok(n) => machine.commit_read(n)?, + Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, + Err(e) => return Err(e), + } + } + Ok(()) + } + + fn fail(&mut self, poll: &mut Poll, on_complete: &mut F, msg: &str) + where + F: FnMut(u64, Result<&mut [u8], EngineError>), + { + tracing::warn!("{msg}"); + let err = msg.to_string(); + if let Some(rpc_id) = self.in_flight.take() { + on_complete(rpc_id, Err(EngineError::Http(err.clone()))); + } + if let Some(rpc_id) = self.pending_id.take() { + on_complete(rpc_id, Err(EngineError::Http(err.clone()))); + } + self.machine.reset(); + let old = std::mem::replace(&mut self.conn, Conn::Disconnected); + if let Conn::Connecting(mut stream) | Conn::Connected(mut stream) = old { + let _ = poll.registry().deregister(&mut stream); + } + } + + fn update_interest(&mut self, poll: &mut Poll) { + let interest = if self.pending_id.is_none() { + Interest::READABLE + } else { + Interest::READABLE | Interest::WRITABLE + }; + let stream = match &mut self.conn { + Conn::Connecting(s) | Conn::Connected(s) => s, + Conn::Disconnected => return, + }; + let _ = poll.registry().reregister(stream, self.token, interest); + } +} + +fn parse_addr(endpoint: &str) -> io::Result { + let hostport = endpoint.trim_start_matches("http://").split('/').next().unwrap_or(endpoint); + hostport + .to_socket_addrs()? + .next() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no address resolved")) +} + +pub(crate) struct HttpPool { + connections: Vec, + endpoint: Endpoint, + jwt: JwtSecret, + max_connections: usize, +} + +impl HttpPool { + pub(crate) fn new(endpoint: Endpoint, jwt: JwtSecret, max_connections: usize) -> Self { + let connections = vec![PooledConnection::new(endpoint.clone(), jwt.clone(), Token(0))]; + Self { connections, endpoint, jwt, max_connections } + } + + /// `enqueue` never refuses work; every caller gates on this before + /// submitting. The first-run healthcheck trio issues three requests + /// against one gate check, so the pool can overshoot `max_connections` + /// by at most two connections, once. + pub(crate) fn has_capacity(&self) -> bool { + self.connections.iter().any(PooledConnection::is_free) || + self.connections.len() < self.max_connections + } + + pub(crate) fn enqueue(&mut self, rpc_id: u64, body: &[u8], poll: &mut Poll) { + if let Some(conn) = self.connections.iter_mut().find(|c| c.is_free()) { + conn.enqueue(rpc_id, body, poll); + } else { + let mut new_conn = PooledConnection::new( + self.endpoint.clone(), + self.jwt.clone(), + Token(self.connections.len()), + ); + new_conn.enqueue(rpc_id, body, poll); + self.connections.push(new_conn); + } + } + + pub(crate) fn poll_events(&mut self, events: &Events, poll: &mut Poll, on_complete: &mut F) + where + F: FnMut(u64, Result<&mut [u8], EngineError>), + { + for conn in &mut self.connections { + // Disconnected with a request pending means connect() could not + // even start (resolve/connect/register error): no event will ever + // arrive for it, so fail the rpc here or it is stranded forever. + if matches!(conn.conn, Conn::Disconnected) && conn.pending_id.is_some() { + conn.fail(poll, on_complete, "connect failed to start"); + } + conn.handle_events(events, poll, on_complete); + } + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use tempfile::TempDir; + + use crate::{ + EngineClient, + client::{ReqKind, poll, send_fcu}, + test_el::{FCU_VALID_RESULT, FakeEl, write_jwt}, + types::ForkchoiceState, + }; + + fn fcu_state(byte: u8) -> ForkchoiceState { + ForkchoiceState { + head_block_hash: [byte; 32], + safe_block_hash: [byte; 32], + finalized_block_hash: [byte; 32], + } + } + + fn spin_until(deadline_msg: &str, mut done: impl FnMut() -> bool) { + let deadline = Instant::now() + Duration::from_secs(10); + while !done() { + assert!(Instant::now() < deadline, "timeout: {deadline_msg}"); + std::thread::sleep(Duration::from_millis(1)); + } + } + + #[test] + fn uds_round_trip_resolves_correlation_with_jwt() { + let dir = TempDir::new().unwrap(); + let jwt_path = write_jwt(dir.path()); + let socket = dir.path().join("engine.sock"); + let mut el = FakeEl::uds(&socket); + + let mut client = EngineClient::new_uds(&socket, jwt_path.to_str().unwrap(), 32); + let block_root = [7u8; 32]; + send_fcu(&mut client, block_root, fcu_state(1), None); + + let mut responded = false; + let mut completed: Option<([u8; 32], Vec)> = None; + spin_until("fcu round trip over uds", || { + poll(&mut client, |kind, response| { + let ReqKind::Fcu(root) = kind else { panic!("unexpected completion") }; + completed = Some((root, response.expect("fcu response").to_vec())); + }); + el.pump(); + if !responded && !el.requests.is_empty() { + let request = &el.requests[0]; + assert_eq!(request.method, "engine_forkchoiceUpdatedV3"); + let auth = request.authorization.as_deref().expect("JWT header sent over UDS"); + let token = auth.strip_prefix("Bearer ").expect("bearer scheme"); + assert_eq!(token.split('.').count(), 3, "three-part JWT"); + assert!( + request.body.contains(&format!("\"headBlockHash\":\"0x{}\"", "01".repeat(32))) + ); + el.respond(0, FCU_VALID_RESULT); + responded = true; + } + completed.is_some() + }); + + let (root, body) = completed.unwrap(); + assert_eq!(root, block_root, "completion correlated to the issued request"); + assert!(String::from_utf8(body).unwrap().contains("VALID")); + } + + #[test] + fn connect_failure_fails_rpc_and_frees_connection() { + let dir = TempDir::new().unwrap(); + let jwt_path = write_jwt(dir.path()); + let missing_socket = dir.path().join("missing.sock"); + + // max_connections = 1: after the failure, has_capacity() can only be + // true again if the zombie connection was actually freed. + let mut client = EngineClient::new_uds(&missing_socket, jwt_path.to_str().unwrap(), 1); + let block_root = [3u8; 32]; + send_fcu(&mut client, block_root, fcu_state(3), None); + assert!(!client.has_capacity(), "request occupies the only connection"); + + let mut failed: Option<[u8; 32]> = None; + spin_until("connect failure surfaces as rpc error", || { + poll(&mut client, |kind, response| { + let ReqKind::Fcu(root) = kind else { panic!("unexpected completion") }; + assert!(response.is_err(), "unstartable connect must fail the rpc"); + failed = Some(root); + }); + failed.is_some() + }); + + assert_eq!(failed.unwrap(), block_root); + assert!(client.has_capacity(), "failed connection must be reusable"); + } + + #[test] + fn transport_error_fails_in_flight_request() { + let dir = TempDir::new().unwrap(); + let jwt_path = write_jwt(dir.path()); + let socket = dir.path().join("engine.sock"); + let mut el = FakeEl::uds(&socket); + + let mut client = EngineClient::new_uds(&socket, jwt_path.to_str().unwrap(), 32); + let block_root = [9u8; 32]; + send_fcu(&mut client, block_root, fcu_state(2), None); + + let mut request_seen = false; + let mut failure: Option<[u8; 32]> = None; + spin_until("in-flight request failed on connection close", || { + poll(&mut client, |kind, response| { + let ReqKind::Fcu(root) = kind else { panic!("unexpected completion") }; + assert!(response.is_err(), "closed connection must fail the rpc"); + failure = Some(root); + }); + el.pump(); + if !request_seen && !el.requests.is_empty() { + el.close_connection_of(0); + request_seen = true; + } + failure.is_some() + }); + + assert_eq!(failure.unwrap(), block_root); + } +} diff --git a/crates/engine/src/req_handlers.rs b/crates/engine_api/src/req_handlers.rs similarity index 100% rename from crates/engine/src/req_handlers.rs rename to crates/engine_api/src/req_handlers.rs diff --git a/crates/engine/src/resp_handlers.rs b/crates/engine_api/src/resp_handlers.rs similarity index 100% rename from crates/engine/src/resp_handlers.rs rename to crates/engine_api/src/resp_handlers.rs diff --git a/crates/engine_api/src/test_el.rs b/crates/engine_api/src/test_el.rs new file mode 100644 index 00000000..8fbbb9d2 --- /dev/null +++ b/crates/engine_api/src/test_el.rs @@ -0,0 +1,186 @@ +use std::{ + io::{self, Read, Write}, + net::{TcpListener, TcpStream}, + os::unix::net::{UnixListener, UnixStream}, + path::{Path, PathBuf}, +}; + +use simd_json::prelude::{ValueAsScalar, ValueObjectAccess}; + +pub const FCU_VALID_RESULT: &str = r#"{"payloadStatus":{"status":"VALID","latestValidHash":null,"validationError":null},"payloadId":null}"#; + +pub fn write_jwt(dir: &Path) -> PathBuf { + let path = dir.join("jwt.hex"); + std::fs::write(&path, "0000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + path +} + +enum ElListener { + Tcp(TcpListener), + Uds(UnixListener), +} + +enum ElStream { + Tcp(TcpStream), + Uds(UnixStream), +} + +impl Read for ElStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Self::Tcp(s) => s.read(buf), + Self::Uds(s) => s.read(buf), + } + } +} + +impl Write for ElStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + Self::Tcp(s) => s.write(buf), + Self::Uds(s) => s.write(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Self::Tcp(s) => s.flush(), + Self::Uds(s) => s.flush(), + } + } +} + +pub struct ElRequest { + conn: usize, + pub id: u64, + pub method: String, + pub authorization: Option, + pub body: String, +} + +/// Deterministic single-threaded fake execution client: accepts connections +/// and buffers requests on `pump`, answers only when the test says so. +pub struct FakeEl { + listener: ElListener, + conns: Vec>, + read_bufs: Vec>, + pub requests: Vec, +} + +impl FakeEl { + pub fn tcp() -> (Self, String) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + (Self::new(ElListener::Tcp(listener)), endpoint) + } + + pub fn uds(path: &Path) -> Self { + let listener = UnixListener::bind(path).unwrap(); + listener.set_nonblocking(true).unwrap(); + Self::new(ElListener::Uds(listener)) + } + + fn new(listener: ElListener) -> Self { + Self { listener, conns: Vec::new(), read_bufs: Vec::new(), requests: Vec::new() } + } + + pub fn pump(&mut self) { + loop { + let accepted = match &self.listener { + ElListener::Tcp(l) => l.accept().map(|(s, _)| { + s.set_nonblocking(true).unwrap(); + ElStream::Tcp(s) + }), + ElListener::Uds(l) => l.accept().map(|(s, _)| { + s.set_nonblocking(true).unwrap(); + ElStream::Uds(s) + }), + }; + match accepted { + Ok(stream) => { + self.conns.push(Some(stream)); + self.read_bufs.push(Vec::new()); + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, + Err(e) => panic!("accept: {e}"), + } + } + + for i in 0..self.conns.len() { + let Some(stream) = self.conns[i].as_mut() else { continue }; + let mut chunk = [0u8; 65536]; + let mut closed = false; + loop { + match stream.read(&mut chunk) { + Ok(0) => { + closed = true; + break; + } + Ok(n) => self.read_bufs[i].extend_from_slice(&chunk[..n]), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, + Err(e) => panic!("read: {e}"), + } + } + if closed { + self.conns[i] = None; + } + while let Some((consumed, request)) = parse_request(i, &self.read_bufs[i]) { + self.requests.push(request); + self.read_bufs[i].drain(..consumed); + } + } + } + + pub fn respond(&mut self, request_index: usize, result_json: &str) { + let request = &self.requests[request_index]; + let body = format!(r#"{{"jsonrpc":"2.0","id":{},"result":{result_json}}}"#, request.id); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + let stream = self.conns[request.conn].as_mut().expect("respond on closed connection"); + let mut bytes = response.as_bytes(); + while !bytes.is_empty() { + match stream.write(bytes) { + Ok(n) => bytes = &bytes[n..], + Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, + Err(e) => panic!("write: {e}"), + } + } + } + + pub fn close_connection_of(&mut self, request_index: usize) { + self.conns[self.requests[request_index].conn] = None; + } +} + +fn parse_request(conn: usize, buf: &[u8]) -> Option<(usize, ElRequest)> { + let mut headers = [httparse::EMPTY_HEADER; 32]; + let mut request = httparse::Request::new(&mut headers); + let header_end = match request.parse(buf) { + Ok(httparse::Status::Complete(n)) => n, + _ => return None, + }; + let content_length: usize = headers + .iter() + .find(|h| h.name.eq_ignore_ascii_case("content-length")) + .and_then(|h| std::str::from_utf8(h.value).ok()?.trim().parse().ok()) + .expect("request without Content-Length"); + if buf.len() < header_end + content_length { + return None; + } + let authorization = headers + .iter() + .find(|h| h.name.eq_ignore_ascii_case("authorization")) + .map(|h| String::from_utf8(h.value.to_vec()).unwrap()); + + let body = String::from_utf8(buf[header_end..header_end + content_length].to_vec()).unwrap(); + let mut json = body.clone().into_bytes(); + let json = simd_json::to_borrowed_value(&mut json).expect("request body is JSON"); + let id = json.get("id").and_then(|v| v.as_u64()).expect("rpc id"); + let method = json.get("method").and_then(|v| v.as_str()).expect("rpc method").to_string(); + + Some((header_end + content_length, ElRequest { conn, id, method, authorization, body })) +} diff --git a/crates/engine/src/types.rs b/crates/engine_api/src/types.rs similarity index 100% rename from crates/engine/src/types.rs rename to crates/engine_api/src/types.rs diff --git a/crates/engine/testdata/empty_var_payload.ssz b/crates/engine_api/testdata/empty_var_payload.ssz similarity index 100% rename from crates/engine/testdata/empty_var_payload.ssz rename to crates/engine_api/testdata/empty_var_payload.ssz diff --git a/crates/engine/testdata/get_payload_tcache.bin b/crates/engine_api/testdata/get_payload_tcache.bin similarity index 100% rename from crates/engine/testdata/get_payload_tcache.bin rename to crates/engine_api/testdata/get_payload_tcache.bin diff --git a/crates/engine/testdata/large_extra_payload.ssz b/crates/engine_api/testdata/large_extra_payload.ssz similarity index 100% rename from crates/engine/testdata/large_extra_payload.ssz rename to crates/engine_api/testdata/large_extra_payload.ssz diff --git a/crates/engine/testdata/many_tx_payload.ssz b/crates/engine_api/testdata/many_tx_payload.ssz similarity index 100% rename from crates/engine/testdata/many_tx_payload.ssz rename to crates/engine_api/testdata/many_tx_payload.ssz diff --git a/crates/engine/testdata/sample_payload.ssz b/crates/engine_api/testdata/sample_payload.ssz similarity index 100% rename from crates/engine/testdata/sample_payload.ssz rename to crates/engine_api/testdata/sample_payload.ssz diff --git a/crates/engine/testdata/signed_block.ssz b/crates/engine_api/testdata/signed_block.ssz similarity index 100% rename from crates/engine/testdata/signed_block.ssz rename to crates/engine_api/testdata/signed_block.ssz diff --git a/crates/engine/testdata/signed_block_params.json b/crates/engine_api/testdata/signed_block_params.json similarity index 100% rename from crates/engine/testdata/signed_block_params.json rename to crates/engine_api/testdata/signed_block_params.json diff --git a/crates/engine/testdata/tx_multi.bin b/crates/engine_api/testdata/tx_multi.bin similarity index 100% rename from crates/engine/testdata/tx_multi.bin rename to crates/engine_api/testdata/tx_multi.bin diff --git a/crates/engine/testdata/tx_single.bin b/crates/engine_api/testdata/tx_single.bin similarity index 100% rename from crates/engine/testdata/tx_single.bin rename to crates/engine_api/testdata/tx_single.bin diff --git a/crates/engine/testdata/withdrawals.bin b/crates/engine_api/testdata/withdrawals.bin similarity index 100% rename from crates/engine/testdata/withdrawals.bin rename to crates/engine_api/testdata/withdrawals.bin diff --git a/crates/engine_api/tests/newpayload_alloc.rs b/crates/engine_api/tests/newpayload_alloc.rs new file mode 100644 index 00000000..61e696a2 --- /dev/null +++ b/crates/engine_api/tests/newpayload_alloc.rs @@ -0,0 +1,113 @@ +//! Pins the newPayload hot-path invariant: once every buffer is warm (scratch, +//! connection write buffer, JWT token cache, pending-request map), the SSZ→JSON +//! transcode + frame + enqueue path performs zero heap allocations. + +use std::{ + alloc::{GlobalAlloc, Layout, System}, + cell::Cell, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use silver_engine_api::{ + EngineClient, ReqKind, poll, send_new_payload, + test_el::{FakeEl, write_jwt}, +}; + +thread_local! { + static ALLOCATION_EVENTS: Cell = const { Cell::new(0) }; +} + +fn allocation_events() -> u64 { + ALLOCATION_EVENTS.with(Cell::get) +} + +struct CountingAllocator; + +unsafe impl GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + ALLOCATION_EVENTS.with(|c| c.set(c.get() + 1)); + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + ALLOCATION_EVENTS.with(|c| c.set(c.get() + 1)); + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + ALLOCATION_EVENTS.with(|c| c.set(c.get() + 1)); + unsafe { System.realloc(ptr, layout, new_size) } + } +} + +#[global_allocator] +static GLOBAL: CountingAllocator = CountingAllocator; + +const SIGNED_BLOCK_SSZ: &[u8] = include_bytes!("../testdata/signed_block.ssz"); +const NEW_PAYLOAD_VALID: &str = + r#"{"status":"VALID","latestValidHash":null,"validationError":null}"#; + +fn unix_secs() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() +} + +fn complete_round_trip(client: &mut EngineClient, el: &mut FakeEl, request_index: usize) { + let deadline = Instant::now() + Duration::from_secs(10); + let mut responded = false; + let mut done = false; + while !done { + assert!(Instant::now() < deadline, "timeout: newPayload round trip {request_index}"); + el.pump(); + if !responded && el.requests.len() > request_index { + assert_eq!(el.requests[request_index].method, "engine_newPayloadV4"); + el.respond(request_index, NEW_PAYLOAD_VALID); + responded = true; + } + poll(client, |kind, response| { + assert!(matches!(kind, ReqKind::NewPayload(_))); + response.expect("newPayload response"); + done = true; + }); + std::thread::sleep(Duration::from_millis(1)); + } +} + +#[test] +fn warm_new_payload_send_allocates_nothing() { + let dir = tempfile::tempdir().unwrap(); + let jwt_path = write_jwt(dir.path()); + let socket = dir.path().join("engine.sock"); + let mut el = FakeEl::uds(&socket); + let mut client = EngineClient::new_uds(&socket, jwt_path.to_str().unwrap(), 4); + + send_new_payload(&mut client, SIGNED_BLOCK_SSZ, [0u8; 32]).unwrap(); + complete_round_trip(&mut client, &mut el, 0); + let mut request_index = 1; + assert!(allocation_events() > 0, "counting allocator must observe the cold path"); + + // The JWT bearer token is cached per wall-clock second, so a warm send and + // the measured send must land in the same second for the token recompute + // to stay out of the measured window; retry on the rare rollover. + for _ in 0..5 { + let second = unix_secs(); + send_new_payload(&mut client, SIGNED_BLOCK_SSZ, [1u8; 32]).unwrap(); + complete_round_trip(&mut client, &mut el, request_index); + request_index += 1; + + let before = allocation_events(); + send_new_payload(&mut client, SIGNED_BLOCK_SSZ, [2u8; 32]).unwrap(); + let events = allocation_events() - before; + + complete_round_trip(&mut client, &mut el, request_index); + request_index += 1; + if unix_secs() == second { + assert_eq!(events, 0, "warm newPayload send performed {events} heap allocations"); + return; + } + } + panic!("wall clock crossed a second boundary on every attempt"); +} diff --git a/crates/httpcore/Cargo.toml b/crates/httpcore/Cargo.toml new file mode 100644 index 00000000..237e1c7a --- /dev/null +++ b/crates/httpcore/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "silver_httpcore" +edition.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +httparse.workspace = true +mio.workspace = true +tracing.workspace = true + +[dev-dependencies] +tempfile = "3" + +[lints] +workspace = true diff --git a/crates/httpcore/src/client.rs b/crates/httpcore/src/client.rs new file mode 100644 index 00000000..293c6bf3 --- /dev/null +++ b/crates/httpcore/src/client.rs @@ -0,0 +1,343 @@ +use std::io::{self, Write}; + +// 4096 covers any realistic HTTP response header block; once headers are +// parsed, reads are sized to exactly the remaining Content-Length. +const HEADER_READ_LEN: usize = 4096; + +pub struct ClientConnection { + write_buf: Vec, + write_pos: usize, + read_buf: Vec, + read_end: usize, + read_offset: usize, + response_header_end: usize, + response_total: usize, +} + +impl ClientConnection { + pub fn with_capacity(read_capacity: usize, write_capacity: usize) -> Self { + Self { + write_buf: Vec::with_capacity(write_capacity), + write_pos: 0, + read_buf: Vec::with_capacity(read_capacity), + read_end: 0, + read_offset: 0, + response_header_end: 0, + response_total: 0, + } + } + + pub fn begin_request(&mut self) -> &mut Vec { + debug_assert!( + self.pending_write().is_empty(), + "one request in flight per connection: previous request not fully written" + ); + self.write_buf.clear(); + self.write_pos = 0; + &mut self.write_buf + } + + pub fn pending_write(&self) -> &[u8] { + &self.write_buf[self.write_pos..] + } + + pub fn commit_write(&mut self, n: usize) { + debug_assert!(self.write_pos + n <= self.write_buf.len()); + self.write_pos += n; + } + + pub fn read_space(&mut self) -> &mut [u8] { + if self.read_offset != 0 && self.read_offset == self.read_end { + self.read_end = 0; + self.read_offset = 0; + } + let want = if self.response_total > 0 { + self.response_total - (self.read_end - self.read_offset) + } else { + HEADER_READ_LEN + }; + debug_assert!(want > 0, "complete response pending: take_response before reading more"); + if self.read_buf.len() < self.read_end + want { + self.read_buf.resize(self.read_end + want, 0); + } + &mut self.read_buf[self.read_end..self.read_end + want] + } + + pub fn commit_read(&mut self, n: usize) -> io::Result<()> { + debug_assert!(self.read_end + n <= self.read_buf.len()); + self.read_end += n; + if self.response_total == 0 { + if let Some((header_end, content_length)) = + parse_response_head(&self.read_buf[self.read_offset..self.read_end])? + { + self.response_header_end = header_end; + self.response_total = header_end + content_length; + } + } + Ok(()) + } + + pub fn take_response(&mut self) -> Option<&mut [u8]> { + if self.response_total == 0 || self.read_end - self.read_offset < self.response_total { + return None; + } + let start = self.read_offset + self.response_header_end; + let end = self.read_offset + self.response_total; + self.read_offset = end; + self.response_header_end = 0; + self.response_total = 0; + Some(&mut self.read_buf[start..end]) + } + + pub fn reset(&mut self) { + self.write_buf.clear(); + self.write_pos = 0; + self.read_end = 0; + self.read_offset = 0; + self.response_header_end = 0; + self.response_total = 0; + } +} + +// Returns (header_end, content_length) when headers are complete, None if +// partial. Content-Length framing only: a response without it is an error, +// chunked transfer encoding is unsupported. +fn parse_response_head(buf: &[u8]) -> io::Result> { + let mut headers = [httparse::EMPTY_HEADER; 32]; + let mut resp = httparse::Response::new(&mut headers); + let header_end = match resp.parse(buf) { + Ok(httparse::Status::Complete(n)) => n, + Ok(httparse::Status::Partial) => return Ok(None), + Err(e) => return Err(io::Error::new(io::ErrorKind::InvalidData, format!("httparse: {e}"))), + }; + match headers.iter().find(|h| h.name.eq_ignore_ascii_case("content-length")) { + Some(h) if !h.value.is_empty() && h.value.iter().all(|b| b.is_ascii_digit()) => { + let cl = h.value.iter().copied().fold(0usize, |acc, b| acc * 10 + (b - b'0') as usize); + Ok(Some((header_end, cl))) + } + Some(_) => Err(io::Error::new(io::ErrorKind::InvalidData, "invalid Content-Length")), + None => Err(io::Error::new(io::ErrorKind::InvalidData, "missing Content-Length")), + } +} + +pub fn frame_request( + out: &mut Vec, + host: &str, + body: &[u8], + authorization: Option<&str>, + keep_alive: bool, +) { + let connection = if keep_alive { "keep-alive" } else { "close" }; + match authorization { + Some(bearer) => write!( + out, + "POST / HTTP/1.1\r\nHost: {host}\r\nContent-Type: application/json\r\n\ + Content-Length: {len}\r\nAuthorization: {bearer}\r\nConnection: {connection}\r\n\r\n", + len = body.len(), + ), + None => write!( + out, + "POST / HTTP/1.1\r\nHost: {host}\r\nContent-Type: application/json\r\n\ + Content-Length: {len}\r\nConnection: {connection}\r\n\r\n", + len = body.len(), + ), + } + .unwrap(); + out.extend_from_slice(body); +} + +#[cfg(test)] +mod tests { + use super::*; + + const BODY: &[u8] = br#"{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}"#; + const BEARER: &str = "Bearer aGVhZGVy.cGF5bG9hZA.c2ln"; + + fn machine() -> ClientConnection { + ClientConnection::with_capacity(4096, 4096) + } + + fn make_response(body: &[u8]) -> Vec { + let mut buf = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .into_bytes(); + buf.extend_from_slice(body); + buf + } + + fn feed(conn: &mut ClientConnection, bytes: &[u8]) -> io::Result<()> { + let space = conn.read_space(); + let n = bytes.len().min(space.len()); + assert_eq!(n, bytes.len(), "test chunk exceeds offered read space"); + space[..n].copy_from_slice(bytes); + conn.commit_read(n) + } + + // Captured verbatim from the engine crate's `build_request_into` before the + // extraction (2026-08-17); the framed request must stay byte-identical. + #[test] + fn golden_request_bytes_keep_alive() { + let mut conn = machine(); + frame_request(conn.begin_request(), "localhost:8551", BODY, Some(BEARER), true); + let expected: Vec = [ + b"POST / HTTP/1.1\r\nHost: localhost:8551\r\nContent-Type: application/json\r\n\ + Content-Length: 59\r\nAuthorization: Bearer aGVhZGVy.cGF5bG9hZA.c2ln\r\n\ + Connection: keep-alive\r\n\r\n" + .as_ref(), + BODY, + ] + .concat(); + assert_eq!(conn.pending_write(), expected); + } + + #[test] + fn golden_request_bytes_connection_close() { + let mut conn = machine(); + frame_request(conn.begin_request(), "localhost:8551", BODY, Some(BEARER), false); + let expected: Vec = [ + b"POST / HTTP/1.1\r\nHost: localhost:8551\r\nContent-Type: application/json\r\n\ + Content-Length: 59\r\nAuthorization: Bearer aGVhZGVy.cGF5bG9hZA.c2ln\r\n\ + Connection: close\r\n\r\n" + .as_ref(), + BODY, + ] + .concat(); + assert_eq!(conn.pending_write(), expected); + } + + #[test] + fn frame_request_without_authorization_omits_header() { + let mut out = Vec::new(); + frame_request(&mut out, "localhost:8551", b"{}", None, true); + let text = String::from_utf8(out).unwrap(); + assert!(!text.contains("Authorization")); + assert!(text.contains("Content-Length: 2\r\n")); + } + + #[test] + fn request_drained_in_small_chunks() { + let mut conn = machine(); + frame_request(conn.begin_request(), "localhost:8551", BODY, Some(BEARER), true); + let expected = conn.pending_write().to_vec(); + + let mut wire = Vec::new(); + while !conn.pending_write().is_empty() { + let chunk_len = conn.pending_write().len().min(3); + wire.extend_from_slice(&conn.pending_write()[..chunk_len]); + conn.commit_write(chunk_len); + } + assert_eq!(wire, expected); + } + + #[test] + fn response_fed_one_byte_at_a_time() { + let mut conn = machine(); + let body = br#"{"jsonrpc":"2.0","id":1,"result":false}"#; + let response = make_response(body); + + for (i, byte) in response.iter().enumerate() { + assert!(conn.take_response().is_none(), "byte {i}"); + feed(&mut conn, &[*byte]).unwrap(); + } + assert_eq!(conn.take_response().unwrap(), body); + assert!(conn.take_response().is_none()); + } + + #[test] + fn headers_complete_body_incomplete_returns_none() { + let mut conn = machine(); + let mut response = make_response(br#"{"result":1}"#); + response.truncate(response.len() - 3); + feed(&mut conn, &response).unwrap(); + assert!(conn.take_response().is_none()); + feed(&mut conn, br#":1}"#).unwrap(); + assert_eq!(conn.take_response().unwrap(), br#"{"result":1}"#.as_ref()); + } + + #[test] + fn partial_headers_return_none_without_error() { + let mut conn = machine(); + feed(&mut conn, b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n").unwrap(); + assert!(conn.take_response().is_none()); + } + + #[test] + fn missing_content_length_is_error() { + let mut conn = machine(); + let err = feed(&mut conn, b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{}") + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + assert_eq!(err.to_string(), "missing Content-Length"); + } + + #[test] + fn invalid_content_length_is_error() { + let mut conn = machine(); + let err = feed(&mut conn, b"HTTP/1.1 200 OK\r\nContent-Length: abc\r\n\r\n{}").unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + assert_eq!(err.to_string(), "invalid Content-Length"); + } + + #[test] + fn body_larger_than_header_read_arrives_in_exact_sized_reads() { + let mut conn = machine(); + let body = vec![b'x'; 3 * HEADER_READ_LEN]; + let response = make_response(&body); + + let mut sent = 0; + while sent < response.len() { + assert!(conn.take_response().is_none()); + let space = conn.read_space(); + let n = space.len().min(response.len() - sent); + space[..n].copy_from_slice(&response[sent..sent + n]); + conn.commit_read(n).unwrap(); + sent += n; + } + assert_eq!(conn.take_response().unwrap(), body); + } + + #[test] + fn keep_alive_connection_serves_second_request() { + let mut conn = machine(); + for body in [br#"{"id":1}"#.as_ref(), br#"{"id":2}"#.as_ref()] { + frame_request(conn.begin_request(), "h", body, None, true); + while !conn.pending_write().is_empty() { + let n = conn.pending_write().len(); + conn.commit_write(n); + } + feed(&mut conn, &make_response(body)).unwrap(); + assert_eq!(conn.take_response().unwrap(), body); + } + } + + #[test] + fn two_connections_complete_out_of_order() { + let mut first = machine(); + let mut second = machine(); + frame_request(first.begin_request(), "h", br#"{"id":1}"#, None, true); + frame_request(second.begin_request(), "h", br#"{"id":2}"#, None, true); + + feed(&mut second, &make_response(br#"{"id":2,"result":"b"}"#)).unwrap(); + assert!(first.take_response().is_none()); + assert_eq!(second.take_response().unwrap(), br#"{"id":2,"result":"b"}"#.as_ref()); + + feed(&mut first, &make_response(br#"{"id":1,"result":"a"}"#)).unwrap(); + assert_eq!(first.take_response().unwrap(), br#"{"id":1,"result":"a"}"#.as_ref()); + } + + #[test] + fn reset_clears_partial_state_but_keeps_capacity() { + let mut conn = machine(); + frame_request(conn.begin_request(), "h", b"{}", None, true); + feed(&mut conn, b"HTTP/1.1 200 OK\r\nContent-Le").unwrap(); + + conn.reset(); + assert!(conn.pending_write().is_empty()); + assert!(conn.take_response().is_none()); + + feed(&mut conn, &make_response(b"{}")).unwrap(); + assert_eq!(conn.take_response().unwrap(), b"{}"); + } +} diff --git a/crates/httpcore/src/lib.rs b/crates/httpcore/src/lib.rs new file mode 100644 index 00000000..7efa53d4 --- /dev/null +++ b/crates/httpcore/src/lib.rs @@ -0,0 +1,7 @@ +mod client; +mod server; +mod stream; + +pub use client::{ClientConnection, frame_request}; +pub use server::{AfterResponse, ParsedRequest, ServerConnection, frame_response}; +pub use stream::{Bind, Listener, Stream}; diff --git a/crates/httpcore/src/server.rs b/crates/httpcore/src/server.rs new file mode 100644 index 00000000..540b0a8c --- /dev/null +++ b/crates/httpcore/src/server.rs @@ -0,0 +1,500 @@ +use std::io::{self, Write}; + +// Hard cap on the read buffer. Raw SSZ, uncompressed. 16 MiB matches observed +// production maximums (21 blobs × 128 KiB plus block fields). +const READ_BUF_MAX: usize = 16 << 20; +const READ_BUF_INIT: usize = 4096; +const WRITE_BUF_INIT: usize = 4096; + +pub struct ParsedRequest<'a> { + pub method: &'a str, + pub path: &'a str, + pub query: &'a str, + pub body: &'a [u8], + pub version: u8, + pub keep_alive: bool, +} + +impl<'a> ParsedRequest<'a> { + fn parse(buf: &'a [u8]) -> Option<(usize, Self)> { + let mut headers = [httparse::EMPTY_HEADER; 64]; + let mut req = httparse::Request::new(&mut headers); + let headers_end = match req.parse(buf) { + Ok(httparse::Status::Complete(n)) => n, + _ => return None, + }; + let method = req.method?; + let raw_path = req.path?; + let (path, query) = raw_path.split_once('?').unwrap_or((raw_path, "")); + let version = req.version?; + let keep_alive = version == 1 && + !headers.iter().any(|h| { + h.name.eq_ignore_ascii_case("connection") && h.value.eq_ignore_ascii_case(b"close") + }); + let content_length: usize = + match headers.iter().find(|h| h.name.eq_ignore_ascii_case("content-length")) { + None => 0, + Some(h) => std::str::from_utf8(h.value).ok().and_then(|v| v.trim().parse().ok())?, + }; + let total = headers_end + content_length; + if buf.len() < total { + return None; + } + Some((total, Self { + method, + path, + query, + body: &buf[headers_end..total], + version, + keep_alive, + })) + } +} + +#[derive(Debug, PartialEq)] +#[must_use] +pub enum AfterResponse { + Close, + ResponsePending, + AwaitRequest, +} + +pub struct ServerConnection { + read_buf: Vec, + read_pos: usize, + read_end: usize, + write_buf: Vec, + write_pos: usize, + keep_alive: bool, +} + +impl ServerConnection { + pub fn new() -> Self { + Self { + read_buf: vec![0u8; READ_BUF_INIT], + read_pos: 0, + read_end: 0, + write_buf: Vec::with_capacity(WRITE_BUF_INIT), + write_pos: 0, + keep_alive: true, + } + } + + pub fn read_space(&mut self) -> io::Result<&mut [u8]> { + // Compact the partial tail to the front: without this, a long-lived + // pipelined keep-alive connection whose buffer never fully drains + // creeps read_end toward the cap and spuriously rejects small requests. + if self.read_pos > 0 { + self.read_buf.copy_within(self.read_pos..self.read_end, 0); + self.read_end -= self.read_pos; + self.read_pos = 0; + } + if self.read_end == READ_BUF_MAX { + return Err(io::Error::new(io::ErrorKind::InvalidData, "request too large")); + } + if self.read_end == self.read_buf.len() { + self.read_buf.resize((self.read_buf.len() * 2).min(READ_BUF_MAX), 0); + } + Ok(&mut self.read_buf[self.read_end..]) + } + + pub fn commit_read(&mut self, n: usize) { + debug_assert!(self.read_end + n <= self.read_buf.len()); + self.read_end += n; + } + + pub fn dispatch, &mut Vec)>(&mut self, handler: &F) -> bool { + let Some((consumed, req)) = + ParsedRequest::parse(&self.read_buf[self.read_pos..self.read_end]) + else { + return false; + }; + if req.version != 1 { + tracing::warn!("rejecting HTTP/1.0 request"); + self.keep_alive = false; + frame_response(&mut self.write_buf, "505 HTTP Version Not Supported", None, b""); + } else { + self.keep_alive = req.keep_alive; + handler(&req, &mut self.write_buf); + } + self.read_pos += consumed; + if self.read_pos == self.read_end { + self.read_pos = 0; + self.read_end = 0; + } + true + } + + pub fn pending_write(&self) -> &[u8] { + &self.write_buf[self.write_pos..] + } + + pub fn commit_write(&mut self, n: usize) { + debug_assert!(self.write_pos + n <= self.write_buf.len()); + self.write_pos += n; + } + + pub fn after_response, &mut Vec)>( + &mut self, + handler: &F, + ) -> AfterResponse { + debug_assert!(self.write_pos == self.write_buf.len()); + if !self.keep_alive { + return AfterResponse::Close; + } + self.write_buf.clear(); + self.write_pos = 0; + // A request pipelined behind the one just answered is already in + // read_buf — the transport will never feed those bytes again, so it + // must be dispatched here or it never will be. + if self.dispatch(handler) { + AfterResponse::ResponsePending + } else { + AfterResponse::AwaitRequest + } + } +} + +impl Default for ServerConnection { + fn default() -> Self { + Self::new() + } +} + +pub fn frame_response(out: &mut Vec, status: &str, content_type: Option<&str>, body: &[u8]) { + match content_type { + Some(ct) => write!( + out, + "HTTP/1.1 {status}\r\nContent-Type: {ct}\r\nContent-Length: {}\r\n\r\n", + body.len() + ), + None => write!(out, "HTTP/1.1 {status}\r\nContent-Length: {}\r\n\r\n", body.len()), + } + .unwrap(); + out.extend_from_slice(body); +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + + use super::*; + + fn get_req(path: &str, version: &str) -> Vec { + format!("GET {path} {version}\r\nHost: localhost\r\n\r\n").into_bytes() + } + + fn feed(conn: &mut ServerConnection, bytes: &[u8]) { + let space = conn.read_space().unwrap(); + space[..bytes.len()].copy_from_slice(bytes); + conn.commit_read(bytes.len()); + } + + fn feed_all(conn: &mut ServerConnection, mut bytes: &[u8]) { + while !bytes.is_empty() { + let space = conn.read_space().unwrap(); + let n = space.len().min(bytes.len()); + space[..n].copy_from_slice(&bytes[..n]); + conn.commit_read(n); + bytes = &bytes[n..]; + } + } + + fn fill_with_junk_until_reject(conn: &mut ServerConnection) -> io::Error { + loop { + match conn.read_space() { + Ok(space) => { + let n = space.len(); + space.fill(b'j'); + conn.commit_read(n); + } + Err(e) => return e, + } + assert!(!conn.dispatch(&|_, _: &mut Vec| { + panic!("incomplete request must not dispatch") + })); + } + } + + fn drain(conn: &mut ServerConnection) -> Vec { + let out = conn.pending_write().to_vec(); + conn.commit_write(out.len()); + out + } + + fn echo_path(req: &ParsedRequest<'_>, out: &mut Vec) { + frame_response(out, "200 OK", None, req.path.as_bytes()); + } + + #[test] + fn parse_http11_defaults_keep_alive() { + let req = get_req("/eth/v1/node/identity", "HTTP/1.1"); + let (_, r) = ParsedRequest::parse(&req).unwrap(); + assert_eq!(r.path, "/eth/v1/node/identity"); + assert!(r.keep_alive); + } + + #[test] + fn parse_http11_connection_close() { + let req = b"GET /metrics HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"; + let (_, r) = ParsedRequest::parse(req).unwrap(); + assert_eq!(r.path, "/metrics"); + assert!(!r.keep_alive); + } + + #[test] + fn parse_http10_defaults_close() { + let req = get_req("/", "HTTP/1.0"); + let (_, r) = ParsedRequest::parse(&req).unwrap(); + assert!(!r.keep_alive); + } + + #[test] + fn parse_partial_returns_none() { + assert!(ParsedRequest::parse(b"GET /eth/v1/node/identity HTTP/1.1\r\n").is_none()); + } + + #[test] + fn parse_query_string_split() { + let req = get_req("/eth/v1/beacon/states/head/validators?status=active", "HTTP/1.1"); + let (_, r) = ParsedRequest::parse(&req).unwrap(); + assert_eq!(r.path, "/eth/v1/beacon/states/head/validators"); + assert_eq!(r.query, "status=active"); + } + + #[test] + fn parse_post_body_buffered() { + let body = b"{\"slot\":\"1\"}"; + let req = format!( + "POST /eth/v1/beacon/blocks HTTP/1.1\r\nHost: localhost\r\nContent-Length: {}\r\n\r\n", + body.len() + ); + let mut buf = req.into_bytes(); + // incomplete — body not yet arrived + assert!(ParsedRequest::parse(&buf).is_none()); + buf.extend_from_slice(body); + let (consumed, r) = ParsedRequest::parse(&buf).unwrap(); + assert_eq!(r.method, "POST"); + assert_eq!(r.body, body.as_ref()); + assert_eq!(consumed, buf.len()); + } + + #[test] + fn parse_returns_consumed_byte_count() { + let req1 = b"GET /metrics HTTP/1.1\r\nHost: localhost\r\n\r\n"; + let req2 = b"GET /eth/v1/node/identity HTTP/1.1\r\nHost: localhost\r\n\r\n"; + let mut buf = req1.to_vec(); + buf.extend_from_slice(req2); + let (consumed, r) = ParsedRequest::parse(&buf).unwrap(); + assert_eq!(r.path, "/metrics"); + assert_eq!(consumed, req1.len()); + let (_, r2) = ParsedRequest::parse(&buf[consumed..]).unwrap(); + assert_eq!(r2.path, "/eth/v1/node/identity"); + } + + #[test] + fn parse_invalid_content_length_returns_none() { + let req = b"POST /foo HTTP/1.1\r\nHost: localhost\r\nContent-Length: abc\r\n\r\n"; + assert!(ParsedRequest::parse(req).is_none()); + } + + #[test] + fn frame_response_without_content_type_omits_header() { + let mut out = Vec::new(); + frame_response(&mut out, "404 Not Found", None, b""); + assert_eq!(out, b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n"); + } + + #[test] + fn frame_response_content_length_matches_body() { + let mut out = Vec::new(); + frame_response(&mut out, "200 OK", Some("application/json"), b"{\"data\":1}"); + assert_eq!( + out, + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 10\r\n\r\n{\"data\":1}" + ); + } + + #[test] + fn dispatch_http10_writes_version_not_supported_then_closes() { + let mut conn = ServerConnection::new(); + feed(&mut conn, b"GET /metrics HTTP/1.0\r\nHost: localhost\r\n\r\n"); + + assert!(conn.dispatch(&|_, out: &mut Vec| { + out.extend_from_slice(b"should not appear"); + })); + assert_eq!( + conn.pending_write(), + b"HTTP/1.1 505 HTTP Version Not Supported\r\nContent-Length: 0\r\n\r\n" + ); + + drain(&mut conn); + assert_eq!(conn.after_response(&echo_path), AfterResponse::Close); + } + + #[test] + fn connection_close_request_closes_after_response() { + let mut conn = ServerConnection::new(); + feed(&mut conn, b"GET /metrics HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"); + + assert!(conn.dispatch(&echo_path)); + drain(&mut conn); + assert_eq!(conn.after_response(&echo_path), AfterResponse::Close); + } + + #[test] + fn request_fed_one_byte_at_a_time() { + let mut conn = ServerConnection::new(); + let req = get_req("/metrics", "HTTP/1.1"); + + for (i, byte) in req.iter().enumerate() { + feed(&mut conn, &[*byte]); + assert_eq!(conn.dispatch(&echo_path), i == req.len() - 1, "byte {i}"); + } + assert_eq!(conn.pending_write(), b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\n/metrics"); + + drain(&mut conn); + assert_eq!(conn.after_response(&echo_path), AfterResponse::AwaitRequest); + } + + #[test] + fn pipelined_requests_split_across_feeds_respond_in_order() { + let mut conn = ServerConnection::new(); + + feed(&mut conn, b"GET /first HTTP/1.1\r\nHost: x\r\n\r\nGET /sec"); + assert!(conn.dispatch(&echo_path)); + assert_eq!(drain(&mut conn), b"HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\n/first"); + assert_eq!(conn.after_response(&echo_path), AfterResponse::AwaitRequest); + + feed(&mut conn, b"ond HTTP/1.1\r\nHost: x\r\n\r\n"); + assert!(conn.dispatch(&echo_path)); + assert_eq!(drain(&mut conn), b"HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\n/second"); + assert_eq!(conn.after_response(&echo_path), AfterResponse::AwaitRequest); + } + + #[test] + fn buffered_pipelined_request_dispatched_after_drain() { + let mut conn = ServerConnection::new(); + let calls = RefCell::new(Vec::new()); + let handler = |req: &ParsedRequest<'_>, out: &mut Vec| { + calls.borrow_mut().push(req.path.to_string()); + echo_path(req, out); + }; + + feed( + &mut conn, + b"GET /first HTTP/1.1\r\nHost: x\r\n\r\nGET /second HTTP/1.1\r\nHost: x\r\n\r\n", + ); + assert!(conn.dispatch(&handler)); + assert_eq!(*calls.borrow(), ["/first"]); + + let mut written = Vec::new(); + while !conn.pending_write().is_empty() { + let chunk_len = conn.pending_write().len().min(3); + written.extend_from_slice(&conn.pending_write()[..chunk_len]); + conn.commit_write(chunk_len); + assert_eq!(*calls.borrow(), ["/first"], "no dispatch mid-drain"); + } + assert_eq!(written, b"HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\n/first"); + + assert_eq!(conn.after_response(&handler), AfterResponse::ResponsePending); + assert_eq!(*calls.borrow(), ["/first", "/second"]); + assert_eq!(conn.pending_write(), b"HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\n/second"); + } + + #[test] + fn read_space_exhausted_rejects_request_too_large() { + let mut conn = ServerConnection::new(); + feed( + &mut conn, + b"POST /big HTTP/1.1\r\nHost: localhost\r\nContent-Length: 33554432\r\n\r\n", + ); + + let err = fill_with_junk_until_reject(&mut conn); + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + assert_eq!(err.to_string(), "request too large"); + } + + #[test] + fn body_just_over_cap_rejects_with_identical_error() { + let mut conn = ServerConnection::new(); + let header = format!( + "POST /big HTTP/1.1\r\nHost: localhost\r\nContent-Length: {READ_BUF_MAX}\r\n\r\n" + ); + feed(&mut conn, header.as_bytes()); + + let err = fill_with_junk_until_reject(&mut conn); + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + assert_eq!(err.to_string(), "request too large"); + } + + #[test] + fn body_near_cap_dispatches() { + let mut conn = ServerConnection::new(); + let body_len = READ_BUF_MAX - 128; + let header = + format!("POST /big HTTP/1.1\r\nHost: localhost\r\nContent-Length: {body_len}\r\n\r\n"); + feed_all(&mut conn, header.as_bytes()); + let chunk = vec![b'b'; 1 << 16]; + let mut remaining = body_len; + while remaining > 0 { + let n = remaining.min(chunk.len()); + feed_all(&mut conn, &chunk[..n]); + remaining -= n; + } + + let seen = RefCell::new(0usize); + assert!(conn.dispatch(&|req: &ParsedRequest<'_>, out: &mut Vec| { + *seen.borrow_mut() = req.body.len(); + assert!(req.body.iter().all(|&b| b == b'b')); + frame_response(out, "200 OK", None, b""); + })); + assert_eq!(*seen.borrow(), body_len); + } + + #[test] + fn pipelined_keep_alive_partial_tails_never_creep_into_cap() { + let mut conn = ServerConnection::new(); + let mut request = b"POST /r HTTP/1.1\r\nHost: x\r\nContent-Length: 65536\r\n\r\n".to_vec(); + request.extend_from_slice(&vec![b'p'; 65536]); + let split = 16; + + // Feed twice the cap in total; every dispatch leaves a partial + // successor in the buffer, so the pre-compaction offsets would reach + // READ_BUF_MAX about halfway through and reject with "request too + // large". + let rounds = 2 * READ_BUF_MAX / request.len(); + feed_all(&mut conn, &request[..split]); + for _ in 0..rounds { + feed_all(&mut conn, &request[split..]); + feed_all(&mut conn, &request[..split]); + assert!(conn.dispatch(&echo_path)); + drain(&mut conn); + assert_eq!(conn.after_response(&echo_path), AfterResponse::AwaitRequest); + } + } + + #[test] + fn request_split_across_growth_boundary_not_corrupted() { + let mut conn = ServerConnection::new(); + let body: Vec = (0..6000u32).map(|i| (i % 251) as u8).collect(); + let mut request = + format!("POST /grow HTTP/1.1\r\nHost: x\r\nContent-Length: {}\r\n\r\n", body.len()) + .into_bytes(); + let header_len = request.len(); + request.extend_from_slice(&body); + + feed_all(&mut conn, &request[..READ_BUF_INIT]); + assert!( + !conn.dispatch(&|_, _: &mut Vec| panic!("incomplete request must not dispatch")) + ); + feed_all(&mut conn, &request[READ_BUF_INIT..]); + + let seen = RefCell::new(Vec::new()); + assert!(conn.dispatch(&|req: &ParsedRequest<'_>, out: &mut Vec| { + seen.borrow_mut().extend_from_slice(req.body); + frame_response(out, "200 OK", None, b""); + })); + assert_eq!(*seen.borrow(), request[header_len..]); + } +} diff --git a/crates/httpcore/src/stream.rs b/crates/httpcore/src/stream.rs new file mode 100644 index 00000000..e63eda53 --- /dev/null +++ b/crates/httpcore/src/stream.rs @@ -0,0 +1,293 @@ +use std::{ + io::{self, Read, Write}, + net::SocketAddr, + path::{Path, PathBuf}, +}; + +use mio::{ + Interest, Registry, Token, + event::Source, + net::{TcpListener, TcpStream, UnixListener, UnixStream}, +}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Bind { + Tcp(SocketAddr), + Unix(PathBuf), +} + +impl Bind { + pub fn parse(text: &str) -> Self { + match text.parse() { + Ok(addr) => Self::Tcp(addr), + Err(_) => Self::Unix(PathBuf::from(text)), + } + } +} + +pub enum Listener { + Tcp(TcpListener), + Unix(UnixListener), +} + +impl Listener { + pub fn bind(bind: &Bind) -> io::Result { + match bind { + Bind::Tcp(addr) => TcpListener::bind(*addr).map(Self::Tcp), + Bind::Unix(path) => UnixListener::bind(path).map(Self::Unix), + } + } + + pub fn accept(&self) -> io::Result { + match self { + Self::Tcp(listener) => { + let (stream, peer) = listener.accept()?; + tracing::info!("accepted connection from {peer}"); + Ok(Stream::Tcp(stream)) + } + Self::Unix(listener) => { + let (stream, _) = listener.accept()?; + tracing::info!("accepted connection on unix socket"); + Ok(Stream::Uds(stream)) + } + } + } + + /// The resolved bind: for TCP the actual listening address (a port-0 bind + /// reports the ephemeral port the OS assigned), for Unix the socket path. + pub fn local_addr(&self) -> Bind { + match self { + Self::Tcp(listener) => Bind::Tcp(listener.local_addr().expect("tcp local_addr")), + Self::Unix(listener) => Bind::Unix( + listener + .local_addr() + .ok() + .and_then(|addr| addr.as_pathname().map(Path::to_path_buf)) + .expect("unix listener bound to a path"), + ), + } + } +} + +impl Source for Listener { + fn register( + &mut self, + registry: &Registry, + token: Token, + interests: Interest, + ) -> io::Result<()> { + match self { + Self::Tcp(l) => l.register(registry, token, interests), + Self::Unix(l) => l.register(registry, token, interests), + } + } + + fn reregister( + &mut self, + registry: &Registry, + token: Token, + interests: Interest, + ) -> io::Result<()> { + match self { + Self::Tcp(l) => l.reregister(registry, token, interests), + Self::Unix(l) => l.reregister(registry, token, interests), + } + } + + fn deregister(&mut self, registry: &Registry) -> io::Result<()> { + match self { + Self::Tcp(l) => l.deregister(registry), + Self::Unix(l) => l.deregister(registry), + } + } +} + +pub enum Stream { + Tcp(TcpStream), + Uds(UnixStream), +} + +impl Stream { + pub fn connect_tcp(addr: SocketAddr) -> io::Result { + Ok(Self::Tcp(TcpStream::connect(addr)?)) + } + + pub fn connect_uds(path: &Path) -> io::Result { + Ok(Self::Uds(UnixStream::connect(path)?)) + } + + /// After the writable event that ends a non-blocking connect, distinguishes + /// success from failure: TCP has a peer address only once connected; a Unix + /// socket reports connect failure through SO_ERROR (mio's readiness flags + /// are not reliable for it). + pub fn connect_complete(&self) -> io::Result<()> { + match self { + Self::Tcp(s) => s.peer_addr().map(|_| ()), + Self::Uds(s) => match s.take_error()? { + Some(e) => Err(e), + None => Ok(()), + }, + } + } +} + +impl Read for Stream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Self::Tcp(s) => s.read(buf), + Self::Uds(s) => s.read(buf), + } + } +} + +impl Write for Stream { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + Self::Tcp(s) => s.write(buf), + Self::Uds(s) => s.write(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Self::Tcp(s) => s.flush(), + Self::Uds(s) => s.flush(), + } + } +} + +impl Source for Stream { + fn register( + &mut self, + registry: &Registry, + token: Token, + interests: Interest, + ) -> io::Result<()> { + match self { + Self::Tcp(s) => s.register(registry, token, interests), + Self::Uds(s) => s.register(registry, token, interests), + } + } + + fn reregister( + &mut self, + registry: &Registry, + token: Token, + interests: Interest, + ) -> io::Result<()> { + match self { + Self::Tcp(s) => s.reregister(registry, token, interests), + Self::Uds(s) => s.reregister(registry, token, interests), + } + } + + fn deregister(&mut self, registry: &Registry) -> io::Result<()> { + match self { + Self::Tcp(s) => s.deregister(registry), + Self::Uds(s) => s.deregister(registry), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::client::{ClientConnection, frame_request}; + + #[test] + fn parse_socket_addr_is_tcp() { + assert_eq!(Bind::parse("0.0.0.0:5051"), Bind::Tcp("0.0.0.0:5051".parse().unwrap())); + assert_eq!(Bind::parse("127.0.0.1:0"), Bind::Tcp("127.0.0.1:0".parse().unwrap())); + assert_eq!(Bind::parse("[::1]:5051"), Bind::Tcp("[::1]:5051".parse().unwrap())); + } + + #[test] + fn parse_non_addr_is_unix_path() { + assert_eq!(Bind::parse("/run/beacon.sock"), Bind::Unix("/run/beacon.sock".into())); + assert_eq!(Bind::parse("beacon.sock"), Bind::Unix("beacon.sock".into())); + // Hostnames don't parse as SocketAddr (no resolution here), so they + // fall through to a path. + assert_eq!(Bind::parse("localhost:5051"), Bind::Unix("localhost:5051".into())); + } + + #[test] + fn tcp_listener_reports_ephemeral_port() { + let listener = Listener::bind(&Bind::parse("127.0.0.1:0")).unwrap(); + let Bind::Tcp(addr) = listener.local_addr() else { panic!("tcp bind") }; + assert_ne!(addr.port(), 0); + } + + #[test] + fn unix_listener_reports_bound_path() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("api.sock"); + let listener = Listener::bind(&Bind::Unix(path.clone())).unwrap(); + assert_eq!(listener.local_addr(), Bind::Unix(path)); + } + + #[test] + fn uds_pair_round_trip_through_client_connection() { + let (client_half, mut server_half) = UnixStream::pair().unwrap(); + let mut stream = Stream::Uds(client_half); + let mut conn = ClientConnection::with_capacity(4096, 4096); + + let body = br#"{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":7}"#; + frame_request(conn.begin_request(), "localhost", body, Some("Bearer t.t.t"), true); + while !conn.pending_write().is_empty() { + match stream.write(conn.pending_write()) { + Ok(n) => conn.commit_write(n), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, + Err(e) => panic!("write: {e}"), + } + } + + let mut request = vec![0u8; 4096]; + let n = blocking_read(&mut server_half, &mut request); + let request = String::from_utf8(request[..n].to_vec()).unwrap(); + assert!(request.starts_with("POST / HTTP/1.1\r\n")); + assert!(request.contains("Authorization: Bearer t.t.t\r\n")); + assert!(request.ends_with(std::str::from_utf8(body).unwrap())); + + let response_body = br#"{"jsonrpc":"2.0","id":7,"result":false}"#; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{}", + response_body.len(), + std::str::from_utf8(response_body).unwrap() + ); + blocking_write(&mut server_half, response.as_bytes()); + + loop { + if let Some(got) = conn.take_response() { + assert_eq!(got, response_body); + break; + } + match stream.read(conn.read_space()) { + Ok(n) => conn.commit_read(n).unwrap(), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, + Err(e) => panic!("read: {e}"), + } + } + } + + fn blocking_read(stream: &mut UnixStream, buf: &mut [u8]) -> usize { + use std::io::Read as _; + loop { + match stream.read(buf) { + Ok(n) => return n, + Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, + Err(e) => panic!("read: {e}"), + } + } + } + + fn blocking_write(stream: &mut UnixStream, mut bytes: &[u8]) { + use std::io::Write as _; + while !bytes.is_empty() { + match stream.write(bytes) { + Ok(n) => bytes = &bytes[n..], + Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, + Err(e) => panic!("write: {e}"), + } + } + } +} diff --git a/docs/adr/0001-single-api-tile.md b/docs/adr/0001-single-api-tile.md new file mode 100644 index 00000000..425964a6 --- /dev/null +++ b/docs/adr/0001-single-api-tile.md @@ -0,0 +1,26 @@ +--- +status: proposed +--- + +# One tile hosts all API access + +Every tile is an OS thread pinned to a dedicated CPU core, and API traffic — +serving the beacon API, calling the engine API — is latency-tolerant work +dominated by network round-trips that cannot justify two pinned cores. All API +access is consolidated into a single `client_server` tile hosting two +transport-free crates: `beacon_api` (HTTP server) and `engine_api` (HTTP +client, renamed from `engine`). Hosted crates are hardcoded and composed by +plain function calls in the tile's `loop_body` — no plugin registry, no +hosting trait; adding a future hosted crate (e.g. a builder-API client or a +`health`/`log_tail` endpoint family) edits the tile, which is a deliberate, +cheap cost. The spine contract is unchanged: producers and consumers of +`engine_reqs`/`engine_resps`/`engine_health` see no difference. + +## Considered options + +Separate tiles per API surface (status quo — wastes a core per surface); a +`Hosted` trait + registry (speculative generality for exactly two crates); +per-crate transport ownership behind a port trait (generics leak into every +hosted crate's signatures). Four independent designs were produced and +compared; see `.local/client-server-design.md` (untracked design notes) for +the full comparison. diff --git a/docs/adr/0002-hand-rolled-http.md b/docs/adr/0002-hand-rolled-http.md new file mode 100644 index 00000000..a3593756 --- /dev/null +++ b/docs/adr/0002-hand-rolled-http.md @@ -0,0 +1,21 @@ +--- +status: proposed +--- + +# Hand-rolled HTTP over mio; no async runtime, no TLS + +API I/O uses the same idiom as the rest of the node: non-blocking mio polled +from a busy-poll loop with `httparse` framing — one shared connection state +machine (crate `httpcore`) serving both roles, server and client — rather +than hyper/axum/reqwest and the tokio runtime they drag in. The node has no +async runtime and will not grow one for its coldest path; the machine already +existed twice (engine `http.rs` and the beacon_api prototype, plus a dead +474-line UDS copy) and, once shared, is small and testable at the byte level. + +Transports are a closed set we control, so they are an enum +(`Tcp | Uds`), not a trait. Unix sockets are supported on both sides: the +beacon_api server bind and the execution endpoint. TLS is a non-goal — all +API connections run over trusted local LAN or VPN. Auth is protocol-layer, +not transport-layer: `engine_api` owns the JWT Authorization header; UDS +relies on socket path permissions, and JWT-over-UDS can be added later as an +`engine_api` config flag without touching the transport layer. diff --git a/docs/adr/0003-dispatch-asymmetry.md b/docs/adr/0003-dispatch-asymmetry.md new file mode 100644 index 00000000..39b87de5 --- /dev/null +++ b/docs/adr/0003-dispatch-asymmetry.md @@ -0,0 +1,21 @@ +--- +status: proposed +--- + +# Dispatch: table for server routes, enum match for client methods + +Beacon-api request routing is a const data table — (method, parameterised +path pattern) → handler function, compiled to segments at init and linearly +scanned. Engine-api call dispatch stays a Rust `match` on closed enums +(`EngineReq` inbound, `ReqKind` on completion). The asymmetry is deliberate: +the server-side endpoint set is open and keyed by runtime wire strings, so a +table earns its keep; the client-side protocol set is closed and minted by +us, where a match is already a compile-time-exhaustive jump table, and a +runtime table would force type erasure over encoders with genuinely +different shapes (TCache handles, the hand-written newPayload envelope), +trading compile errors for runtime failures. + +Do not "fix" this inconsistency by making the client side table-driven: four +independently-produced designs each converged on exactly this split. The +governing principle, which also chose the transport enum in ADR-0002: +**closed set we control → enum; open set from the wire → table.** diff --git a/docs/adr/0004-sync-materialized-api.md b/docs/adr/0004-sync-materialized-api.md new file mode 100644 index 00000000..63e1f0c1 --- /dev/null +++ b/docs/adr/0004-sync-materialized-api.md @@ -0,0 +1,23 @@ +--- +status: proposed +--- + +# Synchronous handlers, materialized responses, no streaming + +Beacon-api handlers are synchronous compute — no I/O, no blocking — invoked +only once a request has fully arrived; responses are materialized in the +connection's write buffer and drained incrementally. All transport pumps are +non-blocking (`poll(Duration::ZERO)`), so serving and engine traffic +interleave per readiness event: a slow API consumer never stalls engine +calls, and vice versa. + +This holds for the whole surface v1 targets: verified against the +beacon-APIs spec and five validator clients (see +`.local/beacon-api-vc-surface.md`, untracked), nothing a validator client +requires streams or long-polls except the optional `/eth/v1/events` SSE +stream, which every surveyed client can replace with polling. v1 answers it +with a clean 404 and tolerates client reconnect retries. If subscriptions +are ever wanted, they may be served out-of-process (e.g. a circular-buffer +export read by a separate serving process) rather than by adding streaming +here. Endpoints whose response cannot be materialized in a bounded buffer +are out of scope by construction; revisit this ADR before accepting one. diff --git a/docs/spine-message-flow.md b/docs/spine-message-flow.md index 25b9eb4d..a86ba26b 100644 --- a/docs/spine-message-flow.md +++ b/docs/spine-message-flow.md @@ -8,7 +8,8 @@ them (see [tcaches](#tcaches)). The tiles: **Network** (QUIC + discv5), **Control** (`PeerManager` + `SyncEngine` + `GossipHandler` — gossipsub decode/encode runs in-tile, not as its own tile), **BeaconState** (state transition + fork choice), **Storage** (disk + backfill), -**Engine** (EL / engine API). +**ClientServer** (hosting the `engine_api` client and the `beacon_api` server; the +server talks HTTP only, so it has no spine edges of its own). ```mermaid flowchart LR @@ -16,7 +17,7 @@ flowchart LR CTL["Control
PeerManager + SyncEngine + GossipHandler"] BS["BeaconState
state · fork choice"] ST["Storage
disk · backfill"] - EN["Engine
EL / engine API"] + EN["ClientServer
engine_api client · beacon_api server"] %% ---- inbound ---- NET -.->|"incoming_gossip (tcache)"| CTL @@ -69,8 +70,8 @@ output is `new_gossip`). The gossip handler's other traffic is in-tile, not on t spine: its `PeerEvent`s (gossipsub scoring/misbehaviour) go straight to the `PeerManager`, `PeerControl` is forwarded to the handler directly, and its fork digest is set from the `Status` Control already consumes. `engine_health` is omitted -from the diagram: Engine produces it but no tile currently consumes it. Both -Storage↔Engine edges carry only the `GetBlobs` variants (EL-mempool blob fetch); the +from the diagram: ClientServer produces it but no tile currently consumes it. Both +Storage↔ClientServer edges carry only the `GetBlobs` variants (EL-mempool blob fetch); the queues are broadcast, so Storage sees every `EngineResp` and ignores the rest. ## Spine queues @@ -87,9 +88,9 @@ queues are broadcast, so Storage sees every `EngineResp` and ignores the rest. | `sync_target` | `SyncUpdate` | Control | BeaconState, Storage | inline | | `replay_blocks` | `ReplayBlock` | Storage | BeaconState | ref → `replay_blocks` tcache | | `syncing_strategy` | `SyncingStrategy` | Control | Storage | inline | -| `engine_reqs` | `EngineReq` | BeaconState, Storage _(GetBlobs)_ | Engine | refs → `ssz_gossip` / `incoming_rpc`; GetBlobs inline | -| `engine_resps` | `EngineResp` | Engine | BeaconState, Storage _(GetBlobs)_ | ref → `incoming_engine_resp` | -| `engine_health` | `EngineHealthEvent` | Engine | _none (currently unconsumed)_ | inline | +| `engine_reqs` | `EngineReq` | BeaconState, Storage _(GetBlobs)_ | ClientServer | refs → `ssz_gossip` / `incoming_rpc`; GetBlobs inline | +| `engine_resps` | `EngineResp` | ClientServer | BeaconState, Storage _(GetBlobs)_ | ref → `incoming_engine_resp` | +| `engine_health` | `EngineHealthEvent` | ClientServer | _none (currently unconsumed)_ | inline | ## TCaches @@ -98,12 +99,12 @@ Bulk-byte rings that the queue messages reference, so payloads cross tiles witho | TCache | Producer | Consumer(s) | Payload | |--------|----------|-------------|---------| | `incoming_gossip` | Network | Control _(gossip)_ | raw gossipsub protobuf from the wire | -| `ssz_gossip` | Control _(gossip)_ | BeaconState, Storage (live + persist), Engine | decompressed gossip SSZ | +| `ssz_gossip` | Control _(gossip)_ | BeaconState, Storage (live + persist), ClientServer | decompressed gossip SSZ | | `outgoing_gossip` | Control _(gossip)_ | Network | gossip protobuf: mcache copies of incoming messages, local publishes, IDONTWANT/IWANT control frames | -| `incoming_rpc` | Network | BeaconState, Storage (live + persist), Engine, Control (column republish) | RPC response bodies (BeaconBlock / DataColumnSidecar) | +| `incoming_rpc` | Network | BeaconState, Storage (live + persist), ClientServer, Control (column republish) | RPC response bodies (BeaconBlock / DataColumnSidecar) | | `outgoing_rpc` _(multi-producer)_ | Control, Storage | Network | RPC request bodies (we ask) + served response bodies (we answer) | | `replay_blocks` | Storage | BeaconState | persisted block SSZ replayed at startup | -| `incoming_engine_resp` | Engine | BeaconState, Storage (GetBlobs) | EL responses (payloads, blobs, bodies) | +| `incoming_engine_resp` | ClientServer | BeaconState, Storage (GetBlobs) | EL responses (payloads, blobs, bodies) | ---