fix(adapters): subscriptions escape the 30s total timeout + total SSE byte cap (FWD-15, FWD-14)

FWD-15: forward_stream now sends through SharedHttpClient::stream_client
— a client derived from the same config with the total request timeout
removed and connect + read timeouts retained. reqwest 0.13's per-request
override can lengthen a client-level total timeout but never clear it
(request-scoped None falls back to the client default), so the derived
client is the only correct mechanism. Both clients rebuild-and-swap
together atomically (FWD-12). A healthy >30s subscription survives; the
read timeout stays as the staleness guard, matching the gateway's
deadline: None dispatch contract (alkcall ADR-021).

FWD-14: the streaming branch enforces a total streamed-bytes cap per
subscription (HttpClientConfig::stream_total_byte_cap, default 1 GiB),
accumulated across every chunk fed to the SSE parser; exceeding it
terminates with a single terminal HTTP_413 error envelope. The SSE
line-cap check moved before extend_from_slice so the reassembly buffer
can never exceed the cap. Removing the total timeout without this cap
would open an unbounded-memory window, so both land together.

Wire tests: keepalive trickle past a scaled total-timeout deadline keeps
delivering; over-cap stream terminates with exactly one terminal error;
parser boundary tests for pre-extend cap checks.

Verified: cargo test (302+5), --all-features (373+41), clippy
--all-targets -D warnings (default + all-features), fmt --check,
doc --no-deps clean.
This commit is contained in:
2026-08-30 12:01:55 +00:00
parent e2c255d40c
commit 7f89db1058
2 changed files with 448 additions and 54 deletions
+117 -31
View File
@@ -34,8 +34,17 @@
//!
//! `HttpClientConfig::default()` carries a 30 s overall request timeout
//! (anchored to the gateway's 30 s deadline), a 10 s connect timeout and
//! a 30 s read timeout, so a stalled upstream cannot hold a caller open
//! indefinitely. `Retry-After` deadlines are clamped to
//! a 30 s read timeout. The overall timeout bounds the request/response
//! forwarding halves (Once-op forwards); it never covers a streaming
//! send — subscriptions are unbounded in *time* by contract (alkcall
//! ADR-021 sets `deadline: None` for the streaming dispatch), so the
//! derived client exposed by
//! [`SharedHttpClient::stream_client`] is built from the same config
//! minus the total timeout while keeping the connect + read timeouts,
//! whose read timeout is the upstream-staleness guard (FWD-15). The
//! byte-wise bound for streaming responses is
//! [`HttpClientConfig::stream_total_byte_cap`], enforced by the SSE
//! forwarding handler (FWD-14). Retry-After deadlines are clamped to
//! [`HttpClientConfig::retry_after_ceiling`] (default 300 s) — a hostile
//! backend cannot park the client on a 10-year deadline.
//!
@@ -84,6 +93,14 @@ const DEFAULT_MAX_TOTAL_RETRY_DURATION: Duration = Duration::from_secs(10);
/// upstream (seconds and HTTP-date forms alike).
const DEFAULT_RETRY_AFTER_CEILING: Duration = Duration::from_secs(300);
/// Default total streamed-bytes cap for one streaming (SSE)
/// subscription forward: 1 GiB. Subscriptions are unbounded in *time*
/// by contract (alkcall ADR-021), so the only agent of a hostile
/// upstream is the total number of bytes it may push into envelope
/// allocation per subscription; past this cap the forward terminates
/// with a single terminal error envelope (FWD-14).
const DEFAULT_STREAM_TOTAL_BYTE_CAP: u64 = 1024 * 1024 * 1024;
/// Default retry count: attempts beyond the first failure of an
/// idempotent request.
const DEFAULT_MAX_RETRIES: u32 = 3;
@@ -137,17 +154,32 @@ impl Default for RetryConfig {
/// (`SharedHttpClient`). Defaults satisfy the review-001 request-policy
/// findings (FWD-03/04/05): same-host-only redirects, idempotent-only
/// retries with a wall-clock budget, 30 s request + 10 s connect
/// timeouts, and a 300 s `Retry-After` ceiling.
/// timeouts, and a 300 s `Retry-After` ceiling. Streaming (SSE)
/// forwards ride a derived client with the total request timeout
/// removed (FWD-15) and enforce a total streamed-bytes cap (FWD-14).
#[derive(Debug, Clone)]
pub struct HttpClientConfig {
/// Idle connections kept per host; `None` uses the reqwest default.
pub pool_max_idle_per_host: Option<usize>,
/// Overall per-request timeout (default 30 s, off with `None`).
/// Anchored to the gateway's 30 s Once-op deadline: a forwarded
/// call must fail before its caller does. Applies only to
/// request/response forwards — streaming sends ride
/// [`SharedHttpClient::stream_client`], which is built without it
/// (FWD-15).
pub request_timeout: Option<Duration>,
/// TCP connect timeout (default 10 s, off with `None`).
pub connect_timeout: Option<Duration>,
/// Idle timeout between body bytes (default 30 s, off with `None`).
/// The stall guard: it bounds upstream *staleness*, not stream
/// lifetime.
pub read_timeout: Option<Duration>,
/// Total bytes a single streaming (SSE) forward may pull from its
/// upstream before the forward terminates with one terminal error
/// envelope (default 1 GiB, 0 = uncapped — un-recommended outside
/// tests). Bounded *bytes* per subscription; unbounded *time* is the
/// contract (alkcall ADR-021) (FWD-14).
pub stream_total_byte_cap: u64,
/// Retry backoff shape; only idempotent methods are ever retried
/// (see `RetryGateMiddleware`).
pub retry: RetryConfig,
@@ -168,6 +200,7 @@ impl Default for HttpClientConfig {
request_timeout: Some(DEFAULT_REQUEST_TIMEOUT),
connect_timeout: Some(DEFAULT_CONNECT_TIMEOUT),
read_timeout: Some(DEFAULT_READ_TIMEOUT),
stream_total_byte_cap: DEFAULT_STREAM_TOTAL_BYTE_CAP,
retry: RetryConfig::default(),
max_total_retry_duration: DEFAULT_MAX_TOTAL_RETRY_DURATION,
retry_after_ceiling: DEFAULT_RETRY_AFTER_CEILING,
@@ -230,16 +263,28 @@ pub enum HttpClientBuildError {
/// A hot-reloadable, middleware-stacked outbound HTTP client shared by
/// consumer adapters. Clone-cheap (`ArcSwap` inner); callers reach the
/// current stack through [`SharedHttpClient::client`].
///
/// Streaming (SSE) subscription forwards reach for
/// [`SharedHttpClient::stream_client`]: a client derived from the same
/// config minus the total request timeout (connect + read timeouts
/// intact; FWD-15) — a healthy subscription longer than 30 s must
/// survive, and
/// reqwest 0.13's per-request override can lengthen but never clear a
/// client-level total timeout. Both rebuild-and-swap together so a
/// reload can never pair one's config with the other's transport
/// (FWD-12).
pub struct SharedHttpClient {
inner: ArcSwap<SharedHttpInner>,
}
/// Joint holder for the client and its config so a reload swaps both in
/// one atomic `ArcSwap::store` — a reader can never observe the new
/// config paired with the previous client (FWD-12).
/// Joint holder for both derived clients and the config so a reload
/// swaps all three in one atomic `ArcSwap::store` — a reader can never
/// observe the new config paired with the previous clients, nor the
/// request client paired with the stale stream client (FWD-12, FWD-15).
#[derive(Clone)]
struct SharedHttpInner {
client: Arc<ClientWithMiddleware>,
stream_client: Arc<ClientWithMiddleware>,
config: Arc<HttpClientConfig>,
}
@@ -258,37 +303,49 @@ impl SharedHttpClient {
/// hot-reload path; use [`SharedHttpClient::reload`] for async
/// rebuilds.
pub fn new(config: HttpClientConfig) -> Result<Self, HttpClientBuildError> {
let client = build_client_sync(&config)?;
let (client, stream_client) = build_clients_sync(&config)?;
Ok(Self {
inner: ArcSwap::from_pointee(SharedHttpInner {
client: Arc::new(client),
stream_client: Arc::new(stream_client),
config: Arc::new(config),
}),
})
}
/// The current middleware-stacked client. Every call loads the
/// latest stack — after a [`reload`](Self::reload), new requests
/// ride the rebuilt client.
/// The current middleware-stacked client for request/response
/// forwarding. Every call loads the latest stack — after a
/// [`reload`](Self::reload), new requests ride the rebuilt client.
pub fn client(&self) -> Arc<ClientWithMiddleware> {
Arc::clone(&self.inner.load().client)
}
/// The config the current client was built from (swapped together
/// with the client; FWD-12).
/// The current middleware-stacked client for streaming (SSE)
/// subscription forwards: built from the same config with the total
/// request timeout removed and the connect + read timeouts retained
/// (FWD-15 — a subscription is unbounded in *time* by contract,
/// alkcall ADR-021; the read timeout remains the stall guard).
/// Swapped atomically with [`client`](Self::client) (FWD-12).
pub fn stream_client(&self) -> Arc<ClientWithMiddleware> {
Arc::clone(&self.inner.load().stream_client)
}
/// The config the current clients were built from (swapped together
/// with them; FWD-12).
pub fn config(&self) -> Arc<HttpClientConfig> {
Arc::clone(&self.inner.load().config)
}
/// Rebuild the underlying client and swap it in for new callers
/// (in-flight requests complete on the previous client). PEM reads
/// Rebuild the underlying clients and swap them in for new callers
/// (in-flight requests complete on the previous clients). PEM reads
/// use `tokio::fs`, so this is safe to call from async contexts
/// without blocking a worker. Client and config swap together in a
/// without blocking a worker. Clients and config swap together in a
/// single atomic store (FWD-12).
pub async fn reload(&self, config: HttpClientConfig) -> Result<(), HttpClientBuildError> {
let client = build_client(&config).await?;
let (client, stream_client) = build_clients(&config).await?;
self.inner.store(Arc::new(SharedHttpInner {
client: Arc::new(client),
stream_client: Arc::new(stream_client),
config: Arc::new(config),
}));
Ok(())
@@ -394,10 +451,35 @@ impl<P: RetryPolicy> RetryPolicy for TotalRetryBudget<P> {
}
}
async fn build_client(
/// Builds the pair (request client, streaming client) from one config:
/// identical stacks except the streaming client carries no total
/// request timeout (FWD-15).
async fn build_clients(
config: &HttpClientConfig,
) -> Result<ClientWithMiddleware, HttpClientBuildError> {
let ca_pem = match &config.ca_bundle {
) -> Result<(ClientWithMiddleware, ClientWithMiddleware), HttpClientBuildError> {
let pems = read_pems(config).await?;
let client = build_client_with_pems(config, pems.clone(), false)?;
let stream_client = build_client_with_pems(config, pems, true)?;
Ok((client, stream_client))
}
fn build_clients_sync(
config: &HttpClientConfig,
) -> Result<(ClientWithMiddleware, ClientWithMiddleware), HttpClientBuildError> {
let pems = read_pems_sync(config)?;
let client = build_client_with_pems(config, pems.clone(), false)?;
let stream_client = build_client_with_pems(config, pems, true)?;
Ok((client, stream_client))
}
#[derive(Clone)]
struct ClientPems {
ca: Option<Vec<u8>>,
client: Option<(Vec<u8>, Vec<u8>)>,
}
async fn read_pems(config: &HttpClientConfig) -> Result<ClientPems, HttpClientBuildError> {
let ca = match &config.ca_bundle {
Some(path) => Some(tokio::fs::read(path).await.map_err(|source| {
HttpClientBuildError::CaBundleRead {
path: path.clone(),
@@ -406,7 +488,7 @@ async fn build_client(
})?),
None => None,
};
let client_pems = match &config.client_cert {
let client = match &config.client_cert {
Some(cfg) => {
let cert_pem = tokio::fs::read(&cfg.cert_pem).await.map_err(|source| {
HttpClientBuildError::ClientCertRead {
@@ -424,13 +506,11 @@ async fn build_client(
}
None => None,
};
build_client_with_pems(config, ca_pem, client_pems)
Ok(ClientPems { ca, client })
}
fn build_client_sync(
config: &HttpClientConfig,
) -> Result<ClientWithMiddleware, HttpClientBuildError> {
let ca_pem = match &config.ca_bundle {
fn read_pems_sync(config: &HttpClientConfig) -> Result<ClientPems, HttpClientBuildError> {
let ca = match &config.ca_bundle {
Some(path) => {
Some(
std::fs::read(path).map_err(|source| HttpClientBuildError::CaBundleRead {
@@ -441,7 +521,7 @@ fn build_client_sync(
}
None => None,
};
let client_pems = match &config.client_cert {
let client = match &config.client_cert {
Some(cfg) => {
let cert_pem = std::fs::read(&cfg.cert_pem).map_err(|source| {
HttpClientBuildError::ClientCertRead {
@@ -459,21 +539,27 @@ fn build_client_sync(
}
None => None,
};
build_client_with_pems(config, ca_pem, client_pems)
Ok(ClientPems { ca, client })
}
fn build_client_with_pems(
config: &HttpClientConfig,
ca_pem: Option<Vec<u8>>,
client_pems: Option<(Vec<u8>, Vec<u8>)>,
pems: ClientPems,
streaming: bool,
) -> Result<ClientWithMiddleware, HttpClientBuildError> {
let ClientPems {
ca: ca_pem,
client: client_pems,
} = pems;
let mut builder = ClientBuilder::new();
builder = builder.redirect(same_host_redirect_policy());
if let Some(pool_max_idle) = config.pool_max_idle_per_host {
builder = builder.pool_max_idle_per_host(pool_max_idle);
}
if let Some(timeout) = config.request_timeout {
builder = builder.timeout(timeout);
if !streaming {
if let Some(timeout) = config.request_timeout {
builder = builder.timeout(timeout);
}
}
if let Some(timeout) = config.connect_timeout {
builder = builder.connect_timeout(timeout);