fix(adapters): subscriptions escape the 30s total timeout + total SSE byte cap (FWD-15, FWD-14)
FWD-15: forward_stream now sends through SharedHttpClient::stream_client — a client derived from the same config with the total request timeout removed and connect + read timeouts retained. reqwest 0.13's per-request override can lengthen a client-level total timeout but never clear it (request-scoped None falls back to the client default), so the derived client is the only correct mechanism. Both clients rebuild-and-swap together atomically (FWD-12). A healthy >30s subscription survives; the read timeout stays as the staleness guard, matching the gateway's deadline: None dispatch contract (alkcall ADR-021). FWD-14: the streaming branch enforces a total streamed-bytes cap per subscription (HttpClientConfig::stream_total_byte_cap, default 1 GiB), accumulated across every chunk fed to the SSE parser; exceeding it terminates with a single terminal HTTP_413 error envelope. The SSE line-cap check moved before extend_from_slice so the reassembly buffer can never exceed the cap. Removing the total timeout without this cap would open an unbounded-memory window, so both land together. Wire tests: keepalive trickle past a scaled total-timeout deadline keeps delivering; over-cap stream terminates with exactly one terminal error; parser boundary tests for pre-extend cap checks. Verified: cargo test (302+5), --all-features (373+41), clippy --all-targets -D warnings (default + all-features), fmt --check, doc --no-deps clean.
This commit is contained in:
+321
-13
@@ -5,6 +5,41 @@
|
||||
//! SSE frame parser for streaming (SSE → `ResponseEnvelope`) handlers
|
||||
//! (ADR-049).
|
||||
//!
|
||||
//! # Timeout / byte-bound policy for streaming forwards (FWD-15, FWD-14)
|
||||
//!
|
||||
//! The two send paths impose deliberately different bounds, mirroring
|
||||
//! the gateway's dispatch contract (alkcall ADR-021: Once-op invokes
|
||||
//! carry a 30 s deadline; `HandlerKind::Stream` invokes set
|
||||
//! `deadline: None` — subscriptions are unbounded in *time* by design):
|
||||
//!
|
||||
//! - **`forward`** (request/response) sends through the shared client's
|
||||
//! total request timeout (30 s default), so a forwarded call fails
|
||||
//! before its caller does.
|
||||
//! - **`forward_stream`** (subscriptions) sends through
|
||||
//! [`SharedHttpClient::stream_client`] — the same config minus the
|
||||
//! total request timeout, connect + read timeouts retained. A healthy
|
||||
//! subscription longer than 30 s survives; the read timeout remains
|
||||
//! the stall guard on upstream staleness (it resets on every body
|
||||
//! byte, so a keep-alive-emitting source lives as long as it keeps
|
||||
//! the connection warm — and quiet silence past the read timeout
|
||||
//! still terminates it). reqwest 0.13's per-request timeout override
|
||||
//! can lengthen but never clear a client-level total timeout, hence
|
||||
//! the derived client rather than an extension override.
|
||||
//!
|
||||
//! Unbounded time without a byte bound would let a hostile upstream
|
||||
//! stream well-formed 1-MiB-line events forever (the 1 MiB SSE line cap
|
||||
//! bounds one line, not the stream), so the streaming branch also
|
||||
//! enforces a total streamed-bytes cap per subscription —
|
||||
//! [`crate::client::HttpClientConfig::stream_total_byte_cap`], 1 GiB by
|
||||
//! default, accumulated across every chunk fed to the SSE parser.
|
||||
//! Exceeding it terminates the stream with a single terminal error
|
||||
//! envelope (the stream-ends semantics: one error frame, then end —
|
||||
//! matching the other terminal arms). The line-cap check runs before
|
||||
//! the buffer grows, so the reassembly buffer can never exceed the
|
||||
//! line cap.
|
||||
//!
|
||||
//! # Credentials and input routing
|
||||
//!
|
||||
//! The forwarding handler is the no-env-vars credential injection point
|
||||
//! (ADR-014): it reads `OperationContext.capabilities`, never
|
||||
//! `std::env::var`. Imported error codes are `HTTP_<status>` to avoid
|
||||
@@ -820,10 +855,11 @@ pub(crate) fn forward_stream(
|
||||
|
||||
let request_id_stream = request_id.clone();
|
||||
let error_status_codes_stream = error_status_codes.clone();
|
||||
let stream_byte_cap = http_client.config().stream_total_byte_cap;
|
||||
|
||||
let init = async move {
|
||||
let request_builder = http_client
|
||||
.client()
|
||||
.stream_client()
|
||||
.request(http_method, url.as_str())
|
||||
.headers(headers)
|
||||
.header(ACCEPT, "text/event-stream");
|
||||
@@ -879,21 +915,55 @@ pub(crate) fn forward_stream(
|
||||
let request_id_inner = request_id.clone();
|
||||
Box::pin(
|
||||
stream::unfold(
|
||||
(response.bytes_stream(), SseParser::new(), false),
|
||||
move |(mut bytes, mut parser, broken)| {
|
||||
(
|
||||
response.bytes_stream(),
|
||||
SseParser::new(),
|
||||
false,
|
||||
0u64,
|
||||
),
|
||||
move |(mut bytes, mut parser, broken, mut total_bytes)| {
|
||||
let request_id = request_id_inner.clone();
|
||||
async move {
|
||||
if broken {
|
||||
return None;
|
||||
}
|
||||
match bytes.next().await {
|
||||
Some(Ok(chunk)) => match parser.feed(&chunk, false) {
|
||||
Some(Ok(chunk)) => {
|
||||
let chunk_len = chunk.len() as u64;
|
||||
if stream_byte_cap > 0
|
||||
&& total_bytes.saturating_add(chunk_len)
|
||||
> stream_byte_cap
|
||||
{
|
||||
let error = CallError::new(
|
||||
"HTTP_413",
|
||||
format!(
|
||||
"upstream SSE stream exceeded the {stream_byte_cap}-byte total streamed-bytes cap on a subscription operation"
|
||||
),
|
||||
false,
|
||||
);
|
||||
return Some((
|
||||
vec![ResponseEnvelope::error(
|
||||
request_id, error,
|
||||
)],
|
||||
(bytes, parser, true, total_bytes),
|
||||
));
|
||||
}
|
||||
total_bytes = total_bytes.saturating_add(chunk_len);
|
||||
match parser.feed(&chunk, false) {
|
||||
Ok(events) => {
|
||||
let envelopes: Vec<ResponseEnvelope> = events
|
||||
let envelopes: Vec<ResponseEnvelope> =
|
||||
events
|
||||
.into_iter()
|
||||
.map(|e| sse_event_envelope(e, &request_id))
|
||||
.map(|e| {
|
||||
sse_event_envelope(
|
||||
e, &request_id,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
Some((envelopes, (bytes, parser, false)))
|
||||
Some((
|
||||
envelopes,
|
||||
(bytes, parser, false, total_bytes),
|
||||
))
|
||||
}
|
||||
Err(err) => {
|
||||
let error = CallError::internal(format!(
|
||||
@@ -903,9 +973,10 @@ pub(crate) fn forward_stream(
|
||||
vec![ResponseEnvelope::error(
|
||||
request_id, error,
|
||||
)],
|
||||
(bytes, parser, true),
|
||||
(bytes, parser, true, total_bytes),
|
||||
))
|
||||
}
|
||||
}
|
||||
},
|
||||
Some(Err(err)) => {
|
||||
let error = CallError::internal(format!(
|
||||
@@ -913,7 +984,7 @@ pub(crate) fn forward_stream(
|
||||
));
|
||||
Some((
|
||||
vec![ResponseEnvelope::error(request_id, error)],
|
||||
(bytes, parser, true),
|
||||
(bytes, parser, true, total_bytes),
|
||||
))
|
||||
}
|
||||
None => match parser.feed(&[], true) {
|
||||
@@ -922,7 +993,10 @@ pub(crate) fn forward_stream(
|
||||
.into_iter()
|
||||
.map(|e| sse_event_envelope(e, &request_id))
|
||||
.collect();
|
||||
Some((envelopes, (bytes, parser, true)))
|
||||
Some((
|
||||
envelopes,
|
||||
(bytes, parser, true, total_bytes),
|
||||
))
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
@@ -963,7 +1037,12 @@ pub(crate) enum SseParseError {
|
||||
/// buffer. A single event (all `data:` lines plus framing) must fit
|
||||
/// within this budget; a stream emitting a longer partial line — or an
|
||||
/// unterminated event — trips `SseParseError::BufferOverflow` instead
|
||||
/// of buffering without bound.
|
||||
/// of buffering without bound. The check runs *before* the buffer
|
||||
/// takes a chunk's bytes, so the reassembly buffer can never exceed
|
||||
/// the cap (FWD-14). This bounds one *line*, not the
|
||||
/// stream; the per-subscription total is
|
||||
/// [`HttpClientConfig::stream_total_byte_cap`], enforced by the
|
||||
/// `forward_stream` unfold across every `feed`.
|
||||
pub(crate) const SSE_EVENT_BUFFER_CAP: usize = 1024 * 1024;
|
||||
|
||||
/// Incremental byte-level SSE frame parser.
|
||||
@@ -999,10 +1078,10 @@ impl SseParser {
|
||||
/// `eof`, also dispatches a pending event if it carries data
|
||||
/// lines, and flushes the buffer.
|
||||
pub(crate) fn feed(&mut self, chunk: &[u8], eof: bool) -> Result<Vec<SseEvent>, SseParseError> {
|
||||
self.buf.extend_from_slice(chunk);
|
||||
if self.buf.len() > SSE_EVENT_BUFFER_CAP {
|
||||
if self.buf.len().saturating_add(chunk.len()) > SSE_EVENT_BUFFER_CAP {
|
||||
return Err(SseParseError::BufferOverflow);
|
||||
}
|
||||
self.buf.extend_from_slice(chunk);
|
||||
let mut events = Vec::new();
|
||||
let mut start = 0usize;
|
||||
while let Some(nl) = self.buf[start..].iter().position(|&b| b == b'\n') {
|
||||
@@ -1492,6 +1571,66 @@ mod tests {
|
||||
builder.body(body).expect("static response builds")
|
||||
}
|
||||
|
||||
/// Spawns a raw-TCP responder whose response head is written from a
|
||||
/// status/content-type pair, then hands the socket to an async
|
||||
/// writer so a test can trickle SSE frames over time (the
|
||||
/// wire-level seam the FWD-15/FWD-14 tests need: a stream that stays
|
||||
/// open past a deadline, or dribbles bytes toward a cap).
|
||||
async fn spawn_sse_responder_with_writer<F, Fut>(head: &str, writer: F) -> String
|
||||
where
|
||||
F: FnOnce(tokio::net::tcp::OwnedWriteHalf) -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = ()> + Send,
|
||||
{
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind");
|
||||
let addr = listener.local_addr().expect("local addr");
|
||||
let head = head.to_string();
|
||||
tokio::spawn(async move {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
let Ok((mut sock, _)) = listener.accept().await else {
|
||||
return;
|
||||
};
|
||||
let mut buf = vec![0u8; 8192];
|
||||
loop {
|
||||
let read = sock.read(&mut buf).await.unwrap_or(0);
|
||||
if read == 0 || String::from_utf8_lossy(&buf).contains("\r\n\r\n") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let _ = sock.write_all(head.as_bytes()).await;
|
||||
let _ = sock.flush().await;
|
||||
let (_, write_half) = sock.into_split();
|
||||
writer(write_half).await;
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
fn streaming_client(total_byte_cap: u64) -> TestArc<SharedHttpClient> {
|
||||
TestArc::new(
|
||||
SharedHttpClient::new(crate::client::HttpClientConfig {
|
||||
stream_total_byte_cap: total_byte_cap,
|
||||
..crate::client::HttpClientConfig::default()
|
||||
})
|
||||
.expect("client builds"),
|
||||
)
|
||||
}
|
||||
|
||||
/// Builds a client whose configured request/read timeout is the
|
||||
/// (test-scaled) old 30 s deadline: the FWD-15 wire test sends
|
||||
/// through it and asserts the stream outlives that deadline.
|
||||
fn client_with_timeout(timeout: Duration) -> TestArc<SharedHttpClient> {
|
||||
TestArc::new(
|
||||
SharedHttpClient::new(crate::client::HttpClientConfig {
|
||||
request_timeout: Some(timeout),
|
||||
connect_timeout: Some(Duration::from_secs(5)),
|
||||
read_timeout: Some(timeout),
|
||||
..crate::client::HttpClientConfig::default()
|
||||
})
|
||||
.expect("client builds"),
|
||||
)
|
||||
}
|
||||
|
||||
async fn call_forward(base_url: &str, ctx: OperationContext) -> ResponseEnvelope {
|
||||
call_forward_authed(base_url, ctx, &None).await
|
||||
}
|
||||
@@ -1834,4 +1973,173 @@ mod tests {
|
||||
other => panic!("expected HTTP_500, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// FWD-15 wire test (deadline-scaled): the responder trickles
|
||||
/// `: keepalive` comments and one `data:` event past the configured
|
||||
/// total request timeout (a scaled stand-in for the 30 s default
|
||||
/// the streaming path used to inherit); the subscription must still
|
||||
/// be delivering events after that deadline has passed. With the
|
||||
/// fix, `forward_stream` sends through the derived no-total-timeout
|
||||
/// client, so the stream survives; without it, reqwest 0.13's total
|
||||
/// timeout rides into the body stream and kills the subscription at
|
||||
/// the deadline.
|
||||
#[tokio::test]
|
||||
async fn stream_survives_past_the_total_request_timeout() {
|
||||
let timeout = Duration::from_millis(500);
|
||||
let keepalive = Duration::from_millis(200);
|
||||
let total_keepalives = 6u32;
|
||||
let start = std::time::Instant::now();
|
||||
let head = "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\n\r\n";
|
||||
let base = spawn_sse_responder_with_writer(head, move |mut sock| async move {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
for _ in 0..total_keepalives {
|
||||
tokio::time::sleep(keepalive).await;
|
||||
let _ = sock.write_all(b": keepalive\n\n").await;
|
||||
let _ = sock.flush().await;
|
||||
}
|
||||
let _ = sock.write_all(b"data: {\"late\":true}\n\n").await;
|
||||
let _ = sock.flush().await;
|
||||
let _ = sock.shutdown().await;
|
||||
})
|
||||
.await;
|
||||
let client = client_with_timeout(timeout);
|
||||
let stream = forward_stream(
|
||||
&client,
|
||||
&base,
|
||||
"/x",
|
||||
"GET",
|
||||
&None,
|
||||
&TestHashMap::new(),
|
||||
"svc",
|
||||
&serde_json::json!({"type": "object"}),
|
||||
&[],
|
||||
json!({}),
|
||||
noop_context(),
|
||||
);
|
||||
tokio::pin!(stream);
|
||||
let crossed = tokio::time::timeout(Duration::from_secs(5), stream.next())
|
||||
.await
|
||||
.expect("the stream must deliver within the test budget")
|
||||
.expect("stream must not end without the data event");
|
||||
assert!(
|
||||
start.elapsed() > timeout,
|
||||
"the event must arrive after the old total-request deadline (elapsed {:?}, timeout {:?})",
|
||||
start.elapsed(),
|
||||
timeout
|
||||
);
|
||||
match crossed.result {
|
||||
Ok(value) => assert_eq!(value, json!({"late": true})),
|
||||
other => panic!("expected the post-deadline data event, got {other:?}"),
|
||||
}
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(500), stream.next())
|
||||
.await
|
||||
.unwrap_or(None)
|
||||
.is_none(),
|
||||
"responder closed after the data event; stream must end"
|
||||
);
|
||||
}
|
||||
|
||||
/// FWD-14 wire test: a stream whose total bytes exceed the
|
||||
/// configured per-subscription cap terminates with exactly one
|
||||
/// terminal error envelope (the stream-ends semantics: error frame,
|
||||
/// then end).
|
||||
#[tokio::test]
|
||||
async fn stream_exceeding_total_byte_cap_terminates_with_one_error() {
|
||||
let cap = 4096u64;
|
||||
let chunk = vec![b'a'; 1024];
|
||||
let head = "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\n\r\n";
|
||||
let base = spawn_sse_responder_with_writer(head, move |mut sock| async move {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
for _ in 0..64 {
|
||||
let _ = sock.write_all(b"data: ").await;
|
||||
let _ = sock.write_all(&chunk).await;
|
||||
let _ = sock.write_all(b"\n\n").await;
|
||||
let _ = sock.flush().await;
|
||||
}
|
||||
let _ = sock.shutdown().await;
|
||||
})
|
||||
.await;
|
||||
let client = streaming_client(cap);
|
||||
let stream = forward_stream(
|
||||
&client,
|
||||
&base,
|
||||
"/x",
|
||||
"GET",
|
||||
&None,
|
||||
&TestHashMap::new(),
|
||||
"svc",
|
||||
&serde_json::json!({"type": "object"}),
|
||||
&[],
|
||||
json!({}),
|
||||
noop_context(),
|
||||
);
|
||||
let envelopes = collect_stream(stream).await;
|
||||
assert!(!envelopes.is_empty(), "events before the cap still flow");
|
||||
assert!(
|
||||
envelopes[..envelopes.len() - 1]
|
||||
.iter()
|
||||
.all(|e| e.result.is_ok()),
|
||||
"every envelope before the terminal one is an event"
|
||||
);
|
||||
let terminal = envelopes.last().expect("terminal envelope present");
|
||||
match &terminal.result {
|
||||
Err(err) => {
|
||||
assert_eq!(err.code, "HTTP_413");
|
||||
assert!(err.message.contains("total streamed-bytes cap"));
|
||||
}
|
||||
other => panic!("expected the terminal cap error, got {other:?}"),
|
||||
}
|
||||
let ok_count = envelopes[..envelopes.len() - 1].len();
|
||||
assert_eq!(
|
||||
envelopes.len(),
|
||||
ok_count + 1,
|
||||
"exactly one terminal error envelope after the last event"
|
||||
);
|
||||
}
|
||||
|
||||
/// FWD-14 parser-boundary test: for a never-newline feed the
|
||||
/// cap check fires *before* the buffer takes the chunk's bytes, so
|
||||
/// the buffered bytes stay at or below the cap — the pre-fix shape
|
||||
/// (extend first, check after) buffered past the cap before erroring.
|
||||
/// A full-cap buffer remains legal (the cap is inclusive) as long as
|
||||
/// the arriving chunk completes a line.
|
||||
#[test]
|
||||
fn line_cap_trips_before_the_buffer_takes_the_overshooting_chunk() {
|
||||
let mut parser = SseParser::new();
|
||||
let seed = vec![b'x'; SSE_EVENT_BUFFER_CAP + 1];
|
||||
let oversized = parser.feed(&seed, false);
|
||||
assert!(
|
||||
matches!(oversized, Err(SseParseError::BufferOverflow)),
|
||||
"a single over-cap line trips at the pre-extend check"
|
||||
);
|
||||
let mut parser = SseParser::new();
|
||||
let half = vec![b'x'; SSE_EVENT_BUFFER_CAP / 2];
|
||||
let first = parser.feed(&half, false);
|
||||
assert!(first.is_ok(), "partial line under the cap buffers fine");
|
||||
let second = parser.feed(&seed, false);
|
||||
assert!(
|
||||
matches!(second, Err(SseParseError::BufferOverflow)),
|
||||
"the chunk that would push past the cap is rejected before extend"
|
||||
);
|
||||
let mut parser = SseParser::new();
|
||||
let at_cap = vec![b'x'; SSE_EVENT_BUFFER_CAP - 2];
|
||||
let ok = parser.feed(&at_cap, false);
|
||||
assert!(ok.is_ok(), "a partial line under the cap is legal");
|
||||
let framing = parser.feed(b"\n\n", false);
|
||||
assert!(
|
||||
framing.is_ok(),
|
||||
"the newline pair completes the event without tripping the cap"
|
||||
);
|
||||
let next = parser.feed(&vec![b'y'; SSE_EVENT_BUFFER_CAP], false);
|
||||
assert!(
|
||||
next.is_ok(),
|
||||
"the dispatched event drained the buffer; a fresh full-cap line is legal again"
|
||||
);
|
||||
let over = parser.feed(b"z", false);
|
||||
assert!(
|
||||
matches!(over, Err(SseParseError::BufferOverflow)),
|
||||
"one byte past a full buffer still trips before extend"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+115
-29
@@ -34,8 +34,17 @@
|
||||
//!
|
||||
//! `HttpClientConfig::default()` carries a 30 s overall request timeout
|
||||
//! (anchored to the gateway's 30 s deadline), a 10 s connect timeout and
|
||||
//! a 30 s read timeout, so a stalled upstream cannot hold a caller open
|
||||
//! indefinitely. `Retry-After` deadlines are clamped to
|
||||
//! a 30 s read timeout. The overall timeout bounds the request/response
|
||||
//! forwarding halves (Once-op forwards); it never covers a streaming
|
||||
//! send — subscriptions are unbounded in *time* by contract (alkcall
|
||||
//! ADR-021 sets `deadline: None` for the streaming dispatch), so the
|
||||
//! derived client exposed by
|
||||
//! [`SharedHttpClient::stream_client`] is built from the same config
|
||||
//! minus the total timeout while keeping the connect + read timeouts,
|
||||
//! whose read timeout is the upstream-staleness guard (FWD-15). The
|
||||
//! byte-wise bound for streaming responses is
|
||||
//! [`HttpClientConfig::stream_total_byte_cap`], enforced by the SSE
|
||||
//! forwarding handler (FWD-14). Retry-After deadlines are clamped to
|
||||
//! [`HttpClientConfig::retry_after_ceiling`] (default 300 s) — a hostile
|
||||
//! backend cannot park the client on a 10-year deadline.
|
||||
//!
|
||||
@@ -84,6 +93,14 @@ const DEFAULT_MAX_TOTAL_RETRY_DURATION: Duration = Duration::from_secs(10);
|
||||
/// upstream (seconds and HTTP-date forms alike).
|
||||
const DEFAULT_RETRY_AFTER_CEILING: Duration = Duration::from_secs(300);
|
||||
|
||||
/// Default total streamed-bytes cap for one streaming (SSE)
|
||||
/// subscription forward: 1 GiB. Subscriptions are unbounded in *time*
|
||||
/// by contract (alkcall ADR-021), so the only agent of a hostile
|
||||
/// upstream is the total number of bytes it may push into envelope
|
||||
/// allocation per subscription; past this cap the forward terminates
|
||||
/// with a single terminal error envelope (FWD-14).
|
||||
const DEFAULT_STREAM_TOTAL_BYTE_CAP: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
/// Default retry count: attempts beyond the first failure of an
|
||||
/// idempotent request.
|
||||
const DEFAULT_MAX_RETRIES: u32 = 3;
|
||||
@@ -137,17 +154,32 @@ impl Default for RetryConfig {
|
||||
/// (`SharedHttpClient`). Defaults satisfy the review-001 request-policy
|
||||
/// findings (FWD-03/04/05): same-host-only redirects, idempotent-only
|
||||
/// retries with a wall-clock budget, 30 s request + 10 s connect
|
||||
/// timeouts, and a 300 s `Retry-After` ceiling.
|
||||
/// timeouts, and a 300 s `Retry-After` ceiling. Streaming (SSE)
|
||||
/// forwards ride a derived client with the total request timeout
|
||||
/// removed (FWD-15) and enforce a total streamed-bytes cap (FWD-14).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HttpClientConfig {
|
||||
/// Idle connections kept per host; `None` uses the reqwest default.
|
||||
pub pool_max_idle_per_host: Option<usize>,
|
||||
/// Overall per-request timeout (default 30 s, off with `None`).
|
||||
/// Anchored to the gateway's 30 s Once-op deadline: a forwarded
|
||||
/// call must fail before its caller does. Applies only to
|
||||
/// request/response forwards — streaming sends ride
|
||||
/// [`SharedHttpClient::stream_client`], which is built without it
|
||||
/// (FWD-15).
|
||||
pub request_timeout: Option<Duration>,
|
||||
/// TCP connect timeout (default 10 s, off with `None`).
|
||||
pub connect_timeout: Option<Duration>,
|
||||
/// Idle timeout between body bytes (default 30 s, off with `None`).
|
||||
/// The stall guard: it bounds upstream *staleness*, not stream
|
||||
/// lifetime.
|
||||
pub read_timeout: Option<Duration>,
|
||||
/// Total bytes a single streaming (SSE) forward may pull from its
|
||||
/// upstream before the forward terminates with one terminal error
|
||||
/// envelope (default 1 GiB, 0 = uncapped — un-recommended outside
|
||||
/// tests). Bounded *bytes* per subscription; unbounded *time* is the
|
||||
/// contract (alkcall ADR-021) (FWD-14).
|
||||
pub stream_total_byte_cap: u64,
|
||||
/// Retry backoff shape; only idempotent methods are ever retried
|
||||
/// (see `RetryGateMiddleware`).
|
||||
pub retry: RetryConfig,
|
||||
@@ -168,6 +200,7 @@ impl Default for HttpClientConfig {
|
||||
request_timeout: Some(DEFAULT_REQUEST_TIMEOUT),
|
||||
connect_timeout: Some(DEFAULT_CONNECT_TIMEOUT),
|
||||
read_timeout: Some(DEFAULT_READ_TIMEOUT),
|
||||
stream_total_byte_cap: DEFAULT_STREAM_TOTAL_BYTE_CAP,
|
||||
retry: RetryConfig::default(),
|
||||
max_total_retry_duration: DEFAULT_MAX_TOTAL_RETRY_DURATION,
|
||||
retry_after_ceiling: DEFAULT_RETRY_AFTER_CEILING,
|
||||
@@ -230,16 +263,28 @@ pub enum HttpClientBuildError {
|
||||
/// A hot-reloadable, middleware-stacked outbound HTTP client shared by
|
||||
/// consumer adapters. Clone-cheap (`ArcSwap` inner); callers reach the
|
||||
/// current stack through [`SharedHttpClient::client`].
|
||||
///
|
||||
/// Streaming (SSE) subscription forwards reach for
|
||||
/// [`SharedHttpClient::stream_client`]: a client derived from the same
|
||||
/// config minus the total request timeout (connect + read timeouts
|
||||
/// intact; FWD-15) — a healthy subscription longer than 30 s must
|
||||
/// survive, and
|
||||
/// reqwest 0.13's per-request override can lengthen but never clear a
|
||||
/// client-level total timeout. Both rebuild-and-swap together so a
|
||||
/// reload can never pair one's config with the other's transport
|
||||
/// (FWD-12).
|
||||
pub struct SharedHttpClient {
|
||||
inner: ArcSwap<SharedHttpInner>,
|
||||
}
|
||||
|
||||
/// Joint holder for the client and its config so a reload swaps both in
|
||||
/// one atomic `ArcSwap::store` — a reader can never observe the new
|
||||
/// config paired with the previous client (FWD-12).
|
||||
/// Joint holder for both derived clients and the config so a reload
|
||||
/// swaps all three in one atomic `ArcSwap::store` — a reader can never
|
||||
/// observe the new config paired with the previous clients, nor the
|
||||
/// request client paired with the stale stream client (FWD-12, FWD-15).
|
||||
#[derive(Clone)]
|
||||
struct SharedHttpInner {
|
||||
client: Arc<ClientWithMiddleware>,
|
||||
stream_client: Arc<ClientWithMiddleware>,
|
||||
config: Arc<HttpClientConfig>,
|
||||
}
|
||||
|
||||
@@ -258,37 +303,49 @@ impl SharedHttpClient {
|
||||
/// hot-reload path; use [`SharedHttpClient::reload`] for async
|
||||
/// rebuilds.
|
||||
pub fn new(config: HttpClientConfig) -> Result<Self, HttpClientBuildError> {
|
||||
let client = build_client_sync(&config)?;
|
||||
let (client, stream_client) = build_clients_sync(&config)?;
|
||||
Ok(Self {
|
||||
inner: ArcSwap::from_pointee(SharedHttpInner {
|
||||
client: Arc::new(client),
|
||||
stream_client: Arc::new(stream_client),
|
||||
config: Arc::new(config),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// The current middleware-stacked client. Every call loads the
|
||||
/// latest stack — after a [`reload`](Self::reload), new requests
|
||||
/// ride the rebuilt client.
|
||||
/// The current middleware-stacked client for request/response
|
||||
/// forwarding. Every call loads the latest stack — after a
|
||||
/// [`reload`](Self::reload), new requests ride the rebuilt client.
|
||||
pub fn client(&self) -> Arc<ClientWithMiddleware> {
|
||||
Arc::clone(&self.inner.load().client)
|
||||
}
|
||||
|
||||
/// The config the current client was built from (swapped together
|
||||
/// with the client; FWD-12).
|
||||
/// The current middleware-stacked client for streaming (SSE)
|
||||
/// subscription forwards: built from the same config with the total
|
||||
/// request timeout removed and the connect + read timeouts retained
|
||||
/// (FWD-15 — a subscription is unbounded in *time* by contract,
|
||||
/// alkcall ADR-021; the read timeout remains the stall guard).
|
||||
/// Swapped atomically with [`client`](Self::client) (FWD-12).
|
||||
pub fn stream_client(&self) -> Arc<ClientWithMiddleware> {
|
||||
Arc::clone(&self.inner.load().stream_client)
|
||||
}
|
||||
|
||||
/// The config the current clients were built from (swapped together
|
||||
/// with them; FWD-12).
|
||||
pub fn config(&self) -> Arc<HttpClientConfig> {
|
||||
Arc::clone(&self.inner.load().config)
|
||||
}
|
||||
|
||||
/// Rebuild the underlying client and swap it in for new callers
|
||||
/// (in-flight requests complete on the previous client). PEM reads
|
||||
/// Rebuild the underlying clients and swap them in for new callers
|
||||
/// (in-flight requests complete on the previous clients). PEM reads
|
||||
/// use `tokio::fs`, so this is safe to call from async contexts
|
||||
/// without blocking a worker. Client and config swap together in a
|
||||
/// without blocking a worker. Clients and config swap together in a
|
||||
/// single atomic store (FWD-12).
|
||||
pub async fn reload(&self, config: HttpClientConfig) -> Result<(), HttpClientBuildError> {
|
||||
let client = build_client(&config).await?;
|
||||
let (client, stream_client) = build_clients(&config).await?;
|
||||
self.inner.store(Arc::new(SharedHttpInner {
|
||||
client: Arc::new(client),
|
||||
stream_client: Arc::new(stream_client),
|
||||
config: Arc::new(config),
|
||||
}));
|
||||
Ok(())
|
||||
@@ -394,10 +451,35 @@ impl<P: RetryPolicy> RetryPolicy for TotalRetryBudget<P> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_client(
|
||||
/// Builds the pair (request client, streaming client) from one config:
|
||||
/// identical stacks except the streaming client carries no total
|
||||
/// request timeout (FWD-15).
|
||||
async fn build_clients(
|
||||
config: &HttpClientConfig,
|
||||
) -> Result<ClientWithMiddleware, HttpClientBuildError> {
|
||||
let ca_pem = match &config.ca_bundle {
|
||||
) -> Result<(ClientWithMiddleware, ClientWithMiddleware), HttpClientBuildError> {
|
||||
let pems = read_pems(config).await?;
|
||||
let client = build_client_with_pems(config, pems.clone(), false)?;
|
||||
let stream_client = build_client_with_pems(config, pems, true)?;
|
||||
Ok((client, stream_client))
|
||||
}
|
||||
|
||||
fn build_clients_sync(
|
||||
config: &HttpClientConfig,
|
||||
) -> Result<(ClientWithMiddleware, ClientWithMiddleware), HttpClientBuildError> {
|
||||
let pems = read_pems_sync(config)?;
|
||||
let client = build_client_with_pems(config, pems.clone(), false)?;
|
||||
let stream_client = build_client_with_pems(config, pems, true)?;
|
||||
Ok((client, stream_client))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ClientPems {
|
||||
ca: Option<Vec<u8>>,
|
||||
client: Option<(Vec<u8>, Vec<u8>)>,
|
||||
}
|
||||
|
||||
async fn read_pems(config: &HttpClientConfig) -> Result<ClientPems, HttpClientBuildError> {
|
||||
let ca = match &config.ca_bundle {
|
||||
Some(path) => Some(tokio::fs::read(path).await.map_err(|source| {
|
||||
HttpClientBuildError::CaBundleRead {
|
||||
path: path.clone(),
|
||||
@@ -406,7 +488,7 @@ async fn build_client(
|
||||
})?),
|
||||
None => None,
|
||||
};
|
||||
let client_pems = match &config.client_cert {
|
||||
let client = match &config.client_cert {
|
||||
Some(cfg) => {
|
||||
let cert_pem = tokio::fs::read(&cfg.cert_pem).await.map_err(|source| {
|
||||
HttpClientBuildError::ClientCertRead {
|
||||
@@ -424,13 +506,11 @@ async fn build_client(
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
build_client_with_pems(config, ca_pem, client_pems)
|
||||
Ok(ClientPems { ca, client })
|
||||
}
|
||||
|
||||
fn build_client_sync(
|
||||
config: &HttpClientConfig,
|
||||
) -> Result<ClientWithMiddleware, HttpClientBuildError> {
|
||||
let ca_pem = match &config.ca_bundle {
|
||||
fn read_pems_sync(config: &HttpClientConfig) -> Result<ClientPems, HttpClientBuildError> {
|
||||
let ca = match &config.ca_bundle {
|
||||
Some(path) => {
|
||||
Some(
|
||||
std::fs::read(path).map_err(|source| HttpClientBuildError::CaBundleRead {
|
||||
@@ -441,7 +521,7 @@ fn build_client_sync(
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let client_pems = match &config.client_cert {
|
||||
let client = match &config.client_cert {
|
||||
Some(cfg) => {
|
||||
let cert_pem = std::fs::read(&cfg.cert_pem).map_err(|source| {
|
||||
HttpClientBuildError::ClientCertRead {
|
||||
@@ -459,22 +539,28 @@ fn build_client_sync(
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
build_client_with_pems(config, ca_pem, client_pems)
|
||||
Ok(ClientPems { ca, client })
|
||||
}
|
||||
|
||||
fn build_client_with_pems(
|
||||
config: &HttpClientConfig,
|
||||
ca_pem: Option<Vec<u8>>,
|
||||
client_pems: Option<(Vec<u8>, Vec<u8>)>,
|
||||
pems: ClientPems,
|
||||
streaming: bool,
|
||||
) -> Result<ClientWithMiddleware, HttpClientBuildError> {
|
||||
let ClientPems {
|
||||
ca: ca_pem,
|
||||
client: client_pems,
|
||||
} = pems;
|
||||
let mut builder = ClientBuilder::new();
|
||||
builder = builder.redirect(same_host_redirect_policy());
|
||||
if let Some(pool_max_idle) = config.pool_max_idle_per_host {
|
||||
builder = builder.pool_max_idle_per_host(pool_max_idle);
|
||||
}
|
||||
if !streaming {
|
||||
if let Some(timeout) = config.request_timeout {
|
||||
builder = builder.timeout(timeout);
|
||||
}
|
||||
}
|
||||
if let Some(timeout) = config.connect_timeout {
|
||||
builder = builder.connect_timeout(timeout);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user