fix(gateway): SSE terminality, deadline, error-mapping fidelity (GW-03..GW-07, GW-12..GW-14)

- GW-04: /subscribe error events are terminal — scan-based emission of
  the error frame, then end of stream (matches call.error semantics).
- GW-05: enforce the 30 s gateway deadline via tokio::time::timeout in
  GatewayDispatch::invoke; hung handlers surface as TIMEOUT (504),
  streaming/sink stay unbounded per ADR-021.
- GW-07: gateway error paths route through the new identity-aware
  call_error_to_http_response_with_identity; retryable HTTP_429/HTTP_503
  now carry Retry-After on /call, /batch, /search, /schema, /publish.
- GW-13: SSE keep-alive (15 s comment frames) + retry: 15000 field.
- GW-14: module doc fixed (6 endpoints; /publish lives in routes.rs).
- GW-03/GW-12: mapping rides d7ee302's INVALID_OPERATION_TYPE mapper
  (documented in http-server.md table); 200-on-stream asymmetry
  documented.

Verification: cargo test (243 passed); cargo clippy --all-targets -- -D
warnings clean; cargo fmt --check clean.
This commit is contained in:
2026-08-29 10:13:03 +00:00
parent 713f1eba46
commit 9bc9e669e1
5 changed files with 422 additions and 45 deletions
+77 -2
View File
@@ -17,6 +17,17 @@
//! The root `OperationContext` is constructed identically for both
//! gateways, making them provably identical on the security axis (auth,
//! authority, ACL); they diverge only on wire-framing.
//!
//! # Deadline
//!
//! Once-op invokes ([`GatewayDispatch::invoke`]) are bounded by
//! `DEFAULT_TIMEOUT` (30 s): the registry invoke is wrapped in
//! `tokio::time::timeout` and a hung handler surfaces as a `TIMEOUT`
//! error envelope (`504` under the gateway's error mapping), not an
//! indefinitely-held HTTP request. Streaming and sink dispatch set
//! `deadline: None` (subscriptions are unbounded per alkcall ADR-021,
//! and a `/publish` body is bounded by the client's upload, not a
//! fixed window).
use std::collections::HashMap;
use std::sync::Arc;
@@ -24,7 +35,7 @@ use std::time::{Duration, Instant};
use alkcall::core::auth::{AuthToken, Identity, IdentityProvider};
use alkcall::core::types::Capabilities;
use alkcall::protocol::wire::ResponseEnvelope;
use alkcall::protocol::wire::{CallError, ResponseEnvelope};
use alkcall::registry::context::{AbortPolicy, OperationContext, ScopedPeerEnv};
use alkcall::registry::env::LocalOperationEnv;
use alkcall::registry::registration::OperationRegistry;
@@ -70,7 +81,20 @@ impl GatewayDispatch {
let operation_name = strip_leading_slash(op).to_string();
let request_id = uuid::Uuid::new_v4().to_string();
let context = self.build_root_context(&request_id, &operation_name, identity);
self.registry.invoke(&operation_name, input, context).await
let result = tokio::time::timeout(
DEFAULT_TIMEOUT,
self.registry.invoke(&operation_name, input, context),
)
.await;
match result {
Ok(envelope) => envelope,
Err(_elapsed) => ResponseEnvelope::error(
request_id,
CallError::timeout(format!(
"operation did not complete within the {DEFAULT_TIMEOUT:?} gateway deadline"
)),
),
}
}
pub fn invoke_streaming(
@@ -186,6 +210,9 @@ mod tests {
use futures::StreamExt;
use std::sync::Mutex as StdMutex;
// The tests below run against the module's real 30 s deadline; only
// the elapsed-time bound (< 60 s, anti-flake) is asserted by hand.
struct StaticIdentityProvider {
tokens: StdMutex<HashMap<String, Identity>>,
}
@@ -287,6 +314,54 @@ mod tests {
}
}
#[tokio::test]
async fn invoke_enforces_the_default_deadline_on_a_hung_handler() {
use std::time::Duration;
let mut registry = OperationRegistry::new();
registry
.register(HandlerRegistration::new(
spec("hung/op", Visibility::External, OperationType::Query),
HandlerKind::Once(make_handler(|_input, ctx| async move {
tokio::time::sleep(Duration::from_secs(120)).await;
ResponseEnvelope::ok(ctx.request_id, serde_json::json!({}))
})),
OperationProvenance::Local,
None,
None,
Capabilities::new(),
))
.unwrap();
let dispatch =
GatewayDispatch::new(Arc::new(registry), Arc::new(StaticIdentityProvider::new()));
let started = std::time::Instant::now();
let envelope = dispatch
.invoke(None, "/hung/op", serde_json::json!({}))
.await;
let elapsed = started.elapsed();
assert!(
elapsed < Duration::from_secs(60),
"the 30 s deadline must fire well before the 120 s handler sleep, took {elapsed:?}"
);
match envelope.result {
Err(error) => {
assert_eq!(error.code, "TIMEOUT");
assert!(error.retryable, "the deadline error is retryable");
}
Ok(v) => panic!("expected a TIMEOUT error, got {v:?}"),
}
}
#[tokio::test]
async fn invoke_completes_within_the_deadline_for_a_fast_handler() {
let dispatch =
GatewayDispatch::new(echo_registry(), Arc::new(StaticIdentityProvider::new()));
let envelope = dispatch
.invoke(None, "/echo/run", serde_json::json!({}))
.await;
assert!(envelope.result.is_ok(), "a fast handler must not time out");
}
#[tokio::test]
async fn streaming_sub_op_streams_envelopes() {
let mut registry = OperationRegistry::new();
+48 -1
View File
@@ -6,6 +6,14 @@
//! imported from external HTTP APIs are prefixed `HTTP_<status>` and map
//! to their declared status.
//!
//! The identity-aware variant maps the ambiguous protocol codes
//! (`FORBIDDEN`, `INVALID_OPERATION_TYPE`) to `401` when no token
//! resolved and `403`/`422` when one did — the HTTP-specific refinement
//! the neutral [`call_error_to_http_response`] cannot express (the
//! gateway's auth middleware has already run by the time it is invoked,
//! so live gateway paths always route through the identity-aware
//! variant).
//!
//! [ADR-023]: https://docs.rs/alkhttp (docs/architecture/decisions)
use alkcall::core::auth::Identity;
@@ -67,7 +75,19 @@ pub fn call_error_to_http_status_with_identity(
}
pub fn call_error_to_http_response(error: &CallError) -> Response {
let status_code = call_error_to_http_status(error);
call_error_to_http_response_with_identity(error, None)
}
/// Identity-aware variant of [`call_error_to_http_response`]: the
/// ambiguous protocol codes (`FORBIDDEN`, `INVALID_OPERATION_TYPE`) map
/// to `401` without a caller identity and `403`/`422` with one. Live
/// gateway error paths resolve the identity first, so they route
/// through this variant.
pub fn call_error_to_http_response_with_identity(
error: &CallError,
identity: Option<&Identity>,
) -> Response {
let status_code = call_error_to_http_status_with_identity(error, identity);
let status = status_code_from_u16(status_code);
let body = serde_json::to_value(error).unwrap_or(Value::Null);
@@ -206,6 +226,33 @@ mod tests {
assert_eq!(retry_after.as_deref(), Some("30"));
}
#[test]
fn retry_after_present_on_identity_aware_path_for_429() {
let error = CallError::new("HTTP_429", "rate limited", true)
.with_details(serde_json::json!({ "retry_after": 12 }));
let resp = call_error_to_http_response_with_identity(&error, None);
assert_eq!(resp.status(), 429);
let retry_after = resp
.headers()
.get(header::RETRY_AFTER)
.map(|v| v.to_str().unwrap().to_string());
assert_eq!(retry_after.as_deref(), Some("12"));
}
#[test]
fn identity_aware_response_keeps_401_vs_403_split() {
let error = CallError::forbidden("insufficient scopes");
assert_eq!(
call_error_to_http_response_with_identity(&error, None).status(),
401
);
let id = identity();
assert_eq!(
call_error_to_http_response_with_identity(&error, Some(&id)).status(),
403
);
}
#[test]
fn non_retryable_503_does_not_set_retry_after() {
let error = CallError::new("HTTP_503", "overloaded", false)
+195 -26
View File
@@ -1,6 +1,17 @@
//! The 6 fixed gateway endpoints (`/search`, `/schema`, `/call`,
//! `/batch`, `/subscribe`, `/publish`) — the sole HTTP invoke path
//! (ADR-042, ADR-047; `/publish` per ADR-068).
//!
//! Each endpoint delegates to `GatewayDispatch` (the shared dispatch
//! spine); auth is the shared `bearer_auth_middleware`; error mapping is
//! `gateway::error`. There is no per-operation `POST /{service}/{op}`
//! direct-call surface (ADR-047). `/publish` lives in this module too
//! (the review-001 "separate module" note is stale).
use std::collections::VecDeque;
use std::convert::Infallible;
use std::sync::Arc;
use std::time::Duration;
use alkcall::core::auth::{Identity, IdentityProvider};
use alkcall::protocol::wire::{CallError, ResponseEnvelope};
@@ -9,7 +20,7 @@ use alkcall::registry::spec::{AccessResult, Visibility};
use axum::body::Bytes;
use axum::extract::{FromRef, Query, State};
use axum::http::StatusCode;
use axum::response::sse::Event;
use axum::response::sse::{Event, KeepAlive};
use axum::response::{IntoResponse, Json, Response, Sse};
use axum::routing::{get, post};
use axum::Router;
@@ -20,7 +31,7 @@ use serde::Deserialize;
use serde_json::{json, Value};
use super::dispatch::GatewayDispatch;
use super::error::{call_error_to_http_response, call_error_to_http_status_with_identity};
use super::error::call_error_to_http_response_with_identity;
use crate::server::auth::ResolvedIdentity;
use crate::server::state::RouterState;
@@ -29,6 +40,12 @@ const SERVICES_SCHEMA: &str = "services/schema";
const MAX_BATCH_OPERATIONS: usize = 100;
const MAX_PUBLISH_LINE_BYTES: usize = 2 * 1024 * 1024;
/// SSE keep-alive interval on `/subscribe` (GW-13). Shared comment
/// frames (axum's KeepAlive::default) plus a `retry:` field on the
/// stream's first event reconnect the client on drops; 15 s sits under
/// the common LB/proxy idle timeouts (30-60 s).
const SSE_KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(15);
type ByteStream = futures::stream::BoxStream<'static, Result<Bytes, axum::Error>>;
#[derive(Clone)]
@@ -168,7 +185,7 @@ pub(crate) async fn subscribe_handler(
State(state): State<GatewayState>,
ResolvedIdentity(identity): ResolvedIdentity,
Json(request): Json<CallRequest>,
) -> Sse<SubscribeStream> {
) -> Response {
let stream = if is_internal_op(&state.registry, &request.operation) {
subscribe_stream_internal_error(request.operation)
} else {
@@ -178,6 +195,12 @@ pub(crate) async fn subscribe_handler(
subscribe_stream_from_envelope_stream(envelope_stream)
};
Sse::new(stream)
.keep_alive(
KeepAlive::new()
.interval(SSE_KEEP_ALIVE_INTERVAL)
.event(keep_alive_event()),
)
.into_response()
}
pub type SubscribeStream = BoxStream<'static, Result<Event, Infallible>>;
@@ -415,23 +438,56 @@ impl futures::Stream for NdjsonChunkStream {
}
}
/// The SSE projection of a streaming envelope (GW-04): each `Ok`
/// envelope becomes a `data:` frame carrying the output JSON plus the
/// `retry:` reconnect hint; the first `Err` envelope becomes an
/// `event:error` frame carrying the serialized `CallError` **and ends
/// the stream** — matching the wire dispatcher's `call.error`-is-
/// terminal semantics and http-server.md's documented contract
/// (an `Err` is terminal; the stream does not continue after it).
/// A quiet-but-alive stream is kept alive by axum comment frames at
/// `SSE_KEEP_ALIVE_INTERVAL` (GW-13).
///
/// The `scan` closure yields the error frame and flags the stream done
/// (subsequent polls return `None`), so the frame that ends the stream
/// is still written — the error event is emitted, not swallowed.
fn subscribe_stream_from_envelope_stream(
stream: BoxStream<'static, ResponseEnvelope>,
) -> SubscribeStream {
Box::pin(stream.map(|envelope| match envelope.result {
Ok(output) => {
let data = serde_json::to_string(&output).unwrap_or_else(|_| "null".to_string());
Ok(Event::default().data(data))
}
Err(error) => {
let payload = serde_json::to_value(&error).unwrap_or(Value::Null);
let data = serde_json::to_string(&payload).unwrap_or_else(|_| "null".to_string());
Ok(Event::default().event("error").data(data))
}
Box::pin(stream.scan(false, |done, envelope| {
std::future::ready(if *done {
None
} else {
let item = match envelope.result {
Ok(output) => {
let data =
serde_json::to_string(&output).unwrap_or_else(|_| "null".to_string());
Event::default().retry(SSE_KEEP_ALIVE_INTERVAL).data(data)
}
Err(error) => {
*done = true;
let payload = serde_json::to_value(&error).unwrap_or(Value::Null);
let data =
serde_json::to_string(&payload).unwrap_or_else(|_| "null".to_string());
Event::default()
.event("error")
.retry(SSE_KEEP_ALIVE_INTERVAL)
.data(data)
}
};
Some(Ok::<_, Infallible>(item))
})
}))
}
fn subscribe_stream_internal_error(operation: String) -> SubscribeStream {
/// The keep-alive comment frame: an SSE comment (colon-prefixed, no
/// event/data) plus a `retry:` hint, emitted on quiet streams so
/// LB/proxy idle timeouts do not kill the connection.
fn keep_alive_event() -> Event {
Event::default().retry(SSE_KEEP_ALIVE_INTERVAL)
}
pub(crate) fn subscribe_stream_internal_error(operation: String) -> SubscribeStream {
Box::pin(stream::once(async move { error_event(&operation) }))
}
@@ -441,13 +497,7 @@ fn envelope_to_response(envelope: ResponseEnvelope, identity: Option<&Identity>)
let body = envelope_to_ok_json(&envelope.request_id, &output);
(StatusCode::OK, Json(body)).into_response()
}
Err(error) => {
let status_code = call_error_to_http_status_with_identity(&error, identity);
let status =
StatusCode::from_u16(status_code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
let body = serde_json::to_value(&error).unwrap_or(Value::Null);
(status, Json(body)).into_response()
}
Err(error) => call_error_to_http_response_with_identity(&error, identity),
}
}
@@ -485,15 +535,12 @@ fn not_found_envelope_json(operation: &str) -> Value {
fn not_found_response(operation: &str) -> Response {
let error = CallError::not_found(operation);
call_error_to_http_response(&error)
call_error_to_http_response_with_identity(&error, None)
}
fn forbidden_response(message: String, identity: Option<&Identity>) -> Response {
let error = CallError::forbidden(message);
let status_code = call_error_to_http_status_with_identity(&error, identity);
let status = StatusCode::from_u16(status_code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
let body = serde_json::to_value(&error).unwrap_or(Value::Null);
(status, Json(body)).into_response()
call_error_to_http_response_with_identity(&error, identity)
}
fn access_check_for_op(
@@ -799,6 +846,10 @@ mod tests {
.with_state(state)
}
// The gateway deadline is the module's real 30 s constant (GW-05
// asserts enforcement in dispatch.rs); these SSE tests only touch
// streaming ops, which are exempt from it.
fn auth_header(token: &str) -> (&'static str, String) {
("authorization", format!("Bearer {token}"))
}
@@ -1192,6 +1243,86 @@ mod tests {
);
}
#[tokio::test]
async fn subscribe_stream_is_terminal_after_an_error_event() {
let router = build_router(
registry_with_subscription_stream_continuing_after_error(
"events/continue",
CallError::internal("mid-stream failure"),
),
unused_provider(),
);
let req = json_request(
"POST",
"/subscribe",
json!({ "operation": "events/continue", "input": {} }),
);
let resp = router.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let bytes = resp.into_body().collect().await.unwrap().to_bytes();
let body = String::from_utf8_lossy(&bytes);
let error_events = body.matches("event:").count();
assert_eq!(error_events, 1, "exactly one error event, got: {body}");
let data_frames = body.matches("data:").count();
assert_eq!(
data_frames, 1,
"the error frame is the last event — no post-error data frames, got: {body}"
);
assert!(
!body.contains("\"after\":true"),
"the post-error envelope must not reach the wire, got: {body}"
);
}
#[tokio::test]
async fn subscribe_stream_carries_retry_field_and_keep_alive_comment() {
let router = build_router(
registry_with_subscription_stream("events/quiet", vec![json!({ "n": 1 })]),
unused_provider(),
);
let req = json_request(
"POST",
"/subscribe",
json!({ "operation": "events/quiet", "input": {} }),
);
let resp = router.oneshot(req).await.unwrap();
let bytes = resp.into_body().collect().await.unwrap().to_bytes();
let body = String::from_utf8_lossy(&bytes);
assert!(
body.contains("retry: 15000"),
"expected a retry: hint on stream events, got: {body}"
);
assert!(
body.contains(':'),
"expected a keep-alive comment frame, got: {body}"
);
}
fn registry_with_subscription_stream_continuing_after_error(
name: &str,
error: CallError,
) -> Arc<OperationRegistry> {
let mut registry = OperationRegistry::new();
registry
.register(HandlerRegistration::new(
subscription_spec(name, Visibility::External, AccessControl::default()),
HandlerKind::Stream(make_streaming_handler(move |_input, ctx| {
let request_id = ctx.request_id.clone();
let error = error.clone();
futures::stream::iter(vec![
ResponseEnvelope::error(request_id.clone(), error),
ResponseEnvelope::ok(request_id, json!({ "after": true })),
])
})),
OperationProvenance::Local,
None,
None,
Capabilities::new(),
))
.unwrap();
Arc::new(registry)
}
#[tokio::test]
async fn subscribe_response_content_type_is_text_event_stream() {
let router = build_router(
@@ -1344,6 +1475,44 @@ mod tests {
);
}
#[tokio::test]
async fn call_error_envelope_carries_retry_after_on_retryable_503() {
let mut registry = OperationRegistry::new();
registry
.register(HandlerRegistration::new(
external_spec("flaky/op", AccessControl::default()),
HandlerKind::Once(make_handler(|_input, ctx| async move {
ResponseEnvelope::error(
ctx.request_id,
CallError::new("HTTP_503", "overloaded", true)
.with_details(json!({ "retry_after": "30" })),
)
})),
OperationProvenance::Local,
None,
None,
Capabilities::new(),
))
.unwrap();
let router = build_router(Arc::new(registry), unused_provider());
let req = json_request(
"POST",
"/call",
json!({ "operation": "flaky/op", "input": {} }),
);
let resp = router.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
let retry_after = resp
.headers()
.get(axum::http::header::RETRY_AFTER)
.map(|v| v.to_str().unwrap().to_string());
assert_eq!(
retry_after.as_deref(),
Some("30"),
"a retryable HTTP_503 from a handler must carry Retry-After on the gateway error path"
);
}
#[tokio::test]
async fn call_with_leading_slash_in_operation_dispatches() {
let router = build_router(registry_with_echo(), unused_provider());