feat(store): the put seam — two internal paths, the three numbered rules, Batch, kv-only rejection
- `put(len: Option<u64>, stream: impl AsyncRead + Unpin + Send) -> (Key, Pin)`: known-length one-pass flow; unknown-length buffer-to-threshold flow with the mid-stream fresh-stage flush (ADR-003 option (d), three numbered rules: flush-and-continue, derivation over the eventual stored artifact as a whole via buffer form and StagingCursor read-back form, zero residue on stream error) - staged-put seam amended (operator sign-off): `put_staged()` keyless handout, `StagingCursor::commit(self, key)` publishes under the store-core-resolved key — the canonical key completes at stream end and the wave-1 handout shape could not express the commit-rename flow - pin-before-publish rides the arbitration lock on both paths and both tiers; dedup re-puts refcount the existing entry - Batch seam: `Store::batch()` + `put_in_batch()`, members publish pin- before-publish, group pins drop together on `Batch::commit` (new public verb) or drop - kv-only rejection (ADR-012 §6.2): known-length over-threshold rejects immediately (stream never read), unknown-length at mid-stream overflow; zero residue; error carrier `Io(String)` (named design point) - declared-len mismatch rejects `Verification` zero-residue (named design point); stream shape generic-no-dyn (named design point) - incremental BlobHasher (pinned byte-equal to Algorithm::derive); AnyStaging cursor dispatch; git-interop put-path leg against git hash-object; eleven put-path behavior tests Verification: cargo test (default/--features mem/--all-features) green; clippy --all-targets -D warnings green in all three feature shapes; cargo fmt --check green; cargo doc --no-deps at the 24-warning baseline.
This commit is contained in:
1 parent
889be4cada
commit
539dabcbf7
8 files changed
+1723
-105
No files matched your search
+36
-23
@@ -108,6 +108,17 @@ pub(crate) trait ReadCursor: Send {
|
||||
/// on discard with zero residue (the stage-hygiene invariant,
|
||||
/// store-api.md §Write path; exact-count tested per engine).
|
||||
///
|
||||
/// **The key resolves at commit, not at handout (named design point,
|
||||
/// the wave-2 amendment of the wave-1 handout shape).** The canonical
|
||||
/// key is the derivation over preamble + content, and it completes at
|
||||
/// stream end — after the staging cursor already exists (put paths
|
||||
/// stream content once into the cursor while hashing; ADR-003's put
|
||||
/// paths). So `commit` carries the store-core-resolved key and the
|
||||
/// stage's identity before that point is only the engine-internal
|
||||
/// stage file/tx (unique per `put_staged` handout). Publish is the
|
||||
/// CAS first-writer-wins entry under that committed key exactly as
|
||||
/// [`Backend::put`].
|
||||
///
|
||||
/// Every engine implements the staged form: kv-tier bodies are the
|
||||
/// degenerate in-memory stage committed through the engine's whole-value
|
||||
/// row write (the store core selects the form by tier policy, ADR-010
|
||||
@@ -121,15 +132,18 @@ pub(crate) trait StagingCursor: ReadCursor {
|
||||
/// "restart the stream" requirement anywhere).
|
||||
async fn append(&mut self, chunk: &[u8]) -> Result<(), Error>;
|
||||
|
||||
/// Publish the entry: the stage's bytes become the CAS entry in the
|
||||
/// Publish the entry: the stage's bytes become the CAS entry under
|
||||
/// `key` — the store-core-resolved canonical key (the derivation
|
||||
/// completes at stream end; see the type's design point) — in the
|
||||
/// engine's put shape (commit-rename on `local`, the in-tx `lowrite`
|
||||
/// batch on pg-lo, the row write on kv bodies). This is the staged
|
||||
/// form's terminal step — the store core never re-uses the cursor
|
||||
/// after the call. On failure the stage is **already discarded** —
|
||||
/// zero residue by the stage-hygiene invariant; there is no
|
||||
/// post-failure discard to drive (the consuming call took the
|
||||
/// cursor).
|
||||
async fn commit(self) -> Result<(), Error>;
|
||||
/// batch on pg-lo, the row write on kv bodies). CAS first-writer-wins:
|
||||
/// a losing race against an existing entry is dedup (`Ok`), never a
|
||||
/// failure. This is the staged form's terminal step — the store core
|
||||
/// never re-uses the cursor after the call. On failure the stage is
|
||||
/// **already discarded** — zero residue by the stage-hygiene
|
||||
/// invariant; there is no post-failure discard to drive (the
|
||||
/// consuming call took the cursor).
|
||||
async fn commit(self, key: &[u8]) -> Result<(), Error>;
|
||||
|
||||
/// Discard the stage: every non-commit path converges here, zero
|
||||
/// residue. Driven explicitly by the store core (stream error,
|
||||
@@ -189,11 +203,12 @@ pub(crate) trait Backend: Send + Sync {
|
||||
async fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Error>;
|
||||
|
||||
/// The staged put's cursor handout (ADR-010 §1). The store core
|
||||
/// selects the form before content flows: the large tier's
|
||||
/// stage-then-commit-rename contract, kv engines' degenerate
|
||||
/// in-memory stage. See [`StagingCursor`] for the contract every
|
||||
/// body converges on.
|
||||
async fn put_staged(&self, key: &[u8]) -> Result<Self::Staging, Error>;
|
||||
/// selects the form before content flows (the cursor's identity
|
||||
/// before commit is the engine-internal stage only — the canonical
|
||||
/// key resolves at [`StagingCursor::commit`], the type's design
|
||||
/// point). See [`StagingCursor`] for the contract every body
|
||||
/// converges on.
|
||||
async fn put_staged(&self) -> Result<Self::Staging, Error>;
|
||||
|
||||
/// Unconditional delete — arbitration has already happened above the
|
||||
/// trait (the facade decides and owns the refusal: `GcRefuse` never
|
||||
@@ -442,7 +457,6 @@ mod tests {
|
||||
}
|
||||
|
||||
struct TestStaging {
|
||||
key: Vec<u8>,
|
||||
staged: Vec<u8>,
|
||||
rows: Arc<Mutex<BTreeMap<Vec<u8>, Vec<u8>>>>,
|
||||
fail_commit: bool,
|
||||
@@ -464,13 +478,13 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn commit(mut self) -> Result<(), Error> {
|
||||
async fn commit(mut self, key: &[u8]) -> Result<(), Error> {
|
||||
if self.fail_commit {
|
||||
self.staged.clear();
|
||||
return Err(Error::Io("engine commit failure (test double)".to_owned()));
|
||||
}
|
||||
let value = std::mem::take(&mut self.staged);
|
||||
self.rows.lock().entry(self.key.clone()).or_insert(value);
|
||||
self.rows.lock().entry(key.to_vec()).or_insert(value);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -508,9 +522,8 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_staged(&self, key: &[u8]) -> Result<Self::Staging, Error> {
|
||||
async fn put_staged(&self) -> Result<Self::Staging, Error> {
|
||||
Ok(TestStaging {
|
||||
key: key.to_vec(),
|
||||
staged: Vec::new(),
|
||||
rows: self.rows.clone(),
|
||||
fail_commit: self.fail_commit,
|
||||
@@ -603,7 +616,7 @@ mod tests {
|
||||
let backend = TestBackend::default();
|
||||
let key = b"\x01staged-key".to_vec();
|
||||
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(&CONTENT[..8]).await.unwrap();
|
||||
staging.append(&CONTENT[8..16]).await.unwrap();
|
||||
staging.append(&CONTENT[16..]).await.unwrap();
|
||||
@@ -620,7 +633,7 @@ mod tests {
|
||||
assert_eq!(bytes, CONTENT, "staged artifact readable as a whole");
|
||||
assert_eq!(digest, slice_digest(CONTENT));
|
||||
|
||||
staging.commit().await.expect("commit publishes");
|
||||
staging.commit(&key).await.expect("commit publishes");
|
||||
assert!(backend.has(&key).await.unwrap(), "entry published");
|
||||
|
||||
let cursor = backend
|
||||
@@ -650,7 +663,7 @@ mod tests {
|
||||
let backend = TestBackend::default();
|
||||
let key = b"\x01discard-key".to_vec();
|
||||
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(b"residue").await.unwrap();
|
||||
staging.discard().await;
|
||||
|
||||
@@ -676,9 +689,9 @@ mod tests {
|
||||
};
|
||||
let key = b"\x01failure-key".to_vec();
|
||||
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(CONTENT).await.unwrap();
|
||||
let err = staging.commit().await.unwrap_err();
|
||||
let err = staging.commit(&key).await.unwrap_err();
|
||||
assert!(matches!(err, Error::Io(_)), "engine media failure");
|
||||
|
||||
assert!(
|
||||
|
||||
@@ -178,6 +178,15 @@ impl Pin {
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for Pin {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("Pin")
|
||||
.field("key", &self.key.digest())
|
||||
.field("armed", &self.armed)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Pin {
|
||||
fn drop(&mut self) {
|
||||
if self.armed {
|
||||
@@ -239,6 +248,17 @@ impl Batch {
|
||||
self.keys.is_empty()
|
||||
}
|
||||
|
||||
/// **Batch commit**: the group's pins drop together in one ordered
|
||||
/// step — protection spans the whole batch or none of it, and ends
|
||||
/// when the batch ends (ADR-005 §Marks 2; the same release batch
|
||||
/// drop drives). A committed batch's members are no longer
|
||||
/// batch-protected; durable protection beyond the write group is
|
||||
/// the consumer's liveness seam (`Store::register_liveness_source`
|
||||
/// or the pin-conversion ordering).
|
||||
pub fn commit(mut self) {
|
||||
self.release_members();
|
||||
}
|
||||
|
||||
/// The one-step release: every member refcount decremented inside
|
||||
/// a single arbitration-lock section, sorted into the keys' total
|
||||
/// order. Driven by [`Drop::drop`] (batch drop) and available for
|
||||
|
||||
+104
-1
@@ -27,7 +27,7 @@
|
||||
//! is absent from the crate entirely (ADR-002 Decision class 3).
|
||||
|
||||
use sha1::Sha1;
|
||||
use sha2::Sha256;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
use crate::Error;
|
||||
|
||||
@@ -129,6 +129,61 @@ where
|
||||
hasher.finalize().to_vec()
|
||||
}
|
||||
|
||||
/// The streaming form of the derivation: one hasher fed the preamble
|
||||
/// at initialization (`"blob <len>\0"`, the length known up front on
|
||||
/// the known-length put path) and the content as it flows. The finish
|
||||
/// is byte-identical to [`Algorithm::derive`] over the same input —
|
||||
/// this is the same derivation, driven incrementally so content can
|
||||
/// stream once (ADR-003's put paths; the wave-2 facade logic this
|
||||
/// module's doc anticipated).
|
||||
///
|
||||
/// Crate-internal: the derivation is a store-core concern; this type
|
||||
/// is the put path's tool, not public API.
|
||||
pub(crate) struct BlobHasher {
|
||||
algorithm: Algorithm,
|
||||
inner: InnerHasher,
|
||||
}
|
||||
|
||||
enum InnerHasher {
|
||||
Sha256(Sha256),
|
||||
Sha1(Sha1),
|
||||
}
|
||||
|
||||
impl BlobHasher {
|
||||
/// A hasher initialized with the preamble for an artifact of `len`
|
||||
/// bytes — the known-length path's one-pass form (the preamble is
|
||||
/// known before content flows).
|
||||
pub(crate) fn new(algorithm: Algorithm, len: u64) -> BlobHasher {
|
||||
let mut hasher = BlobHasher {
|
||||
algorithm,
|
||||
inner: match algorithm {
|
||||
Algorithm::GitBlobSha256 => InnerHasher::Sha256(Sha256::new()),
|
||||
Algorithm::GitBlobSha1 => InnerHasher::Sha1(Sha1::new()),
|
||||
},
|
||||
};
|
||||
hasher.update(&blob_preamble(len));
|
||||
hasher
|
||||
}
|
||||
|
||||
/// Feed the next chunk of content.
|
||||
pub(crate) fn update(&mut self, chunk: &[u8]) {
|
||||
match &mut self.inner {
|
||||
InnerHasher::Sha256(inner) => inner.update(chunk),
|
||||
InnerHasher::Sha1(inner) => inner.update(chunk),
|
||||
}
|
||||
}
|
||||
|
||||
/// The derivation's answer over preamble + everything fed so far —
|
||||
/// pinned equal to [`Algorithm::derive`] by the in-module tests.
|
||||
pub(crate) fn finish(self) -> Digest {
|
||||
let bytes = match self.inner {
|
||||
InnerHasher::Sha256(inner) => inner.finalize().to_vec(),
|
||||
InnerHasher::Sha1(inner) => inner.finalize().to_vec(),
|
||||
};
|
||||
Digest::from_parts(self.algorithm, bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// An algorithm + digest pair — the hash of a blob under one of this
|
||||
/// crate's derivations (ADR-002; hashing-and-keys.md §The
|
||||
/// Key/Digest surface).
|
||||
@@ -294,6 +349,54 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The streaming form is the same derivation: a `BlobHasher` fed
|
||||
/// the content in arbitrary chunk boundaries finishes byte-identical
|
||||
/// to [`Algorithm::derive`] — both algorithms, boundaries around the
|
||||
/// known-answer vectors, and a multi-chunk bulk body.
|
||||
#[test]
|
||||
fn streaming_hasher_matches_known_length_derivation() {
|
||||
for algorithm in [Algorithm::GitBlobSha256, Algorithm::GitBlobSha1] {
|
||||
let bodies: [&[u8]; 4] = [b"", b"hello", &[0u8; 100_003], &[7u8; 9]];
|
||||
|
||||
for body in bodies {
|
||||
let expected = algorithm.derive(body, body.len() as u64);
|
||||
|
||||
// whole-body update.
|
||||
let mut hasher = BlobHasher::new(algorithm, body.len() as u64);
|
||||
hasher.update(body);
|
||||
assert_eq!(hasher.finish(), expected, "whole-body, {algorithm:?}");
|
||||
|
||||
// byte-at-a-time.
|
||||
let mut hasher = BlobHasher::new(algorithm, body.len() as u64);
|
||||
for byte in body {
|
||||
hasher.update(std::slice::from_ref(byte));
|
||||
}
|
||||
assert_eq!(hasher.finish(), expected, "byte-at-a-time, {algorithm:?}");
|
||||
|
||||
// awkward fixed chunks (7, including past the end).
|
||||
let mut hasher = BlobHasher::new(algorithm, body.len() as u64);
|
||||
for chunk in body.chunks(7) {
|
||||
hasher.update(chunk);
|
||||
}
|
||||
assert_eq!(hasher.finish(), expected, "chunks of 7, {algorithm:?}");
|
||||
}
|
||||
|
||||
// The canonical 'hello' and empty vectors, streamed.
|
||||
let expected = algorithm.derive(b"hello", 5).bytes().to_vec();
|
||||
let mut hasher = BlobHasher::new(algorithm, 5);
|
||||
hasher.update(b"hel");
|
||||
hasher.update(b"lo");
|
||||
assert_eq!(hasher.finish().bytes().to_vec(), expected);
|
||||
|
||||
let hasher = BlobHasher::new(algorithm, 0);
|
||||
assert_eq!(
|
||||
hasher.finish().bytes().to_vec(),
|
||||
algorithm.derive(b"", 0).bytes().to_vec(),
|
||||
"empty content streams with no update"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Algorithm ordering equals registry-byte ordering, so
|
||||
/// digest-derived sort is the keys' byte total order.
|
||||
#[test]
|
||||
|
||||
+33
-29
@@ -266,16 +266,18 @@ impl ReadCursor for LocalCursor {
|
||||
/// The staged put's staging cursor (ADR-010 §1): backed by a stage file
|
||||
/// in the staging area, unique per stage — appends write through, the
|
||||
/// staged length tracks completed appends (the sole-writer protocol),
|
||||
/// and `commit` renames the stage file into its sharded committed path —
|
||||
/// publish is the rename. Every failure path converges on discard with
|
||||
/// zero residue: a commit that cannot complete (media failure, CAS
|
||||
/// dedup, parent creation) removes the stage file itself; [`StagingCursor::discard`]
|
||||
/// removes it; and a cursor dropped without either is best-effort
|
||||
/// removed by the [`Drop`] body — the engine's cancellation story,
|
||||
/// covered by the in-module hygiene tests.
|
||||
/// and `commit(key)` renames the stage file into its sharded committed
|
||||
/// path under the store-core-resolved canonical key (the staged-put
|
||||
/// seam's key-at-commit design point: the derivation completes at stream
|
||||
/// end, so until commit the stage's identity is the engine-internal
|
||||
/// stage file alone) — publish is the rename. Every failure path
|
||||
/// converges on discard with zero residue: a commit that cannot complete
|
||||
/// (media failure, CAS dedup, parent creation) removes the stage file
|
||||
/// itself; [`StagingCursor::discard`] removes it; and a cursor dropped
|
||||
/// without either is best-effort removed by the [`Drop`] body — the
|
||||
/// engine's cancellation story, covered by the in-module hygiene tests.
|
||||
pub(crate) struct LocalStaging {
|
||||
root: Arc<PathBuf>,
|
||||
key: Vec<u8>,
|
||||
stage_path: Option<PathBuf>,
|
||||
file: File,
|
||||
staged_len: AtomicU64,
|
||||
@@ -321,7 +323,7 @@ impl StagingCursor for LocalStaging {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn commit(mut self) -> Result<(), Error> {
|
||||
async fn commit(mut self, key: &[u8]) -> Result<(), Error> {
|
||||
let stage = match self.stage_path.take() {
|
||||
Some(stage) => stage,
|
||||
None => {
|
||||
@@ -329,7 +331,7 @@ impl StagingCursor for LocalStaging {
|
||||
}
|
||||
};
|
||||
let root = Arc::clone(&self.root);
|
||||
let key = std::mem::take(&mut self.key);
|
||||
let key = key.to_vec();
|
||||
let cleanup = stage.clone();
|
||||
let result = blocking(move || {
|
||||
let committed = committed_path(&root, &key)?;
|
||||
@@ -454,10 +456,8 @@ impl Backend for LocalBackend {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_staged(&self, key: &[u8]) -> Result<Self::Staging, Error> {
|
||||
let root = Arc::clone(&self.root);
|
||||
let key = key.to_vec();
|
||||
let inner_root = Arc::clone(&root);
|
||||
async fn put_staged(&self) -> Result<Self::Staging, Error> {
|
||||
let inner_root = Arc::clone(&self.root);
|
||||
let name = format!("stage-{}", stage_id());
|
||||
#[cfg(test)]
|
||||
let fail_commit = self.fail_commit;
|
||||
@@ -473,7 +473,6 @@ impl Backend for LocalBackend {
|
||||
.map_err(io_err)?;
|
||||
Ok(LocalStaging {
|
||||
root: inner_root,
|
||||
key,
|
||||
stage_path: Some(stage_path),
|
||||
file,
|
||||
staged_len: AtomicU64::new(0),
|
||||
@@ -590,9 +589,9 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn staged(backend: &LocalBackend, key: &[u8], content: &[u8]) -> Result<(), Error> {
|
||||
let mut staging = backend.put_staged(key).await?;
|
||||
let mut staging = backend.put_staged().await?;
|
||||
staging.append(content).await?;
|
||||
staging.commit().await
|
||||
staging.commit(key).await
|
||||
}
|
||||
|
||||
/// The Backend method contract: the name, the seven methods'
|
||||
@@ -770,10 +769,13 @@ mod tests {
|
||||
assert_eq!(cursor.read_at(0, 8).await.unwrap(), Vec::<u8>::new());
|
||||
|
||||
let staged_key = key_for(b"empty-staged");
|
||||
let mut staging = backend.put_staged(&staged_key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(b"").await.unwrap();
|
||||
assert_eq!(staging.len(), 0, "empty append stays empty");
|
||||
staging.commit().await.expect("empty commit publishes");
|
||||
staging
|
||||
.commit(&staged_key)
|
||||
.await
|
||||
.expect("empty commit publishes");
|
||||
assert!(
|
||||
backend.has(&staged_key).await.unwrap(),
|
||||
"empty entry landed"
|
||||
@@ -800,7 +802,7 @@ mod tests {
|
||||
let (_root, backend) = new_backend().await;
|
||||
let key = key_for(b"staged-round-trip");
|
||||
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(&CONTENT[..8]).await.unwrap();
|
||||
staging.append(&CONTENT[8..24]).await.unwrap();
|
||||
staging.append(&CONTENT[24..]).await.unwrap();
|
||||
@@ -819,7 +821,7 @@ mod tests {
|
||||
let tail = staging.read_at(CONTENT.len() as u64, 32).await.unwrap();
|
||||
assert_eq!(tail, Vec::<u8>::new(), "staged read clamps at staged EOF");
|
||||
|
||||
staging.commit().await.expect("commit publishes");
|
||||
staging.commit(&key).await.expect("commit publishes");
|
||||
assert!(backend.has(&key).await.unwrap(), "entry published");
|
||||
|
||||
let cursor = backend
|
||||
@@ -857,7 +859,7 @@ mod tests {
|
||||
let (root, backend) = new_backend().await;
|
||||
let key = key_for(b"discard-key");
|
||||
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(b"residue").await.unwrap();
|
||||
staging.discard().await;
|
||||
|
||||
@@ -887,9 +889,9 @@ mod tests {
|
||||
};
|
||||
let key = key_for(b"failure-key");
|
||||
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(CONTENT).await.unwrap();
|
||||
let err = staging.commit().await.unwrap_err();
|
||||
let err = staging.commit(&key).await.unwrap_err();
|
||||
assert!(matches!(err, Error::Io(_)), "engine media failure");
|
||||
|
||||
assert!(
|
||||
@@ -913,7 +915,7 @@ mod tests {
|
||||
let key = key_for(b"cancelled-key");
|
||||
|
||||
{
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(b"never published").await.unwrap();
|
||||
// Dropped without commit or discard.
|
||||
}
|
||||
@@ -953,21 +955,23 @@ mod tests {
|
||||
for i in 0..N {
|
||||
let content = format!("discard-{i}");
|
||||
let key = key_for(content.as_bytes());
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let _ = &key;
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(content.as_bytes()).await.unwrap();
|
||||
staging.discard().await;
|
||||
}
|
||||
for i in 0..N {
|
||||
let content = format!("commit-failure-{i}");
|
||||
let key = key_for(content.as_bytes());
|
||||
let mut staging = failure_backend.put_staged(&key).await.unwrap();
|
||||
let mut staging = failure_backend.put_staged().await.unwrap();
|
||||
staging.append(content.as_bytes()).await.unwrap();
|
||||
staging.commit().await.unwrap_err();
|
||||
staging.commit(&key).await.unwrap_err();
|
||||
}
|
||||
for i in 0..N {
|
||||
let content = format!("cancelled-{i}");
|
||||
let key = key_for(content.as_bytes());
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let _ = &key;
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(content.as_bytes()).await.unwrap();
|
||||
drop(staging);
|
||||
}
|
||||
|
||||
+32
-23
@@ -10,9 +10,12 @@
|
||||
//! form: [`get`][Backend::get] hands out a read cursor that materializes
|
||||
//! the whole value (in-memory slices; `len()` from the stored value), and
|
||||
//! [`put_staged`][Backend::put_staged] hands out the degenerate in-memory
|
||||
//! stage whose `commit` publishes through the same whole-value CAS row
|
||||
//! write as `put` — `entry().or_insert()`, first writer wins, a re-put is
|
||||
//! dedup and never an overwrite. `commit` cannot fail in normal operation
|
||||
//! stage whose `commit(key)` publishes through the same whole-value CAS
|
||||
//! row write as `put` — `entry().or_insert()`, first writer wins, a re-put
|
||||
//! is dedup and never an overwrite. The key resolves at commit (the
|
||||
//! staged-put seam's wave-2 amendment: the canonical derivation completes
|
||||
//! at stream end, so the stage's identity before that point is the
|
||||
//! engine-internal buffer alone). `commit` cannot fail in normal operation
|
||||
//! (no media can fail here); the test-only hook noted on [`MemStaging`]
|
||||
//! exercises the failure form's self-discard hygiene, standing in for the
|
||||
//! media-failure injection later engines need.
|
||||
@@ -75,9 +78,8 @@ impl Backend for MemBackend {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_staged(&self, key: &[u8]) -> Result<Self::Staging, Error> {
|
||||
async fn put_staged(&self) -> Result<Self::Staging, Error> {
|
||||
Ok(MemStaging {
|
||||
key: key.to_vec(),
|
||||
staged: Vec::new(),
|
||||
rows: Arc::clone(&self.rows),
|
||||
#[cfg(test)]
|
||||
@@ -122,14 +124,15 @@ impl ReadCursor for MemCursor {
|
||||
}
|
||||
|
||||
/// The degenerate in-memory staging cursor (ADR-010 §1): appends extend
|
||||
/// the buffer; `commit` publishes the whole staged bytes through the same
|
||||
/// CAS clause as [`Backend::put`] — first writer wins, losing the race is
|
||||
/// dedup (`Ok`), never a failure — and every non-commit path, including
|
||||
/// the failure form, converges on zero residue. `commit` takes the
|
||||
/// buffer via `std::mem::take` before the table lock, so a failed publish
|
||||
/// cannot strand the staged bytes in the map.
|
||||
/// the buffer; `commit(key)` publishes the whole staged bytes through the
|
||||
/// same CAS clause as [`Backend::put`] — under the store-core-resolved
|
||||
/// canonical key (the staged-put seam's key-at-commit design point),
|
||||
/// first writer wins, losing the race is dedup (`Ok`), never a failure —
|
||||
/// and every non-commit path, including the failure form, converges on
|
||||
/// zero residue. `commit` takes the buffer via `std::mem::take` before
|
||||
/// the table lock, so a failed publish cannot strand the staged bytes
|
||||
/// in the map.
|
||||
pub(crate) struct MemStaging {
|
||||
key: Vec<u8>,
|
||||
staged: Vec<u8>,
|
||||
rows: Arc<Mutex<BTreeMap<Vec<u8>, Vec<u8>>>>,
|
||||
/// Test-only injection, shared with the issuing [`MemBackend`] — see
|
||||
@@ -155,7 +158,7 @@ impl StagingCursor for MemStaging {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn commit(mut self) -> Result<(), Error> {
|
||||
async fn commit(mut self, key: &[u8]) -> Result<(), Error> {
|
||||
#[cfg(test)]
|
||||
if self.fail_commit {
|
||||
self.staged.clear();
|
||||
@@ -164,7 +167,7 @@ impl StagingCursor for MemStaging {
|
||||
));
|
||||
}
|
||||
let value = std::mem::take(&mut self.staged);
|
||||
self.rows.lock().entry(self.key).or_insert(value);
|
||||
self.rows.lock().entry(key.to_vec()).or_insert(value);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -338,10 +341,13 @@ mod tests {
|
||||
assert_eq!(cursor.read_at(0, 8).await.unwrap(), Vec::<u8>::new());
|
||||
|
||||
let staged_key = b"\x01empty-staged-key".to_vec();
|
||||
let mut staging = backend.put_staged(&staged_key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(b"").await.unwrap();
|
||||
assert_eq!(staging.len(), 0, "empty append stays empty");
|
||||
staging.commit().await.expect("empty commit publishes");
|
||||
staging
|
||||
.commit(&staged_key)
|
||||
.await
|
||||
.expect("empty commit publishes");
|
||||
assert!(
|
||||
backend.has(&staged_key).await.unwrap(),
|
||||
"empty entry landed"
|
||||
@@ -369,7 +375,7 @@ mod tests {
|
||||
let backend = MemBackend::default();
|
||||
let key = b"\x01staged-key".to_vec();
|
||||
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(&CONTENT[..8]).await.unwrap();
|
||||
staging.append(&CONTENT[8..16]).await.unwrap();
|
||||
staging.append(&CONTENT[16..]).await.unwrap();
|
||||
@@ -386,7 +392,7 @@ mod tests {
|
||||
assert_eq!(bytes, CONTENT, "staged artifact readable as a whole");
|
||||
assert_eq!(digest, crate::backend::slice_digest(CONTENT));
|
||||
|
||||
staging.commit().await.expect("commit publishes");
|
||||
staging.commit(&key).await.expect("commit publishes");
|
||||
assert!(backend.has(&key).await.unwrap(), "entry published");
|
||||
|
||||
let cursor = backend
|
||||
@@ -410,9 +416,12 @@ mod tests {
|
||||
Some(CONTENT),
|
||||
"whole-value re-put does not overwrite"
|
||||
);
|
||||
let mut second = backend.put_staged(&key).await.unwrap();
|
||||
let mut second = backend.put_staged().await.unwrap();
|
||||
second.append(b"overwritten-by-stage").await.unwrap();
|
||||
second.commit().await.expect("commit is dedup, not failure");
|
||||
second
|
||||
.commit(&key)
|
||||
.await
|
||||
.expect("commit is dedup, not failure");
|
||||
assert_eq!(
|
||||
stored_value(&backend, &key).await.as_deref(),
|
||||
Some(CONTENT),
|
||||
@@ -427,7 +436,7 @@ mod tests {
|
||||
let backend = MemBackend::default();
|
||||
let key = b"\x01discard-key".to_vec();
|
||||
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(b"residue").await.unwrap();
|
||||
staging.discard().await;
|
||||
|
||||
@@ -455,9 +464,9 @@ mod tests {
|
||||
};
|
||||
let key = b"\x01failure-key".to_vec();
|
||||
|
||||
let mut staging = backend.put_staged(&key).await.unwrap();
|
||||
let mut staging = backend.put_staged().await.unwrap();
|
||||
staging.append(CONTENT).await.unwrap();
|
||||
let err = staging.commit().await.unwrap_err();
|
||||
let err = staging.commit(&key).await.unwrap_err();
|
||||
assert!(matches!(err, Error::Io(_)), "engine media failure");
|
||||
|
||||
assert!(
|
||||
|
||||
+1314
-19
File diff suppressed because it is too large.
Load diff
+110
-10
@@ -1,7 +1,7 @@
|
||||
---
|
||||
id: store-put-paths
|
||||
name: Store put seam — two internal paths, the three numbered rules, Batch, kv-only rejection
|
||||
status: pending
|
||||
status: completed
|
||||
depends_on: [gc-protocol]
|
||||
scope: moderate
|
||||
risk: medium
|
||||
@@ -67,22 +67,22 @@ internal paths behind one signature, with pin-before-publish from the
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- [ ] `put(len, stream) → (Key, Pin)` with both internal paths behind
|
||||
- [x] `put(len, stream) → (Key, Pin)` with both internal paths behind
|
||||
the one signature; routing is the pure length function (no
|
||||
content ever migrates tiers)
|
||||
- [ ] The three numbered rules each behavior-tested: mid-stream
|
||||
- [x] The three numbered rules each behavior-tested: mid-stream
|
||||
overflow flushes into a fresh stage and the stream continues;
|
||||
the derivation pass runs over the eventual stored artifact as a
|
||||
whole (buffer form and staged-bytes form); stream error leaves
|
||||
no entry and no stage file
|
||||
- [ ] Pin-before-publish holds on both paths and both tiers; dedup
|
||||
- [x] Pin-before-publish holds on both paths and both tiers; dedup
|
||||
re-puts refcount the existing entry
|
||||
- [ ] `Batch` groups multi-put writes; member pins drop together
|
||||
- [ ] kv-only mode: known-length over-threshold rejects immediately;
|
||||
- [x] `Batch` groups multi-put writes; member pins drop together
|
||||
- [x] kv-only mode: known-length over-threshold rejects immediately;
|
||||
unknown-length rejects at mid-stream overflow; zero residue on
|
||||
both
|
||||
- [ ] Stream-shape and len-mismatch design points decided and recorded
|
||||
- [ ] `cargo test` green; `cargo test --features mem` and `cargo test
|
||||
- [x] Stream-shape and len-mismatch design points decided and recorded
|
||||
- [x] `cargo test` green; `cargo test --features mem` and `cargo test
|
||||
--all-features` green; clippy/fmt gates green
|
||||
|
||||
## References
|
||||
@@ -96,8 +96,108 @@ internal paths behind one signature, with pin-before-publish from the
|
||||
|
||||
## Notes
|
||||
|
||||
> To be filled by implementation agent
|
||||
- **The staged-put seam was amended: the key resolves at commit, not
|
||||
at handout.** Implementing the two large-tier flows surfaced a
|
||||
contradiction in the wave-1 pin: `put_staged(key)` hands the staging
|
||||
cursor out with the key, but both large-tier put paths only know the
|
||||
canonical key when the derivation completes — at stream end (the
|
||||
preamble covers a length, and the digest covers content that has
|
||||
not streamed yet). Content streams once (non-restartable sources are
|
||||
the normal case), so `commit(self)` publishing under the handout
|
||||
key cannot express the specced flow ("the derivation completes at
|
||||
stream end; commit-rename publishes"). Amended with operator
|
||||
sign-off: `put_staged()` is a keyless handout and
|
||||
`StagingCursor::commit(self, key)` publishes under the
|
||||
store-core-resolved key (CAS first-writer-wins as the whole-value
|
||||
put). No ADR text pins the key's arrival point (ADR-003/010 pin
|
||||
handout/commit/discard mechanics only); the wave-1 task notes'
|
||||
`put_staged(key)` shape was pre-flow realism. Recorded for
|
||||
review-wave-2. Engines updated in place (mem, local, the wave-1
|
||||
test double) with their hygiene suites re-run green.
|
||||
- **Put-path named design points (recorded in `src/store.rs` module
|
||||
docs, for review-wave-2):**
|
||||
- *Stream parameter shape:* `put(len, stream: impl AsyncRead + Unpin
|
||||
+ Send)` — generic, no boxing; the family's no-`dyn` posture
|
||||
carried to the write seam (a boxed reader is itself `AsyncRead +
|
||||
Unpin + Send` under tokio's blanket impl, so callers needing
|
||||
erasure box at their call site). `Send` keeps spawn-site bounds
|
||||
resolvable for the ops wave; no `'static` (nothing spawns).
|
||||
- *Declared-len mismatch:* a known-length stream yielding fewer or
|
||||
more bytes than `len` rejects `Error::Verification` on either
|
||||
tier, zero residue either side — the preamble carries the
|
||||
*declared* length, so a mismatched artifact's canonical address
|
||||
can never equal the stored key (storing it would break pool
|
||||
integrity by construction; `Verification` is the put-path
|
||||
content-≠-key family member, ADR-006's whole-blob posture applied
|
||||
at put time).
|
||||
- *kv-only rejection carrier (a named decision beyond the two the
|
||||
task listed):* `Error::Io(String)` with a descriptive message —
|
||||
the ADR-pinned family (ADR-012 §6.5) holds no dedicated variant
|
||||
and the rejection is an operational/capability fact about the
|
||||
medium posture, the bucket `Io(String)` covers. A dedicated
|
||||
variant for typed matchability would be a new failure family —
|
||||
a new ADR.
|
||||
- **Flows as implemented** (`src/store.rs`, write-path `impl Store`):
|
||||
known-length initializes the derivation with the preamble before
|
||||
content flows and streams once (kv: buffer + hash in one pass →
|
||||
mint pin → whole-value put; large: stream into the staging cursor
|
||||
while hashing → mint pin → `commit(key)`). Unknown-length buffers
|
||||
to the threshold; a mid-stream overflow (strictly past it —
|
||||
exactly-at stays kv) flushes the buffered prefix into a fresh stage
|
||||
and the stream continues into it; at EOF the derivation pass runs
|
||||
over the eventual stored artifact as a whole — the buffer (kv form)
|
||||
or preamble + staged bytes read back through the
|
||||
`StagingCursor: ReadCursor` seam (staged form) — then pin, then
|
||||
commit. Every failure path (read error, mismatch, under-read)
|
||||
discards: no entry, no stage file.
|
||||
- **API shapes:** `Store::put(&self, len: Option<u64>, stream) →
|
||||
Result<(Key, Pin), Error>`; the batching seam is `Store::batch() →
|
||||
Batch` + `Store::put_in_batch(&mut Batch, len, stream) →
|
||||
Result<Key, Error>` (members run the lone-put flow; the minted guard
|
||||
is contributed to the group via crate-internal `Batch::add`); a
|
||||
public `Batch::commit()` was added consuming the batch with the
|
||||
same one-step release as drop (store-api.md's "pins drop together
|
||||
on batch commit or drop", as a named verb). `hashing.rs` gained
|
||||
crate-internal `BlobHasher` — the same derivation driven
|
||||
incrementally, pinned byte-equal to `Algorithm::derive` by tests
|
||||
(chunk-boundary invariance, both algorithms, the git vectors).
|
||||
- **git-interop leg added** (`tests/git_interop.rs`): the put seam
|
||||
keys entries git-identically through both put paths on both tiers
|
||||
against live `git hash-object` (INV-1's source-of-truth posture
|
||||
extended from the derivation constructor to the facade seam).
|
||||
- **Wave-1 note updates:** `put_staged`/`commit` doc text in
|
||||
`backend.rs`, `mem.rs`, `local.rs` now carries the key-at-commit
|
||||
design point; the store.rs module doc's remaining-deferral note
|
||||
shrunk to the actual remaining gaps (fleet clauses, wave-3 kv
|
||||
engine).
|
||||
|
||||
## Summary
|
||||
|
||||
> To be filled on completion
|
||||
Implemented the facade's put seam (`src/store.rs`): `put(len:
|
||||
Option<u64>, stream: impl AsyncRead + Unpin + Send) →
|
||||
Result<(Key, Pin), Error>` with the two internal paths behind the one
|
||||
signature — the known-length one-pass flow and the unknown-length
|
||||
buffer-then-maybe-flush flow implementing ADR-003's option (d) three
|
||||
numbered rules — plus `batch()`/`put_in_batch()` (ADR-005 §Marks 2's
|
||||
batch-scoped pin group) and kv-only's put-time rejection at its two
|
||||
pinned points (ADR-012 §6.2). Pin-before-publish rides
|
||||
`gc().acquire` under the arbitration lock before both tiers'
|
||||
publishes; dedup re-puts refcount the existing entry; every failure
|
||||
path converges on discard with zero stage files. The staged-put seam
|
||||
was amended (keyless handout; `commit(self, key)`) after operator
|
||||
sign-off — the one blocking finding of the task. Supporting changes:
|
||||
staging-cursor dispatch enum `AnyStaging` beside the read path's
|
||||
`AnyCursor`; incremental `BlobHasher` in `hashing.rs`; public
|
||||
`Batch::commit`; a `Pin`/`Debug` impl; the git-interop put-seam leg.
|
||||
Eleven put-path tests in `src/store.rs` (plus four fixtures) pin the
|
||||
three rules, both-tier round-trips, the threshold boundary, pin-
|
||||
before-publish under a racing sweep, dedup refcounting across paths
|
||||
and tiers, batch grouping/commit/failure, kv-only's two rejection
|
||||
points, len-mismatch zero-residue, mem-only's threshold-free posture,
|
||||
and the empty-content degenerate.
|
||||
|
||||
Verification: `cargo test` (default), `cargo test --features mem`,
|
||||
`cargo test --all-features` all green (97 lib tests under mem plus
|
||||
conformance and git-interop); `cargo clippy --all-targets --
|
||||
-D warnings` green in all three feature shapes; `cargo fmt --check`
|
||||
green; `cargo doc --no-deps` at the pre-existing 24-warning baseline.
|
||||
@@ -162,6 +162,80 @@ fn external_oid_round_trips_into_pool_keys() {
|
||||
}
|
||||
}
|
||||
|
||||
/// The facade put seam against git (store-api.md §Invariants 1's
|
||||
/// source-of-truth leg): both internal put paths (known-length,
|
||||
/// unknown-length) key entries git-identically on both tiers, and the
|
||||
/// get path round-trips git's content byte-exactly. mem-gated: the
|
||||
/// wave-2 constructible shape (`dual_tier_mem_local`).
|
||||
#[cfg(feature = "mem")]
|
||||
#[tokio::test]
|
||||
async fn put_seam_keys_match_git_hash_object_on_both_tiers_and_paths() {
|
||||
use std::io::Cursor;
|
||||
|
||||
use alkblobs::Store;
|
||||
|
||||
let root = scratch_dir("store-root");
|
||||
let store = Store::dual_tier_mem_local(root.join("root"), 32)
|
||||
.await
|
||||
.expect("dual-tier store constructs");
|
||||
let small = b"small git-interop blob";
|
||||
let big = (0..200).map(|i| (i % 253) as u8).collect::<Vec<u8>>();
|
||||
|
||||
let (repo, oid) = git_hash_object(small, true);
|
||||
drop_dir(&repo);
|
||||
let expected_small: Key = Digest::from_git_oid_hex(&oid)
|
||||
.expect("git oid decodes")
|
||||
.into();
|
||||
let (repo, oid) = git_hash_object(&big, true);
|
||||
drop_dir(&repo);
|
||||
let expected_big: Key = Digest::from_git_oid_hex(&oid)
|
||||
.expect("git oid decodes")
|
||||
.into();
|
||||
|
||||
let (key, pin) = store
|
||||
.put(Some(small.len() as u64), Cursor::new(&small[..]))
|
||||
.await
|
||||
.expect("known-length put");
|
||||
assert_eq!(key, expected_small, "kv put keys git-identical");
|
||||
drop(pin);
|
||||
let (key, pin) = store
|
||||
.put(None, Cursor::new(&small[..]))
|
||||
.await
|
||||
.expect("unknown-length put");
|
||||
assert_eq!(key, expected_small, "buffered form keys git-identical");
|
||||
let (len, blob) = store
|
||||
.get(&expected_small.as_bytes())
|
||||
.await
|
||||
.expect("get succeeds")
|
||||
.expect("entry present");
|
||||
assert_eq!(blob.read_at(0, len).await.unwrap(), &small[..]);
|
||||
drop(pin);
|
||||
|
||||
let (key, pin) = store
|
||||
.put(Some(big.len() as u64), Cursor::new(&big[..]))
|
||||
.await
|
||||
.expect("known-length large put");
|
||||
assert_eq!(key, expected_big, "staged put keys git-identical");
|
||||
drop(pin);
|
||||
let (key, pin) = store
|
||||
.put(None, Cursor::new(&big[..]))
|
||||
.await
|
||||
.expect("unknown-length large put");
|
||||
assert_eq!(
|
||||
key, expected_big,
|
||||
"staged-bytes derivation keys git-identical"
|
||||
);
|
||||
let (len, blob) = store
|
||||
.get(&expected_big.as_bytes())
|
||||
.await
|
||||
.expect("get succeeds")
|
||||
.expect("entry present");
|
||||
assert_eq!(blob.read_at(0, len).await.unwrap(), big);
|
||||
drop(pin);
|
||||
|
||||
let _ = fs::remove_dir_all(root);
|
||||
}
|
||||
|
||||
/// Tagged-key coexistence (POC finding 5): git oids of both formats
|
||||
/// address the same flat key universe from one external producer.
|
||||
#[test]
|
||||
|
||||
Reference in new issue
Block a user